معالجة اللغة الطبيعية: خط أنابيب تصنيف النصوص

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import ComplementNB
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.metrics import classification_report
import numpy as np

# ── Sample text data ───────────────────────────────────────────────────
corpus = [
    "machine learning algorithms data science python",
    "neural network deep learning pytorch tensorflow",
    "natural language processing text classification bert",
    "computer vision image recognition convolutional",
    "reinforcement learning reward policy agent",
    "data preprocessing feature engineering pipeline",
] * 40   # 240 samples, 6 classes
labels = list(range(6)) * 40
X_text = corpus
y = np.array(labels)
X_train, X_test, y_train, y_test = train_test_split(
    X_text, y, test_size=0.2, stratify=y, random_state=42)
# indices for sentence-transformer section
train_idx = np.arange(len(X_train))
test_idx  = np.arange(len(X_test))

# ── Baseline: TF-IDF + Logistic Regression ────────────────────────
pipe_lr = Pipeline([
    ('tfidf', TfidfVectorizer(
        ngram_range=(1,2),
        max_features=100_000,
        sublinear_tf=True,          # log(1+tf) dampens high frequencies
        strip_accents='unicode',
        analyzer='word',
        token_pattern=r'\w{2,}',  # ignore single-char tokens
        min_df=2,                   # ignore very rare words
    )),
    ('clf', LogisticRegression(C=1.0, max_iter=1000, class_weight='balanced')),
])

# ── Alternative: TF-IDF + LinearSVC (fast, great for text) ────────
pipe_svm = Pipeline([
    ('tfidf', TfidfVectorizer(ngram_range=(1,2), max_features=100_000, sublinear_tf=True)),
    ('clf', LinearSVC(C=0.5, class_weight='balanced', max_iter=2000)),
])

# ── Evaluate both with cross-validation ───────────────────────────
for name, pipe in [('LR', pipe_lr), ('SVM', pipe_svm)]:
    scores = cross_val_score(pipe, X_text, y, cv=5, scoring='f1_macro', n_jobs=-1)
    print(f"{name}: macro-F1 = {scores.mean():.3f} ± {scores.std():.3f}")

# ── Modern approach: sentence embeddings ──────────────────────────
# pip install sentence-transformers
from sentence_transformers import SentenceTransformer
from sklearn.linear_model import LogisticRegression

encoder = SentenceTransformer('all-MiniLM-L6-v2')
X_emb = encoder.encode(X_text, batch_size=256, show_progress_bar=True)
clf = LogisticRegression(max_iter=1000).fit(X_emb[train_idx], y[train_idx])
print(f"Sentence-BERT accuracy: {clf.score(X_emb[test_idx], y[test_idx]):.3f}")