Notes

def train(self, texts, labels, use_topic_modeling=True):
"""Train the classifier on the provided data"""
# Preprocess texts
preprocessed_texts = [self.preprocess_text(text) for text in texts]

# Split data for training and evaluation
X_train, X_test, y_train, y_test = train_test_split(
preprocessed_texts, labels, test_size=0.2, random_state=42
)

# Create and fit vectorizer ONLY on training data
self.vectorizer = TfidfVectorizer(
max_features=5000,
min_df=2,
max_df=0.85,
ngram_range=(1, 2)
)
X_train_vec = self.vectorizer.fit_transform(X_train)

# Transform test data with the SAME vectorizer (no fitting)
X_test_vec = self.vectorizer.transform(X_test)

# Topic modeling features (optional)
if use_topic_modeling:
print("Building topic model...")
# Important: Build topic model on ALL data to get consistent topics
self.build_topic_model(texts)

# Create consistent indices for extracting topic features
train_indices = [i for i, _ in enumerate(preprocessed_texts) if _ in X_train]
test_indices = [i for i, _ in enumerate(preprocessed_texts) if _ in X_test]

# Extract topic features using the SAME topic model
all_topic_features = self.extract_topic_features(texts)
train_topic_features = np.array([all_topic_features[i] for i in train_indices])
test_topic_features = np.array([all_topic_features[i] for i in test_indices])

# Combine with TF-IDF features
X_train_combined = np.hstack((X_train_vec.toarray(), train_topic_features))
X_test_combined = np.hstack((X_test_vec.toarray(), test_topic_features))
else:
X_train_combined = X_train_vec
X_test_combined = X_test_vec

# Train models
print("Training machine learning models...")
models = {
'Naive Bayes': MultinomialNB(),
'SVM': LinearSVC(C=1.0, class_weight='balanced', max_iter=10000),
'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42)
}

best_model = None
best_score = 0

for name, model in models.items():
print(f"Training {name}...")
model.fit(X_train_combined, y_train)

# Always use transform() (not fit_transform()) on test data
y_pred = model.predict(X_test_combined)
accuracy = accuracy_score(y_test, y_pred)

print(f"{name} - Accuracy: {accuracy:.4f}")

if accuracy > best_score:
best_score = accuracy
best_model = model

# Save the best model
self.ml_model = best_model
print(f"Best model selected: {type(best_model).__name__} with accuracy {best_score:.4f}")

# Evaluate on test set - use the built models for prediction
self.evaluate(X_test, y_test)

return self