Metric generation done

model_selection/cv_metric_gen.py

@@ -190,8 +190,8 @@ if __name__ == "__main__":
     # Metric generation through cv for tuned models3
     # --------------------------------------------------------------------------------------------------------
     scores_sheets = {} # To store score dfs as sheets in the same excel file
-    for i, group in enumerate(['pre']):
-        for j, method in enumerate(['']): 
+    for i, group in enumerate(['pre', 'post']):
+        for j, method in enumerate(['', '', 'over_', 'under_']): 
             # Get train dataset based on group and method
             X_train = data_dic['X_train_' + method + group]
             y_train = data_dic['y_train_' + method + group]

model_selection/test_models.py

@@ -149,6 +149,21 @@ def negative_recall_scorer(clf, X, y):
     cm = confusion_matrix(y, y_pred)
     TN_prop = cm[0,0]/(cm[0,1]+cm[0,0])
     return TN_prop
+
+# Custom scorers for AUROC and AUPRC
+def AUROC_scorer(clf, X, y):
+    if hasattr(clf, "decision_function"):
+        y_score = clf.decision_function(X)
+    else:
+        y_score = clf.predict_proba(X)[:, 1]
+    return roc_auc_score(y, y_score)
+
+def AUPRC_scorer(clf, X, y):
+    if hasattr(clf, "decision_function"):
+        y_score = clf.decision_function(X)
+    else:
+        y_score = clf.predict_proba(X)[:, 1]
+    return average_precision_score(y, y_score)
 # --------------------------------------------------------------------------------------------------------
 
 if __name__ == "__main__":
@@ -168,8 +183,8 @@ if __name__ == "__main__":
         'FN':FN_scorer, 
         'FP':FP_scorer, 
         'TP':TP_scorer,
-        'AUROC': make_scorer(roc_auc_score),  # AUROC requires decision function or probability outputs
-        'AUPRC': make_scorer(average_precision_score)  # AUPRC requires probability outputs
+        'AUROC': AUROC_scorer,
+        'AUPRC': AUPRC_scorer
         } 
     method_names = {
         0: "ORIG",