final-training.py

# The aim of this code is to get an ovrview of the performances of the selected models on the filtered data set whetehr thery are clustered of not and for both outputs 
# We train with a 10-folds scenario and get as an output the following metrics for each fold :
# -AUROC
# -F1
# -LogLoss

# Import of databases for training
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from imblearn.pipeline import Pipeline
from imblearn.combine import SMOTETomek
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score, make_scorer, precision_score, recall_score
from sklearn.model_selection import StratifiedKFold, cross_validate
from sklearn.ensemble import RandomForestClassifier, BaggingClassifier, AdaBoostClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from sklearn.linear_model import  LogisticRegression
from sklearn.tree import DecisionTreeClassifier

def output_datasets(dataset,filtering): 
    """Gives the features and labels to train the model on depending on the dataset considered"""
    if dataset == "Dropout_1":
        # Import of csv database from the feature engineering R code
        db_cluster = pd.read_csv("dropout_cluster.csv", sep=",")
        db = pd.read_csv("dropout_sin_cluster.csv", sep=",")
        # Features to be selected from the feature filtering step
        features_cluster = (pd.read_csv("dropout_cluster_FSS.csv",sep=",")).columns.values
        features = (pd.read_csv("dropout_sin_cluster_FSS.csv",sep=",")).columns.values

    else:
        # Import of csv database from the feature engineering R code
        db_cluster = pd.read_csv("relapse_cluster.csv", sep=",")
        db = pd.read_csv("relapse_sin_cluster.csv", sep=",")
        # Features to be selected from the feature filtering step
        features_cluster = (pd.read_csv("relapse_cluster_FSS.csv",sep=",")).columns.values
        features = (pd.read_csv("relapse_sin_cluster_FSS.csv",sep=",")).columns.values

    # Creation of train and test sets for the dataset without cluster
    sin_cluster_data_label = db[dataset]
    sin_cluster_data_features = db.drop(dataset, axis=1) #elimination of the output from the training set
    columns_to_be_changed = sin_cluster_data_features.select_dtypes(exclude='number').columns.values
    sin_cluster_data_features = pd.get_dummies(sin_cluster_data_features, columns=columns_to_be_changed) #use of one hot encoding for categorical variables
    if filtering == "FSS" : #selection of features in case the filtering is activated for the dataset
        sin_cluster_data_features = sin_cluster_data_features.filter(features, axis=1)
    sin_cluster_data_features.replace({False: 0, True: 1}, inplace=True) #convertion of boolean by integers for the features set (necessary for newest numpy versions)
    sin_cluster_data_label.replace({False: 0, True: 1}, inplace=True) #convertion of boolean by integers for the features set (necessary for newest numpy versions)

    # Creation of train and test sets for the dataset with cluster (same steps)
    cluster_data_label = db_cluster[dataset]
    cluster_data_features = db_cluster.drop(dataset, axis=1)
    columns_to_be_changed = cluster_data_features.select_dtypes(exclude='number').columns.values
    cluster_data_features = pd.get_dummies(cluster_data_features, columns=columns_to_be_changed)
    if filtering == "FSS" :
        cluster_data_features = cluster_data_features.filter(features_cluster, axis=1)
    cluster_data_features.replace({False: 0, True: 1}, inplace=True)
    cluster_data_label.replace({False: 0, True: 1}, inplace=True)

    return sin_cluster_data_features, sin_cluster_data_label, cluster_data_features, cluster_data_label

def models_input(dataset, filtering) :
    """Gives a dictionnary of models to train with as a tuple model_name:(model optimized without cluster, model optimized with clusters)"""
    if filtering == "FSS" :
        if dataset == "Dropout_1" :
            models = {"Tree" : (DecisionTreeClassifier(splitter='best', max_features='sqrt', criterion='gini'), DecisionTreeClassifier(splitter='random', max_features='log2', criterion='log_loss')), 
                       "RF" : (RandomForestClassifier(criterion='entropy', max_features='sqrt', n_estimators=111), RandomForestClassifier(criterion='gini', max_features='sqrt', n_estimators=194)), 
                       "Boost" : (AdaBoostClassifier(learning_rate= 1.9061, n_estimators= 62),AdaBoostClassifier(learning_rate= 1.9184, n_estimators= 83)), 
                       "Bag" :(BaggingClassifier(max_features= 0.8, max_samples= 1.0, n_estimators= 13, warm_start= True), BaggingClassifier(max_features= 0.8, max_samples= 0.8, n_estimators= 67, warm_start= True)), 
                       "LR" : (LogisticRegression(solver='lbfgs', penalty='l2'), LogisticRegression(solver='newton-cholesky', penalty='l2')), 
                       "SVM" : (SVC(C=1.6663, kernel='linear'), SVC(C=0.9894, kernel='linear')),
                       "NN" : (MLPClassifier(activation='logistic', hidden_layer_sizes=116, learning_rate='invscaling'), MLPClassifier(activation='identity', hidden_layer_sizes=94, learning_rate='adaptive'))}
        if dataset == "Relapse_1" :
            models = {"Tree" : (DecisionTreeClassifier(splitter='random', max_features='log2', criterion='log_loss'), DecisionTreeClassifier(splitter='best', max_features='sqrt', criterion='gini')), 
                       "RF" : (RandomForestClassifier(criterion='gini', max_features='sqrt', n_estimators=158), RandomForestClassifier(criterion='gini', max_features='sqrt', n_estimators=242)), 
                       "Boost" : (AdaBoostClassifier(learning_rate= 0.994, n_estimators= 117),AdaBoostClassifier(learning_rate= 1.672, n_estimators= 144)), 
                       "Bag" :(BaggingClassifier(max_features= 0.8, max_samples= 1, n_estimators= 18, warm_start= False), BaggingClassifier(max_features= 0.8, max_samples= 0.8, n_estimators= 67, warm_start= False)), 
                       "LR" : (LogisticRegression(solver='sag', penalty='l2'), LogisticRegression(solver='saga', penalty='l2')), 
                       "SVM" : (SVC(C=1.511, kernel='rbf'), SVC(C=1.033, kernel='rbf')),
                       "NN" : (MLPClassifier(activation='relu', hidden_layer_sizes=125, learning_rate='invscaling'), MLPClassifier(activation='tanh', hidden_layer_sizes=67, learning_rate='constant'))}
    if filtering == "noFSS" :
        if dataset == "Dropout_1" :
            models = {"Tree" : (DecisionTreeClassifier(splitter='random', max_features='log2', criterion='gini'), DecisionTreeClassifier(splitter='random', max_features='sqrt', criterion='entropy')), 
                       "RF" : (RandomForestClassifier(criterion='entropy', max_features='log2', n_estimators=134), RandomForestClassifier(criterion='entropy', max_features='log2', n_estimators=237)), 
                       "Boost" : (AdaBoostClassifier(learning_rate= 0.9249, n_estimators= 54),AdaBoostClassifier(learning_rate= 0.9984, n_estimators= 91)), 
                       "Bag" :(BaggingClassifier(max_features= 0.8, max_samples= 0.8, n_estimators= 11, warm_start= True), BaggingClassifier(max_features= 0.8, max_samples= 0.8, n_estimators= 16, warm_start= False)), 
                       "LR" : (LogisticRegression(solver='sag', penalty='l2'), LogisticRegression(solver='lbfgs', penalty='l2')), 
                       "SVM" : (SVC(C=0.9152, kernel='linear'), SVC(C=1.3079, kernel='linear')),
                       "NN" : (MLPClassifier(activation='identity', hidden_layer_sizes=114, learning_rate='constant'), MLPClassifier(activation='identity', hidden_layer_sizes=71, learning_rate='constant'))}
        if dataset == "Relapse_1" :
            models = {"Tree" : (DecisionTreeClassifier(splitter='best', max_features='sqrt', criterion='entropy'), DecisionTreeClassifier(splitter='random', max_features='sqrt', criterion='entropy')), 
                       "RF" : (RandomForestClassifier(criterion='entropy', max_features='sqrt', n_estimators=128), RandomForestClassifier(criterion='entropy', max_features='sqrt', n_estimators=131)), 
                       "Boost" : (AdaBoostClassifier(learning_rate= 1.259, n_estimators= 127), AdaBoostClassifier(learning_rate= 1.393, n_estimators= 135)), 
                       "Bag" :(BaggingClassifier(max_features= 0.8, max_samples= 0.8, n_estimators= 80, warm_start= False), BaggingClassifier(max_features= 0.8, max_samples= 0.8, n_estimators= 51, warm_start= False)), 
                       "LR" : (LogisticRegression(solver='saga', penalty='l1'), LogisticRegression(solver='saga', penalty='l1')), 
                       "SVM" : (SVC(C=1.974, kernel='rbf'), SVC(C=1.503, kernel='rbf')),
                       "NN" : (MLPClassifier(activation='relu', hidden_layer_sizes=125, learning_rate='invscaling'), MLPClassifier(activation='tanh', hidden_layer_sizes=100, learning_rate='constant'))}
    return models

def negative_recall_scorer(clf, X, y):
    """Gives the negative recall defined as the (number of true_negative_samples)/(total number of negative samples)"""
    y_pred = clf.predict(X)
    cm = confusion_matrix(y, y_pred)
    TN_prop = cm[0,0]/(cm[0,1]+cm[0,0])
    return TN_prop

def TN_scorer(clf, X, y):
    """Gives the number of samples predicted as true negatives"""
    y_pred = clf.predict(X)
    cm = confusion_matrix(y, y_pred)
    TN = cm[0,0]
    return TN
def FN_scorer(clf, X, y):
    """Gives the number of samples predicted as false negatives"""
    y_pred = clf.predict(X)
    cm = confusion_matrix(y, y_pred)
    FN = cm[0,1]
    return FN
def FP_scorer(clf, X, y):
    """Gives the number of samples predicted as false positive"""
    y_pred = clf.predict(X)
    cm = confusion_matrix(y, y_pred)
    FP = cm[1,0]
    return FP
def TP_scorer(clf, X, y):
    """Gives the number of samples predicted as true positive"""
    y_pred = clf.predict(X)
    cm = confusion_matrix(y, y_pred)
    TP = cm[1,1]
    return TP

if __name__ == '__main__':
    datasets = ["Dropout_1"] #select the dataset to train on
    filtering = ["FSS","noFSS"] #select whether the dataset has been through the filtering step or not 
    scorings = {'f1':make_scorer(f1_score), 'negative_recall': negative_recall_scorer, 'recall':make_scorer(recall_score), 'precision':make_scorer(precision_score), 'TN':TN_scorer, 'FN':FN_scorer, 'FP':FP_scorer, 'TP':TP_scorer} #scorings to be used for model evaluation
    resample = SMOTETomek() #the method used to balance the output classes
    cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1) #the cross-validation protocole used
    for f in filtering : 
        for d in datasets:
            sin_cluster_data_features, sin_cluster_data_label, cluster_data_features, cluster_data_label = output_datasets(d,f)
            models = models_input(d,f) #models selected for training
            cluster_df = pd.DataFrame(columns=range(1,11), index=['SVM_F1','SVM_Precision','SVM_Recall','SVM_TN-prop','NN_F1','NN_Precision','NN_Recall','NN_TN-prop',
                                                              'LR_F1','LR_Precision','LR_Recall','LR_TN-prop','Bag_F1','Bag_Precision','Bag_Recall','Bag_TN-prop',
                                                              'RF_F1','RF_Precision','RF_Recall','RF_TN-prop','Boost_F1','Boost_Precision','Boost_Recall','Boost_TN-prop',
                                                              'Tree_F1','Tree_Precision','Tree_Recall','Tree_TN-prop']) #dataframe to save the results in for the cluster dataset
            sin_cluster_df = cluster_df.copy(deep=True) #dataframe to save the results in for the cluster dataset
            cluster_cm = pd.DataFrame(columns=range(1,11), index=['SVM_TN','SVM_FN','SVM_FP','SVM_TP','NN_TN','NN_FN','NN_FP','NN_TP',
                                                              'LR_TN','LR_FN','LR_FP','LR_TP','Bag_TN','Bag_FN','Bag_FP','Bag_TP',
                                                              'RF_TN','RF_FN','RF_FP','RF_TP','Boost_TN','Boost_FN','Boost_FP','Boost_TP',
                                                              'Tree_TN','Tree_FN','Tree_FP','Tree_TP']) #dataframe to save the results in for the cluster dataset
            sin_cluster_cm = cluster_cm.copy(deep=True) #dataframe to save the results in for the cluster dataset
            for k in models :
                model = models[k][0] #selection of the first model of the tuple which is the one without clusters
                pipeline = Pipeline(steps=[('r', resample), ('m', model)])
                #training of the model for the dataset without clusters 
                sin_cluster_scores = cross_validate(pipeline, sin_cluster_data_features.values, sin_cluster_data_label.values, scoring=scorings, cv=cv, return_train_score=True, n_jobs=1)
                sin_cluster_df.loc[k+'_F1']=list(np.around(np.array(sin_cluster_scores["test_f1"]),4)) #the F1 score for the database without cluster is stored in a dataframe 
                sin_cluster_df.loc[k+'_Precision']=list(np.around(np.array(sin_cluster_scores["test_precision"]),4)) #same for precision
                sin_cluster_df.loc[k+'_Recall']=list(np.around(np.array(sin_cluster_scores["test_recall"]),4)) #same for recall 
                sin_cluster_df.loc[k+'_TN-prop']=list(np.around(np.array(sin_cluster_scores["test_negative_recall"]),4)) #same for negative_recall
                sin_cluster_cm.loc[k+'_TN']=sin_cluster_scores["test_TN"] #the number of true negative samples for the database without cluster is stored in a dataframe 
                sin_cluster_cm.loc[k+'_FN']=sin_cluster_scores["test_FN"] #same for false negative
                sin_cluster_cm.loc[k+'_FP']=sin_cluster_scores["test_FP"] #same for false positive
                sin_cluster_cm.loc[k+'_TP']=sin_cluster_scores["test_TP"] #same for true positive

                model = models[k][1] #selection of the second model of the tuple which is the one with clusters
                pipeline = Pipeline(steps=[('r', resample), ('m', model)])
                #training of the model for the dataset with clusters
                cluster_scores = cross_validate(pipeline, cluster_data_features.values, cluster_data_label.values, scoring=scorings, cv=cv, return_train_score=True, n_jobs=1)
                cluster_df.loc[k+'_F1']=list(np.around(np.array(cluster_scores["test_f1"]),4)) #the F1 score for the database without cluster is stored in a dataframe 
                cluster_df.loc[k+'_Precision']=list(np.around(np.array(cluster_scores["test_precision"]),4)) #same for precision
                cluster_df.loc[k+'_Recall']=list(np.around(np.array(cluster_scores["test_recall"]),4)) #same for recall 
                cluster_df.loc[k+'_TN-prop']=list(np.around(np.array(cluster_scores["test_negative_recall"]),4)) #same for negative_recall
                cluster_cm.loc[k+'_TN']=cluster_scores["test_TN"] #the number of true negative samples for the database without cluster is stored in a dataframe 
                cluster_cm.loc[k+'_FN']=cluster_scores["test_FN"] #same for false negative
                cluster_cm.loc[k+'_FP']=cluster_scores["test_FP"] #same for false positive
                cluster_cm.loc[k+'_TP']=cluster_scores["test_TP"] #same for true positive
        
            #Download of results as csv files 
            cluster_df.to_csv("Results_3_"+d+"_Cluster_"+f+".csv")
            sin_cluster_df.to_csv("Results_3_"+d+"_sin_Cluster_"+f+".csv")
            cluster_cm.to_csv("Results_3_confusion_matrix_"+d+"_Cluster_"+f+".csv")
            sin_cluster_cm.to_csv("Results_3_confusion_matrix"+d+"_sin_Cluster_"+f+".csv")