From f128a515c142d35bae8887e63cfd18163f71ba8f Mon Sep 17 00:00:00 2001 From: Joaquin Torres Bravo Date: Thu, 23 May 2024 12:48:19 +0200 Subject: [PATCH] Testing PR curve --- model_selection/cv_metric_gen.py | 100 +- .../output_cv_metrics/curves/pre_ORIG.svg | 15042 ++++++++++++++-- .../output_cv_metrics/metrics.xlsx | Bin 8036 -> 8129 bytes 3 files changed, 14062 insertions(+), 1080 deletions(-) diff --git a/model_selection/cv_metric_gen.py b/model_selection/cv_metric_gen.py index fea2f36..6918460 100644 --- a/model_selection/cv_metric_gen.py +++ b/model_selection/cv_metric_gen.py @@ -16,7 +16,7 @@ from sklearn.svm import SVC from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from sklearn.model_selection import StratifiedKFold, cross_validate -from sklearn.metrics import RocCurveDisplay, roc_curve, auc +from sklearn.metrics import RocCurveDisplay, auc from sklearn.metrics import PrecisionRecallDisplay, precision_recall_curve import matplotlib.pyplot as plt import ast # String to dictionary @@ -185,49 +185,64 @@ if __name__ == "__main__": # Scores df -> one column per cv split, one row for each model-metric scores_df = pd.DataFrame(columns=range(1,11), index=[f"{model_name}_{metric_name}" for model_name in models.keys() for metric_name in scorings.keys()]) # Create a figure for all models in this group-method - fig, axes = plt.subplots(len(models), 1, figsize=(10, 8 * len(models))) + fig, axes = plt.subplots(len(models), 2, figsize=(10, 8 * len(models))) if len(models) == 1: # Adjustment if there's only one model (axes indexing issue) axes = [axes] # Metric generation for each model for model_idx, (model_name, model) in enumerate(models.items()): - print(f"{group}-{method_names[j]}-{model_name}") - # Retrieve cv scores for our metrics of interest - scores = cross_validate(model, X_train, y_train, scoring=scorings, cv=cv, return_train_score=True, n_jobs=10) - # Save results of each fold - for metric_name in scorings.keys(): - scores_df.loc[model_name + f'_{metric_name}']=list(np.around(np.array(scores[f"test_{metric_name}"]),4)) - # ---------- Generate ROC curves ---------- - mean_fpr = np.linspace(0, 1, 100) - tprs, aucs = [], [] - cmap = plt.get_cmap('tab10') # Colormap - # Loop through each fold in the cross-validation (redoing cv for simplicity) - for fold_idx, (train, test) in enumerate(cv.split(X_train, y_train)): - # Fit the model on the training data - model.fit(X_train[train], y_train[train]) - # Use RocCurveDisplay to generate the ROC curve - roc_display = RocCurveDisplay.from_estimator(model, X_train[test], y_train[test], - name=f"ROC fold {fold_idx}", alpha=0.6, lw=2, - ax=axes[model_idx], color=cmap(fold_idx % 10)) - # Interpolate the true positive rates to get a smooth curve - interp_tpr = np.interp(mean_fpr, roc_display.fpr, roc_display.tpr) - interp_tpr[0] = 0.0 - # Append the interpolated TPR and AUC for this fold - tprs.append(interp_tpr) - aucs.append(roc_display.roc_auc) - # Plot the diagonal line representing random guessing - axes[model_idx].plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', alpha=.8, label='Random guessing') - # Compute the mean of the TPRs - mean_tpr = np.mean(tprs, axis=0) - mean_tpr[-1] = 1.0 - mean_auc = auc(mean_fpr, mean_tpr) # Calculate the mean AUC - # Plot the mean ROC curve with a thicker line and distinct color - axes[model_idx].plot(mean_fpr, mean_tpr, color='b', lw=4, - label=r'Mean ROC (AUC = %0.2f)' % mean_auc, alpha=.8) - # Set plot limits and title - axes[model_idx].set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05], - title=f"ROC Curve - {model_name} ({group}-{method_names[j]})") - axes[model_idx].legend(loc="lower right") - # ---------- END ROC curves Generation ---------- + if model_name == 'XGB': + print(f"{group}-{method_names[j]}-{model_name}") + # Retrieve cv scores for our metrics of interest + scores = cross_validate(model, X_train, y_train, scoring=scorings, cv=cv, return_train_score=True, n_jobs=10) + # Save results of each fold + for metric_name in scorings.keys(): + scores_df.loc[model_name + f'_{metric_name}']=list(np.around(np.array(scores[f"test_{metric_name}"]),4)) + # ---------------------------------------- Generate curves ---------------------------------------- + mean_fpr = np.linspace(0, 1, 100) + tprs, aucs = [], [] + mean_recall = np.linspace(0, 1, 100) + precisions, pr_aucs = [], [] + cmap = plt.get_cmap('tab10') # Colormap + # Loop through each fold in the cross-validation + for fold_idx, (train, test) in enumerate(cv.split(X_train, y_train)): + # Fit the model on the training data + model.fit(X_train[train], y_train[train]) + # Generate ROC curve for the fold + roc_display = RocCurveDisplay.from_estimator(model, X_train[test], y_train[test], + name=f"ROC fold {fold_idx}", alpha=0.6, lw=2, + ax=axes[model_idx][0], color=cmap(fold_idx % 10)) + interp_tpr = np.interp(mean_fpr, roc_display.fpr, roc_display.tpr) + interp_tpr[0] = 0.0 + tprs.append(interp_tpr) + aucs.append(roc_display.roc_auc) + # Generate Precision-Recall curve for the fold + pr_display = PrecisionRecallDisplay.from_estimator(model, X_train[test], y_train[test], + name=f"PR fold {fold_idx}", alpha=0.6, lw=2, + ax=axes[model_idx][1], color=cmap(fold_idx % 10)) + interp_precision = np.interp(mean_recall, pr_display.recall[::-1], pr_display.precision[::-1]) + precisions.append(interp_precision) + pr_aucs.append(pr_display.average_precision) + # Plot diagonal line for random guessing in ROC curve + axes[model_idx][0].plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', alpha=.8, label='Random guessing') + # Compute mean ROC curve + mean_tpr = np.mean(tprs, axis=0) + mean_tpr[-1] = 1.0 + mean_auc = auc(mean_fpr, mean_tpr) + axes[model_idx][0].plot(mean_fpr, mean_tpr, color='b', lw=4, label=r'Mean ROC (AUC = %0.2f)' % mean_auc, alpha=.8) + # Set ROC plot limits and title + axes[model_idx][0].set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05], title=f"ROC Curve - {model_name} ({group}-{method_names[j]})") + axes[model_idx][0].legend(loc="lower right") + # Compute mean Precision-Recall curve + mean_precision = np.mean(precisions, axis=0) + mean_pr_auc = np.mean(pr_aucs) + axes[model_idx][1].plot(mean_recall, mean_precision, color='b', lw=4, label=r'Mean PR (AUC = %0.2f)' % mean_pr_auc, alpha=.8) + # # Plot baseline precision (proportion of positive samples) + # baseline = np.sum(y_train) / len(y_train) + # axes[model_idx][1].plot([0, 1], [baseline, baseline], linestyle='--', lw=2, color='r', alpha=.8, label='Baseline') + # Set Precision-Recall plot limits and title + axes[model_idx][1].set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05], title=f"Precision-Recall Curve - {model_name} ({group}-{method_names[j]})") + axes[model_idx][1].legend(loc="lower right") + # ---------------------------------------- End Generate Curves ---------------------------------------- # Store the DataFrame in the dictionary with a unique key for each sheet sheet_name = f"{group}_{method_names[j]}" scores_sheets[sheet_name] = scores_df @@ -239,7 +254,4 @@ if __name__ == "__main__": with pd.ExcelWriter('./output_cv_metrics/metrics.xlsx') as writer: for sheet_name, data in scores_sheets.items(): data.to_excel(writer, sheet_name=sheet_name) - print("Successful cv metric generation for tuned models") - - - \ No newline at end of file + print("Successful cv metric generation for tuned models") \ No newline at end of file diff --git a/model_selection/output_cv_metrics/curves/pre_ORIG.svg b/model_selection/output_cv_metrics/curves/pre_ORIG.svg index 79ee48f..3c10346 100644 --- a/model_selection/output_cv_metrics/curves/pre_ORIG.svg +++ b/model_selection/output_cv_metrics/curves/pre_ORIG.svg @@ -6,7 +6,7 @@ - 2024-05-23T12:10:22.372119 + 2024-05-23T12:46:28.660631 image/svg+xml @@ -30,10 +30,10 @@ z - @@ -41,17 +41,17 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - - + - - + - - - + + + - + + - - + @@ -1160,44 +6707,45 @@ z - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + - - + - - - + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1402,20 +6937,20 @@ L 492.535 394.494062 - - + + - - + - - - + + + @@ -1436,20 +6971,20 @@ L 492.535 409.172187 - - + + - - + - - - + + + - - + + - - + - - - + + + @@ -1538,20 +7073,47 @@ L 492.535 438.528437 - - + + - - + - - - + + + + + + @@ -1572,20 +7134,20 @@ L 492.535 453.206562 - - + + - - + - - - + + + @@ -1606,32 +7168,20 @@ L 492.535 467.884687 - - + + - - + - - - - - - + + + @@ -1652,20 +7202,20 @@ z - - + + - - + - - - + + + @@ -1686,20 +7236,20 @@ L 492.535 497.240937 - - + + - - + - - - + + + @@ -1720,20 +7270,20 @@ L 492.535 511.919062 - - + + - - + - + - + - - + - - - + + + - - + + - - - + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2203,91 +15170,91 @@ z - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2295,121 +15262,121 @@ z - - + - - + - - + - - + - - - + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2417,91 +15384,91 @@ z - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2509,121 +15476,121 @@ z - - + - - + - - + - - + - - - + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2631,91 +15598,91 @@ z - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2723,121 +15690,121 @@ z - - + - - + - - + - - + - - - + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2845,91 +15812,91 @@ z - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -2937,121 +15904,121 @@ z - - + - - + - - + - - + - - - + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -3059,91 +16026,91 @@ z - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -3151,119 +16118,119 @@ z - - + - - + - - + - - + - - - + + - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + @@ -3273,91 +16240,91 @@ z - - - + + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + @@ -3365,31 +16332,34 @@ z - - + - + - - + - - + - - + + + + + diff --git a/model_selection/output_cv_metrics/metrics.xlsx b/model_selection/output_cv_metrics/metrics.xlsx index 003029e62d816579d0620a65a15a76e07d107fef..8680af8a26424fb75abaec8df07c2d3d8c0bcb15 100644 GIT binary patch delta 4195 zcmZu#c|4n07Y?mb+UjCyYi-rOPiq^5>9qCLUbI?bZ>%8|m8k9Qf+~XDNXmiMKMuPr(#^bu6EPJly(-srASykPyI{Y?1RZ(uxovT zBl8dL)Ayx~OFaBq2eNc78+Je27`Or(8MtuN>Cq!fjk>W3WWjw)ovx1g;`r-CnZd-V z{S&mo(WynNCj`ny{1$cG(4ek*wRRj}Q)sl_H8N!lm$!&?sT9l!ry7opGum9FjGVv` zcc^3Gx+=E;6cM8Gm9z%nC=cTcu@4*SB5Fg4YaReow4IkMoAKhAlLR{CEmqPTU0u~8^R)m^|qDhpbVl`(SAin8i zNxNHnF@aKSC2!P_7SIK+8C~pE-)t(3Zz;E-)XcAK+&-Z*1pq)mF#(0@y_itAwAi4w zeApCRHYPrvx?F{i3p}N1lVD(T^VJ- zUpa#p>gz)?0_OWkE4_knyq^*EYv5`M^@)8KiDcBK9Jw_ZOJ~PI!RNb7%GrE|Qm;Er z(%7+TQh?ix%-8TJT+;O!nf37IUz7MgAHjuRczLE$_YhHh4;Wt?ZpmI=y7`#$a8{Dt zyBXL=3B0yqKQ{wB)?D7GTK)2H>T2B$5dm{F+uz1oDDIM`f=J7=kd(Ve{gR=>l4US5 zl+DVlf0aAzp02=uG3U5Y{0D0TQ{zWk9q%C$y$~SN>{uhjEsXfr z;(mIqaHu6`Z_0JF3TZNpc8k?tn@+oXBU?cz!5y$xiTjzUz$H6yhJ*-uiwM#t5h_%4 z(q!Y#_bBXZ$<`H0I%@51QL2J#(R|Bn3rbq@P47hl;)ng1i_{F z1+k(xuqdFo{z2bn!rWcTD_IPWQG@-|6miA59OIPYyIcPdkTpD>YB-ILm{-o@+4N_v9qeIHAL) zg!(j12<`cUIi^GN4*qwc!W3RywVez$U-P;*{vcLSuI2gT186A1JWEldMTzzX@6g{4 z)cGF3IQTa8$6eO+k!!m7SRC!eZ~of!PHFy4Ed=Q#x=3dr#!uv5@l5(ow`%zd_XW{9 zTfUEHU1GDZ4ue=pM-(B8Wl`!Js}k`G6)81?Ukt3tsqvy{fP+UXaB_0|VEJX^Ok_t( zJV#9B`5~%g5p_nn*@Clq+~u<;?B)aP(5#YX#@F^FSfmZG5x$&}+9 zT->%5bL{kI=s79Xn5jWI!kV(kg(+f7XF})+sE)2*zTXdfCE5l!>qqlMQaxY9P!NV5 zHsi$SEXycd7P4ivVFPc4So#e1P5cTw)%cF zr1n063IYvq<_vwPkK{naCcLqk@+YQnoZY6S9m+S;hxh##C@+NRbWGAyd_P$j6>q$H9mRu6odP!Fj9P&ewiC zw4JaKC_3(s0e&Wm1?gowVQMtVCPFoWbasW1z9McoHgYIqT^Z?zF?nq8NghxS!~7Mk(W2W*(H^lG|5*FvzX``2PC z`+X><#6N0c7T0~Xt7{{^GWgDly`Ccxl?FrW6lV%*e`!gUii*^(p6|S)PZ$W)b{}o5 z&6TT8#{F>){*GynTX%jBk3o#Z9ii8&&lHGjhS_D-96c-6L!jBE%8XzZ!2y+~{w|hR z^|En*VY8>qweW;MN!y5l>OO=%k!D$T2`+s*o-xv?-xEAb>$0cat@h7X$adco zrnMwiZi?yC@C$fqNq9uWjNC{r4nr=A4jQzZwYmFLkLP8Nc3fQ3;OfVZR>%+9ftfB| z4&dBEh?kXdayrBw${Kb^HR0|IMmCVu?y#QA6kUD$Wm*&6mK7Qzzw)W6*TRt156tc@ zrVL9Mtt>q&Hg!pvBn6nNyA+X@m38i!Ep}d3Py>)#>KJvG1QLxmEe@Si)?T+!vj5`Z zquWlb^S%Bt!}LIKwu_3VdV5PO-VqaI10lV8>zopQBx0Cr9(I8^dCsxthft%M*SKIk zJsLDcN48rn=SI`sT-JCdd~3BEFtzU+7{;tEhc`1$*(Q2>k*~93UmJAGy%6t7Y170d zC`g?QsFZ|uEq|uih-|Q-w`m7b>48h0m%u;gQ&9-w)}}P=hTdD@{Luk!g=h$9(UymQ2*JObx-4fWjfsN;AG?zzFbTkk9CymncT!6bal%QpI z&aD?X*Foy=uKE2eNIge^k$S6>T3i9C$9ta|)*^7~ZU>%KfuRhhsC*guCkXuSF7OQ% zrf7+ND~{h{;;zI93<;@(K>qzAvc1Mjf?t3e$GMbQsKQhUB~tZ(z%sH+f9g@bDDBgHM*n{F^tOI{YyIlfiikX}LU6o_a zJLUP&2N43e?4m)+JxAc*lnIDUiy()zU+|LtWm@v5ss3I^f44n0c zSz#sZ&*KJx`~^Uu?I!fwbC*j(1Z^xSw&T@FQk-iM9V&U!X~#m$CDl)7yPsFywi5g{ z9|XF~HFN?53W2Gk;0Ql=INWcizqdBy;uhMo`?mn$`dRu^bqfzZVHUH6)1&Th!B&OsG){DVNixDe57xzIP&r4)H+9i|S zP`b}|I*E>KyKQtz;9!JNuI0TQq=xE*OBZWi3jPFSGT-r^^1rrC>fByq5hbLAl!Bhh zEkyE=!u+#3uR3`(-SZZAC7tYFcy(ro;Y#kfO-gyo#9*IZK7z5XXOA?@Qo}ncj9RX^ z`?{6VB_x-6E~tw;5ld4mD-)~Z zZL4jatsGKY!yzaoVF6w>)7fl3ZtfxvhePLyQ`4^*O{j=#0FLXA53E@4V>IWsa-daJ zjMz3tv$Q$T&t&SaEwqjKap&II3J9J+R7E|?3LM0&xKY}z>CF+C1m<8 z$^tp|34_xR-`v&=*j_FqPY?SSXOuucu{mSx&W>jFtTVF=JeT|V=K9~0XpMCGnBVL* zcz%H9+H`x{+}M<#gSYD{IRY+i3$jZ#@Cr@Zy@U_|jecu8~&^mRsh`Tb0)()FN z&d!V_R1H*dY$;3g4=S(5O?n350+-5HmxlvFOKnNhPj&%=re~Oj%K?kY{Ta4xtfh>R zC?+5qa1z=vra%vGJO+I@u5f8-yC$@QrSLJlQ5QNirm!4-BO^iPlLXRW_yAO5=S?;7 z)32K&cfly_uv))_v3>*7_E#DBgue!FS5;P78m=?`$ew;V(J3t+8`>!kai77K+r>Sr zzAyH(a&99KiBuH!(s8{btEqP*_?lhCZ_2{j6)y=?x4Emyr6*-Z;$BTBe!IyJU{4>6Z%s~Qs~h!SIS*cNu$e_5 zI+fYtqZd~5AG5xjW1|u6>eFi%!!vwus|*i&Q2f+K@vkHrEd73$DRyXuNC@hNte(Ic zsWXf6n-RBG9|90pk97n&By57c$|8ih%7O^P16+0vU{xGZA~VJV&rj|(;mlFE=ZlzxT4hH zeou~gS;VyXY)DBqAAC+&U&*8%FQ=M>dTSa(JEo)5z9$ux_{r5O(Rb1eFDg|7Jb#m? zV|TFW@jX$Z2iEjH>}b;gR(%$#k4~PDQH_XyFri})LTOnCc<6^zbj2)Oa6E1cDJ^m+ zm+wq3sJV$NI-Z zd})wdn}b8jv;R_KLpc8ShhkgRwt!~ zg7x*440a)Eqok;#X)m{AzcXbnO=~O;ut`_h_f_KW3cFhwJ9eq`rK$?Ga=zUa_*Sk( z+39L*4q8;odpap>FDTO0O5L$5tXD?$#GoSS7)by1&j;_8*i#c^dou})S{ z$9LSGtF{)e@8)g+0V_$zcey>uKE;ImassGq+D|+$vZ*GRbw^Kgnq_wCpkWq ziVsj#!^2G=CHkzXmz3kB@qLm(>L9`$SyRt*f(ilq!_QTK0t6_n*UjGf*c2kHis;_9 zPT(IsLvHELLDXb!)YwHI<>7`-Q8rD`efd)U>vI?}8oh3wlb_ANR8&rMX|oN!q!{Lr>m$*o{?;G9U=uCktb z-6>ZPr|V{ryxgFo)yxNSs&KHW>g87-0Lh_(45o}l6{kEJtiF~_(bQF;hEPVp&hLD` z(!KBNjdZRiM68SV`8IE1RFJ*FNNemot87FBT%o0zbBib(&>% zR6bjbSA{v8H10nw;(PtNTCF>DbGv0!*WMgC0=_BHObG3)?&?;;LRF2efvcgLUfVUd zc49y>8OK`w5d^+Aavr*Tg5`jA@!67AUQhIK#<+9_KKN4w4lbn%`PEWd=xmzbj;Hj3 zieQj}30jhcbF^VXI=*cn9@HO7v%~V)18m@l3lM2pH8%MpsSQp&*415elP82s^6_mf zQ1FsD(TtT_NGgOmoGlxl2Z#33YnC@Ejeua_xTtTH_D|-}-YyZ<5oR>YVopV*1`Gi8 zY007*L|vBG(6|Xs5R`Ozw6|+Sbrfe>tfd|+v<1xJfzh1nYWSl2dSIKoT5V&qA!q=` zE8xx54e*|nMQ04_YX|G*;lgykumW&mpP5PK89LV&4bJz!0JVV2WCuGJ4AL-sU24*| z9RhO3()7Kv5cJcaBA8tcMR3a_r6X&%7TpBs*`ub|st5|@J-ly-&F$7w+&PDTFTSOO z>Gd(8+sH|op?dycXUOxMahlh%I#Sr2IyLaIMcD+gIjOHk{t5y={A2hBvGoz&YQ(m1o-%NeLIl1M)HRde}>9$oN^Q@C&1iCyZ}A4u?yzy8fA$oP(|UN IAsYez0|^_w-~a#s -- 2.24.1