gen_train_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Training Data Generation** \\\n",
    "_Author: Joaquín Torres Bravo_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set-up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "# Over/under sampling methods\n",
    "from imblearn.combine import SMOTETomek \n",
    "from imblearn.under_sampling import TomekLinks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load clean datasets\n",
    "df_pre = pd.read_csv('../EDA/output/datasets/pre_dataset.csv')\n",
    "df_post = pd.read_csv('../EDA/output/datasets/post_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22861, 39)\n",
      "(10677, 39)\n",
      "(22861,)\n",
      "(10677,)\n",
      "['Ed_Not_Complete_Primary' 'Ed_Primary' 'Ed_Secondary'\n",
      " 'Ed_Secondary_Technical' 'Ed_Tertiary' 'Social_Protection'\n",
      " 'JobIn_Unstable' 'JobIn_Stable' 'JobIn_Unemployed' 'Hous_Institutional'\n",
      " 'Hous_Stable' 'Hous_Unstable' 'Early_Alterations' 'SocInc_Family_Friends'\n",
      " 'SocInc_Alone' 'SocInc_Instit' 'Risk_Stigma' 'Structural_Conflict' 'Age'\n",
      " 'Sex' 'Num_Children' 'Smoking' 'Bio_Vulner' 'Opiods_DXCIE'\n",
      " 'Cannabis_DXCIE' 'BZD_DXCIE' 'Cocaine_DXCIE' 'Hallucin_DXCIE'\n",
      " 'Tobacco_DXCIE' 'Freq_1dpw' 'Freq_2-3dpw' 'Freq_4-6dpw' 'Freq_l1dpw'\n",
      " 'Freq_None' 'Freq_Everyday' 'Years_Drug_Use' 'Other_Psychiatric_DX'\n",
      " 'Previous_Treatments' 'Treatment_Adherence']\n"
     ]
    }
   ],
   "source": [
    "# Creating a numpy matrix (X) without the target variable and a list with the target variable (y) \n",
    "X_pre, y_pre = df_pre.loc[:, df_pre.columns != \"Treatment_Outcome\"].to_numpy(), df_pre.Treatment_Outcome\n",
    "X_post, y_post = df_post.loc[:, df_post.columns != \"Treatment_Outcome\"].to_numpy(), df_post.Treatment_Outcome\n",
    "feat = np.delete(df_pre.columns.to_numpy(),-1) # Get labels and remove target \n",
    "\n",
    "print(X_pre.shape)\n",
    "print(X_post.shape)\n",
    "print(y_pre.shape)\n",
    "print(y_post.shape)\n",
    "print(feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training-Test Split & Sampling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pipelines 1 and 2: ORIG and ORIG_CW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 90-10 split\n",
    "X_train_pre, X_test_pre, y_train_pre, y_test_pre = train_test_split(X_pre, y_pre, test_size=0.1, random_state=42) \n",
    "X_train_post, X_test_post, y_train_post, y_test_post = train_test_split(X_post, y_post, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save test data\n",
    "np.save('./output/pre/X_test_pre.npy', X_test_pre)\n",
    "np.save('./output/pre/y_test_pre.npy', y_test_pre)\n",
    "np.save('./output/post/X_test_post.npy', X_test_post)\n",
    "np.save('./output/post/y_test_post.npy', y_test_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save ORIGINAL training data\n",
    "np.save('./output/pre/X_train_pre.npy', X_train_pre)\n",
    "np.save('./output/pre/y_train_pre.npy', y_train_pre)\n",
    "np.save('./output/post/X_train_post.npy', X_train_post)\n",
    "np.save('./output/post/y_train_post.npy', y_train_post)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pipeline 3: OVER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OVERSAMPLED training data\n",
    "smote_tomek = SMOTETomek()\n",
    "X_train_over_pre, y_train_over_pre = smote_tomek.fit_resample(X_train_pre, y_train_pre)\n",
    "X_train_over_post, y_train_over_post = smote_tomek.fit_resample(X_train_post, y_train_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save oversampled training data\n",
    "np.save('./output/pre/X_train_over_pre.npy', X_train_over_pre)\n",
    "np.save('./output/pre/y_train_over_pre.npy', y_train_over_pre)\n",
    "np.save('./output/post/X_train_over_post.npy', X_train_over_post)\n",
    "np.save('./output/post/y_train_over_post.npy', y_train_over_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. UNDERSAMPLING: TOMEK-LINKS \n",
    "tomek = TomekLinks()\n",
    "X_train_under_pre, y_train_under_pre = tomek.fit_resample(X_train_pre, y_train_pre)\n",
    "X_train_under_post, y_train_under_post = tomek.fit_resample(X_train_post, y_train_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save undersampled training data\n",
    "np.save('./output/pre/X_train_under_pre.npy', X_train_under_pre)\n",
    "np.save('./output/pre/y_train_under_pre.npy', y_train_under_pre)\n",
    "np.save('./output/post/X_train_under_post.npy', X_train_under_post)\n",
    "np.save('./output/post/y_train_under_post.npy', y_train_under_post)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}