Reorganizing files and directories

data/raw/data_01_neuro_diseases_final_disease_selected.tsv

+cui	disease_name	gene_id	drug_id	symptom_id	Genes in LCC
+C0001206	Acromegaly	178.0	3.0	18.0	38.0
+C0001973	Alcoholic Intoxication, Chronic	1008.0	22.0	34.0	249.0
+C0002395	Alzheimer's Disease	5736.0	38.0	7.0	2697.0
+C0002622	Amnesia	200.0	41.0	14.0	74.0
+C0002736	Amyotrophic Lateral Sclerosis	1791.0	11.0	23.0	847.0
+C0003467	Anxiety	1335.0	0.0	16.0	684.0
+C0003537	Aphasia	94.0	3.0	8.0	27.0
+C0004135	Ataxia Telangiectasia	481.0	0.0	24.0	235.0
+C0004352	Autistic Disorder	1710.0	35.0	5.0	706.0
+C0005586	Bipolar Disorder	2040.0	78.0	12.0	727.0
+C0007282	Carotid Stenosis	163.0	2.0	3.0	23.0
+C0007766	Intracranial Aneurysm	399.0	5.0	13.0	124.0
+C0007785	Cerebral Infarction	891.0	15.0	32.0	444.0
+C0007786	Brain Ischemia	449.0	48.0	4.0	236.0
+C0007787	Transient Ischemic Attack	410.0	17.0	14.0	233.0
+C0007789	Cerebral Palsy	284.0	5.0	31.0	81.0
+C0007959	Charcot-Marie-Tooth Disease	349.0	1.0	19.0	96.0
+C0009171	Cocaine Abuse	182.0	97.0	16.0	57.0
+C0009207	Cockayne Syndrome	133.0	0.0	7.0	46.0
+C0009952	Febrile Convulsions	267.0	8.0	9.0	30.0
+C0010964	Dandy-Walker Syndrome	155.0	0.0	9.0	53.0
+C0011195	Dejerine-Sottas Disease (disorder)	235.0	0.0	22.0	60.0
+C0011206	Delirium	152.0	23.0	18.0	15.0
+C0011265	Presenile dementia	944.0	38.0	23.0	467.0
+C0011269	Dementia, Vascular	268.0	2.0	26.0	103.0
+C0011570	Mental Depression	2208.0	0.0	7.0	1047.0
+C0011581	Depressive disorder	2544.0	112.0	5.0	1266.0
+C0011633	Dermatomyositis	296.0	8.0	18.0	80.0
+C0013080	Down Syndrome	1066.0	1.0	3.0	511.0
+C0013264	Muscular Dystrophy, Duchenne	467.0	3.0	10.0	196.0
+C0013362	Dysarthria	531.0	7.0	10.0	260.0
+C0013384	Dyskinetic syndrome	400.0	31.0	26.0	145.0
+C0014038	Encephalitis	406.0	8.0	37.0	148.0
+C0014057	Japanese Encephalitis	142.0	0.0	21.0	50.0
+C0014060	Encephalitis, St. Louis	330.0	0.0	13.0	124.0
+C0014065	Congenital cerebral hernia	101.0	0.0	4.0	43.0
+C0014070	Encephalomyelitis	1062.0	0.0	32.0	578.0
+C0014544	Epilepsy	1727.0	48.0	19.0	828.0
+C0014553	Absence Epilepsy	131.0	15.0	11.0	28.0
+C0014556	Epilepsy, Temporal Lobe	470.0	11.0	14.0	181.0
+C0015469	Facial paralysis	186.0	1.0	12.0	25.0
+C0016667	Fragile X Syndrome	255.0	1.0	6.0	86.0
+C0016719	Friedreich Ataxia	111.0	4.0	12.0	22.0
+C0017205	Gaucher Disease	183.0	2.0	17.0	35.0
+C0017921	Glycogen storage disease type II	243.0	0.0	12.0	44.0
+C0018378	Guillain-Barre Syndrome	194.0	0.0	44.0	58.0
+C0018524	Hallucinations	192.0	30.0	10.0	83.0
+C0018784	Sensorineural Hearing Loss (disorder)	989.0	6.0	5.0	432.0
+C0019202	Hepatolenticular Degeneration	200.0	9.0	21.0	43.0
+C0019562	Von Hippel-Lindau Syndrome	240.0	0.0	4.0	91.0
+C0020179	Huntington Disease	1345.0	11.0	16.0	677.0
+C0020255	Hydrocephalus	555.0	3.0	15.0	267.0
+C0022336	Creutzfeldt-Jakob disease	180.0	0.0	24.0	53.0
+C0023264	Leigh Disease	285.0	0.0	18.0	70.0
+C0023524	Leukoencephalopathy, Progressive Multifocal	263.0	1.0	6.0	152.0
+C0024266	Lymphocytic Choriomeningitis	159.0	0.0	22.0	47.0
+C0024408	Machado-Joseph Disease	171.0	0.0	14.0	66.0
+C0024809	Marijuana Abuse	209.0	0.0	13.0	52.0
+C0024814	Marinesco-Sjogren syndrome	167.0	1.0	15.0	62.0
+C0025286	Meningioma	929.0	2.0	14.0	460.0
+C0025289	Meningitis	225.0	20.0	42.0	79.0
+C0025958	Microcephaly	1360.0	0.0	10.0	824.0
+C0026106	Mild Mental Retardation	374.0	0.0	2.0	120.0
+C0026654	Moyamoya Disease	151.0	2.0	13.0	16.0
+C0026769	Multiple Sclerosis	2878.0	21.0	28.0	1284.0
+C0026847	Spinal Muscular Atrophy	440.0	0.0	17.0	194.0
+C0026850	Muscular Dystrophy	453.0	1.0	33.0	124.0
+C0026896	Myasthenia Gravis	447.0	17.0	11.0	133.0
+C0027121	Myositis	301.0	4.0	24.0	109.0
+C0027126	Myotonic Dystrophy	204.0	3.0	8.0	72.0
+C0027809	Neurilemmoma	259.0	2.0	13.0	120.0
+C0027830	neurofibroma	157.0	0.0	12.0	65.0
+C0027831	Neurofibromatosis 1	433.0	0.0	15.0	183.0
+C0027832	Neurofibromatosis 2	160.0	0.0	12.0	82.0
+C0027859	Acoustic Neuroma	159.0	0.0	11.0	74.0
+C0027873	Neuromyelitis Optica	214.0	1.0	25.0	30.0
+C0028043	Nicotine Dependence	227.0	13.0	8.0	38.0
+C0028077	Night Blindness	201.0	1.0	15.0	16.0
+C0028738	Nystagmus	923.0	10.0	12.0	551.0
+C0029124	Optic Atrophy	648.0	0.0	15.0	335.0
+C0030567	Parkinson Disease	3240.0	47.0	32.0	1582.0
+C0032000	Pituitary Adenoma	211.0	7.0	15.0	88.0
+C0033375	Prolactinoma	243.0	5.0	11.0	87.0
+C0035258	Restless Legs Syndrome	174.0	17.0	28.0	28.0
+C0035372	Rett Syndrome	299.0	0.0	25.0	105.0
+C0036341	Schizophrenia	5398.0	71.0	18.0	2171.0
+C0037773	Spastic Paraplegia, Hereditary	209.0	0.0	7.0	41.0
+C0038220	Status Epilepticus	664.0	41.0	23.0	294.0
+C0038379	Strabismus	771.0	1.0	4.0	501.0
+C0038436	Post-Traumatic Stress Disorder	528.0	23.0	8.0	200.0
+C0038525	Subarachnoid Hemorrhage	659.0	14.0	14.0	342.0
+C0038868	Progressive supranuclear palsy	237.0	1.0	19.0	92.0
+C0039483	Giant Cell Arteritis	327.0	5.0	15.0	123.0
+C0040517	Gilles de la Tourette syndrome	232.0	17.0	11.0	37.0
+C0041341	Tuberous Sclerosis	378.0	4.0	18.0	159.0
+C0042170	Uveomeningoencephalitic Syndrome	124.0	5.0	14.0	15.0
+C0043124	West Nile Fever	89.0	1.0	25.0	18.0
+C0043459	Zellweger Syndrome	81.0	0.0	5.0	22.0
+C0080178	Spina Bifida	233.0	1.0	12.0	59.0
+C0085084	Motor Neuron Disease	270.0	3.0	35.0	118.0
+C0085655	Polymyositis	226.0	7.0	10.0	41.0
+C0085762	Alcohol abuse	226.0	22.0	29.0	47.0
+C0086769	Panic Attacks	63.0	27.0	23.0	20.0
+C0149940	Sciatic Neuropathy	121.0	2.0	7.0	44.0
+C0151311	Cranial nerve palsies	81.0	2.0	29.0	26.0
+C0151740	Intracranial Hypertension	72.0	12.0	14.0	27.0
+C0152020	Gastroparesis	102.0	4.0	24.0	17.0
+C0152025	Polyneuropathy	192.0	6.0	14.0	45.0
+C0153633	Malignant neoplasm of brain	330.0	22.0	36.0	161.0
+C0162309	Adrenoleukodystrophy	368.0	0.0	12.0	119.0
+C0162635	Angelman Syndrome	149.0	0.0	17.0	44.0
+C0162666	Mitochondrial Encephalomyopathies	60.0	1.0	15.0	19.0
+C0175754	Agenesis of corpus callosum	767.0	0.0	3.0	417.0
+C0206728	Plexiform Neurofibroma	56.0	1.0	13.0	20.0
+C0220756	Niemann-Pick Disease, Type C	230.0	1.0	19.0	119.0
+C0221056	Adult type dermatomyositis	256.0	8.0	18.0	67.0
+C0221406	Pituitary-dependent Cushing's disease	164.0	3.0	11.0	37.0
+C0234144	Dysgraphia	43.0	0.0	9.0	22.0
+C0236642	Pick Disease of the Brain	289.0	1.0	9.0	137.0
+C0238190	Inclusion Body Myositis (disorder)	101.0	1.0	11.0	35.0
+C0238288	Muscular Dystrophy, Facioscapulohumeral	187.0	0.0	14.0	48.0
+C0242350	Erectile dysfunction	322.0	35.0	2.0	72.0
+C0265219	Miller Dieker syndrome	221.0	0.0	20.0	99.0
+C0266463	Lissencephaly	98.0	0.0	15.0	30.0
+C0266464	Polymicrogyria	239.0	0.0	2.0	109.0
+C0266483	Pachygyria	156.0	0.0	15.0	39.0
+C0270824	Visual seizure	235.0	216.0	25.0	53.0
+C0270972	Cornelia De Lange Syndrome	78.0	0.0	10.0	22.0
+C0271270	Oculovestibuloauditory syndrome	95.0	0.0	13.0	33.0
+C0276226	Herpes encephalitis	68.0	2.0	2.0	16.0
+C0276496	Familial Alzheimer Disease (FAD)	336.0	38.0	7.0	165.0
+C0282527	Infantile Refsum Disease (disorder)	37.0	0.0	2.0	15.0
+C0338451	Frontotemporal dementia	464.0	4.0	9.0	203.0
+C0338508	Optic Atrophy, Autosomal Dominant	161.0	0.0	16.0	59.0
+C0349204	Nonorganic psychosis	528.0	0.0	9.0	184.0
+C0410189	Muscular Dystrophy, Emery-Dreifuss	76.0	0.0	7.0	15.0
+C0410207	Tubular Aggregate Myopathy	78.0	0.0	14.0	31.0
+C0431380	Cortical Dysplasia	139.0	2.0	6.0	31.0
+C0494463	Alzheimer Disease, Late Onset	529.0	38.0	7.0	241.0
+C0496899	Benign neoplasm of brain, unspecified	42.0	22.0	23.0	18.0
+C0497327	Dementia	1153.0	16.0	28.0	522.0
+C0520679	Sleep Apnea, Obstructive	610.0	4.0	26.0	249.0
+C0543859	Amyotrophic Lateral Sclerosis, Guam Form	40.0	11.0	26.0	19.0
+C0546126	Acute Confusional Senile Dementia	100.0	38.0	7.0	48.0
+C0577631	Carotid Atherosclerosis	263.0	2.0	9.0	75.0
+C0600427	Cocaine Dependence	300.0	97.0	16.0	65.0
+C0740391	Middle Cerebral Artery Occlusion	766.0	15.0	7.0	404.0
+C0740392	Infarction, Middle Cerebral Artery	160.0	15.0	7.0	62.0
+C0750900	Alzheimer's Disease, Focal Onset	100.0	38.0	7.0	48.0
+C0750901	Alzheimer Disease, Early Onset	207.0	38.0	7.0	95.0
+C0750974	Brain Tumor, Primary	137.0	22.0	16.0	64.0
+C0750977	Recurrent Brain Neoplasm	39.0	22.0	16.0	18.0
+C0750979	Primary malignant neoplasm of brain	42.0	22.0	16.0	20.0
+C0751265	Learning Disabilities	114.0	38.0	7.0	25.0
+C0751587	CADASIL Syndrome	53.0	0.0	7.0	16.0
+C0751690	Malignant Peripheral Nerve Sheath Tumor	332.0	0.0	9.0	172.0
+C0751713	Inclusion Body Myopathy, Sporadic	93.0	1.0	11.0	38.0
+C0751772	REM Sleep Behavior Disorder	60.0	4.0	13.0	17.0
+C0751781	Dentatorubral-Pallidoluysian Atrophy	123.0	3.0	18.0	37.0
+C0751967	Multiple Sclerosis, Relapsing-Remitting	249.0	7.0	3.0	61.0
+C0752120	Spinocerebellar Ataxia Type 1	126.0	2.0	13.0	33.0
+C0752125	Spinocerebellar Ataxia Type 7	94.0	2.0	13.0	28.0
+C0752166	Bardet-Biedl Syndrome	176.0	0.0	10.0	27.0
+C0752304	Hypoxic-Ischemic Encephalopathy	197.0	2.0	1.0	62.0
+C0752347	Lewy Body Disease	335.0	3.0	32.0	145.0
+C0917798	Cerebral Ischemia	121.0	48.0	4.0	49.0
+C0917816	Mental deficiency	150.0	2.0	1.0	27.0
+C1263846	Attention deficit hyperactivity disorder	1084.0	30.0	14.0	484.0
+C1269683	Major Depressive Disorder	1814.0	56.0	19.0	775.0
+C1306214	ACTH-Secreting Pituitary Adenoma	88.0	0.0	3.0	23.0
+C1510586	Autism Spectrum Disorders	1478.0	0.0	9.0	699.0
+C1839259	Bulbo-Spinal Atrophy, X-Linked	144.0	0.0	13.0	70.0
+C1868675	PARKINSON DISEASE 2, AUTOSOMAL RECESSIVE JUVENILE	89.0	37.0	2.0	27.0
+C1955869	Malformations of Cortical Development	80.0	2.0	6.0	26.0
+C2931689	Dystrophia myotonica 2	144.0	3.0	18.0	21.0
+C3658299	Zellweger Spectrum	35.0	0.0	5.0	17.0

data/raw/top10_genecount_disease.tsv

+cui	disease_name	gene_id	drug_id	symptom_id	Genes in LCC
+C0002395	Alzheimer's Disease	5736.0	38.0	7.0	2697.0
+C0036341	Schizophrenia	5398.0	71.0	18.0	2171.0
+C0030567	Parkinson Disease	3240.0	47.0	32.0	1582.0
+C0026769	Multiple Sclerosis	2878.0	21.0	28.0	1284.0
+C0011581	Depressive disorder	2544.0	112.0	5.0	1266.0
+C0011570	Mental Depression	2208.0	0.0	7.0	1047.0
+C0005586	Bipolar Disorder	2040.0	78.0	12.0	727.0
+C1269683	Major Depressive Disorder	1814.0	56.0	19.0	775.0
+C0002736	Amyotrophic Lateral Sclerosis	1791.0	11.0	23.0	847.0
+C0014544	Epilepsy	1727.0	48.0	19.0	828.0

results/disease_gds.csv

+disease_id,gds_id
+C0221056,GDS3417
+C0221056,GDS2153
+C0221056,GDS2855
+C0221056,GDS1956
+C0038436,GDS1020
+C0032000,GDS2432
+C0032000,GDS1253
+C0013264,GDS2855
+C0013264,GDS1956
+C0013264,GDS612
+C0013264,GDS611
+C0013264,GDS610
+C0013264,GDS609
+C0013264,GDS563
+C0013264,GDS270
+C0013264,GDS265
+C0013264,GDS264
+C0013264,GDS262
+C0013264,GDS214
+C0035372,GDS2613
+C0080178,GDS2470
+C0027121,GDS2153
+C0037773,GDS1956
+C0033375,GDS1253
+C0410189,GDS2855
+C0410189,GDS1956
+C0496899,GDS4470
+C0496899,GDS5181
+C0496899,GDS3069
+C0496899,GDS2374
+C0496899,GDS2432
+C0496899,GDS1962
+C0496899,GDS1816
+C0496899,GDS1815
+C0496899,GDS1976
+C0496899,GDS1975
+C0496899,GDS2853
+C0496899,GDS1253
+C0496899,GDS232
+C1306214,GDS1253
+C0750979,GDS4470
+C0750979,GDS5181
+C0750979,GDS3069
+C0750979,GDS2374
+C0750979,GDS2432
+C0750979,GDS1962
+C0750979,GDS1816
+C0750979,GDS1815
+C0750979,GDS1976
+C0750979,GDS1975
+C0750979,GDS2853
+C0750979,GDS1253
+C0750979,GDS232
+C0085655,GDS2153
+C0085084,GDS4353
+C0085084,GDS3644
+C0085084,GDS2855
+C0085084,GDS1956
+C0085084,GDS412
+C0750974,GDS4470
+C0750974,GDS5181
+C0750974,GDS3069
+C0750974,GDS2374
+C0750974,GDS2432
+C0750974,GDS1962
+C0750974,GDS1816
+C0750974,GDS1815
+C0750974,GDS1976
+C0750974,GDS1975
+C0750974,GDS2853
+C0750974,GDS1253
+C0750974,GDS232
+C0497327,GDS2763
+C0497327,GDS2795
+C0238288,GDS2855
+C0238288,GDS1956
+C0014070,GDS4152
+C0028043,GDS2447
+C0221406,GDS2374
+C0038868,GDS2519
+C0026850,GDS2855
+C0026850,GDS1956
+C0026850,GDS612
+C0026850,GDS611
+C0026850,GDS610
+C0026850,GDS609
+C0026850,GDS563
+C0026850,GDS270
+C0026850,GDS265
+C0026850,GDS264
+C0026850,GDS262
+C0026850,GDS214
+C0013384,GDS4541
+C0013384,GDS2519
+C0013384,GDS1912
+C0013384,GDS1726
+C0013384,GDS1331
+C0151311,GDS2519
+C0151311,GDS1112
+C0751265,GDS1917
+C0020179,GDS4541
+C0020179,GDS1331
+C0011570,GDS2447
+C0002736,GDS1956
+C0002736,GDS412
+C0002395,GDS4136
+C0002395,GDS4135
+C0002395,GDS4128
+C0002395,GDS2519
+C0002395,GDS2795
+C0002395,GDS810
+C0014038,GDS4218
+C0014038,GDS1726
+C0001973,GDS5430
+C0001973,GDS2447
+C0001973,GDS2191
+C0001973,GDS2190
+C0750900,GDS4136
+C0750900,GDS4135
+C0750900,GDS4128
+C0750900,GDS2519
+C0750900,GDS2795
+C0750900,GDS810
+C0004352,GDS4431
+C0085762,GDS2191
+C0085762,GDS2190
+C0024809,GDS2447
+C0349204,GDS2779
+C1868675,GDS5646
+C1868675,GDS4154
+C1868675,GDS2821
+C1868675,GDS2519
+C1868675,GDS1912
+C0003467,GDS4152
+C0003467,GDS4012
+C0003467,GDS2978
+C0014544,GDS1962
+C0014544,GDS1051
+C0014544,GDS1050
+C0546126,GDS4136
+C0546126,GDS4135
+C0546126,GDS4128
+C0546126,GDS2519
+C0546126,GDS2795
+C0546126,GDS810
+C0025286,GDS2865
+C0007789,GDS4353
+C0007789,GDS3644
+C0011195,GDS1956
+C0543859,GDS1956
+C0543859,GDS412
+C0030567,GDS5646
+C0030567,GDS4154
+C0030567,GDS2821
+C0030567,GDS2519
+C0001206,GDS2432
+C0751967,GDS4150
+C0751967,GDS2419
+C0276496,GDS4136
+C0276496,GDS4135
+C0276496,GDS4128
+C0276496,GDS2519
+C0276496,GDS2795
+C0276496,GDS810
+C0162666,GDS1065
+C0026769,GDS4218
+C0026769,GDS3920
+C0026769,GDS4152
+C0026769,GDS4150
+C0026769,GDS2978
+C0026769,GDS2419
+C0270824,GDS4854
+C0270824,GDS3110
+C0270824,GDS968
+C0750901,GDS4136
+C0750901,GDS4135
+C0750901,GDS4128
+C0750901,GDS2519
+C0750901,GDS2795
+C0750901,GDS810
+C0002622,GDS2795
+C0011633,GDS3417
+C0011633,GDS2153
+C0011633,GDS2855
+C0011633,GDS1956
+C0011581,GDS2447
+C0041341,GDS3281
+C0036341,GDS4522
+C0036341,GDS4523
+C0036341,GDS1917
+C0751690,GDS2736
+C0005586,GDS2779
+C0005586,GDS2191
+C0005586,GDS2190
+C0917816,GDS2613
+C0750977,GDS4470
+C0750977,GDS5181
+C0750977,GDS3069
+C0750977,GDS2374
+C0750977,GDS2432
+C0750977,GDS1962
+C0750977,GDS1816
+C0750977,GDS1815
+C0750977,GDS1976
+C0750977,GDS1975
+C0750977,GDS2853
+C0750977,GDS1253
+C0750977,GDS232
+C0494463,GDS4136
+C0494463,GDS4135
+C0494463,GDS4128
+C0494463,GDS2519
+C0494463,GDS2795
+C0494463,GDS810

results/gds_gpl.csv

+gds_id,gpl_id
+GDS1050,GPL96
+GDS4136,GPL570
+GDS4150,GPL570
+GDS563,GPL8300
+GDS5430,GPL570
+GDS4541,GPL96
+GDS612,GPL95
+GDS2978,GPL96
+GDS2190,GPL96
+GDS1112,GPL8300
+GDS232,GPL74
+GDS2613,GPL8300
+GDS214,GPL246
+GDS2447,GPL1426
+GDS3281,GPL96
+GDS1962,GPL570
+GDS1816,GPL97
+GDS2519,GPL96
+GDS2763,GPL96
+GDS1815,GPL96
+GDS1975,GPL96
+GDS3920,GPL570
+GDS2432,GPL570
+GDS2795,GPL570
+GDS265,GPL95
+GDS2853,GPL8300
+GDS610,GPL93
+GDS264,GPL94
+GDS1912,GPL201
+GDS4353,GPL571
+GDS2855,GPL97
+GDS1253,GPL96
+GDS1956,GPL96
+GDS5646,GPL10558
+GDS3417,GPL96
+GDS3069,GPL96
+GDS1726,GPL8300
+GDS4523,GPL570
+GDS611,GPL94
+GDS270,GPL92
+GDS5181,GPL4133
+GDS2419,GPL4191
+GDS4128,GPL570
+GDS2191,GPL96
+GDS2470,GPL570
+GDS2374,GPL570
+GDS3644,GPL96
+GDS2821,GPL570
+GDS1051,GPL97
+GDS4012,GPL10526
+GDS968,GPL8300
+GDS2779,GPL570
+GDS2736,GPL96
+GDS1065,GPL96
+GDS3110,GPL96
+GDS1020,GPL91
+GDS2865,GPL96
+GDS810,GPL96
+GDS1976,GPL97
+GDS4135,GPL570
+GDS1331,GPL96
+GDS609,GPL92
+GDS4431,GPL570
+GDS4470,GPL570
+GDS2153,GPL96
+GDS4154,GPL571
+GDS1917,GPL570
+GDS4152,GPL570
+GDS4854,GPL570
+GDS4218,GPL570
+GDS412,GPL80
+GDS4522,GPL570
+GDS262,GPL91

results/prefilt_disease_gds.csv

+disease_id,gds_id
+C0221056,GDS4841
+C0221056,GDS3417
+C0221056,GDS2153
+C0221056,GDS2855
+C0221056,GDS1956
+C0038436,GDS4879
+C0038436,GDS1020
+C0032000,GDS2432
+C0032000,GDS1253
+C0520679,GDS4857
+C0013080,GDS5211
+C0013080,GDS2941
+C0016667,GDS2824
+C0162309,GDS4559
+C0162309,GDS4451
+C0338451,GDS3459
+C0013264,GDS3027
+C0013264,GDS2855
+C0013264,GDS1956
+C0013264,GDS612
+C0013264,GDS611
+C0013264,GDS610
+C0013264,GDS609
+C0013264,GDS563
+C0013264,GDS270
+C0013264,GDS265
+C0013264,GDS264
+C0013264,GDS262
+C0013264,GDS214
+C0035372,GDS2613
+C0080178,GDS2470
+C0027121,GDS4841
+C0027121,GDS2153
+C0037773,GDS1956
+C0033375,GDS4859
+C0033375,GDS1253
+C0009171,GDS5047
+C0410189,GDS2855
+C0410189,GDS1956
+C0496899,GDS4838
+C0496899,GDS4464
+C0496899,GDS4469
+C0496899,GDS4470
+C0496899,GDS4471
+C0496899,GDS4473
+C0496899,GDS4859
+C0496899,GDS4477
+C0496899,GDS5181
+C0496899,GDS3952
+C0496899,GDS4275
+C0496899,GDS3069
+C0496899,GDS2374
+C0496899,GDS2432
+C0496899,GDS1962
+C0496899,GDS1816
+C0496899,GDS1815
+C0496899,GDS1976
+C0496899,GDS1975
+C0496899,GDS1813
+C0496899,GDS2853
+C0496899,GDS1253
+C0496899,GDS232
+C1306214,GDS1253
+C0750979,GDS4838
+C0750979,GDS4464
+C0750979,GDS4469
+C0750979,GDS4470
+C0750979,GDS4471
+C0750979,GDS4473
+C0750979,GDS4859
+C0750979,GDS4477
+C0750979,GDS5181
+C0750979,GDS3952
+C0750979,GDS4275
+C0750979,GDS3069
+C0750979,GDS2374
+C0750979,GDS2432
+C0750979,GDS1962
+C0750979,GDS1816
+C0750979,GDS1815
+C0750979,GDS1976
+C0750979,GDS1975
+C0750979,GDS1813
+C0750979,GDS2853
+C0750979,GDS1253
+C0750979,GDS232
+C0085655,GDS4841
+C0085655,GDS2153
+C0085084,GDS4353
+C0085084,GDS3644
+C0085084,GDS2855
+C0085084,GDS1956
+C0085084,GDS412
+C0750974,GDS4838
+C0750974,GDS4464
+C0750974,GDS4469
+C0750974,GDS4470
+C0750974,GDS4471
+C0750974,GDS4473
+C0750974,GDS4859
+C0750974,GDS4477
+C0750974,GDS5181
+C0750974,GDS3952
+C0750974,GDS4275
+C0750974,GDS3069
+C0750974,GDS2374
+C0750974,GDS2432
+C0750974,GDS1962
+C0750974,GDS1816
+C0750974,GDS1815
+C0750974,GDS1976
+C0750974,GDS1975
+C0750974,GDS1813
+C0750974,GDS2853
+C0750974,GDS1253
+C0750974,GDS232
+C0497327,GDS4758
+C0497327,GDS4358
+C0497327,GDS3459
+C0497327,GDS2763
+C0497327,GDS2795
+C0238288,GDS4404
+C0238288,GDS2855
+C0238288,GDS1956
+C0014070,GDS4152
+C0025289,GDS3540
+C0025289,GDS1563
+C0017921,GDS4410
+C0017921,GDS4409
+C0028043,GDS2447
+C0221406,GDS2374
+C0038868,GDS2519
+C0026850,GDS4843
+C0026850,GDS4404
+C0026850,GDS3475
+C0026850,GDS3474
+C0026850,GDS3027
+C0026850,GDS2855
+C0026850,GDS1956
+C0026850,GDS612
+C0026850,GDS611
+C0026850,GDS610
+C0026850,GDS609
+C0026850,GDS563
+C0026850,GDS270
+C0026850,GDS265
+C0026850,GDS264
+C0026850,GDS262
+C0026850,GDS214
+C0013384,GDS5242
+C0013384,GDS4314
+C0013384,GDS4541
+C0013384,GDS2887
+C0013384,GDS2519
+C0013384,GDS1912
+C0013384,GDS1726
+C0013384,GDS1331
+C0013384,GDS1330
+C0013384,GDS1332
+C0013384,GDS1235
+C0276226,GDS4540
+C0276226,GDS4539
+C0024266,GDS4161
+C0151311,GDS4224
+C0151311,GDS2519
+C0151311,GDS1112
+C0751265,GDS1917
+C0020179,GDS4541
+C0020179,GDS2887
+C0020179,GDS1331
+C0020179,GDS1332
+C0011570,GDS4879
+C0011570,GDS3345
+C0011570,GDS3268
+C0011570,GDS2447
+C0002736,GDS1956
+C0002736,GDS412
+C0002395,GDS4758
+C0002395,GDS4141
+C0002395,GDS4136
+C0002395,GDS4135
+C0002395,GDS4128
+C0002395,GDS2519
+C0002395,GDS2601
+C0002395,GDS2795
+C0002395,GDS810
+C0014038,GDS4358
+C0014038,GDS4218
+C0014038,GDS4540
+C0014038,GDS4539
+C0014038,GDS1726
+C0001973,GDS5430
+C0001973,GDS4879
+C0001973,GDS2447
+C0001973,GDS2191
+C0001973,GDS2190
+C0750900,GDS4758
+C0750900,GDS4141
+C0750900,GDS4136
+C0750900,GDS4135
+C0750900,GDS4128
+C0750900,GDS2519
+C0750900,GDS2601
+C0750900,GDS2795
+C0750900,GDS810
+C0004352,GDS4431
+C0004352,GDS2824
+C0085762,GDS2191
+C0085762,GDS2190
+C1510586,GDS2824
+C0024809,GDS2447
+C0349204,GDS2779
+C2931689,GDS5276
+C1868675,GDS5646
+C1868675,GDS4141
+C1868675,GDS4154
+C1868675,GDS3129
+C1868675,GDS3128
+C1868675,GDS2821
+C1868675,GDS2519
+C1868675,GDS1912
+C0003467,GDS4838
+C0003467,GDS4464
+C0003467,GDS4559
+C0003467,GDS4296
+C0003467,GDS4152
+C0003467,GDS3886
+C0003467,GDS4012
+C0003467,GDS2978
+C0003467,GDS1960
+C0014544,GDS3864
+C0014544,GDS1962
+C0014544,GDS1051
+C0014544,GDS1050
+C0546126,GDS4758
+C0546126,GDS4141
+C0546126,GDS4136
+C0546126,GDS4135
+C0546126,GDS4128
+C0546126,GDS2519
+C0546126,GDS2601
+C0546126,GDS2795
+C0546126,GDS810
+C0025286,GDS2865
+C0238190,GDS4841
+C0007789,GDS4353
+C0007789,GDS3644
+C0011195,GDS1956
+C0543859,GDS1956
+C0543859,GDS412
+C0030567,GDS5646
+C0030567,GDS4141
+C0030567,GDS4154
+C0030567,GDS3129
+C0030567,GDS3128
+C0030567,GDS2821
+C0030567,GDS2519
+C0001206,GDS2432
+C0751967,GDS4150
+C0751967,GDS4147
+C0751967,GDS2419
+C0007766,GDS3903
+C0276496,GDS4758
+C0276496,GDS4141
+C0276496,GDS4136
+C0276496,GDS4135
+C0276496,GDS4128
+C0276496,GDS2519
+C0276496,GDS2601
+C0276496,GDS2795
+C0276496,GDS810
+C0162666,GDS1065
+C0026769,GDS4218
+C0026769,GDS3920
+C0026769,GDS4152
+C0026769,GDS3886
+C0026769,GDS4150
+C0026769,GDS4147
+C0026769,GDS2978
+C0026769,GDS2419
+C0270824,GDS4854
+C0270824,GDS4279
+C0270824,GDS3085
+C0270824,GDS3110
+C0270824,GDS1563
+C0270824,GDS913
+C0270824,GDS968
+C0750901,GDS4758
+C0750901,GDS4141
+C0750901,GDS4136
+C0750901,GDS4135
+C0750901,GDS4128
+C0750901,GDS2519
+C0750901,GDS2601
+C0750901,GDS2795
+C0750901,GDS810
+C0002622,GDS2795
+C0011269,GDS4758
+C0011633,GDS4841
+C0011633,GDS3417
+C0011633,GDS2153
+C0011633,GDS2855
+C0011633,GDS1956
+C0011581,GDS4879
+C0011581,GDS3345
+C0011581,GDS3268
+C0011581,GDS2447
+C0041341,GDS3281
+C0036341,GDS4522
+C0036341,GDS3938
+C0036341,GDS4523
+C0036341,GDS3502
+C0036341,GDS3345
+C0036341,GDS1917
+C0027126,GDS5276
+C0751690,GDS2736
+C0751713,GDS4841
+C0005586,GDS4837
+C0005586,GDS3502
+C0005586,GDS3345
+C0005586,GDS2779
+C0005586,GDS2191
+C0005586,GDS2190
+C0917816,GDS5211
+C0917816,GDS4559
+C0917816,GDS4451
+C0917816,GDS3830
+C0917816,GDS2941
+C0917816,GDS2824
+C0917816,GDS2613
+C0750977,GDS4838
+C0750977,GDS4464
+C0750977,GDS4469
+C0750977,GDS4470
+C0750977,GDS4471
+C0750977,GDS4473
+C0750977,GDS4859
+C0750977,GDS4477
+C0750977,GDS5181
+C0750977,GDS3952
+C0750977,GDS4275
+C0750977,GDS3069
+C0750977,GDS2374
+C0750977,GDS2432
+C0750977,GDS1962
+C0750977,GDS1816
+C0750977,GDS1815
+C0750977,GDS1976
+C0750977,GDS1975
+C0750977,GDS1813
+C0750977,GDS2853
+C0750977,GDS1253
+C0750977,GDS232
+C0494463,GDS4758
+C0494463,GDS4141
+C0494463,GDS4136
+C0494463,GDS4135
+C0494463,GDS4128
+C0494463,GDS2519
+C0494463,GDS2601
+C0494463,GDS2795
+C0494463,GDS810

scripts/gpl_main.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "41a9395c-0e12-4e7a-85b8-1eac4f02870d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from Bio import Entrez\n",
+    "import GEOparse\n",
+    "import pandas as pd\n",
+    "import os\n",
+    "import preprocess_functions\n",
+    "import insert_tables\n",
+    "import mysql.connector\n",
+    "from mysql.connector import errorcode"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "34bc052f-3e52-4a26-8946-0c12a9317bde",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gpl_path=\"/home/lmasa/GEO_Laura/data/gpl\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4981165e-a2bf-462b-8b1d-8573872e930b",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "06-Jul-2024 13:30:43 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL94.annot: \n",
+      "06-Jul-2024 13:30:43 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:43 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL571.annot: \n",
+      "06-Jul-2024 13:30:43 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:43 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL570.annot: \n",
+      "06-Jul-2024 13:30:43 DEBUG GEOparse - ANNOTATION: \n",
+      "/home/lmasa/miniconda3/lib/python3.12/site-packages/GEOparse/GEOparse.py:401: DtypeWarning: Columns (12) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  return read_csv(StringIO(data), index_col=None, sep=\"\\t\")\n",
+      "06-Jul-2024 13:30:43 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL93.annot: \n",
+      "06-Jul-2024 13:30:43 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:44 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL96.annot: \n",
+      "06-Jul-2024 13:30:44 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:44 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL1426.annot: \n",
+      "06-Jul-2024 13:30:44 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:44 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL201.annot: \n",
+      "06-Jul-2024 13:30:44 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:44 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL8300.annot: \n",
+      "06-Jul-2024 13:30:44 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:44 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL4191.annot: \n",
+      "06-Jul-2024 13:30:44 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:44 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL97.annot: \n",
+      "06-Jul-2024 13:30:44 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:44 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL10526.annot: \n",
+      "06-Jul-2024 13:30:44 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:45 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL92.annot: \n",
+      "06-Jul-2024 13:30:45 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:45 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL95.annot: \n",
+      "06-Jul-2024 13:30:45 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:45 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL74.annot: \n",
+      "06-Jul-2024 13:30:45 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:45 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL4133.annot: \n",
+      "06-Jul-2024 13:30:45 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:45 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL80.annot: \n",
+      "06-Jul-2024 13:30:45 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:45 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL10558.annot: \n",
+      "06-Jul-2024 13:30:46 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:46 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL91.annot: \n",
+      "06-Jul-2024 13:30:46 DEBUG GEOparse - ANNOTATION: \n",
+      "06-Jul-2024 13:30:46 INFO GEOparse - Parsing /home/lmasa/GEO_Laura/data/gpl/GPL246.annot: \n",
+      "06-Jul-2024 13:30:46 DEBUG GEOparse - ANNOTATION: \n"
+     ]
+    }
+   ],
+   "source": [
+    "gpl_data=preprocess_functions.fetch_gpl_annot(gpl_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "23f1d834-caeb-4f0e-a28c-95d258f8ea53",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>gpl_id</th>\n",
+       "      <th>gpl_title</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>GPL94</td>\n",
+       "      <td>[HG_U95D] Affymetrix Human Genome U95D Array</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>GPL571</td>\n",
+       "      <td>[HG-U133A_2] Affymetrix Human Genome U133A 2.0...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>GPL570</td>\n",
+       "      <td>[HG-U133_Plus_2] Affymetrix Human Genome U133 ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>GPL93</td>\n",
+       "      <td>[HG_U95C] Affymetrix Human Genome U95C Array</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>GPL96</td>\n",
+       "      <td>[HG-U133A] Affymetrix Human Genome U133A Array</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   gpl_id                                          gpl_title\n",
+       "0   GPL94       [HG_U95D] Affymetrix Human Genome U95D Array\n",
+       "1  GPL571  [HG-U133A_2] Affymetrix Human Genome U133A 2.0...\n",
+       "2  GPL570  [HG-U133_Plus_2] Affymetrix Human Genome U133 ...\n",
+       "3   GPL93       [HG_U95C] Affymetrix Human Genome U95C Array\n",
+       "4   GPL96     [HG-U133A] Affymetrix Human Genome U133A Array"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "gpl_data.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6389c1a1-c443-4774-bd9d-0fb65af45bee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "##Insert GPL data into database"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "d5785dba-71d0-4269-9ae7-b1968b8e9a19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#The connection details needed to insert data into the database\n",
+    "host = \"{host}\"  # Host where the MySQL server is located. Example: 'localhost' or '127.0.0.1'\n",
+    "user = \"{user_name}\"  # Username for accessing the MySQL database. Example: 'root' or 'my_user'\n",
+    "password = \"{password}\"  # Password for the MySQL user. Ensure to use a secure password for database access.\n",
+    "database = \"{database}\"  # Name of the database to connect to. Example: 'disnet_biolayer'\n",
+    "port = \"{port}\"  # Port number for the MySQL server. Default is 3306, but it may vary depending on the setup."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "b823d154-7c31-4aee-80b1-89bb933edb58",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize the connection variable to None\n",
+    "conn = None\n",
+    "\n",
+    "try:\n",
+    "    # Attempt to establish a connection to the MySQL database\n",
+    "    conn = mysql.connector.connect(\n",
+    "        host=host,\n",
+    "        user=user,\n",
+    "        password=password,\n",
+    "        database=database,\n",
+    "        port=port\n",
+    "    )\n",
+    "        # Insert GPL data into the database\n",
+    "        insert_tables.insert_gpl_main(conn, df)\n",
+    "\n",
+    "# Handle any MySQL errors that occur during the connection or insertion process\n",
+    "except mysql.connector.Error as err:\n",
+    "        print(f\"Error connecting to MySQL: {err}\")\n",
+    "\n",
+    "# Ensure the connection is closed properly even if an error occurs\n",
+    "finally:\n",
+    "        if conn and conn.is_connected():\n",
+    "            conn.close()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

scripts/preprocess_functions.py

+import requests
+from Bio import Entrez
+import GEOparse
+import os
+import gzip
+import shutil
+import pandas as pd
+from os.path import join
+from collections import defaultdict
+import numpy as np
+            
+            
+            
+def fetch_gds_data(gds_path):
+    """
+    Fetches and processes Gene Expression Omnibus (GEO) GDS data files from the specified directory.
+
+    This function iterates through GDS files in the given directory, extracts gene expression data, annotations,
+    and metadata for each file, and consolidates this information into three dataframes.
+
+    Input Parameters:
+        gds_path (str): The path to the directory containing GDS files.
+
+    Returns:
+        tuple: A tuple containing three pandas DataFrames:
+            - expression_df: DataFrame with gene expression data.
+            - annotation_df: DataFrame with annotation data.
+            - metadata_df: DataFrame with metadata about the GDS files.
+    """
+    expression_dfs = []
+    annotation_dfs = []
+    metadata_rows = []
+
+    for filename in os.listdir(gds_path):
+        if not filename.startswith("GDS") or not filename.endswith(".soft"):
+            continue
+
+        filepath = os.path.join(gds_path, filename)
+        gds = GEOparse.get_GEO(filepath=filepath)
+
+        value_types = gds.metadata.get('value_type', [])
+        if 'count' not in value_types:
+            continue
+
+        #Collect metadata
+        metadata_rows.append({
+            'gds_id': gds.name,
+            'gds_title': gds.metadata.get('title', [None])[0],
+            'gds_type': gds.metadata.get('type', [None])[0],
+            'gpl_id': gds.metadata.get('platform', [None])[0],
+            'channel_count': gds.metadata.get('channel_count', [None])[0],
+            'value_type': value_types[0]
+        })
+
+        #Extract gene expression data
+        gsm_columns = [col for col in gds.table.columns if col.startswith("GSM")]
+        non_gsm_columns = [col for col in gds.table.columns if not col.startswith("GSM")]
+
+        melted_df = pd.melt(
+            gds.table,
+            id_vars=non_gsm_columns,
+            value_vars=gsm_columns,
+            var_name='gsm_id',
+            value_name='value'
+        )
+
+        melted_df.dropna(subset=['IDENTIFIER', 'value'], inplace=True)
+        melted_df.rename(columns={'ID_REF': 'id_ref', 'IDENTIFIER': 'gene_symbol'}, inplace=True)
+        melted_df['gds_id'] = gds.name
+
+        #Extract annotation data
+        gds_annot = gds.columns.reset_index().drop(columns=['description'])
+        gds_annot['gds_id'] = gds.name
+
+        #Collect expression data for expression_df
+        expression_dfs.append(melted_df)
+
+        #Collect annotation data for annotation_df
+        annotation_dfs.append(gds_annot)
+
+    #Combine all expression and annotation dataframes
+    expression_df = pd.concat(expression_dfs, ignore_index=True)
+    annotation_df = pd.concat(annotation_dfs, ignore_index=True)
+
+    rename_dict = {
+        'index': 'gsm_id',
+        'disease state': 'disease_state',
+        'cell type': 'cell_type',
+        'development stage': 'development_stage',
+        'genotype/variation': 'genotype'
+    }
+    annotation_df.rename(columns={k: v for k, v in rename_dict.items() if k in annotation_df.columns}, inplace=True)
+
+    metadata_df = pd.DataFrame(metadata_rows)
+
+    return expression_df, annotation_df, metadata_df
+
+
+
+
+# =================================================================================
+
+
+
+def fetch_gpl_annot(gpl_path):
+    """
+    Fetch GPL data from files in the specified directory.
+
+    Input Parameters:
+        gpl_path (str): The directory path where the GPL annotation files are stored.
+
+    Returns:
+        pd.DataFrame: A dataframe containing GPL annotation data.
+                      Columns are 'gpl_id' and 'gpl_title'.
+    """
+    data_rows = []
+    for filename in os.listdir(gpl_path):
+        if filename.startswith("GPL") and filename.endswith(".annot"):
+            filepath = os.path.join(gpl_path, filename)
+            gpl = GEOparse.get_GEO(filepath=filepath)
+
+            #Collect GPL (platform) data
+            data_rows.append({
+                'gpl_id': gpl.name,
+                'gpl_title': gpl.metadata.get('platform_title', [None])[0]
+            })
+    gpl_data = pd.DataFrame(data_rows)
+
+    return gpl_data
+
+# =================================================================================
+
+def process_disease_state(df, fill_value=''):
+    """
+    Processes the 'disease_state' column in the given DataFrame.
+
+    This function fills missing values in the DataFrame, then categorizes the 'disease_state' column into
+    two categories: 'c' for control/normal/healthy states and 'd' for diseased states.
+
+    Input Parameters:
+        df (pandas.DataFrame): The input DataFrame containing a 'disease_state' column.
+        fill_value (str, optional): The value to use for filling missing values in the DataFrame. Default is an empty string.
+
+    Returns:
+        pandas.DataFrame: A new DataFrame with the processed 'disease_state' column.
+    """
+    #Fill missing values and create a copy of the DataFrame
+    new_dataframe = df.fillna(fill_value).copy()
+
+    #Create a mask for control/normal/healthy states using case-insensitive matching
+    mask_control = new_dataframe['disease_state'].str.lower().str.contains('|'.join(['control', 'normal', 'healthy', 'not diseased', 'wild-type']))
+
+    #Assign 'c' to control/normal/healthy states
+    new_dataframe.loc[mask_control, 'disease_state'] = 'c'
+
+    #Assign 'd' to diseased states
+    new_dataframe.loc[~mask_control, 'disease_state'] = 'd'
+
+    return new_dataframe
+
+
+# =================================================================================
+
+def extract_cuis(apikey,dataframe, column_name):
+    """
+    Extracts CUIs (Concept Unique Identifiers) for terms in a specified column of a DataFrame.
+
+    This function interacts with the UMLS API to fetch CUIs for unique terms found in the specified column of the input DataFrame.
+
+    Input Parameters:
+        dataframe (pandas.DataFrame): The input DataFrame containing the terms.
+        column_name (str): The name of the column in the DataFrame containing the terms for which CUIs need to be extracted.
+
+    Returns:
+        dict: A dictionary mapping terms to their respective CUIs.
+    """
+    #UMLS API key and settings
+    apikey = apikey
+    version = 'current'
+    uri = "https://uts-ws.nlm.nih.gov"
+    content_endpoint = "/rest/search/" + version
+    full_url = uri + content_endpoint
+    search_type = 'exact'
+
+    #Get the different terms from the specified column, removing NaNs and empty strings
+    list_different = set(dataframe[column_name].dropna())
+    list_different = {item for item in list_different if item.strip()}
+    different_cuis = {}
+
+    #Fetch CUIs for each different term
+    for item in list_different:
+        page = 0
+        while True:
+            page += 1
+            query = {
+                'string': item,
+                'apiKey': apikey,
+                'pageNumber': page,
+                'searchType': search_type
+            }
+            r = requests.get(full_url, params=query)
+            r.raise_for_status()
+            r.encoding = 'utf-8'
+            outputs = r.json()
+
+            items = ((outputs.get('result', {})).get('results', []))
+
+            if len(items) == 0:
+                if page == 1:
+                    break
+                else:
+                    break
+
+            for result in items:
+                cui = result['ui']
+                different_cuis[item] = cui
+
+    return different_cuis
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+            
+            
+            
+            
+            
+            
+            
+            
+            
\ No newline at end of file