umlsExtractor.py

import es_core_news_md
import mysql.connector
import textdistance
import configparser
from mysql.connector import errorcode
from nltk.corpus import stopwords

nlp = es_core_news_md.load()

#Diccionario con los datos de conexion a la BBDD (se lee desde archivo de configuracion: DEFAULT --> LOCAL, TESTING --> ARES)
configuration = configparser.ConfigParser()
configuration.read('config.ini')

config2 = {'user':configuration['DEFAULT']['DB_USER'],
'password':configuration['DEFAULT']['DB_PASSWORD'],
'port':configuration['DEFAULT']['DB_PORT'],
'host':configuration['DEFAULT']['DB_HOST'],
'db':configuration['DEFAULT']['DB_NAME'],
'auth_plugin':configuration['DEFAULT']['DB_AUTH_PLUGIN']
}

#Función que tokeniza una lista de conceptos dada
#Output: listado de las palabras tokenizadas
def get_words(concepts):

	words = []
	for i in range(0,len(concepts)):

		ent = nlp(concepts[i])

		for i in range(len(ent)):
			if((ent[i].text.lower() not in stopwords.words('spanish')) and (ent[i].text.lower() not in words)):
				words.append(ent[i].text.lower())


	return words

#Función que dada una palabra busca en UMLS conceptos que la contengan y el CUI asociado
#Output: lista pares (concepto,CUI)	
def search_umls(word):
	try:
		#Conectamos con nuestra BD
		cnx = mysql.connector.connect(**config2)
		#Creamos el cursor 
		cursor = cnx.cursor()
		#Nuestra query
		query = "SELECT STR,CUI FROM MRCONSO where LAT='SPA' and STR like '%"+word+"%';"
		cursor.execute(query)		
		lUmls = []
			
		for row in cursor:
			if(row[0].strip!="" and row[1].strip()!=""):
				lUmls.append(str(row[0])+"\t"+str(row[1]))
			
	except mysql.connector.Error as err:
		if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
				print("No pudo conectarse a la BBDD, revisar usuario y password")
		elif err.errno == errorcode.ER_BAD_DB_ERROR:
				print("La BD introducida no existe")
		else:
				print(err)
				
	else:
		cnx.close()
		return lUmls

#Función que dado un cui busca en UMLS conceptos que lo contengan
#Output: lista (concepto)	
def search_umls_cui(cui):
	try:
		#Conectamos con nuestra BD
		cnx = mysql.connector.connect(**config2)
		#Creamos el cursor 
		cursor = cnx.cursor()
		#Nuestra query
		query = "SELECT STR FROM MRCONSO where LAT='SPA' and CUI='"+cui+"';"
		cursor.execute(query)		
		lUmls = []
			
		for row in cursor:
			if(row[0].strip!=""):
				lUmls.append(str(row[0]))
			
	except mysql.connector.Error as err:
		if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
				print("No pudo conectarse a la BBDD, revisar usuario y password")
		elif err.errno == errorcode.ER_BAD_DB_ERROR:
				print("La BD introducida no existe")
		else:
				print(err)
				
	else:
		cnx.close()
		return lUmls
	
	
#Función que busca en umls los conceptos que contengan las palabras asociadas
#Output: Diccionario, key: words, values: (umls_concept,CUI)
def get_umls_concept_cui(words):

	dictConcepts = {}
	
	for i in range(0,len(words)):
		lUmls = search_umls(words[i])
		if (words[i] in dictConcepts.keys()):
			dictConcepts[words[i]] = dictConcepts[words[i]] + lUmls
		else:
			dictConcepts[words[i]] = lUmls
	
	return dictConcepts

#Función que devuelve la similitud entre dos strings	
def similarities(str1,str2):
	
	levenshtein,jaccard,ratcliff = 0,0,0
	
	if(len(str1.strip())+2>=len(str2.split("\t")[0].strip())):
		levenshtein = textdistance.levenshtein.normalized_similarity(str1.strip().lower(),str2.split("\t")[0].strip().lower())
	
	return levenshtein

#Función que devuelve el concepto UMLS más similar a un concepto dado
def get_similarity(concept,lUMLSConcepts):
	
	lJaccard = []
	maxSimilarLevenshtein = 0
	umlsConceptLevenshtein = ""
	cuiConceptLevenshtein = ""

	
	for i in range(0,len(lUMLSConcepts)):
		
		levenshtein = similarities(concept,lUMLSConcepts[i])
		
		if(levenshtein>maxSimilarLevenshtein):
			maxSimilarLevenshtein = levenshtein
			auxUMLS = lUMLSConcepts[i].split("\t")
			umlsConceptLevenshtein = auxUMLS[0]
			cuiConceptLevenshtein = auxUMLS[1]
			
	lJaccard.append((concept,"Levenshtein:",umlsConceptLevenshtein,cuiConceptLevenshtein,maxSimilarLevenshtein,"UMLS"))
	
	return lJaccard


#Función que devuelve una lista de conceptos UMLS más similares a unos conceptos dados y sus CUIS asociadas 
def similarity_concept(concepts, dictConcepts):
	lSimilarConcepts = []
	
	for i in range(0,len(concepts)):
		words = get_words([concepts[i]])
		lAux = []
		
		for j in range(0,len(words)):
			if(words[j] in dictConcepts.keys()):
				lAux = lAux + dictConcepts[words[j]]
				
		lSimilarConcepts = lSimilarConcepts + get_similarity(concepts[i],lAux)
		
	return lSimilarConcepts

def similarity_cui (concepts, jkesCuis):

	listConceptsJKES = []

	for i in range(0,len(jkesCuis)):
		lAux = search_umls_cui(jkesCuis[i])

		if(len(lAux)>0):
			for j in range(0,len(lAux)):
				if(lAux[j] not in listConceptsJKES):
					listConceptsJKES.append(lAux[j]+"\t"+jkesCuis[i])

	lSimilaritiesJKES = []	

	for i in range(0,len(concepts)):
		lSimilaritiesJKES += get_similarity(concepts[i],listConceptsJKES)
		

	return lSimilaritiesJKES	

def get_final_concepts(lConceptsUMLS, lConceptsUMLSJKES):

	listConcepts = []
	for i in range(0,len(lConceptsUMLS)):
		if (lConceptsUMLS[i][4] >= lConceptsUMLSJKES[i][4]):
			listConcepts.append(lConceptsUMLS[i])
		else:
			listConcepts.append(lConceptsUMLSJKES[i])

	return listConcepts
	
#Main
def umls_concept_extractor(concepts,jkesCuis):

	words = get_words(concepts)
	dictConcepts = get_umls_concept_cui(words)
	lSimilarConcepts = similarity_concept(concepts,dictConcepts)
	if(len(jkesCuis)>0):
		lSimilaritiesJKES = similarity_cui(concepts,jkesCuis)
	
	listConcepts = get_final_concepts(lSimilarConcepts,lSimilaritiesJKES)
	return listConcepts


def umls_concept_extractor2(concepts):

	print("Get words")
	words = get_words(concepts)
	print("Tokenized Words")
	dictConcepts = get_umls_concept_cui(words)
	print("Dictionary concepts")
	lSimilarConcepts = similarity_concept(concepts,dictConcepts)
	
	return lSimilarConcepts