application_vnd.jupyter.stderr_output_1.png
from langdetect import detect

from nltk.tokenize import sent_tokenize,word_tokenize

from sklearn.model_selection import train_test_split

from snorkel.labeling import PandasLFApplier,LFAnalysis,LabelingFunction

from snorkel.labeling.model.label_model import LabelModel

from tqdm import tqdm

import pandas as pd

import pandas as pd

keywordlist = ['inhibitor']

def loopsearch(keywordlist, researchpaperfu):
    alldataframeco = pd.DataFrame()
    alldataframenoco = pd.DataFrame()
    allcopid = []
    allnocopid = []
    for i in tqdm(keywordlist):
        covinf = researchpaperfu.covid_related().search(i, num_results=1000, covid_related=False, view='table').results[['cord_uid', 'title', 'abstract']]
        notcovinf = researchpaperfu.not_covid_related().search(i, num_results=10000, covid_related=False, view='table').results[['cord_uid', 'title', 'abstract']]
        covinfpid = list(covinf.cord_uid.values)
        notcovinfpid = list(notcovinf.cord_uid.values)
        alldataframeco = pd.concat([covinf, alldataframeco])
        alldataframenoco = pd.concat([notcovinf, alldataframenoco])
        allcopid.append(covinfpid)
        allnocopid.append(notcovinfpid)
    alldataframeco = alldataframeco.drop_duplicates()
    alldataframenoco = alldataframenoco.drop_duplicates()
    return (allcopid, allnocopid, alldataframeco, alldataframenoco)

fullab = pd.concat([allcoab, allnocoab])
fullab = fullab.rename(columns={'cord_uid': 'pid'})
fullab = fullab[fullab.abstract != '']
lan = []
for i in fullab.abstract:
    lan1 = detect(i)
    lan.append(lan1)
fullab['lan'] = lan
fullab = fullab[fullab.lan == 'en']
fullab = fullab[['pid', 'title', 'abstract']]

question = 'Q1'
question_dir = question + '/'

keylist = pd.read_csv('/kaggle/input/kagglecovid19literature/results/' + question_dir + 'keylist.txt').columns.values
valuelist = pd.read_csv('/kaggle/input/kagglecovid19literature/results/' + question_dir + 'valuelist.txt', header=None).values
viruslist = pd.read_csv('/kaggle/input/kagglecovid19literature/results/' + question_dir + 'viruslist.txt', header=None).values

def build_raw_data(file):
    def retunsb(sentlist, i, lennu):
        sent = sentlist[i]
        if i - 1 < 0:
            present = ''
        else:
            present = sentlist[i - 1]
        if i + 1 >= lennu:
            aftsent = ''
        else:
            aftsent = sentlist[i + 1]
        tempsent = ''
        tempsent = tempsent.join([present, sent, aftsent])
        return tempsent
    allfile = file
    allfile['abstract'] = allfile.abstract.astype(str)
    allsent = []
    allid = []
    allab = []
    for i in tqdm(range(len(allfile))):
        temp = allfile.abstract.iloc[i]
        temp = sent_tokenize(temp)
        for j in range(len(temp)):
            tempab = retunsb(temp, j, len(temp))
            allsent.append(temp[j])
            allid.append(allfile.pid.iloc[i])
            allab.append(tempab)
    allsent = pd.DataFrame(allsent, columns=['sent'])
    allsent['pid'] = allid
    allsent['abstract'] = allab
    return (allfile, allsent)

def loop_labing(keylist, valuelist, virus):
    def keyword_lookup(x, keywords, virus, label):
        if any((word in x.sent.lower() for word in keywords)) and any((word in x.sent.lower() for word in virus)):
            return label
        return Norelevent
    def make_keyword_lf(keywords, virus, name, label=None):
        return LabelingFunction(name=f'keyword_{name}', f=keyword_lookup, resources=dict(keywords=keywords, virus=virus, label=label))
    def keyword_lookup1(x, keywords, virus, label):
        if not any((word in x.sent.lower() for word in keywords)) and any((word in x.sent.lower() for word in virus)):
            return label
        return Norelevent
    def make_keyword_lf1(keywords, virus, name, label=None):
        return LabelingFunction(name=f'keyword_{name}', f=keyword_lookup1, resources=dict(keywords=keywords, virus=virus, label=label))
    def abstract_lookup(x, keywords, virus, label):
        if any((word in x.abstract.lower() for word in keywords)) and any((word in x.abstract.lower() for word in virus)):
            return label
        return Norelevent
    def make_abstract_lf(keywords, virus, name, label=None):
        return LabelingFunction(name=f'abstract_{name}', f=abstract_lookup, resources=dict(keywords=keywords, virus=virus, label=label))
    Norelevent = -1
    allweaklabf = []
    viruselist = virus
    for i in range(len(keylist)):
        labelvalue = 1
        vbname = keylist[i]
        vbnameab = vbname + 'su'
        globals()[vbname] = make_keyword_lf(keywords=valuelist[i], virus=viruselist, name=vbnameab, label=labelvalue)
        vbname1 = keylist[i] + 'ab'
        vbnameab1 = vbname + 'su1'
        globals()[vbname1] = make_abstract_lf(keywords=valuelist[i], virus=viruselist, name=vbnameab1, label=labelvalue)
        vbname2 = keylist[i] + 'sent'
        vbnameab2 = vbname + 'sentno'
        globals()[vbname2] = make_keyword_lf1(keywords=valuelist[i], virus=viruselist, name=vbnameab2, label=0)
        allweaklabf.append(globals()[vbname])
        allweaklabf.append(globals()[vbname1])
        allweaklabf.append(globals()[vbname2])
    return allweaklabf

def snorkel_process(keylist, dataframe, allweaklabf):
    def func(x):
        idx = (-x).argsort()[1:]
        x[idx] = 0
        return x
    cardinalitynu = 2
    applier = PandasLFApplier(lfs=allweaklabf)
    all_train_l = applier.apply(df=dataframe)
    report = LFAnalysis(L=all_train_l, lfs=allweaklabf).lf_summary()
    label_model = LabelModel(cardinality=cardinalitynu, verbose=False)
    label_model.fit(all_train_l)
    predt = label_model.predict(all_train_l)
    dataframe['L_label'] = predt
    dataframe = dataframe[dataframe.L_label >= 0]
    train, test = train_test_split(dataframe, test_size=0.2, random_state=234)
    trainsent = train.sent.values
    trainlabel = train.L_label.values
    testsent = test.sent.values
    testlabel = test.L_label.values
    return (trainsent, trainlabel, testsent, testlabel, keylist, report)

allweaklabf = loop_labing(keylist, valuelist, viruslist)

trainsent, trainlabel, valsent, vallabel, keylist, report = snorkel_process(keylist, allsent, allweaklabf)
testsent = allsent.sent.values
testlabel = allsent.newpid.values