text_plain_output_1.png
from langdetect import detect

from nltk.tokenize import sent_tokenize,word_tokenize

from tqdm import tqdm

import pandas as pd

import pandas as pd

keywordlist = ['inhibitor']

def loopsearch(keywordlist, researchpaperfu):
    alldataframeco = pd.DataFrame()
    alldataframenoco = pd.DataFrame()
    allcopid = []
    allnocopid = []
    for i in tqdm(keywordlist):
        covinf = researchpaperfu.covid_related().search(i, num_results=1000, covid_related=False, view='table').results[['cord_uid', 'title', 'abstract']]
        notcovinf = researchpaperfu.not_covid_related().search(i, num_results=10000, covid_related=False, view='table').results[['cord_uid', 'title', 'abstract']]
        covinfpid = list(covinf.cord_uid.values)
        notcovinfpid = list(notcovinf.cord_uid.values)
        alldataframeco = pd.concat([covinf, alldataframeco])
        alldataframenoco = pd.concat([notcovinf, alldataframenoco])
        allcopid.append(covinfpid)
        allnocopid.append(notcovinfpid)
    alldataframeco = alldataframeco.drop_duplicates()
    alldataframenoco = alldataframenoco.drop_duplicates()
    return (allcopid, allnocopid, alldataframeco, alldataframenoco)

fullab = pd.concat([allcoab, allnocoab])
fullab = fullab.rename(columns={'cord_uid': 'pid'})
fullab = fullab[fullab.abstract != '']
lan = []
for i in fullab.abstract:
    lan1 = detect(i)
    lan.append(lan1)
fullab['lan'] = lan
fullab = fullab[fullab.lan == 'en']
fullab = fullab[['pid', 'title', 'abstract']]

question = 'Q1'
question_dir = question + '/'

keylist = pd.read_csv('/kaggle/input/kagglecovid19literature/results/' + question_dir + 'keylist.txt').columns.values
valuelist = pd.read_csv('/kaggle/input/kagglecovid19literature/results/' + question_dir + 'valuelist.txt', header=None).values
viruslist = pd.read_csv('/kaggle/input/kagglecovid19literature/results/' + question_dir + 'viruslist.txt', header=None).values

def build_raw_data(file):
    def retunsb(sentlist, i, lennu):
        sent = sentlist[i]
        if i - 1 < 0:
            present = ''
        else:
            present = sentlist[i - 1]
        if i + 1 >= lennu:
            aftsent = ''
        else:
            aftsent = sentlist[i + 1]
        tempsent = ''
        tempsent = tempsent.join([present, sent, aftsent])
        return tempsent
    allfile = file
    allfile['abstract'] = allfile.abstract.astype(str)
    allsent = []
    allid = []
    allab = []
    for i in tqdm(range(len(allfile))):
        temp = allfile.abstract.iloc[i]
        temp = sent_tokenize(temp)
        for j in range(len(temp)):
            tempab = retunsb(temp, j, len(temp))
            allsent.append(temp[j])
            allid.append(allfile.pid.iloc[i])
            allab.append(tempab)
    allsent = pd.DataFrame(allsent, columns=['sent'])
    allsent['pid'] = allid
    allsent['abstract'] = allab
    return (allfile, allsent)

allfile, allsent = build_raw_data(fullab)