text_html_output_1.png
from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from nltk.tokenize import word_tokenize

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import string

root_path = '/kaggle/input/CORD-19-research-challenge/'
metadata_path = root_path + 'metadata.csv'
metadata_df = pd.read_csv(metadata_path, dtype={'pubmed_id': str, 'Microsoft Academic Paper ID': str, 'doi': str})
metadata_df = metadata_df.fillna(0)

metadata_df_wt_abs = metadata_df[metadata_df['abstract'] != 0]
metadata_df_wt_abs.shape

lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english') + [i for i in string.punctuation] + ['may', 'also', 'used'])
key_words = []
for word in metadata_df_wt_abs['abstract']:
    temp = word_tokenize(word.lower())
    for txt in temp:
        if txt not in stop_words:
            key_words.append(txt)

def transformations(sentences):
    lemmatizer = WordNetLemmatizer()
    stop_words = set(stopwords.words('english') + [i for i in string.punctuation] + ['may', 'also', 'used'])
    key_words = []
    for word in sentences.split():
        temp = word_tokenize(word.lower())
        for txt in temp:
            txt = lemmatizer.lemmatize(txt)
            if txt not in stop_words:
                key_words.append(txt)
    return key_words

def get_breaks(content, length):
    data = ''
    words = content.split(' ')
    total_chars = 0
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + '<br>' + words[i]
            total_chars = 0
        else:
            data = data + ' ' + words[i]
    return data

Label_df = pd.DataFrame(columns=['Task_text'], data=['What has been published about medical care?', ' What has been published concerning surge capacity and nursing homes?', 'What has been published concerning efforts to inform allocation of scarce resources?', 'What do we know about personal protective equipment?', 'What has been published concerning alternative methods to advise on disease management?', 'What has been published concerning processes of care?', 'What do we know about the clinical characterization and management of the virus?', 'Resources to support skilled nursing facilities and long term care facilities.', 'Mobilization of surge medical staff to address shortages in overwhelmed communities Age-adjusted mortality data for Acute Respiratory Distress Syndrome (ARDS) with/without other organ failure – particularly for viral etiologies', 'Extracorporeal membrane oxygenation (ECMO) outcomes data of COVID-19 patients Outcomes data for COVID-19 after mechanical ventilation adjusted for age.', 'Knowledge of the frequency, manifestations, and course of extrapulmonary manifestations of COVID-19, including, but not limited to, possible cardiomyopathy and cardiac arrest.', 'Application of regulatory standards (e.g., EUA, CLIA) and ability to adapt care to crisis standards of care level.', 'Approaches for encouraging and facilitating the production of elastomeric respirators, which can save thousands of N95 masks. Best telemedicine practices, barriers and faciitators, and specific actions to remove/expand them within and across state boundaries. Guidance on the simple things people can do at home to take care of sick people and manage disease. Oral medications that might potentially work.', 'Use of AI in real-time health care delivery to evaluate interventions, risk factors, and outcomes in a way that could not be done manually. Best practices and critical challenges and innovative solutions and technologies in hospital flow and organization, workforce protection, workforce allocation, community-based support resources, payment, and supply chain management to enhance capacity, efficiency, and outcomes. Efforts to define the natural history of disease to inform clinical care, public health interventions, infection prevention control, transmission, and clinical trials Efforts to develop a core clinical outcome set to maximize usability of data across a range of trials Efforts to determine adjunctive and supportive interventions that can improve the clinical outcomes of infected patients (e.g. steroids, high flow oxygen)'])
Label_df['Bag_of_words'] = Label_df['Task_text'].apply(lambda x: transformations(x))
Label_df

root_path = '/kaggle/input/CORD-19-research-challenge/'
metadata_path = root_path + 'metadata.csv'
metadata_with_pid = pd.read_csv(metadata_path, dtype={'pubmed_id': str, 'Microsoft Academic Paper ID': str, 'doi': str})
metadata_with_pid.drop_duplicates(['abstract'], inplace=True)
metadata_with_pid.dropna(subset=['abstract'], inplace=True)
metadata_with_pid.drop(columns=['WHO #Covidence', 'journal', 'authors', 'full_text_file', 'license'])
metadata_with_pid.shape

for pid in range(metadata_with_pid.shape[0]):
    try:
        if metadata_with_pid.loc[pid, 'sha'] != None:
            metadata_with_pid.loc[pid, 'paper_id'] = metadata_with_pid.loc[pid, 'sha']
        elif metadata_with_pid.loc[pid, 'pmcid'] != None:
            metadata_with_pid.loc[pid, 'paper_id'] = metadata_with_pid.loc[pid, 'pmcid']
    except:
        metadata_with_pid.loc[pid, 'paper_id'] = ''
metadata_with_pid

metadata_with_pid.dropna(subset=['sha', 'pmcid'], how='all')
metadata_with_pid[:200]

dict_ = {'paper_id': [], 'doi': [], 'abstract': [], 'body_text': [], 'authors': [], 'title': [], 'journal': [], 'abstract_summary': []}
for idx, entry in enumerate(all_json):
    try:
        content = FileReader(entry)
    except Exception as e:
        continue
    meta_data = metadata_with_pid.loc[metadata_with_pid['sha'] == content.paper_id]
    if len(meta_data) == 0:
        continue
    dict_['abstract'].append(content.abstract)
    dict_['paper_id'].append(content.paper_id)
    dict_['body_text'].append(content.body_text)
    if len(content.abstract) == 0:
        dict_['abstract_summary'].append('Not provided.')
    elif len(content.abstract.split(' ')) > 100:
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + '...')
    else:
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)
    meta_data = metadata_with_pid.loc[metadata_with_pid['sha'] == content.paper_id]
    try:
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            dict_['authors'].append(get_breaks('. '.join(authors), 40))
        else:
            dict_['authors'].append('. '.join(authors))
    except Exception as e:
        dict_['authors'].append(meta_data['authors'].values[0])
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])
    dict_['journal'].append(meta_data['journal'].values[0])
    dict_['doi'].append(meta_data['doi'].values[0])
df_covid = pd.DataFrame(dict_, columns=['paper_id', 'doi', 'abstract', 'body_text', 'authors', 'title', 'journal', 'abstract_summary'])

df_covid = pd.read_csv('/kaggle/input/cosine-df/cosine_df.csv', index_col=0)

sort_by_q1 = df_covid.sort_values('Q1cosine_similarity', ascending=False)

sort_by_q2 = df_covid.sort_values('Q2cosine_similarity', ascending=False)
sort_by_q2.loc[:, ['paper_id', 'abstract', 'body_text', 'authors', 'title', 'journal', 'abstract_summary', 'Q2cosine_similarity']].head(n=10)