text_plain_output_1.png
from langdetect import detect

from tqdm import tqdm

import glob

import json

import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
pd.set_option('display.expand_frame_repr', False)
pd.options.mode.chained_assignment = None
tqdm.pandas()

def filepath(*args):
    if len(args) < 1:
        return None
    elif len(args) == 1:
        return args[0]
    else:
        return f'{args[0]}/{filepath(*args[1:])}'
def addtimebar(L, threshold=1000):
    if len(L) > threshold:
        return tqdm(L)
    else:
        return L
class FileReader:
    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            try:
                for entry in content['abstract']:
                    self.abstract.append(entry['text'])
            except KeyError:
                pass
            try:
                for entry in content['body_text']:
                    self.body_text.append(entry['text'])
            except KeyError:
                pass
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)
    def __repr__(self):
        return f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'
def get_breaks(content, length):
    data = ''
    words = content.split(' ')
    total_chars = 0
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + '<br>' + words[i]
            total_chars = 0
        else:
            data = data + ' ' + words[i]
    return data
def compose(*funcs):
    *funcs, penultimate, last = funcs
    if funcs:
        penultimate = compose(*funcs, penultimate)
    return lambda *args: penultimate(last(*args))

path = '../input/CORD-19-research-challenge'
meta = 'metadata.csv'
all_jsons = glob.glob(filepath(path, '**', '*.json'), recursive=True)

meta_df = pd.read_csv(filepath(path, meta), dtype={'pubmed_id': str, 'Microsoft Academic Paper ID': str, 'doi': str, 'journal': str}, low_memory=False)

dict_ = {'paper_id': [], 'abstract': [], 'body_text': [], 'authors': [], 'title': [], 'publish_time': [], 'journal': [], 'abstract_summary': []}
for entry in all_jsons[:10]:
    content = FileReader(entry)
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    if len(meta_data) == 0:
        continue
    dict_['paper_id'].append(content.paper_id)
    dict_['abstract'].append(content.abstract)
    dict_['body_text'].append(content.body_text)
    if len(content.abstract) == 0:
        dict_['abstract_summary'].append('Not provided.')
    elif len(content.abstract.split(' ')) > 100:
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + '...')
    else:
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    try:
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            dict_['authors'].append('. '.join(authors[:2]) + '...')
        else:
            dict_['authors'].append('. '.join(authors))
    except Exception as e:
        dict_['authors'].append(meta_data['authors'].values[0])
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])
    try:
        publish_time = get_breaks(meta_data['publish_time'].values[0], 40)
        dict_['publish_time'].append(publish_time)
    except Exception as e:
        dict_['publish_time'].append(meta_data['publish_time'].values[0])
    dict_['journal'].append(meta_data['journal'].values[0])
df_covid = pd.DataFrame(dict_, columns=['paper_id', 'abstract', 'body_text', 'authors', 'title', 'journal', 'publish_time', 'abstract_summary'])

def is_lang(row, item, lang, dropNA=True):
    if row[item] != None and row[item] != '' and (row[item] != 'None') and isinstance(row[item], str):
        try:
            return detect(row[item]) == lang
        except Exception as e:
            return False
    else:
        return not dropNA
def select_article_lang_multi(df, basedon='abstract', lang='en'):
    return df[df.parallel_apply(lambda text: is_lang(text, basedon, lang), axis=1)]
df_covid_eng = select_article_lang_multi(df_covid)
print('Number of English Articles: {}/{}'.format(len(df_covid_eng), len(df_covid)))
df_covid_eng.head(n=2)