text_html_output_1.png
from copy import deepcopy

from tqdm import tqdm

import json

import os

import pandas as pd

import os
import json
from copy import deepcopy
from tqdm import tqdm
import pandas as pd
def format_name(author):
    middle_name = ' '.join(author['middle'])
    if author['middle']:
        return ' '.join([author['first'], middle_name, author['last']])
    else:
        return ' '.join([author['first'], author['last']])
def format_affiliation(affiliation):
    text = []
    location = affiliation.get('location')
    if location:
        text.extend(list(affiliation['location'].values()))
    institution = affiliation.get('institution')
    if institution:
        text = [institution] + text
    return ', '.join(text)
def format_authors(authors, with_affiliation=False):
    name_ls = []
    for author in authors:
        name = format_name(author)
        if with_affiliation:
            affiliation = format_affiliation(author['affiliation'])
            if affiliation:
                name_ls.append(f'{name} ({affiliation})')
            else:
                name_ls.append(name)
        else:
            name_ls.append(name)
    return ', '.join(name_ls)
def format_body(body_text):
    texts = [(di['section'], di['text']) for di in body_text]
    texts_di = {di['section']: '' for di in body_text}
    for section, text in texts:
        texts_di[section] += text
    body = ''
    for section, text in texts_di.items():
        body += section
        body += '\n\n'
        body += text
        body += '\n\n'
    return body
def format_bib(bibs):
    if type(bibs) == dict:
        bibs = list(bibs.values())
    bibs = deepcopy(bibs)
    formatted = []
    for bib in bibs:
        bib['authors'] = format_authors(bib['authors'], with_affiliation=False)
        formatted_ls = [str(bib[k]) for k in ['title', 'authors', 'venue', 'year']]
        formatted.append(', '.join(formatted_ls))
    return '; '.join(formatted)
def load_files(dirname):
    filenames = os.listdir(dirname)
    raw_files = []
    for filename in tqdm(filenames):
        filename = dirname + filename
        file = json.load(open(filename, 'rb'))
        raw_files.append(file)
    return raw_files
def generate_clean_df(all_files):
    cleaned_files = []
    for file in tqdm(all_files):
        features = [file['paper_id'], file['metadata']['title'], format_authors(file['metadata']['authors']), format_authors(file['metadata']['authors'], with_affiliation=True), format_body(file['abstract']) if 'abstract' in file else '', format_body(file['body_text']), format_bib(file['bib_entries'])]
        cleaned_files.append(features)
    col_names = ['paper_id', 'title', 'authors', 'affiliations', 'abstract', 'text', 'bibliography']
    clean_df = pd.DataFrame(cleaned_files, columns=col_names)
    return clean_df

json_dirs = {'biorxiv_pdf': '/kaggle/input/CORD-19-research-challenge/biorxiv_medrxiv/biorxiv_medrxiv/pdf_json/', 'comm_pdf': '/kaggle/input/CORD-19-research-challenge/comm_use_subset/comm_use_subset/pdf_json/', 'comm_pmc': '/kaggle/input/CORD-19-research-challenge/comm_use_subset/comm_use_subset/pmc_json/', 'noncomm_pdf': '/kaggle/input/CORD-19-research-challenge/noncomm_use_subset/noncomm_use_subset/pdf_json/', 'noncomm_pmc': '/kaggle/input/CORD-19-research-challenge/noncomm_use_subset/noncomm_use_subset/pmc_json/', 'custom_pdf': '/kaggle/input/CORD-19-research-challenge/custom_license/custom_license/pdf_json/', 'custom_pmc': '/kaggle/input/CORD-19-research-challenge/custom_license/custom_license/pmc_json/'}
json_dfs = []
for category, json_dir in json_dirs.items():
    json_files = load_files(json_dir)
    json_df = generate_clean_df(json_files)
    json_df['category'] = category
    json_dfs.append(json_df)