text_html_output_1.png
import pandas as pd

def get_data(dir_path):
    """
    Take as input a directory path containing json files from 
    biorxiv_medrxiv, comm_use_subset, noncomm_use_subset or custom_license.
    Four dataframes are returned: papers_df, authors_df, affiliations_df, bib_entries_df
    """
    files = os.listdir(dir_path)
    papers_df = pd.DataFrame(columns=['paper_id', 'title', 'authors', 'abstract', 'body_text', 'bib_titles', 'dataset'])
    authors_df = pd.DataFrame(columns=['author', 'affiliation'])
    affiliations_df = pd.DataFrame(columns=['affiliation', 'country'])
    bib_entries_df = pd.DataFrame(columns=['title', 'authors', 'year', 'venue'])
    line_author_df = 0
    line_affiliations_df = 0
    line_bib_entries_df = 0
    for line, file in enumerate(files):
        n_files = len(files)
        file_path = os.path.join('../data/{}'.format(dir), file)
        with open(file_path) as f:
            data = json.load(f)
        paper_id = data['paper_id']
        title = data['metadata']['title']
        authors, affiliations, countries = ('', '', '')
        for author in data['metadata']['authors']:
            first_last_name = author['first'] + ' ' + author['last']
            authors = authors + ' || ' + first_last_name
            if author['affiliation'] == {}:
                affiliation = 'NA'
                affiliations = affiliations + ' || ' + affiliation.strip()
                country = 'NA'
                countries = countries + ' || ' + country.strip()
                continue
            affiliation = author['affiliation']['laboratory'] + ' ' + author['affiliation']['institution']
            affiliations = affiliations + ' || ' + affiliation.strip()
            if 'country' not in author['affiliation']['location'].keys():
                country = 'NA'
                countries = countries + ' || ' + country
                continue
            country = author['affiliation']['location']['country']
            countries = countries + ' || ' + country
        authors = authors[4:]
        affiliations = affiliations[4:]
        countries = countries[4:]
        abstract = ''
        for info in data['abstract']:
            abstract = abstract + ' ' + info['text']
        abstract = abstract.strip()
        body_text = ''
        for info in data['body_text']:
            body_text = body_text + ' ' + info['text']
        body_text = body_text.strip()
        bib_titles, bib_authors, years, venues = ('', '', '', '')
        for bib in data['bib_entries']:
            bib_titles = bib_titles + ' || ' + data['bib_entries'][bib]['title']
            year = data['bib_entries'][bib]['year']
            years = years + ' || ' + str(year)
            venue = data['bib_entries'][bib]['venue']
            venues = venues + ' || ' + venue
            bib_author = [author['first'] + ' ' + author['last'] for author in data['bib_entries'][bib]['authors']]
            bib_author = ' | '.join(bib_author)
            bib_authors = bib_authors + ' || ' + bib_author
        bib_titles, bib_authors, years, venues = (bib_titles[4:], bib_authors[4:], years[4:], venues[4:])
        papers_df.loc[line, :] = [paper_id, title, authors, abstract, body_text, bib_titles, dir]
        authors_list = authors.split(' || ')
        affiliations_list = affiliations.split(' || ')
        for i in range(len(authors_list)):
            authors_df.loc[line_author_df, :] = (authors_list[i], affiliations_list[i])
            line_author_df += 1
        countries_list = countries.split(' || ')
        for i in range(len(affiliations_list)):
            affiliations_df.loc[line_affiliations_df, :] = (affiliations_list[i], countries_list[i])
            line_affiliations_df += 1
        bib_titles_list = bib_titles.split(' || ')
        bib_authors_list = bib_authors.split(' || ')
        years_list = years.split(' || ')
        venues_list = venues.split(' || ')
        for i in range(len(bib_titles_list)):
            bib_entries_df.loc[line_bib_entries_df, :] = (bib_titles_list[i], bib_authors_list[i], years_list[i], venues_list[i])
            line_bib_entries_df += 1
    authors_df = authors_df.drop_duplicates().reset_index(drop=True)
    affiliations_df = affiliations_df.drop_duplicates().reset_index(drop=True)
    bib_entries_df = bib_entries_df.drop_duplicates().reset_index(drop=True)
    return (papers_df, authors_df, affiliations_df, bib_entries_df)

df = pd.read_csv('/kaggle/input/papers/papers.csv', sep=';', nrows=100)
df.head()