<file_sep># COVID-19-research-paper-analysis
Taking massive amounts of research papers and using machine learning &amp; other analytics to help make them consumable.

## Development Information

### How to install:

Prerequisites:
 - Anaconda
 - Git

Run the following commands:

`git clone https://github.com/The-Lycaeum/COVID-19-research-paper-analysis.git`

`cd COVID-19-research-paper-analysis`

`conda env create -f environment.yml -n covid-papers`

This creates the environment with the correct dependencies.  To access the environment:

`conda activate covid-papers`

During development, more packages might be required to run.  You can use `pip install` to add new packages, but a better way would be to add the package name to the environment.yml file, so everyone will be able to use it. 

After you add the packages to the environment.yml, you can use `pip install`, or you can also sync your environment to yml file with:

`conda env update -f environment.yml -n covid-papers`


## Directory Structure

### Data directory

Data is stored in the `src/data` directory.  There are 4 sub-directories in `src/data`: `data/raw`, `data/interim`, `data/processed`, and `data/other`. More about these divisions below.

Since we don't want to store the data on Github, the data directory & its sub-directories should be empty except for specific .git files, and the `download_data.py` script.  Running this script should automatically download the raw data into your `src/data/raw` directory.

### Data sub-directories

#### data/raw

#### data/interim

#### data/processed

#### data/other<file_sep>import os
import json
from pprint import pprint
from copy import deepcopy

import numpy as np
import pandas as pd
from tqdm.notebook import tqdm


## formats the author's names in a consistent manner

def format_name(author):
    middle_name = " ".join(author['middle'])
    
    if author['middle']:
        return " ".join([author['first'], middle_name, author['last']])
    else:
        return " ".join([author['first'], author['last']])

## parses metadata block to extract text of author affiliations, e.g., university

def format_affiliation(affiliation):
    text = []
    location = affiliation.get('location')
    if location:
        text.extend(list(affiliation['location'].values()))
    
    institution = affiliation.get('institution')
    if institution:
        text = [institution] + text
    return ", ".join(text)


## parses metadata block to extract text of the authors

def format_authors(authors, with_affiliation=False):
    name_ls = []
    
    for author in authors:
        name = format_name(author)
        if with_affiliation:
            affiliation = format_affiliation(author['affiliation'])
            if affiliation:
                name_ls.append(f"{name} ({affiliation})")
            else:
                name_ls.append(name)
        else:
            name_ls.append(name)
    
    return ", ".join(name_ls)

## formats the body content block of the article text

def format_body(body_text):
    texts = [(di['section'], di['text']) for di in body_text]
    texts_di = {di['section']: "" for di in body_text}
    
    for section, text in texts:
        texts_di[section] += text

    body = ""

    for section, text in texts_di.items():
        body += section
        body += "\n\n"
        body += text
        body += "\n\n"
    
    return body

## formats the bibliography metadata

def format_bib(bibs):
    if type(bibs) == dict:
        bibs = list(bibs.values())
    bibs = deepcopy(bibs)
    formatted = []
    
    for bib in bibs:
        bib['authors'] = format_authors(
            bib['authors'], 
            with_affiliation=False
        )
        formatted_ls = [str(bib[k]) for k in ['title', 'authors', 'venue', 'year']]
        formatted.append(", ".join(formatted_ls))

    return "; ".join(formatted)

def load_files(dirname):
    filenames = os.listdir(dirname)
    raw_files = []

    for filename in tqdm(filenames):
        filename = dirname + filename
        file = json.load(open(filename, 'rb'))
        raw_files.append(file)
    
    return raw_files

## pushes the json loaded files into data frame(s)

def generate_clean_df(all_files):
    cleaned_files = []
    
    for file in tqdm(all_files):
        features = [
            file['paper_id'],
            file['metadata']['title'],
            format_authors(file['metadata']['authors']),
            format_authors(file['metadata']['authors'], 
                           with_affiliation=True),
            format_body(file['abstract']),
            format_body(file['body_text']),
            format_bib(file['bib_entries']),
            file['metadata']['authors'],
            file['bib_entries']
        ]

        cleaned_files.append(features)

    col_names = ['paper_id', 'title', 'authors',
                 'affiliations', 'abstract', 'text', 
                 'bibliography','raw_authors','raw_bibliography']

    clean_df = pd.DataFrame(cleaned_files, columns=col_names)
    clean_df.head()
    
    return clean_df


all_files = []

for filename in filenames:
    filename = biorxiv_dir + filename
    file = json.load(open(filename, 'rb'))
    all_files.append(file)

cleaned_files = []

for file in tqdm(all_files):
    features = [
        file['paper_id'],
        file['metadata']['title'],
        format_authors(file['metadata']['authors']),
        format_authors(file['metadata']['authors'], 
                       with_affiliation=True),
        format_body(file['abstract']),
        format_body(file['body_text']),
        format_bib(file['bib_entries']),
        file['metadata']['authors'],
        file['bib_entries']
    ]
    
    cleaned_files.append(features)

## add some column names and throw into pandas dataframe

col_names = [
    'paper_id', 
    'title', 
    'authors',
    'affiliations', 
    'abstract', 
    'text', 
    'bibliography',
    'raw_authors',
    'raw_bibliography'
]

#biorxiv set to CSV
clean_df = pd.DataFrame(cleaned_files, columns=col_names)
clean_df.to_csv('../data/interim/biorxiv_clean.csv', index=False)

#custom license to CSV
pdf_dir = '../data/interim/custom_license/pdf_json/'
pdf_files = load_files(pdf_dir)
pdf_df = generate_clean_df(pdf_files)
pdf_df.to_csv('../data/interim/custom_license_pdf.csv', index=False)

#commercial license to CSV
pdf_dir = '../data/interim/comm_use_subset/pdf_json/'
pdf_files = load_files(pdf_dir)
pdf_df = generate_clean_df(pdf_files)
pdf_df.to_csv('../data/interim/comm_use_subset.csv', index=False)

#noncommercial license to CSV
pdf_dir = '../data/interim/noncomm_use_subset/pdf_json/'
pdf_files = load_files(pdf_dir)
pdf_df = generate_clean_df(pdf_files)
pdf_df.to_csv('../data/interim/noncomm_use_subset.csv', index=False)<file_sep>from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize, sent_tokenize
import re

## Text Cleaning:
    # Convert everything to lowercase
    # Remove HTML tags
    # Contraction mapping (see above)
    # Remove (‘s)
    # Remove any text inside the parenthesis ( )
    # Eliminate punctuations and special characters
    # Remove stopwords
    # Remove short words
## TODO probably need to break this into lots of sub-functions and some way to call 
## them together or separately...

def text_cleaner(text):
    newString = text.lower()
    newString = BeautifulSoup(newString, "lxml").text
    newString = re.sub(r'\([^)]*\)', '', newString)
    newString = re.sub("[\(\[].*?[\)\]]", "", newString)
    newString = re.sub('"','', newString)
    newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(" ")])    
    newString = re.sub(r"'s\b","",newString)
    newString = re.sub("[^a-zA-Z]", " ", newString) 
    tokens = [w for w in newString.split() if not w in stop_words]
    long_words=[]
    for i in tokens:
        if len(i)>=short_word_length:                  #removing short word
            long_words.append(i)   
    return (" ".join(long_words)).strip()


## Basic Text Cleaner
    # just takes lowercase and
    # filters out non-printable characters

def basic_text_cleaner(text):
    text = text.lower()
    printable = set(string.printable)
    text = "".join(list(filter(lambda x: x in printable, text)))
    return text

<file_sep>import re
import requests
from requests.exceptions import RequestException

import os
import sys
import tarfile

import gzip
import shutil

urls = ['https://ai2-semanticscholar-cord-19.s3-us-west-2.amazonaws.com/2020-04-10/biorxiv_medrxiv.tar.gz',
        'https://ai2-semanticscholar-cord-19.s3-us-west-2.amazonaws.com/2020-04-10/custom_license.tar.gz',
        'https://ai2-semanticscholar-cord-19.s3-us-west-2.amazonaws.com/2020-04-10/comm_use_subset.tar.gz',
        'https://ai2-semanticscholar-cord-19.s3-us-west-2.amazonaws.com/2020-04-10/noncomm_use_subset.tar.gz']

## functions for unzipping the downloaded tar.gz files

def unpack(filename, extractPath):
    total = untar(filename)
    if total:
        args = total, total > 1 and 's were' or ' was'
        sys.stdout.write('Report: %s file%s untared.' % args)
    else:
        filename = os.path.basename(sys.argv[0])
        sys.stdout.write('Usage: %s <file_or_dir> ...' % filename)

def untar(paths):
    total = 0
    for path in paths:
        if os.path.isdir(path):
            try:
                dir_list = os.listdir(path)
            except:
                pass
            else:
                total += untar(os.path.join(path, new) for new in dir_list)
        elif os.path.isfile(path):
            try:
                tarfile.open(path).extractall("data/interim/")
            except:
                pass
            else:
                total += 1
    return total


# try to load each dataset from source link
# if URL header has a title for the data, use that as filename
# if not, parse URL for filename

for url in urls:
    print('Attempting to connect to ', url)
    try:
        with requests.get(url) as r:
            
            print('Connection successful.  Downloading content...')
            fname = ''
            if "Content-Disposition" in r.headers.keys():
                fname = re.findall("filename=(.+)", r.headers["Content-Disposition"])[0]
            else:
                fname = url.split("/")[-1]

            print('Content Accessed: ', fname)
            path = "data/raw/" + fname
            extractPath = "data/interim" + fname
            print('Saving ', fname, " to location: ", path)
            with open(path, 'wb') as fd:
                for chunk in r.iter_content(chunk_size=128):
                    fd.write(chunk)
            print("File download complete.", "\n")
            
            print("Extracting file...", "\n")
            #fname = fname.split(".")[0]
            print(fname)
            unpack(fname, extractPath)
            print("Extraction for ", fname, " complete.", "\n")
    except RequestException as e:
        print('Error: ', e, 'with URL: ', url)

print('\n', "All files finished downloading & extracting. Please check /data/raw.")





<file_sep>clean:
	rm -rf data/raw/*
	touch -f data/raw/.gitkeep
	rm raw_files.txt

raw_files.txt:
	python src/download_data.py
	touch raw_files.txt
<file_sep>from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize, sent_tokenize
import re

## Pre-configured word embedding -- GLOVE (Stanford NLP)

def loadEmbeddings(filename):
    vocab2embd = {}

    with open(filename) as infile:
        for line in infile:
            row = line.strip().split(' ')
            word = row[0].lower()
            # print(word)
            if word not in vocab2embd:
                vec = np.asarray(row[1:], np.float32)
                if len(vec) == 100:
                    vocab2embd[word] = vec

    print('Embedding Loaded.')
        
    return vocab2embd