application_vnd.jupyter.stderr_output_1.png
from nltk.corpus import stopwords

from sklearn.feature_extraction.text import CountVectorizer

from tqdm import tqdm

import gensim

import json

import nltk

import numpy as np

import os

import pandas as pd

import pickle

import re

import spacy

import warnings

import os
import pandas as pd
pd.set_option('max_colwidth', 1000)
pd.set_option('max_rows', 100)
import numpy as np
import pickle
import matplotlib.pyplot as plt
from datetime import datetime
import re
import json
from tqdm import tqdm
import textwrap
import importlib as imp
from scipy.spatial.distance import cdist
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
import nltk
nltk.download('stopwords')
import scispacy
import spacy
from sklearn.feature_extraction.text import CountVectorizer
import gensim
from gensim import corpora, models
from gensim.models.coherencemodel import CoherenceModel
import fasttext
import pyLDAvis
import pyLDAvis.gensim
from wordcloud import WordCloud
from IPython.display import display
import ipywidgets as widgets
import warnings
warnings.filterwarnings('ignore')
os.getcwd()

source_column = 'text'
id_colname = 'cord_uid'
split_sentence_by = '(?<=\\.) ?(?![0-9a-z])'
cov_earliest_date = '2019-12-01'
cov_key_terms = ['covid\\W19', 'covid19', 'covid', '2019\\Wncov', '2019ncov', 'ncov\\W2019', 'sars\\Wcov\\W2', 'sars\\Wcov2', '新型冠状病毒']
cov_related_terms = '(novel|new)( beta| )coronavirus'
input_data_path = '/kaggle/input/CORD-19-research-challenge/'
working_data_path = '/kaggle/input/cov19-pickles/'

metadata = pd.read_csv(input_data_path + 'metadata.csv', encoding='utf-8').replace({pd.np.nan: None})
metadata.isnull().sum(axis=0)

def pdf_or_pmc(r):
    if r.has_pdf_parse:
        return 'pdf_json'
    if r.has_pmc_xml_parse:
        return 'pmc_json'
    return ''
metadata['sha_arr'] = metadata.apply(lambda r: r.sha.split(';') if r.sha is not None else [], axis=1)
metadata['full_text_file_path'] = metadata.apply(lambda r: np.unique(['/'.join([r.full_text_file, r.full_text_file, pdf_or_pmc(r), sha.strip()]) if r.has_pdf_parse or r.has_pmc_xml_parse else '' for sha in r.sha_arr]) if len(r.sha_arr) > 0 else [], axis=1)
metadata['publish_time'] = metadata['publish_time'].str.replace(' ([a-zA-Z]{3}-[a-zA-Z]{3})|(Spring)|(Summer)|(Autumn)|(Fall)|(Winter)', '', regex=True).str.strip()
metadata['publish_time_'] = pd.to_datetime(metadata.publish_time, format='%Y-%m-%d', errors='coerce')
mask = metadata.publish_time_.isnull()
metadata.loc[mask, 'publish_time_'] = pd.to_datetime(metadata.publish_time, format='%Y %B', errors='coerce')
mask = metadata.publish_time_.isnull()
metadata.loc[mask, 'publish_time_'] = pd.to_datetime(metadata.publish_time, format='%Y %b', errors='coerce')
mask = metadata.publish_time_.isnull()
metadata.loc[mask, 'publish_time_'] = pd.to_datetime(metadata.publish_time, format='%Y %B %d', errors='coerce')
mask = metadata.publish_time_.isnull()
metadata.loc[mask, 'publish_time_'] = pd.to_datetime(metadata.publish_time, format='%Y %b %d', errors='coerce')
mask = metadata.publish_time_.isnull()
metadata.loc[mask, 'publish_time_'] = pd.to_datetime(metadata.publish_time, format='%Y', errors='coerce')
mask = metadata.publish_time_.isnull()
invalid_dates = metadata.loc[mask, :].shape[0]
metadata.publish_time = metadata.publish_time_
metadata.drop(['publish_time_'], inplace=True, axis=1)

mask = metadata['full_text_file_path'].apply(lambda r: len(r) > 1)

def get_paper_info(json_data):
    return ' '.join([t['text'] for t in json_data['body_text']])
full_text = []
for r in tqdm(metadata.to_dict(orient='records')):
    record = []
    for p in r['full_text_file_path']:
        with open(input_data_path + p + '.json', 'r', encoding='utf-8') as f:
            data = json.load(f)
            record.append(get_paper_info(data))
    full_text_ = '\n'.join(np.unique(record)) if len(record) > 0 else None
    full_text.append(full_text_)
metadata['full_text'] = full_text
meta_full_text = metadata
meta_full_text[source_column] = np.where(meta_full_text['full_text'].isnull(), meta_full_text['abstract'], meta_full_text['full_text'])
meta_full_text = meta_full_text.dropna(subset=[source_column]).reset_index(drop=True)
meta_full_text = meta_full_text.sort_values('publish_time', ascending=False).drop_duplicates(source_column)
meta_full_text = meta_full_text.sort_values('publish_time', ascending=False).drop_duplicates(id_colname)
meta_full_text.drop(['sha', 'pmcid', 'pubmed_id', 'Microsoft Academic Paper ID', 'has_pdf_parse', 'has_pmc_xml_parse', 'full_text_file', 'sha_arr', 'full_text_file_path', 'full_text'], inplace=True, axis=1)
pickle.dump(meta_full_text, open(working_data_path + 'all_papers.pkl', 'wb'))

meta_full_text = pickle.load(open(working_data_path + 'all_papers.pkl', 'rb'))

corpus = meta_full_text[source_column]

stop_words = stopwords.words('english')
cord_stopwords = ['doi', 'preprint', 'copyright', 'peer', 'reviewed', 'org', 'https', 'et', 'al', 'author', 'figure', 'rights', 'reserved', 'permission', 'used', 'using', 'biorxiv', 'medrxiv', 'license', 'fig', 'fig.', 'al.', 'Elsevier', 'PMC', 'CZI', '-PRON-', 'abstract']
for word in tqdm(cord_stopwords):
    if word not in stop_words:
        stop_words.append(word)
    else:
        continue

nlp_lg = spacy.load('en_core_sci_lg', disable=['tagger', 'parser', 'ner'])
nlp_lg.max_length = 2000000

for w in tqdm(stop_words):
    nlp_lg.vocab[w].is_stop = True

def removeParenthesesNumbers(v):
    char_list_rm = ['[(]', '[)]', '[′·]']
    char_list_rm_spc = [' no[nt]-', ' non', ' low-', ' high-']
    v = re.sub('|'.join(char_list_rm), '', v)
    v = re.sub('|'.join(char_list_rm_spc), ' ', v)
    return v

sentence_test = '($2196.8)/case (in)fidelity μg μg/ml a=b2 www.website.org α-gal 2-len a.'
def spacy_tokenizer(sentence):
    sentence = removeParenthesesNumbers(sentence)
    token_rm = ['(www.\\S+)', '(-[1-9.])', '([∼≈≥≤≦⩾⩽→μ]\\S+)', '(\\S+=\\S+)', '(http\\S+)']
    tokenized_list = [word.lemma_ for word in nlp_lg(sentence) if not (word.like_num or word.is_stop or word.is_punct or word.is_space)]
    tokenized_list = [word for word in tokenized_list if not re.search('|'.join(token_rm), word)]
    tokenized_list = [word for word in tokenized_list if len(re.findall('[a-zA-Z]', word)) > 1]
    tokenized_list = [word for word in tokenized_list if re.search('^[a-zA-Z0-9]', word)]
    return tokenized_list
spacy_tokenizer(sentence_test)

vec = CountVectorizer(max_df=0.8, min_df=0.001, tokenizer=spacy_tokenizer)
X = vec.fit_transform(tqdm(corpus))
valid_tokens = vec.get_feature_names()

X = pickle.load(open(working_data_path + 'TM_X.pkl', 'rb'))
valid_tokens = pickle.load(open(working_data_path + 'TM_valid_tokens.pkl', 'rb'))

arr = X.toarray()
texts = []
for i in tqdm(range(arr.shape[0])):
    text = []
    for j in range(arr.shape[1]):
        occurrence = arr[i, j]
        if occurrence > 0:
            text.extend([valid_tokens[j]] * occurrence)
    texts.append(text)

texts = pickle.load(open(working_data_path + 'TM_texts.pkl', 'rb'))

np.random.seed(1)
dictionary = gensim.corpora.Dictionary(texts)
count = 0
for k, v in dictionary.iteritems():
    print(k, v)
    count += 1
    if count > 10:
        break