text_plain_output_1.png image_output_1.png
from langdetect import detect

from nltk.corpus import stopwords

from nltk.stem import PorterStemmer

from nltk.stem import WordNetLemmatizer

from os import path

from pandarallel import pandarallel

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

from tqdm import tqdm

import contractions

import glob

import json

import nltk

import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

import pickle

import re

import re

import seaborn as sns

import seaborn as sns

import seaborn as sns

import spacy

import numpy as np
import pandas as pd
import glob
import json
import re
import itertools
from tqdm import tqdm
import nltk
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
from nltk import tokenize
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from collections import defaultdict
from nltk.corpus import wordnet as wn
from langdetect import detect
from nltk.corpus import stopwords
import contractions
import inflect
from nltk.stem import PorterStemmer
from nltk.stem import LancasterStemmer
from nltk.stem import WordNetLemmatizer
from pandarallel import pandarallel
import pickle
from PIL import Image
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
import seaborn as sns
from sentence_transformers import SentenceTransformer
from sklearn.cluster import KMeans
from sklearn.metrics.pairwise import cosine_similarity
import re
import bs4
import requests
import spacy
from spacy import displacy
nlp = spacy.load('en_core_web_sm')
from spacy.matcher import Matcher
from spacy.tokens import Span
import networkx as nx
pandarallel.initialize(use_memory_fs=False, nb_workers=8)

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
pd.set_option('display.expand_frame_repr', False)
pd.options.mode.chained_assignment = None
tqdm.pandas()

def filepath(*args):
    if len(args) < 1:
        return None
    elif len(args) == 1:
        return args[0]
    else:
        return f'{args[0]}/{filepath(*args[1:])}'
def addtimebar(L, threshold=1000):
    if len(L) > threshold:
        return tqdm(L)
    else:
        return L
class FileReader:
    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            try:
                for entry in content['abstract']:
                    self.abstract.append(entry['text'])
            except KeyError:
                pass
            try:
                for entry in content['body_text']:
                    self.body_text.append(entry['text'])
            except KeyError:
                pass
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)
    def __repr__(self):
        return f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'
def get_breaks(content, length):
    data = ''
    words = content.split(' ')
    total_chars = 0
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + '<br>' + words[i]
            total_chars = 0
        else:
            data = data + ' ' + words[i]
    return data
def compose(*funcs):
    *funcs, penultimate, last = funcs
    if funcs:
        penultimate = compose(*funcs, penultimate)
    return lambda *args: penultimate(last(*args))

path = '../input/CORD-19-research-challenge'
meta = 'metadata.csv'
all_jsons = glob.glob(filepath(path, '**', '*.json'), recursive=True)

meta_df = pd.read_csv(filepath(path, meta), dtype={'pubmed_id': str, 'Microsoft Academic Paper ID': str, 'doi': str, 'journal': str}, low_memory=False)

dict_ = {'paper_id': [], 'abstract': [], 'body_text': [], 'authors': [], 'title': [], 'publish_time': [], 'journal': [], 'abstract_summary': []}
for entry in all_jsons[:10]:
    content = FileReader(entry)
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    if len(meta_data) == 0:
        continue
    dict_['paper_id'].append(content.paper_id)
    dict_['abstract'].append(content.abstract)
    dict_['body_text'].append(content.body_text)
    if len(content.abstract) == 0:
        dict_['abstract_summary'].append('Not provided.')
    elif len(content.abstract.split(' ')) > 100:
        info = content.abstract.split(' ')[:100]
        summary = get_breaks(' '.join(info), 40)
        dict_['abstract_summary'].append(summary + '...')
    else:
        summary = get_breaks(content.abstract, 40)
        dict_['abstract_summary'].append(summary)
    meta_data = meta_df.loc[meta_df['sha'] == content.paper_id]
    try:
        authors = meta_data['authors'].values[0].split(';')
        if len(authors) > 2:
            dict_['authors'].append('. '.join(authors[:2]) + '...')
        else:
            dict_['authors'].append('. '.join(authors))
    except Exception as e:
        dict_['authors'].append(meta_data['authors'].values[0])
    try:
        title = get_breaks(meta_data['title'].values[0], 40)
        dict_['title'].append(title)
    except Exception as e:
        dict_['title'].append(meta_data['title'].values[0])
    try:
        publish_time = get_breaks(meta_data['publish_time'].values[0], 40)
        dict_['publish_time'].append(publish_time)
    except Exception as e:
        dict_['publish_time'].append(meta_data['publish_time'].values[0])
    dict_['journal'].append(meta_data['journal'].values[0])
df_covid = pd.DataFrame(dict_, columns=['paper_id', 'abstract', 'body_text', 'authors', 'title', 'journal', 'publish_time', 'abstract_summary'])

def is_lang(row, item, lang, dropNA=True):
    if row[item] != None and row[item] != '' and (row[item] != 'None') and isinstance(row[item], str):
        try:
            return detect(row[item]) == lang
        except Exception as e:
            return False
    else:
        return not dropNA
def select_article_lang_multi(df, basedon='abstract', lang='en'):
    return df[df.parallel_apply(lambda text: is_lang(text, basedon, lang), axis=1)]
df_covid_eng = select_article_lang_multi(df_covid)

def replace_brackets_with_whitespace(text):
    text = text.replace('(', '')
    text = text.replace(')', '')
    text = text.replace('[', '')
    text = text.replace(']', '')
    return text
def replace_contractions(text):
    return contractions.fix(text)
def strip_characters(text):
    t = re.sub('\\(|\\)|:|,|;|\\.|’||“|\\?|%|>|<', '', text)
    t = re.sub('/', ' ', t)
    t = t.replace("'", '')
    return t
def to_lowercase(word):
    return word.lower()
def do_stemming(stemmer):
    return lambda word: stemmer.stem(word)
def do_lemmatizing(lemmatizer):
    return lambda word: lemmatizer.lemmatize(word, pos='v')
def is_stopword(word):
    return word in stopwords.words('english')
def process_word_by(word_cleanner, uniqueYN):
    def cond(word):
        return len(word) > 1 and (not is_stopword(word)) and (not word.isnumeric()) and word.isalnum() and (word != len(word) * word[0])
    def clean_byword(text):
        return list(take_unique(uniqueYN)((word_cleanner(word) for word in text if cond(word))))
    return clean_byword
def take_unique(YN):
    return set if YN else lambda x: x
text_processor = compose(replace_brackets_with_whitespace, replace_contractions, strip_characters)
word_processor = compose(to_lowercase, do_lemmatizing(WordNetLemmatizer()), do_stemming(PorterStemmer()))
def pre_processing(df, text_tools, word_tools):
    def inner(col, uniqueYN=False):
        return df[col].parallel_apply(text_tools).parallel_apply(nltk.word_tokenize).parallel_apply(process_word_by(word_tools, uniqueYN=uniqueYN))
    return inner

tokenized_df = df_covid_eng.sort_values(by='publish_time', ascending=False)

processor = pre_processing(tokenized_df, text_processor, word_processor)
tokenized_df['abstract_token'] = processor('abstract')
tokenized_df = tokenized_df.reset_index(drop=True)

tokenized_df.head()['abstract_token']

with open('../data/df_kaggle_all_eng_tokenized.pkl', 'rb') as fp:
    tokenized_df = pickle.load(fp)

def get_top_nK_words(corpus, K=1, n=None):
    vec1 = CountVectorizer(max_df=0.7, stop_words=stopwords.words('english'), ngram_range=(K, K), max_features=2000).fit(corpus)
    bag_of_words = vec1.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in vec1.vocabulary_.items()]
    words_freq = sorted(words_freq, key=lambda x: x[1], reverse=True)
    return words_freq[:n]

#Convert most freq words to dataframe for plotting bar plot
top_words = get_top_nK_words(corpus, K=1, n=20)
top_df = pd.DataFrame(top_words)
top_df.columns=["Word", "Freq"]
#Barplot of most freq words
sns.set(rc={'figure.figsize':(13,8)})
g = sns.barplot(x="Word", y="Freq", data=top_df)
g.set_xticklabels(g.get_xticklabels(), rotation=30)

# Top bi-grams
top2_words = get_top_nK_words(corpus, K=2, n=20)
top2_df = pd.DataFrame(top2_words)
top2_df.columns=["Bi-gram", "Freq"]
print(top2_df)
#Barplot of most freq Bi-grams
import seaborn as sns
sns.set(rc={'figure.figsize':(13,8)})
h=sns.barplot(x="Bi-gram", y="Freq", data=top2_df)
h.set_xticklabels(h.get_xticklabels(), rotation=45)
fig = h.get_figure()

top3_words = get_top_nK_words(corpus, K=3, n=20)
top3_df = pd.DataFrame(top3_words)
top3_df.columns=["Tri-gram", "Freq"]
print(top3_df)
#Barplot of most freq Tri-grams
import seaborn as sns
sns.set(rc={'figure.figsize':(13,8)})
j=sns.barplot(x="Tri-gram", y="Freq", data=top3_df)
j.set_xticklabels(j.get_xticklabels(), rotation=45)
fig = j.get_figure()

def tfidf_(df):
    myvectorizer = TfidfVectorizer()
    vectors = myvectorizer.fit_transform(df['abstract_token'].parallel_apply(lambda x: ' '.join(x))).toarray()
    feature_names = myvectorizer.get_feature_names()
    veclist = vectors.tolist()
    out_tfidf = pd.DataFrame(veclist, columns=feature_names)
    return out_tfidf
tfidf_(tokenized_df[:5000]).head()