text_plain_output_1.png
from collections import  Counter

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer,PorterStemmer

from sklearn.cluster import DBSCAN

from sklearn.metrics.pairwise import cosine_similarity

from tqdm import tqdm

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import spacy

path = '../input/CORD-19-research-challenge/'
all_sources = pd.read_csv(path + 'metadata.csv')

all_sources.isna().sum()

headline_length = all_sources['title'].str.len()

headline_length = all_sources['abstract'].str.len()

stop = set(stopwords.words('english'))
def build_list(df, col='title'):
    corpus = []
    lem = WordNetLemmatizer()
    stop = set(stopwords.words('english'))
    new = df[col].dropna().str.split()
    new = new.values.tolist()
    corpus = [lem.lemmatize(word.lower()) for i in new for word in i if word not in stop]
    return corpus

corpus = build_list(all_sources)
counter = Counter(corpus)
most = counter.most_common()
x = []
y = []
for word, count in most[:10]:
    if word not in stop:
        x.append(word)
        y.append(count)

corpus = build_list(all_sources, 'abstract')
counter = Counter(corpus)
most = counter.most_common()
x = []
y = []
for word, count in most[:10]:
    if word not in stop:
        x.append(word)
        y.append(count)

def prepare_similarity(vectors):
    similarity = cosine_similarity(vectors)
    return similarity
def get_top_similar(sentence, sentence_list, similarity_matrix, topN):
    index = sentence_list.index(sentence)
    similarity_row = np.array(similarity_matrix[index, :])
    indices = similarity_row.argsort()[-topN:][::-1]
    return [(i, sentence_list[i]) for i in indices]

nlp = spacy.load('en_core_web_sm')
sent_vecs = {}
docs = []
for i in tqdm(all_sources['title'].fillna('unknown')[:1000]):
    doc = nlp(i)
    docs.append(doc)
    sent_vecs.update({i: doc.vector})

sentences = list(sent_vecs.keys())
vectors = list(sent_vecs.values())

x = np.array(vectors)
dbscan = DBSCAN(eps=0.08, min_samples=2, metric='cosine').fit(x)
df_cluster = pd.DataFrame({'sentences': sentences, 'label': dbscan.labels_})

path = '../input/cord-19-eda-parse-json-and-generate-clean-csv/'

clean_comm = pd.read_csv(path + 'clean_comm_use.csv', nrows=5000)
clean_comm['source'] = 'clean_comm'
biox = pd.read_csv(path + 'biorxiv_clean.csv')
biox['source'] = 'biorx'
all_articles = pd.concat([biox, clean_comm])

tasks = ['What is known about transmission, incubation, and environmental stability', 'What do we know about COVID-19 risk factors', 'What do we know about virus genetics, origin, and evolution', 'What do we know about vaccines and therapeutics', 'What do we know about non-pharmaceutical interventions', 'What do we know about diagnostics and surveillance', 'What has been published about ethical and social science considerations', 'Role of the environment in transmission', 'Range of incubation periods for the disease in humans', 'Prevalence of asymptomatic shedding and transmission', 'Seasonality of transmission', 'Persistence of virus on surfaces of different materials (e,g., copper, stainless steel, plastic)', 'Susceptibility of populations', 'Public health mitigation measures that could be effective for control', 'Transmission dynamics of the virus', 'Evidence that livestock could be infected', 'Socioeconomic and behavioral risk factors for this spill-over', 'Sustainable risk reduction strategies']

task_df = pd.DataFrame({'title': tasks, 'source': 'task'})

task_df.head()