text_html_output_1.png
from collections import  Counter

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer,PorterStemmer

from sklearn.cluster import DBSCAN

from sklearn.metrics.pairwise import cosine_similarity

from tqdm import tqdm

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import spacy

path = '../input/CORD-19-research-challenge/'
all_sources = pd.read_csv(path + 'metadata.csv')

all_sources.isna().sum()

headline_length = all_sources['title'].str.len()

headline_length = all_sources['abstract'].str.len()

stop = set(stopwords.words('english'))
def build_list(df, col='title'):
    corpus = []
    lem = WordNetLemmatizer()
    stop = set(stopwords.words('english'))
    new = df[col].dropna().str.split()
    new = new.values.tolist()
    corpus = [lem.lemmatize(word.lower()) for i in new for word in i if word not in stop]
    return corpus

corpus = build_list(all_sources)
counter = Counter(corpus)
most = counter.most_common()
x = []
y = []
for word, count in most[:10]:
    if word not in stop:
        x.append(word)
        y.append(count)

corpus = build_list(all_sources, 'abstract')
counter = Counter(corpus)
most = counter.most_common()
x = []
y = []
for word, count in most[:10]:
    if word not in stop:
        x.append(word)
        y.append(count)

def prepare_similarity(vectors):
    similarity = cosine_similarity(vectors)
    return similarity
def get_top_similar(sentence, sentence_list, similarity_matrix, topN):
    index = sentence_list.index(sentence)
    similarity_row = np.array(similarity_matrix[index, :])
    indices = similarity_row.argsort()[-topN:][::-1]
    return [(i, sentence_list[i]) for i in indices]

nlp = spacy.load('en_core_web_sm')
sent_vecs = {}
docs = []
for i in tqdm(all_sources['title'].fillna('unknown')[:1000]):
    doc = nlp(i)
    docs.append(doc)
    sent_vecs.update({i: doc.vector})

sentences = list(sent_vecs.keys())
vectors = list(sent_vecs.values())

x = np.array(vectors)
dbscan = DBSCAN(eps=0.08, min_samples=2, metric='cosine').fit(x)
df_cluster = pd.DataFrame({'sentences': sentences, 'label': dbscan.labels_})

path = '../input/cord-19-eda-parse-json-and-generate-clean-csv/'

clean_comm = pd.read_csv(path + 'clean_comm_use.csv', nrows=5000)
clean_comm['source'] = 'clean_comm'
biox = pd.read_csv(path + 'biorxiv_clean.csv')
biox['source'] = 'biorx'
all_articles = pd.concat([biox, clean_comm])

all_articles.shape