text_plain_output_1.png image_output_1.png
from collections import  Counter

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer,PorterStemmer

from nltk.tokenize import word_tokenize

import gensim

import matplotlib.pyplot as plt

import pandas as pd

import seaborn as sns

path = '../input/CORD-19-research-challenge/'
all_sources = pd.read_csv(path + 'metadata.csv')

all_sources.isna().sum()

headline_length = all_sources['title'].str.len()

headline_length = all_sources['abstract'].str.len()

stop = set(stopwords.words('english'))
def build_list(df, col='title'):
    corpus = []
    lem = WordNetLemmatizer()
    stop = set(stopwords.words('english'))
    new = df[col].dropna().str.split()
    new = new.values.tolist()
    corpus = [lem.lemmatize(word.lower()) for i in new for word in i if word not in stop]
    return corpus

corpus = build_list(all_sources)
counter = Counter(corpus)
most = counter.most_common()
x = []
y = []
for word, count in most[:10]:
    if word not in stop:
        x.append(word)
        y.append(count)

corpus = build_list(all_sources, 'abstract')
counter = Counter(corpus)
most = counter.most_common()
x = []
y = []
for word, count in most[:10]:
    if word not in stop:
        x.append(word)
        y.append(count)

def preprocess_news(df):
    corpus = []
    stem = PorterStemmer()
    lem = WordNetLemmatizer()
    for news in df['title'].dropna()[:5000]:
        words = [w for w in word_tokenize(news) if w not in stop]
        words = [lem.lemmatize(w) for w in words if len(w) > 2]
        corpus.append(words)
    return corpus

corpus = preprocess_news(all_sources)
dic = gensim.corpora.Dictionary(corpus)
bow_corpus = [dic.doc2bow(doc) for doc in corpus]

lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=4, id2word=dic, passes=10, workers=2)

lda_model.show_topics()