text_html_output_1.png
from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from nltk.tokenize import word_tokenize

import nltk

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import string

root_path = '/kaggle/input/CORD-19-research-challenge/'
metadata_path = root_path + 'metadata.csv'
metadata_df = pd.read_csv(metadata_path, dtype={'pubmed_id': str, 'Microsoft Academic Paper ID': str, 'doi': str})
metadata_df = metadata_df.fillna(0)

metadata_df_wt_abs = metadata_df[metadata_df['abstract'] != 0]
metadata_df_wt_abs.shape

lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english') + [i for i in string.punctuation] + ['may', 'also', 'used'])
key_words = []
for word in metadata_df_wt_abs['abstract']:
    temp = word_tokenize(word.lower())
    for txt in temp:
        if txt not in stop_words:
            key_words.append(txt)

freq_all = nltk.FreqDist(key_words)
freq_all.plot(25, cumulative=False)