text_plain_output_1.png
from gensim.parsing.preprocessing import remove_stopwords

from gensim.similarities import Similarity

from gensim.test.utils import datapath, get_tmpfile

from nltk.stem import LancasterStemmer

from nltk.stem import PorterStemmer

from nltk.tokenize import word_tokenize, sent_tokenize

import gensim

import pandas as pd

import string

meta = pd.read_csv('/kaggle/input/CORD-19-research-challenge/metadata.csv')

meta_dropped = meta.drop(['Microsoft Academic Paper ID', 'WHO #Covidence'], axis=1)

miss = meta['abstract'].isna().sum()

abstracts_papers = meta[meta['abstract'].notna()]
missing_doi = abstracts_papers['doi'].isna().sum()
missing_url = abstracts_papers['url'].isna().sum()

abstracts_papers = abstracts_papers[abstracts_papers['publish_time'].notna()]
abstracts_papers['year'] = pd.DatetimeIndex(abstracts_papers['publish_time']).year

abstracts_papers = abstracts_papers[abstracts_papers['url'].notna() | abstracts_papers['doi']]

porter = PorterStemmer()
lancaster = LancasterStemmer()
abstracts_only = abstracts_papers['abstract']
tokenized_abs = []
for abst in abstracts_only:
    tokens_without_stop_words = remove_stopwords(abst)
    tokens_cleaned = sent_tokenize(tokens_without_stop_words)
    words = [porter.stem(w.lower()) for text in tokens_cleaned for w in word_tokenize(text) if w.translate(str.maketrans('', '', string.punctuation)).isalnum()]
    tokenized_abs.append(words)

dictionary = []
dictionary = gensim.corpora.Dictionary(tokenized_abs)
corpus = [dictionary.doc2bow(abstract) for abstract in tokenized_abs]
tf_idf = gensim.models.TfidfModel(corpus)

query = 'COVID-19 (corona) non-pharmaceutical interventions, Methods to control the spread in communities, barriers to compliance and how these vary among different populations'
query_without_stop_words = remove_stopwords(query)
tokens = sent_tokenize(query_without_stop_words)
query_doc = [porter.stem(w.lower()) for text in tokens for w in word_tokenize(text) if w.translate(str.maketrans('', '', string.punctuation)).isalnum()]
query_doc_bow = dictionary.doc2bow(query_doc)
query_doc_tf_idf = tf_idf[query_doc_bow]

index_temp = get_tmpfile('index')
index = Similarity(index_temp, tf_idf[corpus], num_features=len(dictionary))
similarities = index[query_doc_tf_idf]
abstracts_papers['similarity'] = similarities
abstracts_papers = abstracts_papers.sort_values(by='similarity', ascending=False)
abstracts_papers.reset_index(inplace=True)

top20 = abstracts_papers.head(20)
norm_range = top20['year'].max() - top20['year'].min()
top20["similarity"] -= (abs(top20['year'] - top20['year'].max()) / norm_range)*0.1
top20 = top20.sort_values(by ='similarity' , ascending=False)
top20.reset_index(inplace = True)

for paper in range(10):
    print(top20.url[paper])