text_plain_output_1.png
from gensim.models import KeyedVectors

from gensim.models.keyedvectors import KeyedVectors

from gensim.utils import simple_preprocess

from nltk import download

from nltk import word_tokenize

from nltk import word_tokenize

from nltk.corpus import stopwords

from nltk.corpus import stopwords

from scipy.spatial import distance

from tqdm.notebook import tqdm

from wordcloud import WordCloud

import gc

import matplotlib.pyplot as plt

import matplotlib.pyplot as plt

import numpy as np

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import matplotlib.pyplot as plt
import json
import requests
import io
import gc
import re
import logging
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from tqdm.notebook import tqdm
pd.set_option('display.max_colwidth', -1)
pd.set_option('display.max_rows', 1000)
plt.rcParams['figure.figsize'] = [12, 8]
from nltk import download
download('stopwords')
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
from nltk import word_tokenize
download('punkt')
plt.rcParams['figure.figsize'] = [12, 8]

MAX_LEN = 3000
research = pd.read_csv('/kaggle/input/CORD-19-research-challenge/metadata.csv')
research['title_abstract'] = [str(research.loc[i, 'title']) + ' ' + str(research.loc[i, 'abstract']) for i in research.index]
research['source'] = 'research'
research
news = pd.read_csv('/kaggle/input/covid19-public-media-dataset/covid19_articles.csv')
del news['Unnamed: 0']
news['source'] = 'news'
news['title_abstract'] = [news.loc[i, 'title'] + '. ' + news.loc[i, 'content'][:MAX_LEN - len(news.loc[i, 'title'])] for i in news.index]
news
data = pd.concat([research[['title_abstract', 'source', 'url']], news[['title_abstract', 'source', 'url']]]).rename(columns={'title_abstract': 'title'}).drop_duplicates().reset_index(drop=True)
del research
del news
gc.collect()
data

import gensim
from gensim.models import Word2Vec
from gensim.utils import simple_preprocess
from gensim.models.keyedvectors import KeyedVectors
filepath = '../input/gnewsvector/GoogleNews-vectors-negative300.bin'
from gensim.models import KeyedVectors
wv_from_bin = KeyedVectors.load_word2vec_format(filepath, binary=True)
embeddings_index = {}
for word, vector in zip(wv_from_bin.vocab, wv_from_bin.vectors):
    coefs = np.asarray(vector, dtype='float32')
    embeddings_index[word] = coefs

def preprocess(doc):
    doc = doc.lower()
    doc = word_tokenize(doc)
    doc = [w for w in doc if not w in stop_words]
    doc = [w for w in doc if w.isalpha()]
    return doc
def avg_feature_vector(sentence, model, num_features):
    words = simple_preprocess(sentence)
    feature_vec = np.zeros((num_features,), dtype='float32')
    n_words = 0
    for word in words:
        if word in embeddings_index.keys():
            n_words += 1
            feature_vec = np.add(feature_vec, model[word])
    if n_words > 0:
        feature_vec = np.divide(feature_vec, n_words)
    return feature_vec
from scipy.spatial import distance
def calc_dist_cosine(s1, target, max_dist=0.5):
    ret = []
    for t in tqdm(target):
        tv = avg_feature_vector(t, model=embeddings_index, num_features=300)
        qv = avg_feature_vector(q, model=embeddings_index, num_features=300)
        dist = distance.cosine(tv, qv)
        if dist <= max_dist:
            ret.append([dist, t])
    df = pd.DataFrame(ret, columns=['dist', 'title']).reset_index(drop=True)
    return pd.merge(df, data, on='title', how='left').sort_values(by='dist', ascending=True).reset_index(drop=True)
def calc_dist_wm(s1, target, max_dist=5.0):
    """
    Word mover distance. Slower than cosine similarity.
    https://markroxor.github.io/gensim/static/notebooks/WMD_tutorial.html
    """
    ret = []
    for t in tqdm(target):
        dist = wv_from_bin.wmdistance(preprocess(s1), preprocess(t))
        if dist <= max_dist:
            ret.append([dist, t])
    df = pd.DataFrame(ret, columns=['dist', 'title']).reset_index(drop=True)
    return pd.merge(df, data, on='title', how='left').sort_values(by='dist', ascending=True).reset_index(drop=True)
def calc_dist(s1, target):
    """
    Dist interface
    """
    return calc_dist_cosine(s1, target)

def make_clickable(link):
    return f'<a target="_blank" href="{link}">{link}</a>'

def search(q, out_prefix='result'):
    res = calc_dist(q, data.title)
    res.to_csv(f'result_{out_prefix}.csv', index=False)
    topn = 20
    wc = WordCloud(background_color='white', stopwords=stop_words).generate(' '.join(res.title.values.tolist()[:topn]).lower())
    plt.axis('off')
    return res

q = 'decontamination'
res = search(q, 'decontamination')
res.head(20)[['title', 'url']].style.format({'url': make_clickable})