text_plain_output_1.png
from gensim.models.phrases import Phraser

from pprint import pprint

from sklearn.preprocessing import normalize

import gensim.models.keyedvectors as word2vec

import numpy as np

import os

import pandas as pd

sentences_df = pd.read_csv('../input/covid19sentencesmetadata/sentences_with_metadata.csv')

bigram_model = Phraser.load('../input/covid19phrasesmodels/covid_bigram_model_v0.pkl')

trigram_model = Phraser.load('../input/covid19phrasesmodels/covid_trigram_model_v0.pkl')

fasttext_model_dir = '../input/fasttext-no-subwords-trigrams'

num_points = 400
first_line = True
index_to_word = []
with open(os.path.join(fasttext_model_dir, 'word-vectors-100d.txt'), 'r') as f:
    for line_num, line in enumerate(f):
        if first_line:
            dim = int(line.strip().split()[1])
            word_vecs = np.zeros((num_points, dim), dtype=float)
            first_line = False
            continue
        line = line.strip()
        word = line.split()[0]
        vec = word_vecs[line_num - 1]
        for index, vec_val in enumerate(line.split()[1:]):
            vec[index] = float(vec_val)
        index_to_word.append(word)
        if line_num >= num_points:
            break
word_vecs = normalize(word_vecs, copy=False, return_norm=False)

from pprint import pprint
import gensim.models.keyedvectors as word2vec
fasttext_model = word2vec.KeyedVectors.load_word2vec_format(os.path.join(fasttext_model_dir, 'word-vectors-100d.txt'))

def print_most_similar(search_term):
    synonyms = fasttext_model.most_similar(search_term)

def create_articles_metadata_mapping(sentences_df: pd.DataFrame) -> dict:
    sentence_id_to_metadata = {}
    for row_count, row in sentences_df.iterrows():
        sentence_id_to_metadata[row_count] = dict(paper_id=row['paper_id'], cord_uid=row['cord_uid'], source=row['source'], url=row['url'], publish_time=row['publish_time'], authors=row['authors'], section=row['section'], sentence=row['sentence'])
    return sentence_id_to_metadata

sentence_id_to_metadata = create_articles_metadata_mapping(sentences_df)

search_engine = SearchEngine(sentence_id_to_metadata, sentences_df, bigram_model, trigram_model, fasttext_model)