text_plain_output_1.png
from datetime import datetime

from gensim.models.phrases import Phraser

from pprint import pprint

from sklearn.preprocessing import normalize

from typing import List

import contractions

import ftfy

import gensim.models.keyedvectors as word2vec

import numpy as np

import operator

import os

import pandas as pd

import re

import string

import re
CURRENCIES = {'$': 'USD', 'zł': 'PLN', '£': 'GBP', '¥': 'JPY', '฿': 'THB', '₡': 'CRC', '₦': 'NGN', '₩': 'KRW', '₪': 'ILS', '₫': 'VND', '€': 'EUR', '₱': 'PHP', '₲': 'PYG', '₴': 'UAH', '₹': 'INR'}
RE_NUMBER = re.compile('(?:^|(?<=[^\\w,.]))[+–-]?(([1-9]\\d{0,2}(,\\d{3})+(\\.\\d*)?)|([1-9]\\d{0,2}([ .]\\d{3})+(,\\d*)?)|(\\d*?[.,]\\d+)|\\d+)(?:$|(?=\\b))')
RE_URL = re.compile('((http://www\\.|https://www\\.|http://|https://)?' + '[a-z0-9]+([\\-.][a-z0-9]+)*\\.[a-z]{2,5}(:[0-9]{1,5})?(/.*)?)')
STOP_WORDS = {'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'if', 'in', 'into', 'is', 'it', 'no', 'not', 'of', 'on', 'or', 'such', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will', 'with'}

import string
from typing import List
import ftfy
import contractions
def clean_tokenized_sentence(tokens: List[str], unicode_normalization='NFC', unpack_contractions=False, replace_currency_symbols=False, remove_punct=True, remove_numbers=False, lowercase=True, remove_urls=True, remove_stop_words=True) -> str:
    if remove_stop_words:
        tokens = [token for token in tokens if token not in STOP_WORDS]
    sentence = ' '.join(tokens)
    if unicode_normalization:
        sentence = ftfy.fix_text(sentence, normalization=unicode_normalization)
    if unpack_contractions:
        sentence = contractions.fix(sentence, slang=False)
    if replace_currency_symbols:
        for currency_sign, currency_tok in CURRENCIES.items():
            sentence = sentence.replace(currency_sign, f'{currency_tok} ')
    if remove_urls:
        sentence = RE_URL.sub('_URL_', sentence)
    if remove_punct:
        sentence = sentence.translate(str.maketrans('', '', string.punctuation))
    sentence = re.sub(' +', ' ', sentence)
    if remove_numbers:
        sentence = RE_NUMBER.sub('_NUMBER_', sentence)
    if lowercase:
        sentence = sentence.lower()
    return sentence

sentences_df = pd.read_csv('../input/covid19sentencesmetadata/sentences_with_metadata.csv')

bigram_model = Phraser.load('../input/covid19phrasesmodels/covid_bigram_model_v0.pkl')

trigram_model = Phraser.load('../input/covid19phrasesmodels/covid_trigram_model_v0.pkl')

fasttext_model_dir = '../input/fasttext-no-subwords-trigrams'

num_points = 400
first_line = True
index_to_word = []
with open(os.path.join(fasttext_model_dir, 'word-vectors-100d.txt'), 'r') as f:
    for line_num, line in enumerate(f):
        if first_line:
            dim = int(line.strip().split()[1])
            word_vecs = np.zeros((num_points, dim), dtype=float)
            first_line = False
            continue
        line = line.strip()
        word = line.split()[0]
        vec = word_vecs[line_num - 1]
        for index, vec_val in enumerate(line.split()[1:]):
            vec[index] = float(vec_val)
        index_to_word.append(word)
        if line_num >= num_points:
            break
word_vecs = normalize(word_vecs, copy=False, return_norm=False)

from pprint import pprint
import gensim.models.keyedvectors as word2vec
fasttext_model = word2vec.KeyedVectors.load_word2vec_format(os.path.join(fasttext_model_dir, 'word-vectors-100d.txt'))

def print_most_similar(search_term):
    synonyms = fasttext_model.most_similar(search_term)

def create_articles_metadata_mapping(sentences_df: pd.DataFrame) -> dict:
    sentence_id_to_metadata = {}
    for row_count, row in sentences_df.iterrows():
        sentence_id_to_metadata[row_count] = dict(paper_id=row['paper_id'], cord_uid=row['cord_uid'], source=row['source'], url=row['url'], publish_time=row['publish_time'], authors=row['authors'], section=row['section'], sentence=row['sentence'])
    return sentence_id_to_metadata

sentence_id_to_metadata = create_articles_metadata_mapping(sentences_df)

import operator
from datetime import datetime
class SearchEngine:
    def __init__(self, sentence_id_to_metadata: dict, sentences_df: pd.DataFrame, bigram_model, trigram_model, fasttext_model):
        self.sentence_id_to_metadata = sentence_id_to_metadata
        self.cleaned_sentences = sentences_df['cleaned_sentence'].tolist()
        self.bigram_model = bigram_model
        self.trigram_model = trigram_model
        self.fasttext_model = fasttext_model
    def _get_search_terms(self, keywords, synonyms_threshold):
        cleaned_terms = [clean_tokenized_sentence(keyword.split(' ')) for keyword in keywords]
        cleaned_terms = [term for term in cleaned_terms if term]
        terms_with_bigrams = self.bigram_model[' '.join(cleaned_terms).split(' ')]
        terms_with_trigrams = self.trigram_model[terms_with_bigrams]
        search_terms = [self.fasttext_model.most_similar(token) for token in terms_with_trigrams]
        search_terms = [synonym[0] for synonyms in search_terms for synonym in synonyms if synonym[1] >= synonyms_threshold]
        search_terms = list(terms_with_trigrams) + search_terms
        return search_terms
    def search(self, keywords: List[str], optional_keywords=None, top_n: int=10, synonyms_threshold=0.7, keyword_weight: float=3.0, optional_keyword_weight: float=0.5) -> List[dict]:
        if optional_keywords is None:
            optional_keywords = []
        search_terms = self._get_search_terms(keywords, synonyms_threshold)
        optional_search_terms = self._get_search_terms(optional_keywords, synonyms_threshold) if optional_keywords else []
        date_today = datetime.today()
        indexes = []
        match_counts = []
        days_diffs = []
        for sentence_index, sentence in enumerate(self.cleaned_sentences):
            sentence_tokens = sentence.split(' ')
            sentence_tokens_set = set(sentence_tokens)
            match_count = sum([keyword_weight if keyword in sentence_tokens_set else 0 for keyword in search_terms])
            if match_count > 0:
                indexes.append(sentence_index)
                if optional_search_terms:
                    match_count += sum([optional_keyword_weight if keyword in sentence_tokens_set else 0 for keyword in optional_search_terms])
                match_counts.append(match_count)
                article_date = self.sentence_id_to_metadata[sentence_index]['publish_time']
                if article_date == '2020':
                    article_date = '2020-01-01'
                article_date = datetime.strptime(article_date, '%Y-%m-%d')
                days_diff = (date_today - article_date).days
                days_diffs.append(days_diff)
        match_counts = [float(match_count) / sum(match_counts) for match_count in match_counts]
        days_diffs = [max(days_diffs) - days_diff for days_diff in days_diffs]
        days_diffs = [float(days_diff) / sum(days_diffs) for days_diff in days_diffs]
        index_to_score = {}
        for index, match_count, days_diff in zip(indexes, match_counts, days_diffs):
            index_to_score[index] = 0.7 * match_count + 0.3 * days_diff
        sorted_indexes = sorted(index_to_score.items(), key=operator.itemgetter(1), reverse=True)
        sorted_indexes = [item[0] for item in sorted_indexes]
        sorted_indexes = sorted_indexes[0:min(top_n, len(sorted_indexes))]
        results = []
        for index in sorted_indexes:
            results.append(self.sentence_id_to_metadata[index])
        return results

search_engine = SearchEngine(sentence_id_to_metadata, sentences_df, bigram_model, trigram_model, fasttext_model)

def search(keywords, optional_keywords=None, top_n=10, synonyms_threshold=0.8, only_sentences=False):
    results = search_engine.search(keywords, optional_keywords=optional_keywords, top_n=top_n, synonyms_threshold=synonyms_threshold)

search(keywords=['spillover', 'bats', 'snakes', 'exotic animals', 'seafood'], optional_keywords=['new coronavirus', 'coronavirus', 'covid19'], top_n=3)