text_plain_output_1.png
from sklearn.metrics.pairwise import cosine_similarity

import ast

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

input_data_path = '/kaggle/input/'
input_data_file = 'filt_merged_text_vector_df_200430.csv'

input_data = pd.read_csv(input_data_path + input_data_file)

rep = {'text': '', 'cite_spans': '', 'ref_spans': '', 'section': '', 'Abstract': '', 'bioRxiv preprint': '', 'medRxiv preprint': '', 'doi:': ''}
rep = dict(((re.escape(k), v) for k, v in rep.items()))
pattern = re.compile('|'.join(rep.keys()))
sentences_temp = [pattern.sub(lambda m: rep[re.escape(m.group(0))], s) for s in input_data.sentence]
pattern = re.compile('.*[A-Za-z].*')
sentences_to_keep = [bool(re.search(pattern, s)) & (len(s.split(' ')) > 2) for s in sentences_temp]
input_processed = input_data.loc[sentences_to_keep, :]
sentences_to_drop = [not i for i in sentences_to_keep]
input_excluded = input_data.loc[sentences_to_drop, :]

input_processed.w2vVector = [re.sub(',+', ',', ','.join(w.replace('\n', '').split(' '))) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub('\\[,', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub(',\\]', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub('\\[', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub('\\]', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = input_processed.w2vVector.apply(lambda s: list(ast.literal_eval(s)))

input_processed.to_csv('cord_titles_abstracts_conclusions.csv')
input_excluded.to_csv('cord_titles_abstracts_conclusions_excluded.csv')

title_data = input_processed.loc[input_processed.section == 'title', :]
abstract_data = input_processed.loc[input_processed.section == 'abstract', :]
conclusion_data = input_processed.loc[(input_processed.section != 'title') & (input_processed.section != 'abstract'), :]

title_data_final = pd.DataFrame(columns=['cord_uid', 'sentence', 'w2vVector'])
for cord_uid in title_data.cord_uid.unique():
    title = ' '.join(title_data.loc[title_data.cord_uid == cord_uid, 'sentence'])
    w2vVector = np.mean(list(title_data.loc[title_data.cord_uid == cord_uid, 'w2vVector']), axis=0)
    title_data_final = title_data_final.append({'cord_uid': cord_uid, 'sentence': title, 'w2vVector': w2vVector}, ignore_index=True)

cosine_similarity(title_data_final.w2vVector[0].reshape(1, -1), title_data_final.w2vVector[1].reshape(1, -1))[0][0]