application_vnd.jupyter.stderr_output_1.png
import ast

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

input_data_path = '/kaggle/input/'
input_data_file = 'filt_merged_text_vector_df_200430.csv'

input_data = pd.read_csv(input_data_path + input_data_file)

rep = {'text': '', 'cite_spans': '', 'ref_spans': '', 'section': '', 'Abstract': '', 'bioRxiv preprint': '', 'medRxiv preprint': '', 'doi:': ''}
rep = dict(((re.escape(k), v) for k, v in rep.items()))
pattern = re.compile('|'.join(rep.keys()))
sentences_temp = [pattern.sub(lambda m: rep[re.escape(m.group(0))], s) for s in input_data.sentence]
pattern = re.compile('.*[A-Za-z].*')
sentences_to_keep = [bool(re.search(pattern, s)) & (len(s.split(' ')) > 2) for s in sentences_temp]
input_processed = input_data.loc[sentences_to_keep, :]
sentences_to_drop = [not i for i in sentences_to_keep]
input_excluded = input_data.loc[sentences_to_drop, :]

input_processed.w2vVector = [re.sub(',+', ',', ','.join(w.replace('\n', '').split(' '))) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub('\\[,', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub(',\\]', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub('\\[', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = [re.sub('\\]', '', w) for w in input_processed.w2vVector]
input_processed.w2vVector = input_processed.w2vVector.apply(lambda s: list(ast.literal_eval(s)))

input_processed.to_csv('cord_titles_abstracts_conclusions.csv')
input_excluded.to_csv('cord_titles_abstracts_conclusions_excluded.csv')

input_processed