text_plain_output_4.png text_plain_output_6.png application_vnd.jupyter.stderr_output_3.png application_vnd.jupyter.stderr_output_5.png text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png
from cleantext import clean

from sentence_transformers import SentenceTransformer

from summarizer import Summarizer

from tqdm import tqdm

import faiss

import numpy as np

import os

import pandas as pd

from sentence_transformers import SentenceTransformer
import faiss
from tqdm import tqdm
from summarizer import Summarizer
import scipy.spatial
import pandas as pd
import numpy as np
import re
import json
import time
import os
from cleantext import clean

def filter_text(text):
    text = text.lower()
    text = text.replace(',', ' ').replace('-', '  ').replace('_', ' ').replace('/', ' ').replace('“', ' ').replace('"', ' ').replace("'", ' ').replace(';', ' ').replace(':', ' ')
    text = text.replace(',', ' ').replace('-', ' ').replace('_', ' ').replace('/', ' ').replace('“', ' ').replace('"', ' ').replace("'", ' ')
    text = text.replace('.', ' ').replace(')', ' ').replace('(', ' ').replace('#', ' ').replace('в®', ' ').replace('®', ' ')
    text = text.replace('$', ' ').replace('*', ' ').replace('”', ' ').replace('+', ' ').replace('&', ' ').replace('>', ' ').replace('=', ' ')
    text = text.replace('[', ' ').replace(']', ' ').replace('?', ' ').replace('\\', ' ').replace('<', ' ')
    text = ' '.join([w for w in text.split() if len(w) > 1])
    text = ' '.join(text.lower().split()[:30])
    return text

def getIndex(corpus, ids):
    index = []
    index2 = []
    index = faiss.IndexFlatIP(corpus.shape[1])
    faiss.normalize_L2(corpus)
    index.train(corpus)
    index2 = faiss.IndexIDMap(index)
    index2.add_with_ids(corpus, ids)
    return index2

def getEmbed(train, embedder):
    corpus = train.astype(str).values.tolist()
    df_embeddings = pd.DataFrame(embedder.encode(corpus), index=train.index)
    df_model = pd.concat([df_embeddings, train], axis=1, join='inner').reindex(train.index)
    return df_model

def writeModel(df_model, index_file_prefix, info_file_prefix):
    corpus_embeddings = np.ascontiguousarray(df_model.filter(regex='\\d+', axis=1)).astype('float32')
    index = getIndex(corpus_embeddings, df_model.index.values)
    faiss.write_index(index, index_file_prefix + df_model.columns[-1])
    df_model.iloc[:, -1].to_json(info_file_prefix + df_model.columns[-1] + '.json')
    return

def writeModelBody(df_model, df_info, index_file_prefix, info_file_prefix):
    corpus_embeddings = np.ascontiguousarray(df_model.filter(regex='\\d+', axis=1)).astype('float32')
    index = getIndex(corpus_embeddings, df_model.index.values)
    faiss.write_index(index, index_file_prefix + df_model.columns[-1])
    df_info.to_json(info_file_prefix + df_model.columns[-1] + '.json')
    return

def prepTitle(Tdata):
    data = Tdata.dropna(subset=['title'])
    data['title'] = data['title'].apply(lambda x: clean(x, no_urls=True, no_emails=True, no_phone_numbers=True))
    data['title'] = data['title'].apply(filter_text)
    data = data.drop_duplicates(subset=['title'])
    return data.title

def prepAbstract(Adata):
    data = Adata.dropna(subset=['abstract'])
    data['abstract'] = data['abstract'].apply(lambda x: clean(x, no_urls=True, no_emails=True, no_phone_numbers=True))
    data['abstract'] = data['abstract'].apply(filter_text)
    data = data.drop_duplicates(subset=['abstract'])
    return data.abstract

def prepBody(Adata, embedder):
    model = Summarizer(model='distilbert-base-uncased')
    tests = pd.DataFrame()
    data = Adata.dropna(subset=['text'])
    for index, row in tqdm(data.iterrows(), total=data.shape[0]):
        ggg = row.text.strip('][').split(', ')
        ct = [len(i.split()) for i in ggg]
        kkk = row.key.strip('][').split(', ')
        kkkk = [int(x.strip("'")) for x in kkk]
        hh = [row.name] * len(kkkk)
        data1 = pd.DataFrame({'text': ggg, 'paraIndex': kkkk, 'paperIndex': hh, 'paraCount': ct})
        tests = tests.append(data1)
    tests = tests.reset_index(drop=True)
    sub1 = tests
    sub1_a = sub1.iloc[250000:275000, :]
    sub = sub1_a
    tqdm.pandas()
    sub['body'] = sub['text'].progress_apply(lambda x: model(clean(x, no_urls=True, no_emails=True, no_phone_numbers=True)))
    df_embed_abs = getEmbed(sub['body'], embedder)
    sub.to_json('body_summary_11.json')
    df_embed_abs.to_json('embed_summary_11.json')
    return sub

def prepQuery(data):
    cdata = clean(data, no_urls=True, no_emails=True, no_phone_numbers=True)
    fdata = filter_text(cdata)
    return fdata

def loadModel(index_path, info_path):
    if os.path.exists(index_path):
        index = faiss.read_index(index_path)
    if os.path.exists(info_path):
        info = pd.read_json(info_path, typ='series')
    return (index, info)

def getLocs(D, I, thresh):
    ii = I[np.where(D > thresh)]
    if len(ii) == 0:
        jj = I[np.where(D > thresh - 0.1)]
    elif len(ii) == D.shape[1]:
        jj = I[np.where(D > thresh + 0.05)]
        if len(jj) == 0:
            jj = ii
    else:
        jj = ii
    return jj

def getRanks(index, info, query, top, thresh):
    faiss.normalize_L2(query)
    D, I = index.search(query, top)
    inds = getLocs(D, I, thresh)
    subset = info.loc[inds]
    return subset

def predict(data_in, embedder, thresh, n_closest, index_file_path, info_file_path):
    index, info = loadModel(index_file_path, info_file_path)
    query = prepQuery(data_in)
    query_embedding = np.ascontiguousarray(embedder.encode([query])).astype('float32')
    ranks = getRanks(index, info, query_embedding, n_closest, thresh)
    return ranks

embedder = SentenceTransformer('/kaggle/input/ethics-transformer/bert-base-ethics_training/')
in_file = '/kaggle/input/covid-processed-data/combined_data3.csv'
index_file_prefix = '/kaggle/working/corpus_index_'
info_file_prefix = '/kaggle/working/corpus_info_'
thresh = 0.6
topRank = 20
isTrainAll = False
isTrainTitle = False
isTrainAbstract = False
isTrainBody = True
if isTrainAll:
    isTrainTitle = True
    isTrainAbstract = True
    isTrainBody = True
isInferAll = False
isInferTitle = False
isInferAbstract = False
isInferBody = False
if isInferAll:
    isInferTitle = True
    isInferAbstract = True
    isInferBody = True

data = pd.read_csv(in_file)
data = data.replace('nan', np.nan, regex=False)

if isTrainBody:
    df_train_body = prepBody(data, embedder)
    print('Body Corpus len:', len(df_train_body))