text_plain_output_1.png
from time import time

import inverness

import pandas as pd

import re

import inverness
model = inverness.Model('/kaggle/input/cord-19-inverness-all-v7/').load(['fun', 'meta', 'phraser', 'dictionary', 'tfidf', 'lsi', 'dense_ann'])

pd.read_csv('/kaggle/input/CORD-19-research-challenge/metadata.csv', nrows=3)

meta_by_sha = {}
meta_by_pmc = {}
t0 = time()
COLS = ['cord_uid', 'sha', 'pmcid', 'publish_time', 'journal', 'url', 'title', 'authors']
df = pd.read_csv('/kaggle/input/CORD-19-research-challenge/metadata.csv')
selected = df[df['full_text_file'] != ''][COLS]
rows = selected.iterrows()
for _, r in rows:
    if type(r['sha']) is str:
        for sha in r['sha'].split(';'):
            sha = sha.strip()
            meta = {k: r[k] for k in COLS}
            meta_by_sha[sha] = meta
    if type(r['pmcid']) is str:
        pmc = r['pmcid']
        meta = {k: r[k] for k in COLS}
        meta_by_pmc[pmc] = meta

def score_text(text, criteria):
    """"""
    total = 0
    value = 1
    for c in criteria:
        if type(c) in (int, float):
            value = c
        else:
            c = c.replace('_', '\\b')
            matches = re.findall(c, text, re.I)
            cnt = len(matches)
            total += value * cnt_to_score(cnt)
    return total
def cnt_to_score(cnt):
    return min(2, cnt)
def score_results(i_d_lists, criteria):
    """"""
    results = []
    for i, d in zip(*i_d_lists):
        score = 0
        paper_id = model.meta[i]['paper_id']
        if paper_id in meta_by_sha:
            meta = meta_by_sha[paper_id]
        else:
            meta = meta_by_pmc[paper_id]
        score += score_text(meta['title'], criteria)
        doc = model.get_doc(i)
        text = model.doc_to_text(doc).replace('\n', ' ').replace('\r', ' ')
        html = highlight(text, criteria, style_by_group_id, default_style)
        score += score_text(text, criteria)
        rec = (score, d, i, html, meta)
        results += [rec]
    results.sort(key=lambda x: (-x[0], x[1]))
    return results
def score_queries(queries, criteria, K=50):
    """"""
    by_score = []
    for query in queries:
        q = model.text_to_dense(query)
        i_d = model.dense_ann_query(q, K)
        results = score_results(i_d, criteria)
        score = agg_results(results)
        by_score += [(score, query)]
    by_score.sort()
    return by_score
def highlight(text, criteria, styles={}, default='w=bold'):
    """"""
    group_id = 0
    for c in criteria:
        if type(c) in (int, float):
            group_id += 1
        else:
            c = c.replace('_', '\\b')
            c = f'({c}\\w*)'
            style = styles.get(group_id, default)
            style_props = []
            for prop in style.split(','):
                k, _, v = prop.partition('=')
                if k == 'w':
                    style_props += [f'font-weight:{v}']
                if k == 'fg':
                    style_props += [f'color:{v}']
                if k == 'bg':
                    style_props += [f'background-color:{v}']
            before = f'''<span style="{';'.join(style_props)}">'''
            after = '</span>'
            text = re.sub(c, before + '\\1' + after, text, flags=re.I)
    return text
def agg_results(results):
    """"""
    scores = [x[0] for x in results]
    return sum([x * x for x in scores]) ** 0.5
def plot_results(results, title=''):
    """"""
    scores = [x[0] for x in results]
    scores.sort(reverse=True)
    score = agg_results(results)
    plt.figtext(0.4, 1, f'total L2 score: {score:.02f}')

criteria = [100, 'mechanical', 'ventilat', 20, 'adjust', '_age', '_years', '_old', 'elder', 'young', 2, '_surviv', 'discharge', 'extubate', 'alive', 2, 'nonsurviv', 'non-surviv', '_died', 'dead', 'death', 'mortality', 'complication', 10, 'Kaplan.Meier', 'APACHE', 'SOFA', 'RIFLE', 'Glasgow.Coma', 'GCS', 'SAPS', '_RESP_', 'RSBI', '1000.person_', 10, 'figure \\d+', '_fig[.]\\s*\\d+', '_table \\d+', 2, 'outcome', 'result', 'occurr', 'cohort', 'median', '_n\\s*=\\s*\\d+', '(?<=[ (])\\d+ patients', '(?<=[ (])\\d+ cases', 150, 'covid|sars-cov|cov-2|cov2|wuhan']
style_by_group_id = {1: 'w=bold', 2: 'bg=#FFFF00', 3: 'bg=#00FF00', 4: 'bg=#FFAAAA', 5: 'bg=#FFCC00', 6: 'bg=#FFAAFF', 7: 'bg=#00FFFF', 9: 'w=bold,fg=#FF0000'}
default_style = ''

K = 50
queries = ['Outcomes data for COVID-19 after mechanical ventilation adjusted for age', 'Outcomes data for COVID-19 / SARS-CoV-2 after mechanical ventilation adjusted for age', 'Results for COVID-19 / SARS-CoV-2 after mechanical ventilation adjusted for age', 'COVID-19 SARS-CoV-2 results after mechnical ventilation discharged dead died', 'COVID-19 SARS-CoV-2 results after mechnical ventilation discharged dead died survived survivors adjusted age years old', 'COVID-19 SARS-CoV-2 results after mechnical ventilation discharged died survived survivors extubated adjusted', 'COVID-19 SARS-CoV-2 results after mechnical ventilation discharged dead died survived survivors adjusted age years old', 'COVID-19 SARS-CoV-2 results after mechnical ventilation discharged died survived extubated adjusted', 'COVID-19 SARS-CoV-2 results after mechnical ventilation discharged died survived extubated adjusted age', 'COVID-19 SARS-CoV-2 outcomes after mechnical ventilation discharged died survived extubated adjusted', 'COVID-19 SARS-CoV-2 results outcomes after mechnical ventilation discharged died survived extubated adjusted age', 'COVID-19 SARS-CoV-2 results outcomes after mechnical ventilation discharged died survived extubated', 'COVID-19 SARS-CoV-2 results outcomes mechnical ventilation discharged died survived extubated', 'COVID-19 results outcomes after mechnical ventilation discharged died survived extubated adjusted', 'Covid-19 results outcomes after mechnical ventilation discharged died survived extubated adjusted', 'COVID-19 covid-19 SARS-CoV-2 results outcomes after mechnical ventilation discharged died survived extubated adjusted', 'COVID-19 SARS-CoV-2 results outcomes after mechnical ventilation discharged died survived extubated adjusted', 'COVID-19 COVID-2019 SARS-CoV-2 SARS-CoV2 results outcomes after mechnical ventilation discharged died survived extubated adjusted', 'COVID-19 SARS-CoV-2 results outcmes data after mechnical ventilation discharged died survived extubated adjusted age']
for score, query in score_queries(queries, criteria, K):
    print(f'{score:10.02f} -- {query}')