text_html_output_10.png text_html_output_16.png text_html_output_4.png text_html_output_6.png text_html_output_2.png text_html_output_15.png text_html_output_5.png image_output_5.png text_html_output_14.png text_html_output_19.png image_output_7.png text_html_output_9.png text_html_output_13.png text_html_output_20.png image_output_4.png text_html_output_1.png text_html_output_17.png text_html_output_18.png image_output_6.png text_html_output_12.png text_html_output_11.png image_output_3.png image_output_2.png image_output_1.png text_html_output_8.png text_html_output_3.png text_html_output_7.png
from IPython.core.display import display, HTML

from nltk.tokenize import sent_tokenize

from nltk.tokenize import sent_tokenize

from wordcloud import WordCloud, STOPWORDS

import json

import json

import json

import matplotlib.pyplot as plt

import numpy as np

import numpy as np

import numpy as np

import os

import os

import requests

import time

import time

import time

import time

import requests
import json
headers = {'accept': 'application/json', 'Content-Type': 'text/plain'}
params = (('annotationTypes', '*'), ('language', 'en'))
def get_json_object(text):
    return requests.post('http://deda1x3026.merckgroup.com:8080/information-discovery/rest/textanalysis/projects/AA-Internal/pipelines/ThemeAnnotator/analyseText', headers=headers, params=params, data=text).json()
def get_json_str(json_obj):
    return json.dumps(json_obj)
def get_pretty_json(json_str):
    return json.dumps(json_str, indent=4)
def get_themes(text):
    json_obj = get_json_object(text)
    json_array = json_obj['annotationDtos']
    return json_array[-1]['themes']

def lst_to_str(word_list):
    return ' '.join(word_list).strip()

import numpy as np
import json
import os
import csv
import time
from nltk.tokenize import sent_tokenize
root = '/kaggle/input/dataset/CORD-19-research-challenge/'
folders = ['biorxiv_medrxiv/biorxiv_medrxiv/', 'comm_use_subset/comm_use_subset/', 'noncomm_use_subset/noncomm_use_subset/', 'custom_license/custom_license/']
def collect_sentences():
    index_in_docs = 0
    num_files_processed = 0
    sentences_np_array = np.empty(100000000, dtype=object)
    start = time.time()
    for folder in folders:
        for filename in os.listdir(root + folder):
            if filename.endswith('.json'):
                input_file_path = root + folder + filename
                with open(input_file_path) as f:
                    data = json.load(f)
                    abstracts = data['abstract']
                    for content in abstracts:
                        abstract_para = content['text']
                        sentences = sent_tokenize(abstract_para)
                        for sentence in sentences:
                            sentences_np_array[index_in_docs] = sentence
                            index_in_docs += 1
                    body_texts = data['body_text']
                    for content in body_texts:
                        body_para = content['text']
                        sentences = sent_tokenize(body_para)
                        for sentence in sentences:
                            sentences_np_array[index_in_docs] = sentence
                            index_in_docs += 1
                num_files_processed += 1
    np.save('sentences.npy', sentences_np_array)

import json
import os
import csv
import time
from nltk.tokenize import sent_tokenize
root = '/kaggle/input/dataset/CORD-19-research-challenge/'
folders = ['biorxiv_medrxiv/biorxiv_medrxiv/', 'comm_use_subset/comm_use_subset/', 'noncomm_use_subset/noncomm_use_subset/', 'custom_license/custom_license/']
def collect_json_docs():
    docs = np.empty(100000000, dtype=np.object)
    index_in_docs = 0
    num_files_processed = 0
    num_docs_collected = 0
    start = time.time()
    for folder in folders:
        for filename in os.listdir(root + folder):
            if filename.endswith('.json'):
                input_file_path = root + folder + filename
                with open(input_file_path) as f:
                    data = json.load(f)
                    paper_title = data['metadata']['title']
                    authors = data['metadata']['authors']
                    authors_names = []
                    for author in authors:
                        first_name = author['first']
                        middle_name = author['middle']
                        last_name = author['last']
                        author_name = first_name + ' ' + lst_to_str(middle_name) + ' ' + last_name
                        authors_names.append(author_name)
                    abstracts = data['abstract']
                    for content in abstracts:
                        abstract_para = content['text']
                        section = content['section']
                        sentences = sent_tokenize(abstract_para)
                        for sentence in sentences:
                            new_doc = {'sentence': sentence, 'section': section, 'paper_title': paper_title, 'authors': authors_names, 'paragraph': abstract_para}
                            docs[index_in_docs] = new_doc
                            index_in_docs += 1
                            num_docs_collected += 1
                    body_texts = data['body_text']
                    for content in body_texts:
                        body_para = content['text']
                        section = content['section']
                        sentences = sent_tokenize(body_para)
                        for sentence in sentences:
                            new_doc = {'sentence': sentence, 'section': section, 'paper_title': paper_title, 'authors': authors_names, 'paragraph': body_para}
                            docs[index_in_docs] = new_doc
                            index_in_docs += 1
                            num_docs_collected += 1
                num_files_processed += 1
    np.save('docs', docs)

import numpy as np
docs = np.load('/kaggle/input/jsondocs/docs.npy', allow_pickle=True)

stopwords = set(STOPWORDS)
for id_index in I[0]:
    doc = docs[id_index]
    html = json2html.convert(doc)
    html = html.replace('<td>', "<td style='text-align:left'>")
    display(HTML(html))
    themes_list = doc['themes']
    final_theme_string = ''
    for theme in themes_list:
        words = theme.replace('-', ' ').split()
        t = '_'.join(words)
        final_theme_string = final_theme_string + ' ' + t
    if doc['themes'] and doc['themes'][0]:
        wordcloud = WordCloud(width=700, height=200, stopwords=stopwords, min_font_size=8, max_font_size=20, background_color='white', prefer_horizontal=1).generate(final_theme_string)
        plt.figure(figsize=(10, 10), linewidth=10, edgecolor='#04253a')
        plt.imshow(wordcloud, interpolation='bilinear')
        plt.axis('off')
        plt.show()
    display(HTML("<hr style='height:3px; color:black'>"))