application_vnd.jupyter.stderr_output_1.png
from nltk.corpus import stopwords

from tqdm import tqdm

import re

import spacy

stop_words = stopwords.words('english')
cord_stopwords = ['doi', 'preprint', 'copyright', 'peer', 'reviewed', 'org', 'https', 'et', 'al', 'author', 'figure', 'rights', 'reserved', 'permission', 'used', 'using', 'biorxiv', 'medrxiv', 'license', 'fig', 'fig.', 'al.', 'Elsevier', 'PMC', 'CZI', '-PRON-', 'abstract']
for word in tqdm(cord_stopwords):
    if word not in stop_words:
        stop_words.append(word)
    else:
        continue

nlp_lg = spacy.load('en_core_sci_lg', disable=['tagger', 'parser', 'ner'])
nlp_lg.max_length = 2000000

for w in tqdm(stop_words):
    nlp_lg.vocab[w].is_stop = True

def removeParenthesesNumbers(v):
    char_list_rm = ['[(]', '[)]', '[′·]']
    char_list_rm_spc = [' no[nt]-', ' non', ' low-', ' high-']
    v = re.sub('|'.join(char_list_rm), '', v)
    v = re.sub('|'.join(char_list_rm_spc), ' ', v)
    return v

sentence_test = '($2196.8)/case (in)fidelity μg μg/ml a=b2 www.website.org α-gal 2-len a.'
def spacy_tokenizer(sentence):
    sentence = removeParenthesesNumbers(sentence)
    token_rm = ['(www.\\S+)', '(-[1-9.])', '([∼≈≥≤≦⩾⩽→μ]\\S+)', '(\\S+=\\S+)', '(http\\S+)']
    tokenized_list = [word.lemma_ for word in nlp_lg(sentence) if not (word.like_num or word.is_stop or word.is_punct or word.is_space)]
    tokenized_list = [word for word in tokenized_list if not re.search('|'.join(token_rm), word)]
    tokenized_list = [word for word in tokenized_list if len(re.findall('[a-zA-Z]', word)) > 1]
    tokenized_list = [word for word in tokenized_list if re.search('^[a-zA-Z0-9]', word)]
    return tokenized_list
spacy_tokenizer(sentence_test)