text_plain_output_1.png image_output_1.png
import pandas as pd

meta = pd.read_csv('/kaggle/input/CORD-19-research-challenge/metadata.csv')

meta_dropped = meta.drop(['Microsoft Academic Paper ID', 'WHO #Covidence'], axis=1)

miss = meta['abstract'].isna().sum()

abstracts_papers = meta[meta['abstract'].notna()]
missing_doi = abstracts_papers['doi'].isna().sum()
missing_url = abstracts_papers['url'].isna().sum()

abstracts_papers = abstracts_papers[abstracts_papers['publish_time'].notna()]
abstracts_papers['year'] = pd.DatetimeIndex(abstracts_papers['publish_time']).year

missing_url_data = abstracts_papers[abstracts_papers['url'].notna()]
print('The total number of papers with abstracts but missing url and missing doi = {:.0f}'.format(missing_url_data.doi.isna().sum()))