text_html_output_1.png text_plain_output_1.png
import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

raw_behaviour = pd.read_csv('/kaggle/input/mind-news-dataset/MINDsmall_train/behaviors.tsv', sep='\t', names=['impressionId', 'userId', 'timestamp', 'click_history', 'impressions'])

news = pd.read_csv('/kaggle/input/mind-news-dataset/MINDsmall_train/news.tsv', sep='\t', names=['itemId', 'category', 'subcategory', 'title', 'abstract', 'url', 'title_entities', 'abstract_entities'])

def process_impression(impression_list):
    list_of_strings = impression_list.split()
    click = [x.split('-')[0] for x in list_of_strings if x.split('-')[1] == '1']
    non_click = [x.split('-')[0] for x in list_of_strings if x.split('-')[1] == '0']
    return (click, non_click)
raw_behaviour['click'], raw_behaviour['noclicks'] = zip(*raw_behaviour['impressions'].map(process_impression))

raw_behaviour['epochhrs'] = pd.to_datetime(raw_behaviour['timestamp']).values.astype(np.int64) / 1000000.0 / 1000 / 3600
raw_behaviour['epochhrs'] = raw_behaviour['epochhrs'].round()

raw_behaviour = raw_behaviour.explode('click').reset_index(drop=True)
click_history = raw_behaviour[['userId', 'click_history']].drop_duplicates().dropna()
click_history['click_history'] = click_history.click_history.map(lambda x: x.split())
click_history = click_history.explode('click_history').rename(columns={'click_history': 'click'})
click_history['epochhrs'] = raw_behaviour.epochhrs.min()
click_history['noclicks'] = pd.Series([[] for _ in range(len(click_history.index))])
raw_behaviour = pd.concat([raw_behaviour, click_history], axis=0).reset_index(drop=True)

min_click_cutoff = 100
print(f'Number of items that have less than {min_click_cutoff} clicks make up', np.round(np.mean(raw_behaviour.groupby('click').size() < min_click_cutoff) * 100, 3), '% of the total, and these will be removed.')