text_plain_output_1.png
import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
import os
import gc
import tensorflow as tf
import tensorflow.keras.models as M
import tensorflow.keras.layers as L
import riiideducation

INPUT_DIR = '/kaggle/input/riiid-test-answer-prediction/'
TRAIN_FILE = os.path.join(INPUT_DIR, 'train.csv')
TEST_FILE = os.path.join(INPUT_DIR, 'test.csv')
QUES_FILE = os.path.join(INPUT_DIR, 'questions.csv')
LEC_FILE = os.path.join(INPUT_DIR, 'lectures.csv')

tr = pd.read_csv(TRAIN_FILE, usecols=[1, 2, 3, 4, 7, 8, 9], dtype={'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', 'prior_question_had_explanation': 'boolean'})

def ds_to_pickle(ds, ds_file, pkl_file):
    ds.to_pickle(pkl_file)
    del ds
    return pd.read_pickle('tr.pkl')

tr = ds_to_pickle(tr, TRAIN_FILE, 'tr.pkl')

total_num_users = tr.user_id.unique().size
unique_user_ids = list(tr.user_id.unique())

total_num_ques = tr.loc[tr.content_type_id == 0].content_id.unique().size
unique_ques = list(tr.loc[tr.content_type_id == 0].content_id.unique())

num_ques_per_user = pd.DataFrame({'user_id': list(tr.loc[tr.content_type_id == 0].user_id.unique()), 'num_ques_answered': list(tr.loc[tr.content_type_id == 0].user_id.value_counts())})
num_ques_answered = num_ques_per_user.sort_values('num_ques_answered')['num_ques_answered'].to_frame(name='num_ques_answered')

new_num_rows = len(tr_user_ques_gt_100.index)
old_num_rows = len(tr.index)
print('Old rows:', old_num_rows, '\nNew rows:', new_num_rows, '\nReduced to:', new_num_rows * 100 / old_num_rows, '% of original dataset size')
print("That's a 70% reduction, YAY!")