text_plain_output_1.png
import pandas as pd

train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=10 ** 6, dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8', 'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', 'prior_question_had_explanation': 'boolean'})

question = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')

lecture = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')

questions_interactions = train_df.merge(question, left_on='content_id', right_on='question_id', how='left')
questions_interactions = questions_interactions[questions_interactions.content_type_id == 0]
questions_interactions.rename(columns={'part': 'test_part'}, inplace=True)
lectures_interactions = train_df.merge(lecture, left_on='content_id', right_on='lecture_id', how='left')
lectures_interactions.rename(columns={'part': 'category'}, inplace=True)
lectures_interactions = lectures_interactions[lectures_interactions.content_type_id == 1]
(questions_interactions.shape, lectures_interactions.shape)

lectures_interactions.isnull().sum()

indeces = questions_interactions[questions_interactions.prior_question_had_explanation.isnull()].index
values = {'prior_question_elapsed_time': -1, 'prior_question_had_explanation': False}
questions_interactions.fillna(value=values, inplace=True)
questions_interactions.prior_question_had_explanation = questions_interactions.prior_question_had_explanation.astype('int8')
questions_interactions.loc[indeces, 'prior_question_had_explanation'] = -1

questions_interactions.prior_question_had_explanation.value_counts()

lectures_interactions.drop(columns=['content_type_id', 'user_answer', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation'], inplace=True)

start_mem_usg1 = questions_interactions.memory_usage().sum() / 1024 ** 2
start_mem_usg2 = lectures_interactions.memory_usage().sum() / 1024 ** 2

questions_interactions.dtypes