text_html_output_1.png
import numpy as np

import pandas as pd

train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=10 ** 6, dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8', 'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', 'prior_question_had_explanation': 'boolean'})

question = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')

lecture = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')

questions_interactions = train_df.merge(question, left_on='content_id', right_on='question_id', how='left')
questions_interactions = questions_interactions[questions_interactions.content_type_id == 0]
questions_interactions.rename(columns={'part': 'test_part'}, inplace=True)
lectures_interactions = train_df.merge(lecture, left_on='content_id', right_on='lecture_id', how='left')
lectures_interactions.rename(columns={'part': 'category'}, inplace=True)
lectures_interactions = lectures_interactions[lectures_interactions.content_type_id == 1]
(questions_interactions.shape, lectures_interactions.shape)

lectures_interactions.isnull().sum()

indeces = questions_interactions[questions_interactions.prior_question_had_explanation.isnull()].index
values = {'prior_question_elapsed_time': -1, 'prior_question_had_explanation': False}
questions_interactions.fillna(value=values, inplace=True)
questions_interactions.prior_question_had_explanation = questions_interactions.prior_question_had_explanation.astype('int8')
questions_interactions.loc[indeces, 'prior_question_had_explanation'] = -1

questions_interactions.prior_question_had_explanation.value_counts()

lectures_interactions.drop(columns=['content_type_id', 'user_answer', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation'], inplace=True)

start_mem_usg1 = questions_interactions.memory_usage().sum() / 1024 ** 2
start_mem_usg2 = lectures_interactions.memory_usage().sum() / 1024 ** 2

questions_interactions.dtypes

questions_interactions.drop(columns=['content_id', 'content_type_id'], inplace=True)

def reduce_mem_usage(props):
    start_mem_usg = props.memory_usage().sum() / 1024**2 
    print("Memory usage of properties dataframe is :",start_mem_usg," MB")
    NAlist = [] # Keeps track of columns that have missing values filled in. 
    for col in props.columns:
        if props[col].dtype != object:  # Exclude strings
            # make variables for Int, max and min
            IsInt = False
            mx = props[col].max()
            mn = props[col].min()
            
            # Integer does not support NA, therefore, NA needs to be filled
            if not np.isfinite(props[col]).all(): 
                NAlist.append(col)
                props[col].fillna(mn-1,inplace=True)  
                   
            # test if column can be converted to an integer
            asint = props[col].fillna(0).astype(np.int64)
            result = (props[col] - asint)
            result = result.sum()
            if result > -0.01 and result < 0.01:
                IsInt = True

            

            
            # Make Integer/unsigned Integer datatypes
            if IsInt:
                if mn >= 0:
                    if mx < 255:
                        props[col] = props[col].astype(np.uint8)
                    elif mx < 65535:
                        props[col] = props[col].astype(np.uint16)
                    elif mx < 4294967295:
                        props[col] = props[col].astype(np.uint32)
                    else:
                        props[col] = props[col].astype(np.uint64)
                else:
                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:
                        props[col] = props[col].astype(np.int8)
                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:
                        props[col] = props[col].astype(np.int16)
                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:
                        props[col] = props[col].astype(np.int32)
                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:
                        props[col] = props[col].astype(np.int64)    
            
            # Make float datatypes 32 bit
            else:
                props[col] = props[col].astype(np.float32)
            
            # Print new column type
           # print("dtype after: ",props[col].dtype)
           # print("******************************")
    
    # Print final result
    print("___MEMORY USAGE AFTER COMPLETION:___")
    mem_usg = props.memory_usage().sum() / 1024**2 
    print("Memory usage is: ",mem_usg," MB")
    print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
    return props, NAlist

questions_interactions, _ = reduce_mem_usage(questions_interactions)