application_vnd.jupyter.stderr_output_1.png
from tqdm import tqdm

import gc

import glob

import numpy as np

import os

import pandas as pd

import pandas as pd
import numpy as np
from tqdm import tqdm
import os
import glob
tqdm.pandas()
import matplotlib.pyplot as plt
import gc

train_path = '../input/ubiquant-market-prediction/train.csv'
test_path = '../input/ubiquant-market-prediction/example_test.csv'

# Lets first try to reduce the size of the dataframe by bringing it to right dtype and saving those chunks.

def reduce_memory_usage(df, chunk):
    start_mem = df.memory_usage().sum() / 1024 ** 2
    print("Initial Memory chunk: {:.3f}".format(start_mem))
    
    for col in df.columns:
        type_ = df[col].dtype
        
        if str(type_) != "object":
            if str(type_)[:3] == "int":
                min_ = df[col].min()
                max_ = df[col].max()

                if min_ > np.iinfo(np.int8).min and max_ < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif min_ > np.iinfo(np.int16).min and max_ < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif min_ > np.iinfo(np.int32).min and max_ < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                else:
                    df[col] = df[col].astype(np.int64)
            else:
                if min_ > np.finfo(np.float16).min and max_ < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif min_ > np.finfo(np.float32).min and max_ < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype("category")
    
    end_mem = df.memory_usage().sum() / 1024 ** 2
    print("Final Memory chunk: {:.3f}".format(end_mem))
    print("Reduced by: {:.2f}".format((start_mem - end_mem) / start_mem))
    
    df.to_pickle(f"chunk_{chunk}.pkl")
    print(f"chunk_{chunk}.pkl","saved!")
                    
    

gc.collect()
chunksize = 10 ** 6
for chunk_id, chunk in enumerate(pd.read_csv(train_path, chunksize=chunksize)):
    reduce_memory_usage(chunk, chunk_id)

appended_list = []
path = glob.glob(os.path.join(os.curdir, 'chunk_*.pkl'), recursive=True)
for item in tqdm(path):
    df = pd.read_pickle(item)
    appended_list.append(df)
final_frame = pd.concat(appended_list, axis=0, ignore_index=True)