application_vnd.jupyter.stderr_output_2.png text_plain_output_1.png
from qlib.data.dataset.handler import DataHandlerLP

from qlib.data.dataset.loader import StaticDataLoader

from sklearn.model_selection import GroupKFold

from typing import Union

import gc

import numpy as np

import pandas as pd

def read_data(path: Union[str, pd.DataFrame]='../input/train.pkl', proc_type='train'):
    """ Read data and turn it into Qlib's format"""
    df = pd.read_pickle(path) if isinstance(path, str) else path
    if proc_type == 'test':
        df['time_id'] = df['row_id'].apply(lambda x: int(x.split('_')[0]))
        del test_df['row_id']
    else:
        assert proc_type == 'train'
    df = df.set_index(['time_id', 'investment_id'])
    df.columns = pd.MultiIndex.from_tuples([('label' if col == 'target' else 'feature', col) for col in df.columns])
    df.index.names = ['datetime', 'instrument']
    df = df.astype(np.float32)
    return df
data_df = read_data('../input/ubiquant-market-prediction-half-precision-pickle/train.pkl')
dh = DataHandlerLP(data_loader=StaticDataLoader(data_df), drop_raw=True)
del data_df
del dh.data_loader

def split_kfold(idx, fold=5):
    kfold = GroupKFold(n_splits=fold)
    cv_index = []
    for fold_id, (train_idx, valid_idx) in enumerate(kfold.split(idx, groups=idx.get_level_values('datetime'))):
        all_seg = {'train': idx[train_idx], 'valid': idx[valid_idx]}
        cv_index.append(all_seg)
    return cv_index
idx = dh.fetch().index
cv_index = split_kfold(idx)
del idx
gc.collect()