text_plain_output_1.png
from finta import TA

import glob

import numpy as np

import os

import pandas as pd

import warnings

import os
import glob
import gc
import yaml
import math
import warnings
from tqdm import tqdm
from functools import reduce
import pandas as pd
from finta import TA
from numba import jit
import numpy as np
from plotly.subplots import make_subplots
import plotly.graph_objects as go
from joblib import Parallel, delayed
warnings.simplefilter('ignore')
pd.set_option('max_column', 300)

DATA_PATH = '../input/optiver-realized-volatility-prediction'
__DATA_DIRS__ = ['book_train.parquet', 'trade_tarin.parquet', 'book_test.parquet', 'trade_test.parquet']
FILE_LIST_MAP = {'book_train': glob.glob(os.path.join(DATA_PATH, 'book_train.parquet/*')), 'trade_train': glob.glob(os.path.join(DATA_PATH, 'trade_train.parquet/*')), 'book_test': glob.glob(os.path.join(DATA_PATH, 'book_test.parquet/*')), 'trade_test': glob.glob(os.path.join(DATA_PATH, 'trade_test.parquet/*'))}
ORDER_PRICE = ['bid_price1', 'bid_price2', 'ask_price1', 'ask_price2']
ORDER_VOLUME = ['bid_size1', 'bid_size2', 'ask_size1', 'ask_size2']
TI_NAMES = [func for func in dir(TA) if callable(getattr(TA, func)) and (not func.startswith('_'))]

def get_wap(df_book):
    """Compute estimated price series.
    Parameters:
        df_book: pd.DataFrame, raw information of book data
    Return: 
        wap: pd.DataFrame, estimated price series with time identifiers
    """
    df_book_ = df_book.copy()
    df_book_['wap1'] = (df_book_['bid_price1'] * df_book_['ask_size1'] + df_book_['ask_price1'] * df_book_['bid_size1']) / (df_book_['bid_size1'] + df_book_['ask_size1'])
    wap = df_book_.loc[:, ['time_id', 'seconds_in_bucket', 'wap1']]
    return wap
def get_ohlcv(prices, volumes, scale=10):
    """Return OHLCV of stock price based on the Kline scale (sec).
    Parameters:
        prices: pd.Series, the estimated price series
        volumes: pd.Series, the trading volume series
        scale: int, the scale of the Kline (sec), default=10
    Return:
        OHLCV: pd.DataFrame, four prices and trading volumes of the stock based on the Kline scale 
    """
    if 600 % scale != 0:
        raise ValueError('Choose scale divisible by 600 seconds...')
    OHLCV = pd.DataFrame(columns=['open', 'high', 'low', 'close', 'volume'])
    for i in range(0, 600, scale):
        p_window = prices[i:i + scale]
        v_window = volumes[i:i + scale]
        p_stick = {'open': p_window.iloc[0], 'high': np.max(p_window), 'low': np.min(p_window), 'close': p_window.iloc[-1], 'volume': np.sum(v_window)}
        OHLCV = OHLCV.append(p_stick, ignore_index=True)
    OHLCV.insert(0, column='sec', value=[scale * (i + 1) for i in range(0, 600 // scale)])
    return OHLCV
def ffill(df):
    """Forward fill information in order book data, followed by bfill to avoid bug in filler data.
    """
    df_ = df.copy()
    df_.set_index(['time_id', 'seconds_in_bucket'], inplace=True)
    df_ = df_.reindex(pd.MultiIndex.from_product([df_.index.levels[0], np.arange(0, 600)], names=['time_id', 'seconds_in_bucket']), method='ffill')
    df_ = df_.reindex(pd.MultiIndex.from_product([df_.index.levels[0], np.arange(0, 600)], names=['time_id', 'seconds_in_bucket']), method='bfill')
    df_.reset_index(inplace=True)
    return df_

df_order = pd.read_parquet(os.path.join(DATA_PATH, 'book_train.parquet/stock_id=0/'))
df_trade = pd.read_parquet(os.path.join(DATA_PATH, 'trade_train.parquet/stock_id=0/'))
df_order = df_order[df_order['time_id'] == 5]
df_order = ffill(df_order)
df_trade = df_trade[df_trade['time_id'] == 5]
df_order.head()