text_plain_output_1.png
from sklearn.model_selection import train_test_split

from tqdm import tqdm

import numpy as np # linear algebra

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

test_dir = '../input/test'
test_files = os.listdir(test_dir)
test_file_0 = pd.read_csv('../input/test/' + test_files[0])

submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={'time_to_failure': np.float32})

chunk_size = 150000
chunks = int(np.floor(train.shape[0] / chunk_size))
X_data = pd.DataFrame(index=range(chunks), dtype=np.float32, columns=['min', 'max', 'std', 'avg', 'sum', 'median', 'mean_diff', 'q05', 'q25', 'q75', 'q95'])
y_data = pd.DataFrame(index=range(chunks), dtype=np.float32, columns=['ttf'])
def create_features(data_chunk, X_df, chunk_no, col_name='acoustic_data'):
    x = data_chunk[col_name]
    X_df.loc[chunk_no, 'min'] = x.min()
    X_df.loc[chunk_no, 'max'] = x.max()
    X_df.loc[chunk_no, 'std'] = x.std()
    X_df.loc[chunk_no, 'avg'] = x.mean()
    X_df.loc[chunk_no, 'sum'] = x.sum()
    X_df.loc[chunk_no, 'median'] = x.median()
    X_df.loc[chunk_no, 'mean_diff'] = np.mean(np.diff(x))
    X_df.loc[chunk_no, 'q05'] = np.quantile(x, 0.05)
    X_df.loc[chunk_no, 'q25'] = np.quantile(x, 0.25)
    X_df.loc[chunk_no, 'q75'] = np.quantile(x, 0.75)
    X_df.loc[chunk_no, 'q95'] = np.quantile(x, 0.95)
    return X_df

for chunk_no in tqdm(range(chunks)):
    data_chunk = train.iloc[chunk_no * chunk_size:chunk_no * chunk_size + chunk_size]
    X_data = create_features(data_chunk, X_data, chunk_no)
    y = data_chunk['time_to_failure'].values[-1]
    y_data.loc[chunk_no, 'ttf'] = y

X_train, X_test, y_train, y_test = train_test_split(X_data.values, y_data.values, test_size=0.2)
X_train.shape