text_plain_output_1.png
from category_encoders import OneHotEncoder, WOEEncoder

from sklearn.impute import SimpleImputer, IterativeImputer, KNNImputer, MissingIndicator

from sklearn.linear_model import BayesianRidge, Ridge, Lasso, HuberRegressor

import gc

import numpy as np

import pandas as pd

def prepreprocessing(df_train, df_test):
    data = pd.concat([df_train, df_test])
    data['m3_missing'] = data['measurement_3'].isnull().astype(np.int8)
    data['m5_missing'] = data['measurement_5'].isnull().astype(np.int8)
    data['area'] = data['attribute_2'] * data['attribute_3']
    feature = [f for f in df_test.columns if f.startswith('measurement') or f == 'loading']
    full_fill_dict = {}
    full_fill_dict['measurement_17'] = {'A': ['measurement_5', 'measurement_6', 'measurement_8'], 'B': ['measurement_4', 'measurement_5', 'measurement_7'], 'C': ['measurement_5', 'measurement_7', 'measurement_8', 'measurement_9'], 'D': ['measurement_5', 'measurement_6', 'measurement_7', 'measurement_8'], 'E': ['measurement_4', 'measurement_5', 'measurement_6', 'measurement_8'], 'F': ['measurement_4', 'measurement_5', 'measurement_6', 'measurement_7'], 'G': ['measurement_4', 'measurement_6', 'measurement_8', 'measurement_9'], 'H': ['measurement_4', 'measurement_5', 'measurement_7', 'measurement_8', 'measurement_9'], 'I': ['measurement_3', 'measurement_7', 'measurement_8']}
    col = [col for col in df_test.columns if 'measurement' not in col] + ['loading', 'm3_missing', 'm5_missing']
    a = []
    b = []
    for x in range(3, 17):
        corr = np.absolute(data.drop(col, axis=1).corr()[f'measurement_{x}']).sort_values(ascending=False)
        a.append(np.round(np.sum(corr[1:4]), 3))
        b.append(f'measurement_{x}')
    c = pd.DataFrame()
    c['Selected columns'] = b
    c['correlation total'] = a
    c = c.sort_values(by='correlation total', ascending=False).reset_index(drop=True)
    for i in range(10):
        measurement_col = 'measurement_' + c.iloc[i, 0][12:]
        fill_dict = {}
        for x in data.product_code.unique():
            corr = np.absolute(data[data.product_code == x].drop(col, axis=1).corr()[measurement_col]).sort_values(ascending=False)
            measurement_col_dic = {}
            measurement_col_dic[measurement_col] = corr[1:5].index.tolist()
            fill_dict[x] = measurement_col_dic[measurement_col]
        full_fill_dict[measurement_col] = fill_dict
    feature = [f for f in data.columns if f.startswith('measurement') or f == 'loading']
    nullValue_cols = [col for col in df_train.columns if df_train[col].isnull().sum() != 0]
    for code in data.product_code.unique():
        for measurement_col in list(full_fill_dict.keys()):
            tmp = data[data.product_code == code]
            column = full_fill_dict[measurement_col][code]
            tmp_train = tmp[column + [measurement_col]].dropna(how='any')
            tmp_test = tmp[(tmp[column].isnull().sum(axis=1) == 0) & tmp[measurement_col].isnull()]
            model = HuberRegressor(epsilon=1.9)
            model.fit(tmp_train[column], tmp_train[measurement_col])
            data.loc[(data.product_code == code) & (data[column].isnull().sum(axis=1) == 0) & data[measurement_col].isnull(), measurement_col] = model.predict(tmp_test[column])
        NA = data.loc[data['product_code'] == code, nullValue_cols].isnull().sum().sum()
        model1 = KNNImputer(n_neighbors=3)
        data.loc[data.product_code == code, feature] = model1.fit_transform(data.loc[data.product_code == code, feature])
    data['measurement_avg'] = data[[f'measurement_{i}' for i in range(3, 17)]].mean(axis=1)
    df_train = data.iloc[:df_train.shape[0], :]
    df_test = data.iloc[df_train.shape[0]:, :]
    features = ['loading', 'attribute_0', 'measurement_17', 'measurement_0', 'measurement_1', 'measurement_2', 'area', 'm3_missing', 'm5_missing', 'measurement_avg']
    encoder = WOEEncoder(cols=['attribute_0'])
    df_train = encoder.fit_transform(df_train, target)
    df_test = encoder.transform(df_test)
    df_train['loading'] = np.log1p(df_train['loading'])
    df_test['loading'] = np.log1p(df_test['loading'])
    return (df_train, df_test, features)

submission = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')
train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv', index_col='id')
test = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv', index_col='id')
target = train['failure'].copy()
gc.collect()
train.head()