text_plain_output_1.png
from sklearn.model_selection import train_test_split

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

X_full = pd.read_csv('../input/30-days-of-ml/train.csv')
X_test_full = pd.read_csv('../input/30-days-of-ml/test.csv')

"""
Checking for missing data
"""
missing_values_count = X_full.isnull().sum()
total_cells = np.product(X_full.shape)
total_missing = missing_values_count.sum()
percent_missing = total_missing / total_cells * 100

X_full.dropna(axis=0, subset=['target'], inplace=True)
y = X_full.target
X_full.drop(['target'], axis=1, inplace=True)
X_train_full, X_valid_full, y_train, y_valid = train_test_split(X_full, y, train_size=0.8, test_size=0.2, random_state=0)

object_cols = [col for col in X_full.columns if X_full[col].dtype == 'object']
good_label_cols = [col for col in object_cols if set(X_valid_full[col]).issubset(set(X_train_full[col]))]
bad_label_cols = list(set(object_cols) - set(good_label_cols))
object_nunique = list(map(lambda col: X_full[col].nunique(), object_cols))
d = dict(zip(object_cols, object_nunique))
sorted(d.items(), key=lambda x: x[1])

label_encoder_cols = [cname for cname in X_train_full.columns if X_train_full[cname].nunique() < 10 and X_train_full[cname].dtype == 'object']
one_hotting_cols = list(set(object_cols) - set(label_encoder_cols))
numerical_cols = [cname for cname in X_train_full.columns if X_train_full[cname].dtype in ['int64', 'float64']]
my_cols = label_encoder_cols + one_hotting_cols + numerical_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()
X_test = X_test_full[my_cols].copy()

X_train_prepared = X_train_full[numerical_cols].copy()
X_valid_prepared = X_valid_full[numerical_cols].copy()
X_test_prepared = X_test_full[numerical_cols].copy()

X_train_prepared = pd.concat([X_train_new_columns, X_train_prepared], axis=1)
X_valid_prepared = pd.concat([X_valid_new_columns, X_valid_prepared], axis=1)
X_test_prepared = pd.concat([X_test_new_columns, X_test_prepared], axis=1)

X_train_prepared = X_train_prepared.drop(['id'], axis=1)
X_valid_prepared = X_valid_prepared.drop(['id'], axis=1)
X_test_prepared = X_test_prepared.drop(['id'], axis=1)

X_train_prepared.head()