text_plain_output_1.png
from sklearn.decomposition import PCA

import pandas as pd

import statistics

train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
train_targets = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')

ALL = pd.concat([train_features, test_features])

ALL.shape

def standardization(l):
    l_mean = statistics.mean(l)
    l_stdev = statistics.stdev(l)
    ret = []
    for x in l:
        y = (x - l_mean) / l_stdev
        ret.append(y)
    return ret

ALL.columns

GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]

ADD_CNT = 12
for x in GENES:
    ALL[x + 'High'] = (ALL[x] ** 2) ** 0.5
    MAX = ALL[x + 'High'].max()
    for y in ALL[x + 'High']:
        if y <= MAX / 2:
            y = 0
        else:
            y = y - MAX / 2
    ALL[x + 'High'] = ALL[x + 'High'] * (abs(ALL[x]) / ALL[x])
    ADD_CNT += 1
for x in CELLS:
    ALL[x + 'High'] = (ALL[x] ** 2) ** 0.5
    MAX = ALL[x + 'High'].max()
    for y in ALL[x + 'High']:
        if y <= MAX / 2:
            y = 0
        else:
            y = y - MAX / 2
    ALL[x + 'High'] = ALL[x + 'High'] * (abs(ALL[x]) / ALL[x])
    ADD_CNT += 1

for x in ALL:
    if x in ALL.columns[-ADD_CNT:]:
        ALL[x] = standardization(ALL[x])

ALL = ALL.drop(['sig_id'], axis=1)

from sklearn.decomposition import PCA
pca = PCA(150)
ALL = pca.fit_transform(ALL)
Columns = []
for i in range(150):
    Columns.append('d' + str(i + 1))
ALL = pd.DataFrame(ALL, columns=Columns)
categorical_features = Columns

ALL.head()