text_html_output_4.png text_html_output_2.png text_html_output_1.png text_html_output_3.png
from scipy.optimize import minimize

from scipy.stats import mode

from tqdm.auto import tqdm

import numpy as np

import pandas as pd

import warnings

def smape_plus_1(y_true, y_pred):
    y_true_plus_1 = y_true + 1
    y_pred_plus_1 = y_pred + 1
    metric = np.zeros(len(y_true_plus_1))
    numerator = np.abs(y_true_plus_1 - y_pred_plus_1)
    denominator = (np.abs(y_true_plus_1) + np.abs(y_pred_plus_1)) / 2
    mask_not_zeros = (y_true_plus_1 != 0) | (y_pred_plus_1 != 0)
    metric[mask_not_zeros] = numerator[mask_not_zeros] / denominator[mask_not_zeros]
    return 100 * np.nanmean(metric)

train_clinical_all = pd.read_csv('/kaggle/input/amp-parkinsons-disease-progression-prediction/train_clinical_data.csv')
proteins = pd.read_csv('/kaggle/input/amp-parkinsons-disease-progression-prediction/train_proteins.csv')
proteins_features = pd.pivot_table(proteins, values='NPX', index='visit_id', columns='UniProt', aggfunc='sum')
train_clinical_all = train_clinical_all.merge(proteins_features, left_on='visit_id', right_index=True, how='left')

import warnings
warnings.filterwarnings('ignore')
train_clinical_all[proteins_features.columns] = train_clinical_all.groupby('patient_id')[proteins_features.columns].transform(lambda x: x.fillna(x.median()))

train_clinical_all['pred_month'] = train_clinical_all['visit_month']
for plus_month in [6, 12, 24]:
    train_shift = train_clinical_all[['patient_id', 'visit_month', 'pred_month', 'updrs_1', 'updrs_2', 'updrs_3', 'updrs_4']].copy()
    train_shift['visit_month'] -= plus_month
    train_shift.rename(columns={f'updrs_{i}': f'updrs_{i}_plus_{plus_month}' for i in range(1, 5)}, inplace=True)
    train_shift.rename(columns={'pred_month': f'pred_month_plus_{plus_month}'}, inplace=True)
    train_clinical_all = train_clinical_all.merge(train_shift, how='left', on=['patient_id', 'visit_month'])
train_clinical_all.rename(columns={f'updrs_{i}': f'updrs_{i}_plus_0' for i in range(1, 5)}, inplace=True)
train_clinical_all.rename(columns={'pred_month': f'pred_month_plus_0'}, inplace=True)
train_clinical_all

from scipy.stats import mode
fill_with_mode = lambda x: x.fillna(mode(x).mode[0])

def calculate_month_trend_predicitons(pred_month, trend):
    if target == 'updrs_4':
        pred_month = pred_month.clip(60, None)
    return trend[0] + pred_month * trend[1]
target_to_trend = {'updrs_1': [5.394793062665313, 0.027091086167821344], 'updrs_2': [5.469498130092747, 0.02824188329658148], 'updrs_3': [21.182145576879183, 0.08897763331790556], 'updrs_4': [-4.434453480103724, 0.07531448585334258]}

def calculate_predicitons_protein(pred_month, protein_shift):
    trend_pred_month = target_to_trend[target]
    pred_month_trend = calculate_month_trend_predicitons(pred_month=pred_month, trend=trend_pred_month)
    return np.round(pred_month_trend + protein_shift)
def function_to_minimize(x):
    metric = smape_plus_1(y_true=y_true_array, y_pred=calculate_predicitons_protein(pred_month=pred_month_array, protein_shift=x[0]))
    return metric

def find_best_const(train_clinical_all_filtered, target):
    columns_with_target = [f'{target}_plus_{plus_month}' for plus_month in [0, 6, 12, 24]]
    columns_with_pred_month = [f'pred_month_plus_{plus_month}' for plus_month in [0, 6, 12, 24]]
    global y_true_array
    global pred_month_array
    global protein_array
    y_true_array = train_clinical_all_filtered[columns_with_target].values.ravel()
    pred_month_array = train_clinical_all_filtered[columns_with_pred_month].values.ravel()
    result = minimize(fun=function_to_minimize, x0=[0.0], method='Powell').x[0]
    return result

feature0 = 'O15240'
quantiles = [0, 0.05, 0.95, 1.0]
df_plot = []
for quantile_low, quantile_high in tqdm(zip(quantiles[:-1], quantiles[1:])):
    item = {'quantile_low': quantile_low, 'quantile_high': quantile_high, 'quantile_middle': (quantile_low + quantile_high) / 2}
    quantile_low_value = train_clinical_all[feature0].quantile(quantile_low)
    quantile_high_value = train_clinical_all[feature0].quantile(quantile_high)
    item['quantile_low_value'] = quantile_low_value
    item['quantile_high_value'] = quantile_high_value
    if quantile_high == 1:
        quantile_high_value += 1e-05
    train_clinical_all_filtered0 = train_clinical_all[(train_clinical_all[feature0] >= quantile_low_value) & (train_clinical_all[feature0] <= quantile_high_value)]
    for target in ['updrs_1', 'updrs_2', 'updrs_3', 'updrs_4']:
        item[f'{target}_shift'] = find_best_const(train_clinical_all_filtered0, target)
    df_plot.append(item)
df_plot = pd.DataFrame(df_plot)

feature1 = 'O00533'
quantiles = [0, 0.05, 0.95, 1.0]
df_plot = []
for quantile_low, quantile_high in tqdm(zip(quantiles[:-1], quantiles[1:])):
    item = {'quantile_low': quantile_low, 'quantile_high': quantile_high, 'quantile_middle': (quantile_low + quantile_high) / 2}
    quantile_low_value = train_clinical_all[feature1].quantile(quantile_low)
    quantile_high_value = train_clinical_all[feature1].quantile(quantile_high)
    item['quantile_low_value'] = quantile_low_value
    item['quantile_high_value'] = quantile_high_value
    if quantile_high == 1:
        quantile_high_value += 1e-05
    train_clinical_all_filtered1 = train_clinical_all[(train_clinical_all[feature1] >= quantile_low_value) & (train_clinical_all[feature1] < quantile_high_value)]
    for target in ['updrs_1', 'updrs_2', 'updrs_3', 'updrs_4']:
        item[f'{target}_shift'] = find_best_const(train_clinical_all_filtered1, target)
    df_plot.append(item)
df_plot = pd.DataFrame(df_plot)

feature2 = 'O15394'
quantiles = [0, 0.05, 0.95, 1.0]
df_plot = []
for quantile_low, quantile_high in tqdm(zip(quantiles[:-1], quantiles[1:])):
    item = {'quantile_low': quantile_low, 'quantile_high': quantile_high, 'quantile_middle': (quantile_low + quantile_high) / 2}
    quantile_low_value = train_clinical_all[feature2].quantile(quantile_low)
    quantile_high_value = train_clinical_all[feature2].quantile(quantile_high)
    item['quantile_low_value'] = quantile_low_value
    item['quantile_high_value'] = quantile_high_value
    if quantile_high == 1:
        quantile_high_value += 1e-05
    train_clinical_all_filtered2 = train_clinical_all[(train_clinical_all[feature2] >= quantile_low_value) & (train_clinical_all[feature2] < quantile_high_value)]
    for target in ['updrs_1', 'updrs_2', 'updrs_3', 'updrs_4']:
        item[f'{target}_shift'] = find_best_const(train_clinical_all_filtered2, target)
    df_plot.append(item)
df_plot = pd.DataFrame(df_plot)