application_vnd.jupyter.stderr_output_2.png text_plain_output_1.png
from pathlib import Path

from tqdm import tqdm

from tsfresh import extract_features

import datetime

import numpy as np

import pandas as pd

import re

import xmltodict

import datetime
import json
import re
from os import listdir
from os.path import isfile, join
from pathlib import Path
import catboost
import numpy as np
import optuna
import pandas as pd
import shap
import xmltodict
from boruta import BorutaPy
from dateutil.relativedelta import relativedelta
from sklearn.ensemble import RandomForestRegressor
from tqdm import tqdm
pd.options.mode.chained_assignment = None

DATA_ROOT = Path('/kaggle/input/kaggle-pog-series-s01e04/')
DF_TRAIN = DATA_ROOT / 'train.csv'
DF_TEST = DATA_ROOT / 'test.csv'
RAW_DATA = DATA_ROOT / 'raw_health_export.xml'
OUTPUT_PATH = '/kaggle/working/'
TSFRESH_PATH = '/kaggle/input/datasets-sleep/'
train = pd.read_csv(DF_TRAIN)
test = pd.read_csv(DF_TEST)

class FeatureExtractor:
    def __init__(self, data_path):
        raw_dict = FeatureExtractor.parse_raw_data(data_path)
        self.records = FeatureExtractor.get_records_df(raw_dict)
    @staticmethod
    def parse_raw_data(data_path):
        with open(RAW_DATA) as file:
            raw_dict = xmltodict.parse(file.read())
        return raw_dict
    @staticmethod
    def get_records_df(raw_dict):
        records = pd.DataFrame(raw_dict['HealthData']['Record'])
        records.columns = [col.replace('@', '') for col in records.columns]
        records['creationDate'] = pd.to_datetime(records['creationDate']).dt.tz_localize(None)
        records['startDate'] = pd.to_datetime(records['startDate']).dt.tz_localize(None)
        records['endDate'] = pd.to_datetime(records['endDate']).dt.tz_localize(None)
        records['type'] = records['type'].str.replace('HKQuantityTypeIdentifier', '')
        return records
    @staticmethod
    def get_type_ts(records, type):
        if type != 'BoostedHeartRate':
            records['is_apple'] = records['sourceName'].str.contains('Apple')
            type_records = records.query(f"type == '{type}' and is_apple")
            type_records = type_records.drop(['type'], axis=1)
            type_records['value'] = type_records['value'].astype(float)
        else:
            hrv = records.query('HeartRateVariabilityMetadataList == HeartRateVariabilityMetadataList')
            extract_list = lambda x: x['InstantaneousBeatsPerMinute']
            hrv['HeartRateVariabilityMetadataList'] = hrv['HeartRateVariabilityMetadataList'].apply(extract_list)
            hrv_exp = hrv.explode('HeartRateVariabilityMetadataList')
            get_bpm = lambda x: x['@bpm']
            get_time = lambda x: x['@time']
            hrv_exp['bpm'] = hrv_exp['HeartRateVariabilityMetadataList'].apply(get_bpm)
            hrv_exp['time'] = hrv_exp['HeartRateVariabilityMetadataList'].apply(get_time)
            hrv_exp['time_date'] = pd.to_datetime(hrv_exp['time'], format='%I:%M:%S.%f %p')
            hrv_exp.loc[hrv_exp['time_date'].dt.hour == 0, 'date'] = hrv_exp[hrv_exp['time_date'].dt.hour == 0]['endDate'].dt.date
            hrv_exp.loc[hrv_exp['time_date'].dt.hour > 0, 'date'] = hrv_exp[hrv_exp['time_date'].dt.hour > 0]['startDate'].dt.date
            hrv_exp['hours'] = hrv_exp['time_date'].astype('datetime64[s]').dt.time
            hrv_exp['new_startDate'] = pd.to_datetime(hrv_exp['date'].astype(str) + ' ' + hrv_exp['hours'].astype(str))
            hrv_exp = hrv_exp.drop(['startDate', 'value'], axis=1)
            hrv_exp = hrv_exp.rename(columns={'new_startDate': 'startDate', 'bpm': 'value'})
            hrv_exp = hrv_exp[['startDate', 'endDate', 'value']]
            hrv_exp['value'] = hrv_exp['value'].astype(float)
            hrv_exp.reset_index(inplace=True, drop=True)
            type = 'HeartRate'
            records['is_apple'] = records['sourceName'].str.contains('Apple')
            type_records = records.query(f"type == '{type}' and is_apple")
            type_records = type_records.drop(['type'], axis=1)
            type_records['value'] = type_records['value'].astype(float)
            type_records = type_records[['startDate', 'endDate', 'value']]
            type_records = pd.concat([type_records, hrv_exp], ignore_index=True)
        type_records = type_records[['startDate', 'endDate', 'value']]
        return type_records
    @staticmethod
    def resample_ts(records, resample='5Min', how='mean'):
        records = records.sort_values('startDate')
        records = records.set_index('startDate')
        records = eval(f"records.resample('{resample}').{how}()")
        records = records.fillna(method='pad')
        records = records.reset_index()
        return records
    @staticmethod
    def trim_outliers(records):
        save_records = records.copy()
        Q1 = np.percentile(records['value'], 25, interpolation='midpoint')
        Q3 = np.percentile(records['value'], 75, interpolation='midpoint')
        IQR = Q3 - Q1
        upper = Q3 + 1.5 * IQR
        lower = Q1 - 1.5 * IQR
        records.loc[records['value'] > upper, 'value'] = upper
        records.loc[records['value'] < lower, 'value'] = lower
        if records['value'].sum() == 0:
            return save_records
        return records
    @staticmethod
    def tsfresh(records, prefix):
        from tsfresh import extract_features
        records = records.sort_values('startDate')
        records['timestep'] = records.groupby(['date']).cumcount() + 1
        extracted_features = extract_features(records[['date', 'timestep', 'value']], column_id='date', column_sort='timestep')
        extracted_features = extracted_features.reset_index()
        extracted_features = extracted_features.rename(columns={'index': 'date'})
        extracted_features.columns = ['date'] + [prefix + '_' + str(col) for col in extracted_features.columns[1:]]
        extracted_features = extracted_features.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '', x))
        return extracted_features
    @staticmethod
    def add_date(records, how):
        if how == '24h':
            records['date'] = (records['startDate'] - datetime.timedelta(hours=12)).dt.date
        elif how == '7days':
            records['date'] = records['startDate'].dt.date
            df_result = records[0:0]
            dates = records['date'].unique()
            for d in dates:
                min_d = d - datetime.timedelta(days=7)
                max_d = d
                df_date = records.query('date >= @min_d and date <= @max_d')
                df_date['date'] = d
                df_result = pd.concat([df_result, df_date])
            records = df_result
        elif how == '3daysAfter':
            records['date'] = records['startDate'].dt.date
            df_result = records[0:0]
            dates = records['date'].unique()
            for d in dates:
                min_d = d
                max_d = d + datetime.timedelta(days=3)
                df_date = records.query('date >= @min_d and date <= @max_d')
                df_date['date'] = d
                df_result = pd.concat([df_result, df_date])
            records = df_result
        return records

aggregation_info = {'BoostedHeartRate': 'mean', 'HeartRate': 'mean', 'ActiveEnergyBurned': 'sum', 'StepCount': 'sum', 'BasalEnergyBurned': 'mean', 'AppleStandTime': 'mean', 'FlightsClimbed': 'sum', 'EnvironmentalAudioExposure': 'mean', 'RespiratoryRate': 'mean', 'AppleExerciseTime': 'sum', 'DistanceWalkingRunning': 'sum', 'RespiratoryRate': 'mean', 'HeadphoneAudioExposure': 'mean', 'StairDescentSpeed': 'mean', 'OxygenSaturation': 'mean', 'StairAscentSpeed': 'mean', 'HeartRateVariabilitySDNN': 'mean'}
aggregation_info = {}
for type in tqdm(aggregation_info.keys()):
    print(type)
    ts_type = FeatureExtractor.get_type_ts(fe.records, type)
    prefix = '24h_' + type
    ts_resample = FeatureExtractor.resample_ts(ts_type, resample='5Min', how=aggregation_info[type])
    ts_resample = FeatureExtractor.trim_outliers(ts_resample)
    print(ts_resample.describe())
    ts_resample = FeatureExtractor.add_date(ts_resample, how='24h')
    ts_resample = FeatureExtractor.tsfresh(ts_resample, prefix)
    ts_resample.to_csv(f'{OUTPUT_PATH}/tsfresh_{prefix}.csv', index=False)
    prefix = '7d_' + type
    ts_resample = FeatureExtractor.resample_ts(ts_type, resample='60min', how=aggregation_info[type])
    print(ts_resample.describe())
    ts_resample = FeatureExtractor.trim_outliers(ts_resample)
    ts_resample = FeatureExtractor.add_date(ts_resample, how='7days')
    ts_resample = FeatureExtractor.tsfresh(ts_resample, prefix)
    ts_resample.to_csv(f'{OUTPUT_PATH}/tsfresh_{prefix}.csv', index=False)
    prefix = '24h_1m_' + type
    ts_resample = FeatureExtractor.resample_ts(ts_type, resample='1Min', how='first')
    ts_resample = FeatureExtractor.trim_outliers(ts_resample)
    print(ts_resample.describe())
    ts_resample = FeatureExtractor.add_date(ts_resample, how='24h')
    ts_resample = FeatureExtractor.tsfresh(ts_resample, prefix)
    ts_resample.to_csv(f'{OUTPUT_PATH}/tsfresh_{prefix}.csv', index=False)
    prefix = '24h_1m_10r_' + type
    ts_resample = FeatureExtractor.resample_ts(ts_type, resample='1Min', how='first')
    ts_resample['value'] = ts_resample['value'].rolling(window=10).var()
    ts_resample['value'] = ts_resample['value'].fillna(0)
    ts_resample = FeatureExtractor.trim_outliers(ts_resample)
    print(ts_resample.describe())
    ts_resample = FeatureExtractor.add_date(ts_resample, how='24h')
    ts_resample = FeatureExtractor.tsfresh(ts_resample, prefix)
    ts_resample.to_csv(f'{OUTPUT_PATH}/tsfresh_{prefix}.csv', index=False)