application_vnd.jupyter.stderr_output_2.png text_plain_output_1.png
from pandas import DataFrame

import datetime as dt

import lowess as lo

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

df_source = pd.read_csv('../input/periodic_traffic.csv')
df_source['rep_date'] = pd.to_datetime(df_source['_time'])
df_source.drop(['_time'], axis=1, inplace=True)
df_source_time = df_source.copy()
df_source_time['rep_time'] = df_source_time['rep_date'].apply(lambda x: dt.datetime.strptime(x.strftime('%H:%M'), '%H:%M'))
df_source_time.drop(['rep_date'], axis=1, inplace=True)
df_source = df_source.set_index('rep_date')
df_source_time = df_source_time.set_index('rep_time')

v_window = 8
k_out = 1.5
k_norm = 1.5
i = df_source_time.index.shape[0]
x = np.linspace(-10, 10, i)
def f_out(x):
    name = x.index[0]
    if x[name] > x[name + '_lo'] + k_out * x[name + '_std_first_step']:
        x[name + '_adj'] = np.nan
    elif x[name] < x[name + '_lo'] - k_out * x[name + '_std_first_step']:
        x[name + '_adj'] = np.nan
    else:
        x[name + '_adj'] = x[name]
    return x
def f_low(df_x):
    df_res = DataFrame(df_x)
    name = df_res.columns[0]
    i = df_x.index.shape[0]
    x = np.linspace(-10, 10, i)
    df_res[name + '_lo'] = lo.lowess(x, df_x.values, x)
    df_res[name + '_std_first_step'] = df_x.rolling(window=v_window, min_periods=0).std().fillna(method='bfill').shift(-int(v_window / 2))
    df_res = df_res.apply(f_out, axis=1)
    df_res[name + '_adj_first_step'] = df_res[name + '_adj'].fillna(method='bfill')
    df_res[name + '_adj'] = lo.lowess(x, np.array(df_res[name + '_adj_first_step']), x)
    df_res[name + '_std'] = df_res[name + '_adj_first_step'].rolling(window=v_window, min_periods=0).std().fillna(method='bfill').shift(-int(v_window / 2))
    return df_res
l = list(df_source_time.columns)
print('Список полученных для анализа фич:\n{}'.format(l))
for name in l:
    df = f_low(df_source_time[name].sort_index(axis=0))
    display(df.head())
    fig, ax = plt.subplots(1, figsize=(12, 9))
    ax.plot(df[name], 'b.', label='Original')
    ax.plot(df[name + '_lo'] + k_out * df[name + '_std_first_step'], 'g', label='Границы фильтрации выбросов')
    ax.plot(df[name + '_lo'] - k_out * df[name + '_std_first_step'], 'g', label='Границы фильтрации выбросов')
    ax.plot(df[name + '_lo'], 'r', label='Восстановленный график на первом шаге')
    ax.plot(df[name + '_adj'] + k_norm * df[name + '_std'], 'k', label='Верхняя граница нормального трафика')
    ax.plot(df[name + '_adj'] - k_norm * df[name + '_std'], 'k', label='Нижняя граница нормального трафика')
    ax.plot(df[name + '_adj'], 'y', label='Восстановленный график на втором шаге')
    ax.set_title(name)
    plt.legend()
    plt.show()