text_plain_output_1.png image_output_1.png
from pandas.plotting import autocorrelation_plot, lag_plot

import matplotlib.gridspec as gridspec

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

cityTable = pd.read_csv('../input/city_attributes.csv')
temperatureDF = pd.read_csv('../input/temperature.csv', index_col=0)
temperatureDF.index = pd.to_datetime(temperatureDF.index)
cityTable

#%% show several temperature plots to get a feel for the dataset
citiesToShow = ['Los Angeles','Chicago','Montreal','Houston']

t0 = temperatureDF.index
t1 = pd.date_range(pd.to_datetime('1/7/2015',dayfirst=True),pd.to_datetime('1/10/2016',dayfirst=True),freq='H')
t2 = pd.date_range(pd.to_datetime('1/7/2015',dayfirst=True),pd.to_datetime('1/9/2015' ,dayfirst=True),freq='H')
t3 = pd.date_range(pd.to_datetime('1/7/2015',dayfirst=True),pd.to_datetime('21/7/2015',dayfirst=True),freq='H')

fig, ax = plt.subplots(nrows=4,ncols=1,figsize=(20,15))
temperatureDF.loc[t0,citiesToShow].plot(ax=ax[0]); 
temperatureDF.loc[t1,citiesToShow].plot(ax=ax[1],legend=False);
temperatureDF.loc[t2,citiesToShow].plot(ax=ax[2],legend=False); 
temperatureDF.loc[t3,citiesToShow].plot(ax=ax[3],legend=False);
ax[0].legend(loc='upper left',fontsize=20,bbox_to_anchor=(0.02,1.3), ncol=len(citiesToShow))
for i in range(len(ax)): ax[i].set_ylabel('Temperature [$^\circ$K]', fontsize=15)
plt.tight_layout()

#%% show autocorr and lag plots

cityToShow = 'Los Angeles'
selectedLagPoints = [1,3,6,9,12,24,36,48,60]
maxLagDays = 7

originalSignal = temperatureDF[cityToShow]

# set grid spec of the subplots
plt.figure(figsize=(12,6))
gs = gridspec.GridSpec(2, len(selectedLagPoints))
axTopRow = plt.subplot(gs[0, :])
axBottomRow = []
for i in range(len(selectedLagPoints)):
    axBottomRow.append(plt.subplot(gs[1, i]))

# plot autocorr
allTimeLags = np.arange(1,maxLagDays*24)
autoCorr = [originalSignal.autocorr(lag=dt) for dt in allTimeLags]
axTopRow.plot(allTimeLags,autoCorr); axTopRow.set_title('Autocorrelation Plot of Temperature Signal')
axTopRow.set_xlabel('time lag [hours]'); axTopRow.set_ylabel('correlation coefficient')
selectedAutoCorr = [originalSignal.autocorr(lag=dt) for dt in selectedLagPoints]
axTopRow.scatter(x=selectedLagPoints, y=selectedAutoCorr, s=50, c='r')

# plot scatter plot of selected points
for i in range(len(selectedLagPoints)):
    lag_plot(originalSignal, lag=selectedLagPoints[i], s=5, ax=axBottomRow[i])    
    if i >= 1:
        axBottomRow[i].set_yticks([],[])
plt.tight_layout()

#%% zoom in and out on the autocorr plot

fig, ax = plt.subplots(nrows=4,ncols=1, figsize=(13,11))

timeLags = np.arange(1,25*24*30)
autoCorr = [originalSignal.autocorr(lag=dt) for dt in timeLags]
ax[0].plot(1.0/(24*30)*timeLags, autoCorr); ax[0].set_title('Autocorrelation Plot')
ax[0].set_xlabel('time lag [months]'); ax[0].set_ylabel('correlation coeff')

timeLags = np.arange(1,20*24*7)
autoCorr = [originalSignal.autocorr(lag=dt) for dt in timeLags]
ax[1].plot(1.0/(24*7)*timeLags, autoCorr);
ax[1].set_xlabel('time lag [weeks]'); ax[1].set_ylabel('correlation coeff')

timeLags = np.arange(1,20*24)
autoCorr = [originalSignal.autocorr(lag=dt) for dt in timeLags]
ax[2].plot(1.0/24*timeLags, autoCorr);
ax[2].set_xlabel('time lag [days]'); ax[2].set_ylabel('correlation coeff')

timeLags = np.arange(1,3*24)
autoCorr = [originalSignal.autocorr(lag=dt) for dt in timeLags]
ax[3].plot(timeLags, autoCorr);
ax[3].set_xlabel('time lag [hours]'); ax[3].set_ylabel('correlation coeff')

#%% apply rolling mean and plot the signal (low pass filter)
windowSize = 5*24

lowPassFilteredSignal = originalSignal.rolling(windowSize, center=True).mean()

t0 = temperatureDF.index
t1 = pd.date_range(pd.to_datetime('1/7/2015',dayfirst=True),pd.to_datetime('1/10/2016',dayfirst=True),freq='H')
t2 = pd.date_range(pd.to_datetime('1/7/2015',dayfirst=True),pd.to_datetime('1/9/2015' ,dayfirst=True),freq='H')
t3 = pd.date_range(pd.to_datetime('1/7/2015',dayfirst=True),pd.to_datetime('21/7/2015',dayfirst=True),freq='H')

fig, ax = plt.subplots(nrows=4,ncols=1,figsize=(20,15))
ax[0].plot(t0,originalSignal)
ax[0].plot(t0,lowPassFilteredSignal)

ax[1].plot(t1,originalSignal[t1])
ax[1].plot(t1,lowPassFilteredSignal[t1])

ax[2].plot(t2,originalSignal[t2])
ax[2].plot(t2,lowPassFilteredSignal[t2])

ax[3].plot(t3,originalSignal[t3])
ax[3].plot(t3,lowPassFilteredSignal[t3])

ax[0].legend(['original','filtered'],fontsize=20,loc='upper left',bbox_to_anchor=(0.02,1.3), ncol=len(citiesToShow))
for i in range(len(ax)): ax[i].set_ylabel('Temperature [$^\circ$K]')

#%% subtract the low pass filtered singal from the original to get high pass filtered signal

highPassFilteredSignal = originalSignal - lowPassFilteredSignal

fig, ax = plt.subplots(nrows=4,ncols=1,figsize=(20,15))
ax[0].plot(t0,highPassFilteredSignal)
ax[1].plot(t1,highPassFilteredSignal[t1])
ax[2].plot(t2,highPassFilteredSignal[t2])
ax[3].plot(t3,highPassFilteredSignal[t3])

ax[0].set_title('deflection of temperature from local mean',fontsize=20)
for i in range(len(ax)): ax[i].set_ylabel('$\Delta$ Temperature [$^\circ$K]')

fig, ax = plt.subplots(nrows=4, ncols=1, figsize=(13, 11))
timeLags = np.arange(1, 25 * 24 * 30)
autoCorr = [lowPassFilteredSignal.autocorr(lag=dt) for dt in timeLags]
ax[0].plot(1.0 / (24 * 30) * timeLags, autoCorr)
ax[0].set_title('Autocorrelation Plot of low Pass Filtered Signal')
ax[0].set_xlabel('time lag [months]')
ax[0].set_ylabel('correlation coeff')
timeLags = np.arange(1, 20 * 24 * 7)
autoCorr = [lowPassFilteredSignal.autocorr(lag=dt) for dt in timeLags]
ax[1].plot(1.0 / (24 * 7) * timeLags, autoCorr)
ax[1].set_xlabel('time lag [weeks]')
ax[1].set_ylabel('correlation coeff')
timeLags = np.arange(1, 20 * 24)
autoCorr = [lowPassFilteredSignal.autocorr(lag=dt) for dt in timeLags]
ax[2].plot(1.0 / 24 * timeLags, autoCorr)
ax[2].set_xlabel('time lag [days]')
ax[2].set_ylabel('correlation coeff')
timeLags = np.arange(1, 3 * 24)
autoCorr = [lowPassFilteredSignal.autocorr(lag=dt) for dt in timeLags]
ax[3].plot(timeLags, autoCorr)
ax[3].set_xlabel('time lag [hours]')
ax[3].set_ylabel('correlation coeff')