application_vnd.jupyter.stderr_output_1.png
import glob

import numpy as np

import pandas as pd

station = 68241
def Datastations(station, path):
    allFiles = glob.glob(path + '/*.csv')
    list_ = []
    array = ['T', 'MinT', 'MaxT', 'Precip', 'AIR_TEMP', 'AIR_TEMP_MIN', 'AIR_TEMP_MAX', 'PRCP']
    for file_ in allFiles:
        df = pd.read_csv(file_, index_col=None, header=0)
        df = df[df.station_number == station]
        df = df.loc[df['parameter'].isin(array)]
        list_.append(df)
    frame = pd.concat(list_, ignore_index=True)
    return frame
df1 = Datastations(station, 'refdata/obs')
df2 = Datastations(station, 'refdata/BoM_ETA_20160501-20170430/obs')

df1['valid_start'] = df1['valid_start'].apply(pd.to_numeric)
df1['valid_end'] = df1['valid_end'].apply(pd.to_numeric)
df1['valid_start'] = df1['valid_start'] + 36000
df1['valid_end'] = df1['valid_end'] + 36000
df1['valid_start'] = pd.to_datetime(df1['valid_start'], unit='s')
df1['valid_end'] = pd.to_datetime(df1['valid_end'], unit='s')
df2['valid_start'] = df2['valid_start'].apply(pd.to_numeric)
df2['valid_end'] = df2['valid_end'].apply(pd.to_numeric)
df2['valid_start'] = df2['valid_start'] + 36000
df2['valid_end'] = df2['valid_end'] + 36000
df2.loc[df2.parameter == 'AIR_TEMP', 'valid_end'] = df2['valid_end'] + 3600
df2.loc[df2.parameter == 'PRCP', 'valid_end'] = df2['valid_end'] + 3000
df2['valid_start'] = pd.to_datetime(df2['valid_start'], unit='s')
df2['valid_end'] = pd.to_datetime(df2['valid_end'], unit='s')

df1['T_Celsius'] = np.where(df1['parameter'] == 'T', df1['value'], '')
df1['MinT_Celsius'] = np.where(df1['parameter'] == 'MinT', df1['value'], '')
df1['MaxT_Celsius'] = np.where(df1['parameter'] == 'MaxT', df1['value'], '')
df1['Precip_mm'] = np.where(df1['parameter'] == 'Precip', df1['value'], '')
df1 = df1.drop(['area_code', 'unit', 'statistic', 'level', 'qc_valid_minutes', 'parameter', 'value', 'qc_valid_start', 'qc_valid_end'], axis=1)
df1 = df1.groupby(['valid_start', 'valid_end', 'station_number'])['T_Celsius', 'MinT_Celsius', 'MaxT_Celsius', 'Precip_mm'].sum().reset_index()
print(len(df1))
print(df1.head())
df2['T_Celsius'] = np.where(df2['parameter'] == 'AIR_TEMP', df2['value'], '')
df2['MinT_Celsius'] = np.where(df2['parameter'] == 'AIR_TEMP_MIN', df2['value'], '')
df2['MaxT_Celsius'] = np.where(df2['parameter'] == 'AIR_TEMP_MAX', df2['value'], '')
df2['Precip_mm'] = np.where(df2['parameter'] == 'PRCP', df2['value'], '')
df2 = df2.drop(['area_code', 'unit', 'statistic', 'level', 'qc_valid_minutes', 'parameter', 'value', 'instantaneous', 'qc_valid_minutes_start', 'qc_valid_minutes_end'], axis=1)
df2 = df2.groupby(['valid_start', 'valid_end', 'station_number'])['T_Celsius', 'MinT_Celsius', 'MaxT_Celsius', 'Precip_mm'].sum().reset_index()
print(len(df2))
print(df2.head())