<repo_name>dinhvanhieu18/gsmap_adjustment<file_sep>/lib/conv2d_gsmap/data_preprocessing_conv2d_gsmap.py
import numpy as np
from pandas import read_csv
from netCDF4 import Dataset
import os
import glob
"""GET NAME, LON, LAT, HEIGHT OF GAUGE DATA"""


def get_lon_lat_gauge_data():

    preprocessed_data_dir = './data/conv2d_gsmap/preprocessed_txt_data/'
    start_index = len(preprocessed_data_dir)
    type_file = '.csv'
    end_index = -len(type_file)
    data_paths_preprocessed_data = glob.glob(preprocessed_data_dir + '*' +
                                             type_file)

    lon_arr = np.empty(shape=(len(data_paths_preprocessed_data)))
    lat_arr = np.empty(shape=(len(data_paths_preprocessed_data)))
    for index, file in enumerate(data_paths_preprocessed_data):
        file_name = file[start_index:end_index]
        file_name_list = file_name.split('_')
        lon = float(file_name_list[1])
        lat = float(file_name_list[2])
        lon_arr[index] = lon
        lat_arr[index] = lat

    lon_arr = np.round(lon_arr, 3)
    lat_arr = np.round(lat_arr, 3)

    return lat_arr, lon_arr


def find_lat_lon_remapnn():
    # find nearest lon_lat
    precip_list = read_csv('./data/conv2d_gsmap/remapnn.csv')
    precip_list = precip_list.to_numpy()

    original_nc = Dataset('data/conv2d_gsmap/gsmap_2011_2018.nc', 'r')

    # check dataset
    gsmap_time = np.array(original_nc['time'][:])
    gsmap_lon = np.array(original_nc['lon'][:])
    gsmap_lat = np.array(original_nc['lat'][:])
    gsmap_precip = np.array(original_nc['precip'][:])
    gsmap_precip = np.round(gsmap_precip, 6)

    index_pos = []
    # precip_list - 2 due to the header and 0-based index
    for i in range(0, len(precip_list)):
        if precip_list[i] > 0:
            index_pos = np.where(gsmap_precip[i, :, :] == np.round(
                precip_list[i, 0], 6))  # 0 because of precip_list is (xxx,1)
            if len(index_pos[0]) == 1 and len(index_pos[1]) == 1:
                break

    # get lat and long
    # 0 is lat
    # 1 is lon
    return gsmap_lat[index_pos[0][0]], gsmap_lon[index_pos[1][0]]


def set_gauge_data_to_gsmap():
    input_lat_arr = []
    input_lon_arr = []

    lat_arr, lon_arr = get_lon_lat_gauge_data()
    for i in range(0, len(lat_arr)):
        os.system(
            'cdo -outputtab,value -remapnn,lon={}_lat={} data/conv2d_gsmap/gsmap_2011_2018.nc > data/conv2d_gsmap/remapnn.csv'
            .format(lon_arr[i], lat_arr[i]))
        lat, lon = find_lat_lon_remapnn()
        input_lat_arr.append(lat)
        input_lon_arr.append(lon)

    input_lat_arr.sort()
    input_lon_arr.sort()

    # get precipitation
    input_precip_arr = np.empty(shape=(1766,
                                       len(input_lat_arr), len(input_lon_arr)))
    for i in range(0, len(input_lat_arr)):
        os.system(
            'cdo -outputtab,value -remapnn,lon={}_lat={} data/conv2d_gsmap/gsmap_2011_2018.nc > data/conv2d_gsmap/precip.csv'
            .format(input_lon_arr[i], input_lat_arr[i]))
        precipitation = read_csv('data/conv2d_gsmap/precip.csv')
        precipitation = precipitation.to_numpy()
        input_precip_arr[:, i, i] = precipitation[:,0]

    return input_lat_arr, input_lon_arr, input_precip_arr


def save_to_npz():
    input_lat, input_lon, input_precip = set_gauge_data_to_gsmap()
    output_nc = Dataset('data/conv2d_gsmap/gsmap_2011_2018.nc', 'r')

    time = np.array(output_nc['time'][:])
    output_lon = np.array(output_nc['lon'][:])
    output_lat = np.array(output_nc['lat'][:])
    output_precip = np.array(output_nc['precip'][:])

    np.savez('data/npz/conv2d_gsmap.npz',
             time=time,
             input_lon=input_lon,
             input_lat=input_lat,
             input_precip=input_precip,
             output_lon=output_lon,
             output_lat=output_lat,
             output_precip=output_precip)

def test():
    input_precip = np.load('data/npz/conv2d_gsmap.npz')['input_precip']
    print(input_precip[4,70,70])

# test()
save_to_npz()<file_sep>/lib/conv2d_gsmap/read_netcdf.py
from netCDF4 import Dataset
from datetime import datetime
import numpy as np

# input_nc = Dataset('data/conv2d_gsmap/tmp.nc', 'r')

# # check dataset
# for i in input_nc.variables:
#     print(i, input_nc.variables[i].shape)

# time = np.array(input_nc['time'][:])
# input_lon = np.array(input_nc['lon'][:])
# input_lat = np.array(input_nc['lat'][:])
# input_precip = np.array(input_nc['precip'][:])

output_nc = Dataset('data/conv2d_gsmap/gsmap_2011_2018.nc', 'r')
output_time = np.array(output_nc['time'][:])
output_lon = np.array(output_nc['lon'][:])
output_lat = np.array(output_nc['lat'][:])
output_precip = np.array(output_nc['precip'][:])

print(output_lat)
print(output_lon)

print(len(output_precip))

# Test nearist grid point
print(output_lat[0], output_lon[1], output_lon[2])
print(output_lat[1], output_lon[1], output_lon[2])

# Neu tu 0.85 den 0.95, lay 0.9 thi no' thien ve 0.95
print(output_precip[2,0,1])
print(output_precip[2,0,2])
print("Goc 3: ",output_precip[0,1,1]) 
print("Goc 4: ",output_precip[9,1,2])
print(output_lon[60])
print(output_lat[92])
np.savez('data/npz/conv2d_gsmap.npz',
         time=time,
         input_lon=input_lon,
         input_lat=input_lat,
         input_precip=input_precip,
         output_lon=output_lon,
         output_lat=output_lat,
         output_precip=output_precip)

# for i in range(0, 5):
#     print(datetime.fromtimestamp(time[i]))<file_sep>/requirements.txt
numpy==1.18.4
pandas==1.0.3
pyyaml==5.3.1
scipy==1.4.1
netcdf4==1.5.3
sklearn==0.0
keras==2.3.1
matplotlib==3.2.1
tensorflow==2.1.0<file_sep>/test.py
print(int(round((23.95-14.75)/0.1)))<file_sep>/lib/conv2d_gsmap/gauge_data_preprocessing.py
import numpy as np
from pandas import read_csv
from netCDF4 import Dataset
import os
import glob

def get_lon_lat_gauge_data():
    
    preprocessed_data_dir = './data/conv2d_gsmap/preprocessed_txt_data/'
    start_index = len(preprocessed_data_dir)
    type_file = '.csv'
    end_index = -len(type_file)
    data_paths_preprocessed_data = glob.glob(preprocessed_data_dir + '*' +
                                             type_file)

    lon_arr = np.empty(shape=(len(data_paths_preprocessed_data)))
    lat_arr = np.empty(shape=(len(data_paths_preprocessed_data)))
    precipitation = []
    for index, file in enumerate(data_paths_preprocessed_data):
        file_name = file[start_index:end_index]
        file_name_list = file_name.split('_')
        lon = float(file_name_list[1])
        lat = float(file_name_list[2])
        lon_arr[index] = lon
        lat_arr[index] = lat
        precip = read_csv(file, usecols=['precipitation'])
        precip = precip.to_numpy()
        precipitation.append(precip[-1766:])
    lon_arr = np.round(lon_arr, 3)
    lat_arr = np.round(lat_arr, 3)

    return lat_arr, lon_arr, precipitation

def find_lat_lon_remapnn():
    # find nearest lon_lat
    precip_list = read_csv('./data/conv2d_gsmap/remapnn.csv')
    precip_list = precip_list.to_numpy()

    original_nc = Dataset('data/conv2d_gsmap/gsmap_2011_2018.nc', 'r')

    # check dataset
    gsmap_time = np.array(original_nc['time'][:])
    gsmap_lon = np.array(original_nc['lon'][:])
    gsmap_lat = np.array(original_nc['lat'][:])
    gsmap_precip = np.array(original_nc['precip'][:])
    gsmap_precip = np.round(gsmap_precip, 6)

    index_pos = []
    # precip_list - 2 due to the header and 0-based index
    for i in range(0, len(precip_list)):
        if precip_list[i] > 0:
            index_pos = np.where(gsmap_precip[i, :, :] == np.round(
                precip_list[i, 0], 6))  # 0 because of precip_list is (xxx,1)
            if len(index_pos[0]) == 1 and len(index_pos[1]) == 1:
                break

    # get lat and long
    # 0 is lat
    # 1 is lon
    return gsmap_lat[index_pos[0][0]], gsmap_lon[index_pos[1][0]]

def set_gauge_data_to_gsmap():
    gauge_lat_arr = []
    gauge_lon_arr = []
    gauge_precip_arr = np.empty(shape=(1766, 72))
    gsmap_precip_arr = np.empty(shape=(1766, 72))
    # get precipitation
    lat_arr, lon_arr, precipitation = get_lon_lat_gauge_data()
    for i in range(0, len(lat_arr)):
        os.system(
            'cdo -outputtab,value -remapnn,lon={}_lat={} data/conv2d_gsmap/gsmap_2011_2018.nc > data/conv2d_gsmap/remapnn.csv'
            .format(lon_arr[i], lat_arr[i]))
        lat, lon = find_lat_lon_remapnn()
        gauge_lat_arr.append(lat)
        gauge_lon_arr.append(lon)
        precip = precipitation[i]
        gauge_precip_arr[:, i] = precip[:, 0]

        gsmap_precipitation = read_csv('data/conv2d_gsmap/remapnn.csv')
        gsmap_precipitation = gsmap_precipitation.to_numpy()
        gsmap_precip_arr[:, i] = gsmap_precipitation[:, 0]

    cal_error(gauge_precip_arr[-353:,:], gsmap_precip_arr[-353:,:])

    return gauge_lat_arr, gauge_lon_arr, gauge_precip_arr

def save_to_npz():
    gauge_lat, gauge_lon, gauge_precip = set_gauge_data_to_gsmap()

    np.savez('data/conv2d_gsmap/npz/gauge_data.npz',
             gauge_lon=gauge_lon,
             gauge_lat=gauge_lat,
             gauge_precip=gauge_precip)

def cal_error(test_arr, prediction_arr):
    from sklearn.metrics import mean_squared_error, mean_absolute_error
    with np.errstate(divide='ignore', invalid='ignore'):
        # cal mse
        error_mae = mean_absolute_error(test_arr, prediction_arr)

        # cal rmse
        error_mse = mean_squared_error(test_arr, prediction_arr)
        error_rmse = np.sqrt(error_mse)

        # cal mape
        y_true, y_pred = np.array(test_arr), np.array(prediction_arr)
        error_mape = np.mean(np.abs((y_true - y_pred) / y_true)) * 100
        error_list = [error_mae, error_rmse, error_mape]
        print("MAE: %.4f" % (error_mae))
        print("RMSE: %.4f" % (error_rmse))
        print("MAPE: %.4f" % (error_mape))
        return error_list

save_to_npz()<file_sep>/lib/conv2d_gsmap/txt_preprocessing.py
import glob
import pandas as pd
from datetime import datetime
import re
import numpy as np

data_dir = 'data/conv2d_gsmap/raw_data_daily/'
data_paths = glob.glob(data_dir + '*.txt')


def satisfy(year, month, day):
    month_31 = [1, 3, 5, 7, 8, 10, 12]
    month_30 = [4, 6, 9, 11]
    if month in month_31:
        return True
    elif month in month_30:
        if day <= 30:
            return True
    else:
        if day <= 28:
            return True
        if day <= 29 and year % 4 == 0:
            return True
    return False


def process(file_pth):
    f = open(file_pth, "r")
    """Read name, lon, lat"""
    name = f.readline()
    name = re.sub(' +', ' ', name).strip()
    address = f.readline()
    address = re.sub(' +', ' ', address).strip().split()
    lon = address[0]
    lat = address[1]
    height = address[2]
    """Preprocessing time and precipitation"""
    time = []
    precipitation = []
    day_arr = []
    month_arr = []
    year_arr = []
    year = 2011
    while year < 2019:
        f.readline()
        for day in range(1, 32):
            data = f.readline()
            data = re.sub(' +', ' ', data).strip().split()
            for month in range(1, len(data)):
                data[month] = float(data[month])
                if satisfy(year, month, day):
                    time.append(datetime(year, month, day).timestamp())
                    day_arr.append(day)
                    month_arr.append(month)
                    year_arr.append(year)
                    if data[month] < 0:
                        data[month] = 0.0
                    precipitation.append(data[month])
        year += 1

    df = pd.DataFrame()
    df['time'] = time
    df['day'] = day_arr
    df['month'] = month_arr
    df['year'] = year_arr
    df['precipitation'] = precipitation
    df = df.sort_values('time', ascending=True)

    df.to_csv(f'./data/conv2d_gsmap/preprocessed_txt_data/{name}_{lon}_{lat}_{height}.csv',
              index=False)


for file in data_paths:
    process(file)
<file_sep>/model/utils/conv2d.py
import numpy as np
from model import common_util
from sklearn.preprocessing import MinMaxScaler

def create_data(**kwargs):

    data_npz = kwargs['data'].get('dataset')
    seq_len = kwargs['model'].get('seq_len')
    horizon = kwargs['model'].get('horizon')

    time = np.load(data_npz)['time']
    # horizon is in seq_len. the last
    T = len(time) - seq_len

    lon = np.load(data_npz)['input_lon']
    lat = np.load(data_npz)['input_lat']
    precip = np.load(data_npz)['input_precip']

    target_lon = np.load(data_npz)['output_lon']
    target_lat = np.load(data_npz)['output_lat']
    target_precip = np.load(data_npz)['output_precip']

    channels = 3  # Two channels are lon, lat and precip

    input_conv2d_gsmap = np.zeros(shape=(T, seq_len,
                                         len(lat), len(lon), channels))
    target_conv2d_gsmap = np.zeros(shape=(T, horizon,
                                          len(target_lat), len(target_lon),
                                          channels))
    """fill input_data"""
    # preprocessing data
    lon_res = lon.reshape(1, lon.shape[0])
    lat_res = lat.reshape(lat.shape[0], 1)
    lon_dup = np.repeat(lon_res, len(lat), axis=0)
    lat_dup = np.repeat(lat_res, len(lon), axis=1)

    # because lon and lat are the same over the period, therefore we duplicate
    lon_dup = np.repeat(lon_dup[np.newaxis, :, :], seq_len, axis=0)
    lat_dup = np.repeat(lat_dup[np.newaxis, :, :], seq_len, axis=0)

    lon_dup = np.repeat(lon_dup[np.newaxis, :, :], T, axis=0)
    lat_dup = np.repeat(lat_dup[np.newaxis, :, :], T, axis=0)

    # fill channels
    input_conv2d_gsmap[:, :, :, :, 0] = lat_dup
    input_conv2d_gsmap[:, :, :, :, 1] = lon_dup
    """fill target_data"""
    # preprocessing data
    target_lon_res = target_lon.reshape(1, target_lon.shape[0])
    target_lat_res = target_lat.reshape(target_lat.shape[0], 1)
    target_lon_dup = np.repeat(target_lon_res, len(target_lat), axis=0)
    target_lat_dup = np.repeat(target_lat_res, len(target_lon), axis=1)

    # because lon and lat are the same over the period, therefore we duplicate
    target_lon_dup = np.repeat(target_lon_dup[np.newaxis, :, :],
                               horizon,
                               axis=0)
    target_lat_dup = np.repeat(target_lat_dup[np.newaxis, :, :],
                               horizon,
                               axis=0)
    target_lon_dup = np.repeat(target_lon_dup[np.newaxis, :, :],
                               T,
                               axis=0)
    target_lat_dup = np.repeat(target_lat_dup[np.newaxis, :, :],
                               T,
                               axis=0)

    # fill channels
    target_conv2d_gsmap[:, :, :, :, 0] = target_lat_dup
    target_conv2d_gsmap[:, :, :, :, 1] = target_lon_dup

    # shape convlstm2d (batch_size, n_frames, height, width, channels)
    _x = np.empty(shape=(seq_len, len(lat), len(lon), channels))
    _y = np.empty(shape=(horizon, 160, 120, channels))
    for i in range(0, T):
        _x = precip[i:i + seq_len]
        _y = target_precip[i + seq_len - horizon]

        input_conv2d_gsmap[i, :, :, :, 2] = _x
        target_conv2d_gsmap[i, :, :, :, 2] = _y
    
    # target_conv2d_gsmap_2 = np.zeros(shape=(T, horizon, 72, 72, 3))
    # target_conv2d_gsmap_2[:,:,:,:,2] = target_conv2d_gsmap[:, :, 0:72, 0:72, 2].copy()
    # target_conv2d_gsmap_2[:, :, :, :, 0:2] = target_conv2d_gsmap[:, :, 0:72, 0:72, 0:2].copy()

    return input_conv2d_gsmap2, target_conv2d_gsmap


def create_data_prediction(**kwargs):
    
    data_npz = kwargs['data'].get('dataset')
    seq_len = kwargs['model'].get('seq_len')
    horizon = kwargs['model'].get('horizon')

    time = np.load(data_npz)['time']
    # horizon is in seq_len. the last
    T = len(time) - seq_len - horizon

    lon = np.load(data_npz)['output_lon']
    lat = np.load(data_npz)['output_lat']
    precip = np.load(data_npz)['output_precip']


    input_conv2d_gsmap = np.zeros(shape=(T, seq_len,
                                         len(lat), len(lon), 1))
    target_conv2d_gsmap = np.zeros(shape=(T, seq_len,
                                          len(lat), len(lon), 1))

    for i in range(T):
        input_conv2d_gsmap[i, :, :, :, 0] = precip[i:i+seq_len]
        target_conv2d_gsmap[i, :, :, :, 0] = precip[i+horizon:i+seq_len+horizon]

    
    # # channel 2 - gauge data
    # gauge_dataset = kwargs['data'].get('gauge_dataset')
    # gauge_lon = np.load(gauge_dataset)['gauge_lon']
    # gauge_lat = np.load(gauge_dataset)['gauge_lat']
    # gauge_precipitation = np.load(gauge_dataset)['gauge_precip']
    # for i in range(len(gauge_lat)):
    #     lat = gauge_lat[i]
    #     lon = gauge_lon[i]
    #     temp_lat = int(round((23.95-lat)/0.1))
    #     temp_lon = int(round((lon-100.05)/0.1))
    #     gauge_precip = gauge_precipitation[:, i]
    #     for j in range(T):
    #         input_conv2d_gsmap[j, :, temp_lat, temp_lon, 1] = gauge_precip[j:j+seq_len]
    #         # remap the target gsmap by gauge data
    #         target_conv2d_gsmap[j, :, temp_lat, temp_lon, 0] = gauge_precip[j:j+seq_len]

    return input_conv2d_gsmap, target_conv2d_gsmap

def load_dataset(**kwargs):
    # get preprocessed input and target
    input_conv2d_gsmap, target_conv2d_gsmap = create_data_prediction(**kwargs)

    # get test_size, valid_size from config
    test_size = kwargs['data'].get('test_size')
    valid_size = kwargs['data'].get('valid_size')

    # split data to train_set, valid_set, test_size
    input_train, input_valid, input_test = common_util.prepare_train_valid_test(
        input_conv2d_gsmap, test_size=test_size, valid_size=valid_size)
    target_train, target_valid, target_test = common_util.prepare_train_valid_test(
        target_conv2d_gsmap, test_size=test_size, valid_size=valid_size)
    data = {}
    for cat in ["train", "valid", "test"]:
        x, y = locals()["input_" + cat], locals()["target_" + cat]
        data["input_" + cat] = x
        data["target_" + cat] = y

    return data
<file_sep>/model/conv2d.py
from keras.models import Sequential
from keras.layers import Flatten, Dense, MaxPooling2D, Conv2D, Conv2DTranspose, UpSampling2D, Cropping2D, Cropping3D, MaxPooling3D, UpSampling3D
from keras.layers.convolutional import Conv3D, Conv3DTranspose
from keras.layers.convolutional_recurrent import ConvLSTM2D
from keras.layers.normalization import BatchNormalization
import numpy as np
from model import common_util
import model.utils.conv2d as utils_conv2d
import os
import yaml
from pandas import read_csv


class Conv2DSupervisor():
    def __init__(self, **kwargs):
        self.config_model = common_util.get_config_model(**kwargs)

        # load_data
        self.data = utils_conv2d.load_dataset(**kwargs)
        self.input_train = self.data['input_train']
        self.input_valid = self.data['input_valid']
        self.input_test = self.data['input_test']
        self.target_train = self.data['target_train']
        self.target_valid = self.data['target_valid']
        self.target_test = self.data['target_test']

        # other configs
        self.log_dir = self.config_model['log_dir']
        self.optimizer = self.config_model['optimizer']
        self.loss = self.config_model['loss']
        self.activation = self.config_model['activation']
        self.batch_size = self.config_model['batch_size']
        self.epochs = self.config_model['epochs']
        self.callbacks = self.config_model['callbacks']
        self.seq_len = self.config_model['seq_len']
        self.horizon = self.config_model['horizon']

        self.model = self.build_model_prediction()

    def build_model_prediction(self):
        model = Sequential()

        # Input
        model.add(
            ConvLSTM2D(filters=16,
                       kernel_size=(3, 3),
                       padding='same',
                       return_sequences=True,
                       activation = self.activation,
                       name = 'input_layer_convlstm2d',
                       input_shape=(self.seq_len, 160, 120, 1)))
        # model.add(BatchNormalization())

        # Max Pooling - Go deeper
        model.add(MaxPooling3D(pool_size=(2, 2, 1)))

        model.add(
            ConvLSTM2D(filters=16,
                       kernel_size=(3, 3),
                       padding='same',
                       activation = self.activation,
                       name='hidden_layer_convlstm2d_1',
                       return_sequences=True))
        # model.add(BatchNormalization())

        model.add(MaxPooling3D(pool_size=(2, 2, 1)))

        model.add(
            ConvLSTM2D(filters=32,
                       kernel_size=(3, 3),
                       padding='same',
                       activation = self.activation,
                       name='hidden_layer_convlstm2d_2',
                       return_sequences=True))
        # model.add(BatchNormalization())

        # Up Sampling
        model.add(UpSampling3D(size=(2, 2, 1)))

        model.add(
            ConvLSTM2D(filters=16,
                       kernel_size=(3, 3),
                       padding='same',
                       activation = self.activation,
                       name='hidden_layer_convlstm2d_3',
                       return_sequences=True))
        # model.add(BatchNormalization())

        model.add(UpSampling3D(size=(2, 2, 1)))

        model.add(
            ConvLSTM2D(filters=16,
                       kernel_size=(3, 3),
                       padding='same',
                       activation = self.activation,
                       name='hidden_layer_convlstm2d_4',
                       return_sequences=True))
        # model.add(BatchNormalization())

        model.add(
            Conv3D(filters=1,
                   kernel_size=(3, 3, 1),
                   padding='same',
                   name='output_layer_conv3d',
                   activation=self.activation))

        print(model.summary())

        # plot model
        from keras.utils import plot_model
        plot_model(model=model,
                   to_file=self.log_dir + '/conv2d_model.png',
                   show_shapes=True)
        return model

    def train(self):
        self.model.compile(optimizer=self.optimizer,
                           loss=self.loss,
                           metrics=['mse', 'mae'])

        training_history = self.model.fit(self.input_train,
                                          self.target_train,
                                          batch_size=self.batch_size,
                                          epochs=self.epochs,
                                          callbacks=self.callbacks,
                                          validation_data=(self.input_valid,
                                                           self.target_valid),
                                          shuffle=True,
                                          verbose=2)

        if training_history is not None:
            common_util._plot_training_history(training_history,
                                               self.config_model)
            common_util._save_model_history(training_history,
                                            self.config_model)
            config = dict(self.config_model['kwargs'])

            # create config file in log again
            config_filename = 'config.yaml'
            config['train']['log_dir'] = self.log_dir
            with open(os.path.join(self.log_dir, config_filename), 'w') as f:
                yaml.dump(config, f, default_flow_style=False)

    def test_prediction(self):
        import sys
        print("Load model from: {}".format(self.log_dir))
        self.model.load_weights(self.log_dir + 'best_model.hdf5')
        self.model.compile(optimizer=self.optimizer, loss=self.loss)

        input_test = self.input_test
        actual_data = self.target_test
        predicted_data = np.zeros(shape=(len(actual_data), self.horizon, 160,
                                         120, 1))
        from tqdm import tqdm
        iterator = tqdm(range(0,len(actual_data)))
        for i in iterator:
            input = np.zeros(shape=(1, self.seq_len, 160, 120, 1))
            input[0] = input_test[i].copy()
            yhats = self.model.predict(input)   
            predicted_data[i, 0] = yhats[0, -1]            
            print("Prediction: ", np.count_nonzero(predicted_data[i, 0] > 0), "Actual: ", np.count_nonzero(actual_data[i, -1]>0))
        
        data_npz = self.config_model['data_kwargs'].get('dataset')
        lon = np.load(data_npz)['input_lon']
        lat = np.load(data_npz)['input_lat']

        gauge_dataset = self.config_model['data_kwargs'].get('gauge_dataset')
        gauge_lon = np.load(gauge_dataset)['gauge_lon']
        gauge_lat = np.load(gauge_dataset)['gauge_lat']
        gauge_precipitation = np.load(gauge_dataset)['gauge_precip']

        actual_arr = []
        preds_arr = []
        gauge_arr = []

        # MAE for 72x72 matrix including gauge data
        for index, lat in np.ndenumerate(lat):
            temp_lat = int(round((23.95-lat)/0.1))

            for index, lon in np.ndenumerate(lon):
                temp_lon = int(round((lon-100.05)/0.1))

                # actual data
                actual_precip = actual_data[:, 0, temp_lat, temp_lon, 0]
                actual_arr.append(actual_precip)

                # prediction data
                preds = predicted_data[:, 0, temp_lat, temp_lon, 0]
                preds_arr.append(preds)
        
        common_util.mae(actual_arr, preds_arr)
        
        preds_arr = []
        # MAE for only gauge data
        for i in range(len(gauge_lat)):
            lat = gauge_lat[i]
            lon = gauge_lon[i]
            temp_lat = int(round((23.95-lat)/0.1))
            temp_lon = int(round((lon-100.05)/0.1))

            # gauge data
            gauge_precip = gauge_precipitation[-353:, i]
            gauge_arr.append(gauge_precip)

            # prediction data
            preds = predicted_data[:, 0, temp_lat, temp_lon, 0]
            preds_arr.append(preds)

        common_util.cal_error(gauge_arr, preds_arr)

    def plot_result(self):
        from matplotlib import pyplot as plt
        preds = np.load(self.log_dir + 'pd.npy')
        gt = np.load(self.log_dir + 'gt.npy')
        plt.plot(preds[:], label='preds')
        plt.plot(gt[:], label='gt')
        plt.legend()
        plt.savefig(self.log_dir + 'result_predict.png')
        plt.close()
