Why is my training error going up? I have made the learning rate very low, but this has not helped.
(I was having difficulty getting the hidden state to work. Maybe there’s a mistake there that is now causing the growing training error?)
import torch
import torch.nn as nn

import seaborn as sns
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error

# formula to create input sequences and scalar labels

def create_inout_sequences(input_data, tw):
    inout_seq = []
    L = len(input_data)
    for i in range(L-tw):
        train_seq = input_data[i:i+tw]
        train_label = input_data[i+tw:i+tw+1]
        inout_seq.append((train_seq ,train_label))
    return inout_seq

# create input sequences and scalar labels from data

# create random data that mimics actual data I am using
sampl = np.random.uniform(low=-3, high=3, size=(50,))
sampl.astype(np.float64)

# data
train_data_normalized = sampl.copy()

# convert data into tensors
train_data_normalized = torch.FloatTensor(train_data_normalized).view(-1)

# seq length
train_window = 10

# generate input sequences and scalar labels
train_inout_seq = create_inout_sequences(train_data_normalized, train_window)

class GRU(nn.Module):
    def __init__(self, input_size=1, hidden_layer_size=32, output_size=1, n_layers=2):
        super().__init__()
        
        self.hidden_layer_size = hidden_layer_size

        self.gru = nn.GRU(input_size, hidden_layer_size, n_layers)

        self.linear = nn.Linear(hidden_layer_size, output_size)

        self.hidden_cell = torch.stack((torch.zeros(1,1,self.hidden_layer_size),
                                        torch.zeros(1,1,self.hidden_layer_size)))

    def forward(self, input_seq):
        gru_out, self.hidden_cell = self.gru(input_seq.view(len(input_seq) ,1, -1), self.hidden_cell)
        predictions = self.linear(gru_out.view(len(input_seq), -1))
        return predictions[-1]

model = GRU()

print(model)

# loss and optimization functions
loss_function = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.0002)

epochs = 10

for i in range(epochs):
    for seq, labels in train_inout_seq:
        
        optimizer.zero_grad()
        
        # reset hidden state
        model.hidden_cell = torch.stack((torch.zeros(1, 1, model.hidden_layer_size),
                                         torch.zeros(1, 1, model.hidden_layer_size)))
        
        # get rid of the 2nd dimension of hidden state to reduce hidden state from 4-d to 3-d
        # from (2, 1, 1, 32) to (2, 1, 32)
        model.hidden_cell = model.hidden_cell.squeeze(2)

        y_pred = model(seq)

        single_loss = loss_function(y_pred, labels)
        
        single_loss.backward()
        
        # clip gradient to prevent exploding gradient
        nn.utils.clip_grad_norm_(model.parameters(), 5)
        
        optimizer.step()

    if i%2 == 0:
        print(f'epoch: {i:3} loss: {single_loss.item():10.8f}')

print(f'epoch: {i:3} loss: {single_loss.item():10.10f}')
print("Finished training")

How do I prevent training error from going up after epoch 4?
epoch:   0 loss: 0.00021652
epoch:   2 loss: 0.00008253
epoch:   4 loss: 0.00004190
epoch:   6 loss: 0.00006440
epoch:   8 loss: 0.00010042
epoch:  10 loss: 0.00012543
epoch:  12 loss: 0.00014364
epoch:  14 loss: 0.00015832
epoch:  16 loss: 0.00017104
epoch:  18 loss: 0.00018215
epoch:  20 loss: 0.00019215
epoch:  22 loss: 0.00020187
epoch:  24 loss: 0.00021184
epoch:  26 loss: 0.00022200
epoch:  28 loss: 0.00023253
epoch:  29 loss: 0.0002379434