I am trying to build a simple encoder - decoder network on time-series data however I am not entirely sure if my implementation is correct.
The training loss (MSE) fluctuates but overall appears to decrease over the first epoch, but then it stalls. Similarly validation error does not decrease after the first epoch
The basic model I’ve started with:
class Encoder(nn.Module):
    def __init__(
        self, number_of_features: int, sequence_length: int, hidden_size: int, num_layers: int, dropout: float
    ):
        super(Encoder, self).__init__()

        self.sequence_length = sequence_length
        self.number_of_features = number_of_features

        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.dropout_amount = dropout

        self.lstm = nn.LSTM(
            input_size=number_of_features,
            hidden_size=hidden_size,
            num_layers=num_layers,
            dropout=dropout
        )

    def init_hidden(self, batch_size, device='cpu'):
        return (torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device),
                torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device))

    def forward(self, data):
        _, hidden = self.lstm(data, self.hidden)

        return hidden

class Decoder(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, num_layers, dropout):
        super(Decoder, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.num_layers = num_layers
        self.dropout = dropout

        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, dropout=dropout)
        self.out = nn.Linear(hidden_size, output_size)

    def forward(self, point_in_time, hidden):
        output, hidden = self.lstm(point_in_time, hidden)
        output = self.out(output[-1])

        return output, hidden

Based on the single feature that I have to predict and the time, I’ve crafted a few extra features, such as sin/cos for the time, and a moving average thus creating a DataFrame with 4 columns.
Afterwards I used sliding window to create sequences of length 20 (for testing) with the following code:
def _create_sequences(data: np.ndarray, window: int = 20) -> (np.array, np.array):
    length = len(data)
    sequences = []

    for index in range(length - window):
        end_window_index = index + window

        sequence = data[index:end_window_index]
        next_in_sequence = data[index + 1:end_window_index + 1][:, 0]

        sequences.append([sequence, next_in_sequence])

    return sequences 

At this point my dataset is compromised of a list of tupples each containing the sequence (X) (20 sequence length x 4 features) and the next sequence (t+1) (Y) (20).
I split the data to train/val sets (80/20) and push them to the DataLoader to get the data in batches of size 25.
The training section is the following:
encoder = Encoder(number_of_features=4, sequence_length=20, hidden_size=256, num_layers=2, dropout=0.3)
decoder = Decoder(input_size=1, hidden_size=256, output_size=1, num_layers=2, dropout=0.3)

def train(loader, current_epoch, teacher_forcing_ratio=0.75):
    encoder.train()
    decoder.train()

    total_loss = 0

    encoder.hidden = encoder.init_hidden(args.batch_size, device=device)

    for i, batch in enumerate(loader):
        encoder_optimizer.zero_grad()
        decoder_optimizer.zero_grad()

        x = batch[0] # (Shape 25 x 20 x 4)
        y = batch[1] # (Shape 25 x 20)

        x = x.to(device)
        y = y.to(device)

        x = x.permute(1, 0, 2) # Change the order to pass it to the LSTM Shape now is 20 x 25 x 4
        y = y.permute(1, 0)    # Change the order to 20 x 25

        max_sequence, batch_size = y.shape

        hidden = encoder(x)

        use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False

        # Create initial start value/token
        input = torch.tensor([[0.0]] * batch_size, dtype=torch.float).to(device)

        # Convert (batch_size, output_size) to (seq_len, batch_size, output_size)
        input = input.unsqueeze(0)

        sequence_total_loss = 0

        for seq_idx in range(max_sequence):
            output, hidden = decoder(input, hidden)

            if use_teacher_forcing:
                input = y[seq_idx].view(1, -1, 1)
            else:
                input = output.unsqueeze(0)

            sequence_total_loss += criterion(output.view(-1), y[seq_idx])

        sequence_total_loss.backward()

        cur_loss = sequence_total_loss / max_sequence

        total_loss += cur_loss

        encoder_optimizer.step()
        decoder_optimizer.step()

I’ve used two Adam optimizers (one for the encoder and one for the decoder). I’ve also tried changing the model size, sequence length, batch size but it didn’t appear to make a difference
If anyone could take the time and give me any hints on where I may have got it wrong I would very much appreciate it.