Hi,
I have an application for a network which must receives multiple inputs (N > 3). So, I have an input tensor of (batch size, no channels, N, w, h). In my network, I have to extract features from each image from those N and to feed them into an LSTM. It is correct to write the network as below (especially the forward method) ?
I suspect that the training process is not performed well. Do you think the forward method is ok?
from networks.ResNet import resnet18
import torch.nn as nn
import torch


class ReccurentNet(nn.Module):

    def __init__(self, seq_len, n_features, hidden_dim=64):
        super(ReccurentNet, self).__init__()

        self.seq_len, self.n_features = seq_len, n_features

        self.rnn1 = nn.LSTM(
            input_size=n_features,
            hidden_size=hidden_dim,
            num_layers=1,
            batch_first=True
        )

    def forward(self, x):
        x, (_, _) = self.rnn1(x)
        x, (hidden_n, _) = self.rnn2(x)

        return hidden_n


class ConvRecNet(nn.Module):
    def __init__(self, hidden_dim=128):
        super(ConvRecNet, self).__init__()
        self.backbone = resnet18() # backbone features extractor, output 512 features

        self.rnn = nn.LSTM(
            input_size=512,
            hidden_size=hidden_dim,
            num_layers=5,
            batch_first=True
        )

        self.activation = nn.LeakyReLU()
        self.fc = nn.Linear(128, 24)

    def forward(self, x):
        windows = []
        for i in range(0, x.shape[2]):
            windows.append(self.backbone(x[:, :, i, :, :]))

        x = torch.stack(windows)
        x, _ = self.rnn(x)
        x = x[-1] # Take the last output from rnn, is it  ok ?

        x = self.activation(x)
        x = self.fc(x)
        return x

Thank you!