Hello @ptrblck, Thanks so much for this. I think I am implementing it differently and may be that’s why I get an increasing loss instead. So will you have a separate loss for the CNN and RNN? can you please demonstrate what you are saying with code. Here is my model. Note: it is an image regression problem:
def CNN(num_outputs):    
    preNet = models.inception_v3(pretrained=True,aux_logits=False)
    num_ftrs = preNet.fc.in_features
    preNet.fc = nn.Linear(num_ftrs, num_outputs)
    return preNet

class BiGRU(nn.Module):
    def __init__(self, input_features, hidden_size,num_layers,output_dim,seq_len,batch_size):
        super(BiGRU, self).__init__()        
        self.cnn = CNN(input_features)         
        self.gru = nn.GRU(input_features, hidden_size, num_layers=num_layers, bidirectional=True)        
        self.out1 = nn.Linear(hidden_size*2, output_dim)      
        
    def forward(self, input):       
        input1 = self.cnn(input.view(batch_size*seq_len,C,H,W))               
        output, hidden = self.gru(input1.view(seq_len,-1,input_features))         
        pred = self.out1(output.view(-1,seq_len,hidden_size*2)) 
        return pred

model = BiGRU(input_features, hidden_size,num_layers,output_dim,seq_len,batch_size)
loss = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=lr)

So the input to the model has the following shape: (batch_size, seq_len, C,H,W) and output is (batch_size, seq_len, output_dim). But the model is not learning as loss and error keep increasing.
Below is my Dataset code to create a sliding sequence of images and the sequence of images are later grouped into batches using : torch.utils.data.DataLoader(Dataset, batch_size = batch_size,shuffle = False,drop_last = False), to produce (batch_size, seq_len, C,H,W) dataloaders.
class CustomImageDataset(Dataset):
    def __init__(self, img_labels, img_dir, seq_length, transform=None):
        self.img_labels = img_labels
        self.img_dir = img_dir
        self.seq_length = seq_length
        self.transform = transform

    def __len__(self):
        return len(self.img_labels)-self.seq_length

    def __getitem__(self, idx):
        start = idx
        end = idx + self.seq_length
        print('Getting images from {} to {}'.format(start, end))
        indices = list(range(start, end))
        images = []
        labels = []
        for i in indices:
            img_path = os.path.join(self.img_dir, self.img_labels.iloc[i, 0])
            image = read_image(img_path)
            if self.transform:
                image = self.transform(image)
            images.append(image)
            labels.append(torch.tensor(self.img_labels.iloc[i, 1:]))
        x = torch.stack(images)
        y = torch.stack(labels)
        return x, y