Sorry for the late response.

My hidden states seem to be working fine. They are not reinitialized at every step.
I have observed that the outputs of CNN are so close. But I don’t think that the problem is either in CNN or RNN. There is another bug probably.

I found out that even with perfectly randomly generated input videos result in the same output. But whenever the parameters of the model changes (with training) the output changes. So I conclude that there might have been some kind of overfitting. But I don’t know why. I am adding the codes for investigation. I am still seeking for help :((
This is my train function.
def train(model, optimizer, epoch, train_loader, args):
    traindebug = True
    model.train()
    total_loss = 0
    outlist = []
    sig = nn.Sigmoid()
    #criterion = nn.MSELoss()
    for batch_idx, (data, target,vname) in tqdm(enumerate(train_loader),total = len(train_loader)):
        data = torch.FloatTensor(data)
        if args["cuda"]:
            data, target = data.cuda(), target.cuda()

        data, target = Variable(data), Variable(target).float()
        optimizer.zero_grad()
        
        # Generating random tensor for trial
        randt = torch.rand(1, 200, 3, 224, 224)
        randt = randt.cuda()
        randt = Variable(randt)
        randout = model(randt)
        
        output = model(data)
        outlist.append([vname,[output[0][0].item()]])
        
        del randt
        del randout
        #loss = criterion(output, target)
        if losstype == 'mae':
            loss = F.l1_loss(output, target)
        elif losstype == 'cross_entropy':
            loss = F.binary_cross_entropy(output, target)
        elif losstype == 'BCEWithLogits':
            pos_weight = torch.ones([class_count])
            criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight).cuda()
            # bunu kullanırken outputu sigmoidden çıkarmak gerekebilir
            loss = criterion(output, target)
        elif losstype == 'sigmoid':
            loss = sig(torch.abs(output-target))
            
        total_loss += loss.item()
        
        loss.backward()
        optimizer.step()
          
    total_loss /= len(train_loader.dataset)
    print('Train epoch {} Average loss: {:.4f})'.format(epoch, total_loss))

    return outlist, [total_loss]

This is my validation function:
def test(model, test_loader, epoch, args, is_cuda):
    testdebug = True
    model.eval()
    test_loss = 0
    outlist = []
    for batch_idx, (data, target,vname) in tqdm(enumerate(test_loader),total = len(test_loader)):
        data = torch.FloatTensor(data)
        if is_cuda:
            data, target = data.cuda(), target.cuda()

        with torch.no_grad():
            data, target = Variable(data), Variable(target).float()
            output = model(data)
            
            randt = torch.rand(1, 200, 3, 224, 224)
            randt = randt.cuda()
            randt = Variable(randt)
            randout = model(randt)
            
            y_pred = torch.round(torch.sigmoid(output))

            outlist.append([vname,[output[0][0].item()]])
            if losstype == 'mae':    
                loss = F.l1_loss(output, target)
            elif losstype == 'cross_entropy' or losstype == 'sigmoid':
                loss = F.l1_loss(output.round(), target)
            else:
                loss = (y_pred == target).float().min()
    
            del randt
            del randout
            
            test_loss += loss.item() / target.size()[0]

            
    test_loss /= len(test_loader.dataset)
    acc = 1 - test_loss
    print('Validation epoch {} Average accuracy: {:.4f}'.format(epoch,acc))

    return outlist, [acc]

This is the forward function of the model:
    def forward(self, x):
            batch_size, timesteps, C, H, W = x.size()
            self.c_in = x.view(batch_size * timesteps, C, H, W)
            self.c_out = self.cnn(self.c_in)
            self.r_in = self.c_out.view(batch_size, timesteps, -1)
            self.r_out, self.h_n = self.rnn(self.r_in)
            return self.linear(self.r_out[:, -1, :])

If someone need more info about the code, I can give it right away.