OK, thanks for the info. I had a bit of spare time and tried to create a starter code:
class MyModel(nn.Module):
    def __init__(self, window=16):
        super(MyModel, self).__init__()
        self.conv_model = nn.Sequential(
            nn.Conv3d(
                in_channels=3,
                out_channels=6,
                kernel_size=3,
                stride=1,
                padding=1),
            nn.MaxPool3d((1, 2, 2)),
            nn.ReLU()
        )
        
        self.rnn = nn.RNN(
            input_size=6*16*12*12,
            hidden_size=1,
            num_layers=1,
            batch_first=True
        )
        
        self.hidden = torch.zeros(1, 1, 1)
        
        self.window = window
        
    def forward(self, x):
        self.hidden = torch.zeros(1, 1, 1) # reset hidden
        
        activations = []
        for idx in range(0, x.size(2), self.window):
            x_ = x[:, :, idx:idx+self.window]
            x_ = self.conv_model(x_)
            x_ = x_.view(x_.size(0), 1, -1)
            activations.append(x_)
        x = torch.cat(activations, 1)
        out, hidden = self.rnn(x, self.hidden)
        
        return out, hidden


class MyDataset(Dataset):
    def __init__(self, frames=512):
        self.data = torch.randn(3, 2048, 24, 24)
        self.frames = frames

    def __getitem__(self, index):
        index = index * self.frames
        x = self.data[:, index:index+self.frames]
        return x
        
    def __len__(self):
        return self.data.size(1) / self.frames


model = MyModel()
dataset = MyDataset()
x = dataset[0]
output, hidden = model(x.unsqueeze(0))

As I’m not that experienced with RNNs, you should definitely check for logical errors.
Let me know, if you find something weird.
Also, currently I’m using the output of the RNN. If you want, you could add a linear layer after the RNN, use a bigger hidden size, and return the linear layer with a sigmoid as the non-linearity. This would probably better fit your use case.