If you don’t want to reduce the depth of your input volume, you could use e.g. a kernel size of 1 for this dimension (or another kernel size with the appropriate padding):
encoder = nn.Sequential(
    nn.Conv3d(in_channels=3, out_channels=16, kernel_size=(1, 3, 3), stride=(1,2,2)), nn.ReLU(), nn.BatchNorm3d(16),
    nn.Conv3d(in_channels=16, out_channels=32, kernel_size=(1, 3, 3), stride=(1,2,2)), nn.ReLU(), nn.BatchNorm3d(32),
    nn.Conv3d(in_channels=32, out_channels=16, kernel_size=(1, 3, 3), stride=(1,2,2)), nn.ReLU(), nn.BatchNorm3d(16),
    nn.Conv3d(in_channels=16, out_channels=8, kernel_size=1, stride=(1,3,4)), nn.ReLU(), nn.BatchNorm3d(8),
    nn.Conv3d(in_channels=8, out_channels=4, kernel_size=1, stride=(1,5,5)), nn.ReLU(), nn.BatchNorm3d(4)
)

x = torch.randn(1,3,25,120,160)
out = encoder(x)
print(out.shape)
> torch.Size([1, 4, 25, 1, 1])