The model itself seems to be correct, as I can perfectly overfit a random input and target batch of 10 samples:
class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3,64,3,2,1)
        self.conv2 = nn.Conv2d(64,128,3,2,1)
        self.conv3 = nn.Conv2d(128,256,3,2,1)
        
        self.deconv1 =nn.ConvTranspose2d(256,128,4,2,1)
        self.deconv2 = nn.ConvTranspose2d(128,64,4,2,1)
        self.deconv3 = nn.ConvTranspose2d(64,64,4,2,1)
        
        self.finalconv = nn.Conv2d(64,22,3,1,1)
            
    def forward(self,x):
        x = self.conv1(x).clamp(min=0)
        x = self.conv2(x).clamp(min=0)
        x = self.conv3(x).clamp(min=0)
        
        x = self.deconv1(x).clamp(min=0)
        x = self.deconv2(x).clamp(min=0)
        x = self.deconv3(x).clamp(min=0)
        
        return self.finalconv(x)

model = Model()
data = torch.randn(10, 3, 24, 24)
target = torch.randint(0, 22, (10, 24, 24))
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()

for epoch in range(200):
    optimizer.zero_grad()
    output = model(data)
    loss = criterion(output, target)
    loss.backward()
    optimizer.step()
    print('epoch {}, loss {}'.format(epoch, loss.item()))


pred = torch.argmax(output, 1)
acc = (pred == target).float().mean()
print(acc)
> tensor(1.)