Hi @Patrice,
*Rule1: If you want to concatenate the layer, it has to have only 1 dimension that is different from the other (i.e. NxDiff1xHxW and NxDiff2xHxW or NxCxDiff1xW and NxCxDiff2xW, etc)
in your case, what you are trying to do:
suppose our input x: 1x3x28x28
what you are doing is:

you do convolution x1, the output is 1x3x28x28
you do convolution x2, the output is 1x3x28x28

(a) you do concatenation cat1 of x1 and x2 at axis 2, the output is 1x56x28x3 (incorrect)
you do convolution x3, the output is 1x3x52x24

(b) you do concatenation cat2 of x2 and x3 at axis 2 again, with this x2 and x3 dimensions, which are 1x56x28x3 (x2) and 1x3x52x24 (x3). These two will violate the *Rule1.

Hence, you need to fix these ones (a and b) to make sure that they conform to *Rule1.
Here is the correct model with dilated convolution that I corrected for you :).
import torch
from torch import nn
from torch.nn import functional as F

class net(nn.Module):   
    def __init__(self):
        super(net, self).__init__()
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=1)
        self.conv2 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3, padding=2, dilation=2)
        self.conv3 = nn.Conv2d(in_channels=6, out_channels=3, kernel_size=5, padding=4, dilation=2)
        self.conv4 = nn.Conv2d(in_channels=6, out_channels=3, kernel_size=7, padding=6, dilation=2)
        self.conv5 = nn.Conv2d(in_channels=12, out_channels=3, kernel_size=3, padding=1)
        self.b = 1

    def forward(self, x, debug=False): 
        
        x1 = F.relu(self.conv1(x))
        if debug: print('x1:', x1.shape)
        x2 = F.relu(self.conv2(x1))
        if debug: print('x2:', x2.shape)
        cat1 = torch.cat((x1, x2), 1)
        if debug: print('cat1:', cat1.shape)
        x3 = F.relu(self.conv3(cat1))
        if debug: print('x3:', x3.shape)
        cat2 = torch.cat((x2, x3), 1)
        if debug: print('cat2:', cat2.shape)
        x4 = F.relu(self.conv4(cat2))
        if debug: print('x4:', x4.shape)
        cat3 = torch.cat((x1, x2, x3, x4),1)
        if debug: print('cat3:', cat3.shape)
        k = F.relu(self.conv5(cat3))
        if debug: print('k:', k.shape)

        if k.size() != x.size():
            raise Exception("k, haze image are different size!")

        output = k * x - k + self.b
        if debug: print('output:', output.shape)
        return F.relu(output)
    
net_instance = net()
b = torch.rand(1, 3, 28, 28)
with torch.no_grad():
    b = net_instance(b, True)

#OUTPUT:    
#x1: torch.Size([1, 3, 28, 28])
#x2: torch.Size([1, 3, 28, 28])
#cat1: torch.Size([1, 6, 28, 28])
#x3: torch.Size([1, 3, 28, 28])
#cat2: torch.Size([1, 6, 28, 28])
#x4: torch.Size([1, 3, 28, 28])
#cat3: torch.Size([1, 12, 28, 28])
#k: torch.Size([1, 3, 28, 28])
#output: torch.Size([1, 3, 28, 28])

Hope it helps, cheers~