I also have this problem. I am trying to reproduce a code I found on a blog post but getting the error RuntimeError: given groups=1, weight of size [512, 1024, 1, 1], expected input[4, 512, 188, 188] to have 1024 channels, but got 512 channels instead…
here is the network:
class ConvRelu(nn.Module):

def __init__(self, in_depth, out_depth):
    super(ConvRelu, self).__init__()
    self.conv = torch.nn.Conv2d(in_depth, out_depth, kernel_size=3, stride=1, padding=1)
    self.activation = nn.ReLU(inplace=True)

def forward(self, x):
    x = self.conv(x)
    x = self.activation(x)
    return x
class DecoderBlock(nn.Module, ABC):
    def __init__(self, in_depth, middle_depth, out_depth):
        super(DecoderBlock, self).__init__()
        self.conv_relu = ConvRelu(in_depth, middle_depth)
        self.conv_transpose = nn.ConvTranspose2d(middle_depth, out_depth, kernel_size=4, stride=2, padding=1)
        self.activation = nn.ReLU(inplace=True)
def forward(self, x):
    x = self.conv_relu(x)
    x = self.conv_transpose(x)
    x = self.activation(x)
    return x
class UNetResNet(nn.Module):
    def __init__(self, num_classes):
        super(UNetResNet, self).__init__()
        self.encoder = torchvision.models.resnet101(pretrained=True)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv1 = nn.Sequential(self.encoder.conv1, self.encoder.bn1, self.encoder.relu, self.pool)
        self.conv2 = self.encoder.layer1
        self.conv3 = self.encoder.layer2
        self.conv4 = self.encoder.layer3
        self.conv4 = self.encoder.layer4

    self.pool = nn.MaxPool2d(2, 2)
    self.center = DecoderBlock(2048, 512, 256)

    self.dec5 = DecoderBlock(2048 + 256, 512, 256)
    self.dec4 = DecoderBlock(1024 + 256, 512, 256)
    self.dec3 = DecoderBlock(512 + 256, 256, 64)
    self.dec2 = DecoderBlock(256 + 64, 128, 128)
    self.dec1 = DecoderBlock(128, 128, 32)
    self.dec0 = ConvRelu(32, 32)
    self.final = nn.Conv2d(32, num_classes, kernel_size=1)

def forward(self, x):
    conv1 = self.conv1(x)
    conv2 = self.conv2(conv1)
    conv3 = self.conv3(conv2)
    conv4 = self.conv4(conv3)
    conv5 = self.conv5(conv4)

    pool = self.pool(conv5)
    center = self.center(pool)

    dec5 = self.dec5(torch.cat([center, conv5], 1))
    dec4 = self.dec4(torch.cat([dec5, conv4], 1))
    dec3 = self.dec4(torch.cat([dec4, conv3], 1))
    dec2 = self.dec4(torch.cat([dec3, conv2], 1))
    dec1 = self.dec1(dec2)
    dec0 = self.dec0(dec1)

    return self.final(dec0)

and the training loop
for epoch_idx in range(2):
    loss_batches = []
    for batch_idx, data in enumerate(tqdm(train_dataloader, desc = "training")):
        imgs = torch.autograd.Variable(data['sat_img'].cuda())
        #imgs = imgs.view(1, -1)
        #imgs.unsqueeze(0)
        #imgs.squeeze()
        masks = torch.autograd.Variable(data['map_img'].cuda())
        masks.unsqueeze_(0)
        #print(imgs.size())


        y = unet_resnet(imgs)
        loss = cross_entropy_loss(y, masks)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        loss_batches.append(loss.data.cpu().numpy())
    print('epoch: ' + str(epoch_idx) + ' training loss: ' + str(np.sum(loss_batches)))

model_file = './unet-' + str(epoch_idx)
unet_resnet = unet_resnet.cpu()
torch.save(unet_resnet.state_dict(), model_file)
unet_resnet = unet_resnet.cuda()
print('model saved')

According to the error, there is something going on in the conv4 layer.
the input is a PIL image with 3 channels and my batch size is 4.
How can I fix this?