This is my D, quite simple:
class PatchDiscriminator(nn.Module):
    """Defines a PatchGAN discriminator"""

    def __init__(self, input_nc=3, ndf=64):
        """Construct a PatchGAN discriminator
        Parameters:
            input_nc (int)  -- the number of channels in input images
            ndf (int)       -- the number of filters in the last conv layer
            n_layers (int)  -- the number of conv layers in the discriminator
            norm_layer      -- normalization layer
        """
        super(PatchDiscriminator, self).__init__()
        sequence= []
        sequence += [nn.Conv2d(input_nc, ndf, kernel_size=4, stride=2, padding=1), nn.LeakyReLU(0.2)]

        sequence += [nn.Conv2d(ndf, ndf*2, kernel_size=4, stride=2, padding=1), nn.LeakyReLU(0.2)]
        sequence += [nn.Conv2d(ndf*2, ndf*4, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(0.2)]
        sequence += [nn.Conv2d(ndf*4, ndf*8, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(0.2)]

        sequence += [nn.Conv2d(ndf*8, 1, kernel_size=3, stride=1, padding=1)]
        self.model = nn.Sequential(*sequence)

    def forward(self, input):
        """Standard forward."""
        return self.model(input)

And the shape of fake_image in my case is varied, I have three different group fake_image, that may be the reason. see code below:
        real_labels = torch.ones((hr.size(0), 1)).type_as(hr)
        fake_labels = torch.zeros((hr.size(0), 1)).type_as(hr)

        ##########################
        #   training generator   #
        ##########################
        optimizer_generator.zero_grad()

        adversarial_loss = 0
        fake_image_group = [lr, pred1, pred2]
        for fake_image in fake_image_group:
            score_real = self.D(hr).mean(dim=(1, 2, 3))
            score_fake = self.D(fake_image).mean(dim=(1, 2, 3))

            discriminator_rf = score_real.unsqueeze(dim=1) - score_fake.mean()
            discriminator_fr = score_fake.unsqueeze(dim=1) - score_real.mean()

            adversarial_loss_rf = self.adversarial_criterion(discriminator_rf, fake_labels)
            adversarial_loss_fr = self.adversarial_criterion(discriminator_fr, real_labels)
            adversarial_loss += (adversarial_loss_fr + adversarial_loss_rf) / 2

        adversarial_loss /= len(fake_image_group)
        
        perceptual_loss = self.perception_criterion(hr, pred1) + self.perception_criterion(hr, pred2) #+ self.perception_criterion(hr, pre3)
        content_loss = self.content_criterion(pred1, hr) + self.content_criterion(pred2, hr) #+ self.content_criterion(pred3, hr)

        generator_loss = adversarial_loss * self.hparams.adversarial_loss_factor + \
                            perceptual_loss * self.hparams.perceptual_loss_factor + \
                            content_loss * self.hparams.content_loss_factor

        self.manual_backward(generator_loss)
        optimizer_generator.step()

        ##########################
        # training discriminator #
        ##########################

        optimizer_discriminator.zero_grad()

        adversarial_loss = 0
        for fake_image in fake_image_group:
            score_real = self.D(hr).mean(dim=(1, 2, 3))
            score_fake = self.D(fake_image.detach()).mean(dim=(1, 2, 3))

            discriminator_rf = score_real.unsqueeze(dim=1) - score_fake.mean()
            discriminator_fr = score_fake.unsqueeze(dim=1) - score_real.mean()

            adversarial_loss_rf = self.adversarial_criterion(discriminator_rf, real_labels)
            adversarial_loss_fr = self.adversarial_criterion(discriminator_fr, fake_labels)
            adversarial_loss += (adversarial_loss_fr + adversarial_loss_rf) / 2

        adversarial_loss /= len(fake_image_group)

        self.manual_backward(adversarial_loss)
        optimizer_discriminator.step()

pred1 and pred2 have same shape, and lr have it’s own shape.