I have made a minimum code (without batchnorms) that seems to show the problem, sorry that it’s a bit long:
import torch
import torch.nn as nn
import matplotlib.pyplot as plt

#%% define the model
class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()
        self.main = nn.Sequential(
            nn.ConvTranspose2d(     100, 2 * 8, 4, 1, 0, bias=False),
            nn.ReLU(True),
            nn.ConvTranspose2d(2 * 8, 2 * 4, 4, 2, 1, bias=False),
            nn.ReLU(True),
            nn.ConvTranspose2d(2 * 4, 2 * 2, 4, 2, 1, bias=False),
            nn.ReLU(True),
            nn.ConvTranspose2d(2 * 2,     2, 4, 2, 1, bias=False),
            nn.ReLU(True),
            nn.ConvTranspose2d(    2,      1, 4, 2, 1, bias=False),
            nn.Tanh()
        )

    def forward(self, input):
        output = self.main(input)
        return output

# make a version of the model on cuda:1 and save the model to file
netG = Generator().to('cuda:1')
torch.save(netG.state_dict(), 'netG.pt')
netG = netG.eval()

# load the model twice onto cuda:0, with .to(device) both before and after loading the state dict
netG_1 = Generator().to('cuda:0')
netG_1.load_state_dict(torch.load('netG.pt'))
netG_1 = netG_1.eval()

netG_2 = Generator()
netG_2.load_state_dict(torch.load('netG.pt'))
netG_2 = netG_2.eval().to('cuda:0')


#%% apply the three versions
z = torch.randn(1,100,1,1) # generator input

with torch.no_grad():
    
    output = netG(z.to('cuda:1'))
    output1 = netG_1(z.to('cuda:0'))
    output2 = netG_2(z.to('cuda:0'))

#%% display output
plt.figure()
plt.subplot(1,3,1)
plt.imshow(torch.squeeze(output.cpu()))
plt.subplot(1,3,2)
plt.imshow(torch.squeeze(output1.cpu()))
plt.subplot(1,3,3)
plt.imshow(torch.squeeze(output2.cpu()))

This is the figure I get when I run the code:

outputs1855×490 20.9 KB

We can see that the middle input, with the model that was sent to cuda:0 before loading the state dict has a different output.