@tom Thank you for your kind explanation. Yet as I am confident that I have applied DataParallel to the generator, may I check if this is indeed a version issue? Namely  torch.save(model.module.state_dict(), PATH) and model.module.load_state_dict(torch.load(PATH)) are new functions only apply to the latest pytorch version?
As reference just in case, the following is my code for training setup and saving checkpoints:
use_gpu = torch.cuda.is_available()
device = torch.device("cuda" if use_gpu else "cpu")
D = model.DiscriminatorACGAN(x_dim=x_dim, c_dim=c_dim, norm=norm, weight_norm=weight_norm).to(device)
G = model.GeneratorACGAN(z_dim=z_dim, c_dim=c_dim).to(device)
ngpu = 2 # I am using a 2 GPU machine
# Handle multi-gpu if desired
if (device.type == 'cuda') and (ngpu > 1):
    G = nn.DataParallel(G, list(range(ngpu))).to(device)
if (device.type == 'cuda') and (ngpu > 1):
    D = nn.DataParallel(D, list(range(ngpu))).to(device)

# gan loss function
d_loss_fn, g_loss_fn = model.get_losses_fn(loss_mode)

# optimizer
d_optimizer = torch.optim.Adam(D.parameters(), lr=d_learning_rate, betas=(0.5, 0.999))
g_optimizer = torch.optim.Adam(G.parameters(), lr=g_learning_rate, betas=(0.5, 0.999))

# run
z_sample = torch.randn(c_dim * 10, z_dim).to(device)
c_sample = torch.tensor(np.concatenate([np.eye(c_dim)] * 10), dtype=z_sample.dtype).to(device)

for ep in range(start_ep, epoch):
    for i, (x, c_dense) in enumerate(train_loader):

        step = ep * len(train_loader) + i + 1
        D.train()
        G.train()

        x = x.to(device)
        c_dense = c_dense.to(device)
        z = torch.randn(batch_size, z_dim).to(device)
        c = torch.tensor(np.eye(c_dim)[c_dense.cpu().numpy()], dtype=z.dtype).to(device)
        x_f = G(z, c)

        # train D
        ...
        # train G
        ...

        if ep % 10 == 0:
            torch.save(G.module.state_dict(), os.path.join(ckpt_dir, 'netG_{}.pth'.format(ep)))
            torch.save(D.module.state_dict(), os.path.join(ckpt_dir, 'netD_{}.pth'.format(ep)))