Thanks for the reply.
For the first one, I see the reason I got std of 0.0099 instead of 0.02, which is because I have spectral_nom on conv layer weights and I print the std after a forward pass (which triggers the spectral norm to modify the weight).
However, for the second point, which weight_init does not work on GPU, I still cannot reason it properly. Please check this new example, which I print out the MSE between generated images:
import torch
from torchvision.utils import save_image
import torch
import torch.nn as nn
from torch.nn.utils import spectral_norm


def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)


class GLU(nn.Module):
    def __init__(self):
        super(GLU, self).__init__()

    def forward(self, x):
        nc = x.size(1)
        assert nc % 2 == 0, 'channels dont divide 2!'
        nc = int(nc/2)
        return x[:, :nc] * torch.sigmoid(x[:, nc:])


def conv1x1(in_planes, out_planes, bias=False):
    "1x1 convolution with padding"
    return spectral_norm(nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1,
                     padding=0, bias=bias))


def conv3x3(in_planes, out_planes):
    "3x3 convolution with padding"
    return spectral_norm(nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,
                     padding=1, bias=False))


# Upsale the spatial size by a factor of 2
def upBlock(in_planes, out_planes):
    block = nn.Sequential(
        nn.Upsample(scale_factor=2, mode='nearest'),
        conv3x3(in_planes, out_planes*2),
        nn.BatchNorm2d(out_planes*2),
        GLU())
    return block


class Generator(nn.Module):
    """docstring for CAN_Generator"""

    def __init__(self, ngf=64, nz=100, nc=3):
        super(Generator, self).__init__()

        self.init = nn.Sequential(
            spectral_norm(nn.ConvTranspose2d(nz, ngf*32, 4, 1, 0)),
            nn.BatchNorm2d(ngf * 32),
            GLU())
        
        self.main = nn.Sequential(
            upBlock(ngf * 16, ngf * 8),
            upBlock(ngf * 8, ngf * 4),
            upBlock(ngf * 4, ngf * 2),
            upBlock(ngf * 2, ngf * 2),
            upBlock(ngf * 2, ngf * 1),
            upBlock(ngf * 1, ngf // 2),
            )
        self.to_256 = nn.Sequential(
            conv1x1(ngf // 2, 3),
            nn.Tanh()
            )

    def forward(self, input):
        feat_256 = self.init(input.view(input.size(0), -1, 1, 1)).view(input.size(0), -1, 4, 4)
        feat_256 = self.main(feat_256)
        img_256 = self.to_256(feat_256)
        return img_256

device = torch.device('cuda:0')
ngf=64
nz=100

torch.manual_seed(8888)

fixed_noise = torch.FloatTensor(8, nz, 1, 1).normal_(0, 1).to(device)

torch.manual_seed(8888)
torch.cuda.manual_seed(8888)
netG1 = Generator(ngf=ngf, nz=nz)
netG1.apply(weights_init)
gimg1 = netG1(fixed_noise.cpu())

torch.manual_seed(8888)
torch.cuda.manual_seed(8888)
netG2 = Generator(ngf=ngf, nz=nz)
gimg2 = netG2(fixed_noise.cpu())


print( torch.nn.functional.mse_loss(gimg1, gimg2) )
save_image( torch.cat([gimg1, gimg2]).add(1).mul(0.5), 'test_cpu.jpg', nrow=8)


torch.manual_seed(8888)
torch.cuda.manual_seed(8888)
netG4 = Generator(ngf=ngf, nz=nz).to(device)
netG4.apply(weights_init)
gimg4 = netG4(fixed_noise)

torch.manual_seed(8888)
torch.cuda.manual_seed(8888)
netG5 = Generator(ngf=ngf, nz=nz).to(device)
gimg5 = netG5(fixed_noise)

print( torch.nn.functional.mse_loss(gimg4, gimg5) )
save_image( torch.cat([gimg4, gimg5]).add(1).mul(0.5), 'test_gpu.jpg', nrow=8)

For image 1 and 2, I run everything on CPU, image 1 is with model weight_init and image 2 is without init, the generated images looks clearly different, and their mse is big (0.37). In contrast, image 4 and 5 are for models on GPU, the generated images looks very same and their mse is only 0.001 (means almost the same).