Hi, I couldn’t reproduce your error. Here’s my minimum working code.
import torch
from torch import nn
from torch.nn.utils import spectral_norm


# weight initialization
def initialize_weights(m):
    if isinstance(m, nn.Conv2d):
        nn.init.kaiming_normal_(m.weight)
        if m.bias is not None:
            nn.init.constant_(m.bias, 0)
    elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.LayerNorm):
        nn.init.constant_(m.weight, 1)
        nn.init.constant_(m.bias, 0)
    elif isinstance(m, nn.Linear):
        nn.init.xavier_uniform_(m.weight)
        nn.init.constant_(m.bias, 0)
        print('No problem with initializing weights for linear....')


# My Conv with spectral norm
class MyConvo2d(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.my_conv_with_sn = nn.Sequential(
            spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1,
                                    padding=1)),
            nn.ReLU(),
        )

    def forward(self, x):
        return self.my_conv_with_sn(x)


# Generator with linear + SN
class Generator(nn.Module):
    def __init__(self):
        super().__init__()

        self.ln1 = spectral_norm(nn.Linear(16, 1))
        self.my_conv_with_sn = MyConvo2d(1, 1)

    def forward(self, x):
        x_size = x.shape
        x = self.my_conv_with_sn(x)
        x = x.view(x_size[0], -1)
        x = self.ln1(x)
        return x


x = torch.rand(size=(1, 1, 4, 4))
Net = Generator()
Net.apply(initialize_weights)  # applying weight initialization
out = Net(x)
print("result from generator: ", out)
print("Linear weight with spectral norm: ", Net.ln1.weight)  # with spectral norm
print("Linear weight - Original: ", Net.ln1.weight_orig)  # no spectral norm


Output:
No problem with initializing weights for linear....
result from generator:  tensor([[-0.2687]], grad_fn=<AddmmBackward>)
Linear weight with spectral norm:  tensor([[-0.1292, -0.3753,  0.3092, -0.1900,  0.3692,  0.0772, -0.3573, -0.1007,
          0.0139, -0.1800,  0.2348, -0.0897, -0.2418,  0.3746,  0.3307, -0.1635]],
       grad_fn=<DivBackward0>)
Linear weight - Original:  Parameter containing:
tensor([[-0.2020, -0.5868,  0.4834, -0.2970,  0.5773,  0.1206, -0.5586, -0.1575,
          0.0217, -0.2814,  0.3671, -0.1402, -0.3781,  0.5857,  0.5170, -0.2556]],
       requires_grad=True)

Can you please check your code?