I’m not familiar with your use case and don’t know why you are adding a constant noise to the conv filters, but these noise tensors might just be too aggressive.
A simple overfitting test shows that the model is properly learning, but the noise seems to disrupt the training:
class Simplenet(nn.Module):                                                                                                                                                                                                                                                 
    def __init__(self):                                                                                                                                                                                                                                                     
        super(Simplenet, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)                                                                                                                                                                                                                                     
        self.relu_conv1 = nn.ReLU()                                                                                                                                                                                                                                         
        self.pool1 = nn.MaxPool2d(2, 2)                                                                                                                                                                                                                                     
        self.conv2 = nn.Conv2d(6, 16, 5)                                                                                                                                                                                                                                    
        self.relu_conv2 = nn.ReLU()                                                                                                                                                                                                                                         
        self.pool2 = nn.MaxPool2d(2, 2)                                                                                                                                                                                                                                     
        self.fc1 = nn.Linear(16 * 5 * 5, 120)                                                                                                                                                                                                                               
        self.relu_fc1 = nn.ReLU()                                                                                                                                                                                                                                           
        self.fc2 = nn.Linear(120, 84)                                                                                                                                                                                                                                       
        self.relu_fc2 = nn.ReLU()                                                                                                                                                                                                                                           
        self.fc3 = nn.Linear(84, 10)                                                                                                                                                                                                                                        
                                                                                                                                                                                                                                                                            
        self.noise_conv1 = torch.randn(self.conv1.weight.size())*0.6 + 0                                                                                                                                                                                      
        self.noise_conv2 = torch.randn(self.conv2.weight.size())*0.6 + 0                                                                                                                                                                                      
                                                                                                                                                                                                                                                                            
    def forward(self, x):
        add_noise(self.conv1.weight, self.noise_conv1)     
        x = self.conv1(x)                                                                                                                                                                                                                                                   
                                                                                                                                                                                      
        x = self.pool1(self.relu_conv1(x))
        add_noise(self.conv2.weight, self.noise_conv2)
        x = self.conv2(x)
        
        x = self.pool2(self.relu_conv2(x))                                                                                                                                                                                                                                  
        # x = self.pool1(self.relu_conv1(self.conv1(x)))                                                                                                                                                                                                                    
        # x = self.pool2(self.relu_conv2(self.conv2(x)))                                                                                                                                                                                                                    
        x = x.view(-1, 16 * 5 * 5)                                                                                                                                                                                                                          
        x = self.relu_fc1(self.fc1(x))                                                                                                                                                                                                                                      
        x = self.relu_fc2(self.fc2(x))                                                                                                                                                                                                                                      
        x = self.fc3(x)                                                                                                                                                                                                                                                     
        return x                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        


def add_noise(weights, noise):                                                                                                                                                                                                                                              
    with torch.no_grad():                                                                                                                                                                                                                                                   
        weights.add_(noise)                                                                                                                                                                                                                                                                                                                                                                                                                                     
    

model = Simplenet()
data = torch.randn(8, 3, 32, 32)
target = torch.randint(0, 10, (8, ))
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)

nb_epochs= 100
for epoch in range(nb_epochs):
    optimizer.zero_grad()
    output = model(data)
    loss = criterion(output, target)
    loss.backward()
    optimizer.step()
    print('epoch {}, loss {:.3f}'.format(epoch, loss.item()))

> epoch 0, loss 11.253
epoch 1, loss 29.167
epoch 2, loss 66.412
epoch 3, loss 35.963
epoch 4, loss 73.107
epoch 5, loss 61.738
epoch 6, loss 57.288
epoch 7, loss 47.320
epoch 8, loss 65.957
epoch 9, loss 0.000
epoch 10, loss 28.883
epoch 11, loss 15.165
epoch 12, loss 31.132
epoch 13, loss 0.000
epoch 14, loss 52.350
epoch 15, loss 8.386
epoch 16, loss 0.004
epoch 17, loss 154.521
epoch 18, loss 0.000
epoch 19, loss 63.121
epoch 20, loss 0.000
epoch 21, loss 0.000
epoch 22, loss 93.031
epoch 23, loss 0.000
epoch 24, loss 23.481
epoch 25, loss 0.000
epoch 26, loss 147.731
epoch 27, loss 67.255
epoch 28, loss 0.000
epoch 29, loss 65.383
epoch 30, loss 0.000
epoch 31, loss 220.798
epoch 32, loss 0.000
epoch 33, loss 67.999
epoch 34, loss 0.000
epoch 35, loss 128.444
epoch 36, loss 0.000
epoch 37, loss 0.000
epoch 38, loss 0.000
epoch 39, loss 0.000
epoch 40, loss 0.000
epoch 41, loss 71.423
epoch 42, loss 0.000
epoch 43, loss 44.055
epoch 44, loss 186.880
epoch 45, loss 0.000
epoch 46, loss 0.000
epoch 47, loss 122.831
epoch 48, loss 97.819
epoch 49, loss 0.000
epoch 50, loss 0.000
epoch 51, loss 0.000
epoch 52, loss 0.000
epoch 53, loss 0.000
epoch 54, loss 278.068
epoch 55, loss 0.000
epoch 56, loss 0.000
epoch 57, loss 622.002
epoch 58, loss 0.000
epoch 59, loss 0.000
epoch 60, loss 0.000
epoch 61, loss 678.884
epoch 62, loss 0.000
epoch 63, loss 0.000
epoch 64, loss 11.639
epoch 65, loss 0.000
epoch 66, loss 449.992
epoch 67, loss 0.000
epoch 68, loss 0.000
epoch 69, loss 0.000
epoch 70, loss 0.000
epoch 71, loss 0.000
epoch 72, loss 131.694
epoch 73, loss 2245.142
epoch 74, loss 0.000
epoch 75, loss 0.000
epoch 76, loss 2373.450
epoch 77, loss 1397.521
epoch 78, loss 0.000
epoch 79, loss 273.361
epoch 80, loss 0.000
epoch 81, loss 492.052
epoch 82, loss 0.000
epoch 83, loss 109.878
epoch 84, loss 0.000
epoch 85, loss 0.000
epoch 86, loss 0.000
epoch 87, loss 110.755
epoch 88, loss 0.000
epoch 89, loss 2689.813
epoch 90, loss 0.000
epoch 91, loss 0.000
epoch 92, loss 0.000
epoch 93, loss 0.000
epoch 94, loss 0.000
epoch 95, loss 1625.105
epoch 96, loss 0.000
epoch 97, loss 0.000
epoch 98, loss 0.000
epoch 99, loss 0.000

If you remove the noise (or reduce it), the training behaves much better.