I tried to add gaussian noise to the parameters using the code below but the network won’t converge. Any though why? I used cifar10 dataset with lr=0.001
 import torch.nn as nn                                                                                                                                                                                                                                                       
 import torch.nn.functional as F                                                                                                                                                                                                                                             
 import torch                                                                                                                                                                                                                                                                
 __all__ = ['simplenet_cifar']                                                                                                                                                                                                                                               
                                                                                                                                                                                                                                                                            
                                                                                                                                                                                                                                                                          
 38 class Simplenet(nn.Module):                                                                                                                                                                                                                                                 
 37     def __init__(self):                                                                                                                                                                                                                                                     
 36         super(Simplenet, self).__init__()                                                                                                                                                                                                                                   
 35         self.conv1 = nn.Conv2d(3, 6, 5)                                                                                                                                                                                                                                     
 34         self.relu_conv1 = nn.ReLU()                                                                                                                                                                                                                                         
 33         self.pool1 = nn.MaxPool2d(2, 2)                                                                                                                                                                                                                                     
 32         self.conv2 = nn.Conv2d(6, 16, 5)                                                                                                                                                                                                                                    
 31         self.relu_conv2 = nn.ReLU()                                                                                                                                                                                                                                         
 30         self.pool2 = nn.MaxPool2d(2, 2)                                                                                                                                                                                                                                     
 29         self.fc1 = nn.Linear(16 * 5 * 5, 120)                                                                                                                                                                                                                               
 28         self.relu_fc1 = nn.ReLU()                                                                                                                                                                                                                                           
 27         self.fc2 = nn.Linear(120, 84)                                                                                                                                                                                                                                       
 26         self.relu_fc2 = nn.ReLU()                                                                                                                                                                                                                                           
 25         self.fc3 = nn.Linear(84, 10)                                                                                                                                                                                                                                        
 24                                                                                                                                                                                                                                                                             
 23         self.noise_conv1 = torch.randn(nn.Parameter(self.conv1.weight).size())*0.6 + 0                                                                                                                                                                                      
 22         self.noise_conv2 = torch.randn(nn.Parameter(self.conv2.weight).size())*0.6 + 0                                                                                                                                                                                      
 21                                                                                                                                                                                                                                                                             
 20     def forward(self, x):                                                                                                                                                                                                                                                   
 19         x = self.conv1(x)                                                                                                                                                                                                                                                   
 18         self.conv1.weight =  add_noise(nn.Parameter(self.conv1.weight), self.noise_conv1)                                                                                                                                                                                   
 17         x = self.pool1(self.relu_conv1(x))                                                                                                                                                                                                                                  
 16         x = self.conv2(x)                                                                                                                                                                                                                                                   
 15         self.conv2.weight =  add_noise(nn.Parameter(self.conv2.weight), self.noise_conv2)                                                                                                                                                                                   
 14         x = self.pool2(self.relu_conv2(x))                                                                                                                                                                                                                                  
 13         # x = self.pool1(self.relu_conv1(self.conv1(x)))                                                                                                                                                                                                                    
 12         # x = self.pool2(self.relu_conv2(self.conv2(x)))                                                                                                                                                                                                                    
 11         x = x.view(-1, 16 * 5 * 5)                                                                                                                                                                                                                          
 10         x = self.relu_fc1(self.fc1(x))                                                                                                                                                                                                                                      
  9         x = self.relu_fc2(self.fc2(x))                                                                                                                                                                                                                                      
  8         x = self.fc3(x)                                                                                                                                                                                                                                                     
  7         return x                                                                                                                                                                                                                                                            
  6                                                                                                                                                                                                                                                                             
  5 def add_noise(weights, noise):                                                                                                                                                                                                                                              
  4     with torch.no_grad():                                                                                                                                                                                                                                                   
  3         weight_noise = nn.Parameter(weights + noise.to("cuda"))                                                                                                                                                                                                                                                                                                                                                                                                                                     
  1     return weight_noise