Can anyone point out why the results for CIFAR10 dataset while running for only 1 epoch is different for below CNN same architecture but defined in different ways !

First way with torch.nn and Sequential:

 class Net(torch.nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, kernel_size_p, stride_p, drop_out):
        super(Net, self).__init__()
        self.Sequence = torch.nn.Sequential(
            torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding),
            torch.nn.BatchNorm2d(out_channels),
            torch.nn.Dropout2d(p=drop_out),
            torch.nn.ReLU(),  
            
            torch.nn.Conv2d(out_channels, out_channels*2, kernel_size, stride, padding),
            torch.nn.BatchNorm2d(out_channels*2),
            torch.nn.Dropout2d(p=drop_out),
            torch.nn.ReLU(),
            torch.nn.MaxPool2d(kernel_size_p, stride_p),
            
            torch.nn.Conv2d(out_channels*2, out_channels*3, kernel_size, stride, padding),
            torch.nn.BatchNorm2d(out_channels*3),
            torch.nn.Dropout2d(p=drop_out),
            torch.nn.ReLU(),
            
    
            torch.nn.Conv2d(out_channels*3, out_channels*4, kernel_size, stride, padding),
            torch.nn.BatchNorm2d(out_channels*4),
            torch.nn.Dropout2dp=(drop_out),
            torch.nn.ReLU(),
            #torch.nn.MaxPool2d(kernel_size_p, stride_p),
            
            torch.nn.Conv2d(out_channels*4, 10, 4, 1, 0),
            torch.nn.Flatten(),
            )

    def forward(self, x):
        return self.Sequence(x)

With above architecture I am getting below results which are acceptable.
Batch     Time     TestLoss     TestAcc     Loss     Acc     avg. Loss     avg. Acc     
0         0.215    2.303        9.89        2.382    8.0     0.238         0.8          
200       1.392    2.303        9.89        1.565    44.0    1.573         42.9         
400       2.769    1.372        49.35       1.392    51.0    1.408         47.0         
490       3.302    1.372        49.35       1.479    42.0    1.408         49.2



Now keeping the rest of the code same I am just defining architecture in below manner. But the results are scary.

def running_mean(out_channels):    
    n_t = torch.randn(out_channels,32,32) #image size is 32*32
    norm_mean = torchvision.transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
    running_mean = norm_mean(n_t)
    return running_mean

def running_var(out_channels):    
    n_t = torch.randn(out_channels,32,32)
    norm_var = torchvision.transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
    running_var = norm_var(n_t)
    return running_var
    

class Net(torch.nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, kernel_size_p, stride_p, drop_out):
        super(Net, self).__init__()
 
        self.conv_1_weight = torch.nn.Parameter(torch.randn(out_channels, in_channels, kernel_size, kernel_size))
        self.bias_1_weight = torch.nn.Parameter(torch.randn(out_channels))
 
        self.conv_2_weight = torch.nn.Parameter(torch.randn(out_channels*2, out_channels, kernel_size, kernel_size))
        self.bias_2_weight = torch.nn.Parameter(torch.randn(out_channels*2))
        
        self.conv_3_weight = torch.nn.Parameter(torch.randn(out_channels*3, out_channels*2, kernel_size, kernel_size))
        self.bias_3_weight = torch.nn.Parameter(torch.randn(out_channels*3))
 
        self.conv_4_weight = torch.nn.Parameter(torch.randn(out_channels*4, out_channels*3, kernel_size, kernel_size))
        self.bias_4_weight = torch.nn.Parameter(torch.randn(out_channels*4))
        
        self.conv_5_weight = torch.nn.Parameter(torch.randn(10, out_channels*4, 4, 4))
        self.bias_5_weight = torch.nn.Parameter(torch.randn(10))

 
    def forward(self, x):
        x = x.view(x.size(0), 3, 32, 32)
        out = torch.nn.functional.conv2d(x, self.conv_1_weight, self.bias_1_weight, stride, padding)
        out = torch.nn.functional.batch_norm(out, running_mean= torch.zeros(out_channels), running_var= torch.ones(out_channels))
        # out = torch.nn.functional.batch_norm(out, running_mean= running_mean(out_channels), running_var= running_var(out_channels))
        out = torch.nn.functional.dropout2d(out, p=drop_out)
        out = torch.nn.functional.relu(out, inplace=False)
        
        out = torch.nn.functional.conv2d(out, self.conv_2_weight, self.bias_2_weight, stride, padding)
        out = torch.nn.functional.batch_norm(out, running_mean= torch.zeros(out_channels*2), running_var= torch.ones(out_channels*2))
        # out = torch.nn.functional.batch_norm(out, running_mean= running_mean(out_channels*2), running_var= running_var(out_channels*2))
        out = torch.nn.functional.dropout2d(out, p=drop_out)
        out = torch.nn.functional.relu(out, inplace=False)
        out = torch.nn.functional.max_pool2d(out, kernel_size_p, stride_p)
        
        out = torch.nn.functional.conv2d(out, self.conv_3_weight, self.bias_3_weight, stride, padding)
        out = torch.nn.functional.batch_norm(out, running_mean= torch.zeros(out_channels*3), running_var= torch.ones(out_channels*3))
        # out = torch.nn.functional.batch_norm(out, running_mean= running_mean(out_channels*3), running_var= running_var(out_channels*3))
        out = torch.nn.functional.dropout2d(out, p=drop_out)
        out = torch.nn.functional.relu(out, inplace=False)
        
        out = torch.nn.functional.conv2d(out, self.conv_4_weight, self.bias_4_weight, stride, padding)
        out = torch.nn.functional.batch_norm(out, running_mean= torch.zeros(out_channels*4), running_var= torch.ones(out_channels*4))
        # out = torch.nn.functional.batch_norm(out, running_mean= running_mean(out_channels*4), running_var= running_var(out_channels*4))
        out = torch.nn.functional.dropout2d(out, p=drop_out)
        out = torch.nn.functional.relu(out, inplace=False)
        
        out = torch.nn.functional.conv2d(out, self.conv_5_weight, self.bias_5_weight, stride, padding)
        out = out.view(out.size(0), -1) 
        return out

Now the results are:
Batch     Time     TestLoss    TestAcc     TrainLoss     TrainAcc         
0         0.226    7996969.105  9.79        8810812.0      7.0               
200       1.407    7996969.105  9.79        606847.938     13.0            
400       2.807    285835.691   11.45       40160.824      9.0                
490       3.329    285835.691   11.45       18792.752      6.0


I think the difference in results is because of this:
In case of torch.nn.BatchNorm2d, there is only 1 argument needed which number of out_channels(say 32). While torch.nn.functional.batch_norm(out, running_mean, running_var) …takes three mandatory arguments.
The first one, I gave “out” which is output of convolutional layer (but I’m not sure if it’s right ). I don’t have any idea about the other 2 arguments running_mean and running_var (I just gave it a try).