I think you are right and there must be something wrong with the initializations or maybe something else I cannot figure out. To get more insights I run some experiments:
These are the models (3 hidden layers of width 10):
Code of the models is here
modelN = TorchNet('No Activation', inp_dim, n_class, lay_size, n_layers, actf='none', track_stats=True, recursive=0)
modelS = TorchNet('Sigmoid', inp_dim, n_class, lay_size, n_layers, actf='sigm', track_stats=True, recursive=0)
modelT = TorchNet('TanH', inp_dim, n_class, lay_size, n_layers, actf='tanh', track_stats=True, recursive=0)
modelR = TorchNet('ReLU', inp_dim, n_class, lay_size, n_layers, actf='relu', track_stats=True, recursive=0)


Screen Shot 2018-11-11 at 8.14.07 PM.jpg1911×1087 667 KB

The results changes from different runs so I guess they are very sensitive to the initialization?
How could I properly initialize them to be sure that the problem is somewhere else? I haven’t manually code any specific initialization but the default random I guess.
I also leave here the code for training:
def train_epoch(model, tr_loader, criterion, optimizer, lr, results):
    
    train_loss = 0     
    correct, total = 0, 0       
    
    # Run minibaches from the training dataset
    for i, (X, labels) in enumerate(tr_loader):
        
        X, labels = Variable(X), Variable(labels)
        
        # Forward pass
        model.zero_grad()
        y_pred = model(X)
        s, preds = torch.max(y_pred.data, 1)
        
        # Compute loss 
        loss = criterion(y_pred, labels)            
        
        # Backward pass
        loss.backward()
        optimizer.step()
        
        # Collect stats 
        train_loss += loss.item()
        model.collect_stats(lr)

        # Compute and store epoch results
        total += y_pred.size(0)
        correct += int(sum(preds == labels)) 
        
    lss = round((train_loss / i+1), 3)
    acc = round((correct / total) * 100, 2)
    results.train_accy.append(acc)    
    results.train_loss.append(lss)
    return lss, acc


def valid_epoch(model, ts_loader, criterion, results):
    
    valid_loss = 0
    correct, total = 0, 0
    
    with torch.no_grad():
        for i, (X, labels) in enumerate(ts_loader):
            
            X, labels = Variable(X), Variable(labels)
            
            # Forward pass
            y_pred = model(X)
            s, preds = torch.max(y_pred.data, 1)
            
            # Compute loss 
            loss = criterion(y_pred, labels)           
            valid_loss += loss.item()
            
            # Compute and store epoch results
            total += y_pred.size(0)
            correct += int(sum(preds == labels)) 
    
    lss = round((valid_loss/i+1), 3)
    acc = round((correct / total) * 100, 3)
    results.valid_loss.append(lss)
    results.valid_accy.append(acc)
    return lss, acc

Then, from the main.py:
models += [modelN, modelS, modelT, modelR]

for model in models:
    
    r = Results()
    optimizer = optim.SGD(model.parameters(), LR, MOMEMTUM, WEIGHT_DECAY, nesterov=NESTEROV)
    model_no_recursive_params = [model, criterion, optimizer, r]
    train_no_recursive_params = [EPOCHS, LR]
    train(*model_no_recursive_params, *train_no_recursive_params)
    results.append(r)