My test error is still much higher than in the paper, I get 14% where they get 5%. I think there might be something wrong in my model. I am trying to implement this model:

Schermafbeelding 2020-03-25 om 15.57.52882×1332 150 KB

As such:
class ConvBNReLu(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=False):
        super(ConvBNReLu, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(in_channels=in_channels,
                      out_channels=out_channels,
                      kernel_size=kernel_size,
                      stride=stride,
                      padding=padding,
                      bias=bias),
            nn.BatchNorm2d(out_channels),
        )

    def forward(self, input):
        return self.main(input)

class Flatten(nn.Module):
    def forward(self, x):
        x = x.view(x.size()[0], -1)
        return x

class ConvNet(nn.Module):
    def __init__(self, n_classes):
        super(ConvNet, self).__init__()
        self.model = nn.Sequential(OrderedDict([
            ('conv1', ConvBNReLu(3, 64, 3, padding=1)),
            ('conv2', ConvBNReLu(64, 64, 3, padding=1)),
            ('max_pool1', nn.MaxPool2d(2, stride=2, ceil_mode=True)),
            ('conv3', ConvBNReLu(64, 128, 3, padding=1)),
            ('conv4', ConvBNReLu(128, 128, 3, padding=1)),
            ('max_pool2', nn.MaxPool2d(2, stride=2, ceil_mode=True)),
            ('conv5', ConvBNReLu(128, 256, 3, padding=1)),
            ('conv6', ConvBNReLu(256, 256, 3, padding=1)),
            ('conv7', ConvBNReLu(256, 256, 3, padding=1)),
            ('conv8', ConvBNReLu(256, 256, 3, padding=1)),
            ('max_pool3', nn.MaxPool2d(2, stride=2, ceil_mode=True)),
            ('flatten', Flatten()),
            ('fc4', nn.Linear(in_features=256 * 4 * 4, out_features=1024, bias=True)),
            ('relu5', nn.ReLU()),
            ('dropout5', nn.Dropout()),
            ('fc5', nn.Linear(1024, 1024)),
            ('relu6', nn.ReLU()),
            ('dropout6', nn.Dropout()),
            ('fc6', nn.Linear(1024, n_classes)),
            ('softmax', nn.LogSoftmax(dim=-1)) # Changed to LogSoftmax from Softmax, negative KL divergence otherwise
        ]))

    def forward(self, inp):
        return self.model(inp)


I am quite new to making model however this works, but not as good as I had hoped. Did I implement the model correctly?
I’m testing the model each epoch like this (no mixing):
criterion = nn.KLDivLoss()
learning_rate = 0.1 # Initial learning rate
optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, nesterov=True, weight_decay=5e-4)
scheduler = MultiStepLR(optimizer, milestones=[100,150,200], gamma=0.1) # Decrease learning rate at milestones

test_error_standard = []
log_test_error = True
nEpoch = 250
for epoch in range(nEpoch):  # Number of epochs (loops over dataset)
  epoch_loss = 0.0
  running_loss = 0.0
  for i, data in enumerate(trainloader):
      # get the inputs; data is a list of [inputs, labels]
      
      label = []
      images_cifar, labels_cifar = data

      for j in range(len(labels_cifar)):
        label.append((torch.eye(10)[int(labels_cifar[j])]))
      labels = torch.stack(label).to(device)
      optimizer.zero_grad()
      # forward + backward + optimize
      outputs = net(images_cifar.to(device))
      loss = criterion(outputs, labels)
      loss.backward()
      optimizer.step()

      # print statistics every 200 loops

      running_loss += loss.item()

  
  print('[%d, %5d] loss: %.3f' %
        (epoch + 1, i, running_loss / i))
  running_loss = 0.0

  scheduler.step()
  if log_test_error: # Test network at each epoch and save test error
    correct = 0
    total = 0
    with torch.no_grad():
        for data in testloader:
            images, labels = data
            images = images.to(device)
            labels = labels.to(device)
            outputs = net(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    test_error = 100 - 100 * correct / total
    test_error_standard.append(test_error)
    print('Test error: %.2f %%' % (
        test_error))


print('Finished Training')

torch.save(net.state_dict(), PATH)

EDIT: The error was not having nn.ReLU() at the end of ConvBNReLu(nn.Module), getting similar validation errors now!