Thank you for such quick and helpful answers, really nice community !
Model (just the initilisation) --> comes from torchvision:
    def _initialize_weights(self):
    for m in self.modules():
        if isinstance(m, nn.Conv2d):
            n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
            m.weight.data.normal_(0, math.sqrt(2. / n))
            if m.bias is not None:
                m.bias.data.zero_()
        elif isinstance(m, nn.BatchNorm2d):
            m.weight.data.fill_(1)
            m.bias.data.zero_()
        elif isinstance(m, nn.Linear):
            m.weight.data.normal_(0, 0.01)
            m.bias.data.zero_()

Optimizer:
optimizer = optim.SGD(net.parameters(),
                  lr=0.00000001,
                  momentum=0.9,
                  weight_decay=0.0005)  

Loss computation:
def runEpoch(self):
    loss_list = np.zeros(100000)
    for it, batch in enumerate(tqdm(self.data_loader)):
        data = Variable(batch['image'])
        target = Variable(batch['class_code'])
        # forward
        if self.mode == 'cuda':
            data = data.cuda()
            target = target.long().cuda()
        output = self.model.forward(data)
        loss = self.criterion(output.float(), target)
        loss.backward()
        self.optimizer.step()
        loss_list[it] = loss.item()
    self.avg_loss = np.mean(loss_list[np.nonzero(loss_list)])

Also, note that I am using 4-band 8-bit encoded images as input.