if use_gpu:
    if torch.cuda.is_available():
        torch.backends.cudnn.enabled = True
        torch.backends.cudnn.benchmark = True
        retinanet = retinanet.cuda()

if torch.cuda.is_available():
    retinanet = torch.nn.DataParallel(retinanet).cuda()
else:
    retinanet = torch.nn.DataParallel(retinanet)

for epoch_num in range(parser.epochs):

    train_loss = train(dataloader_train, retinanet, optimizer, writer, epoch_num, train_hist)

    val_loss = eval(dataloader_val, retinanet, writer, epoch_num, val_hist)

    AP_eval = csv_eval.evaluation(dataset_val, retinanet)

def train(dataloader_train, model, optimizer, writer, epoch, train_hist):

    print("Train")

    model.train()

    model.module.set_compute(True)

    epoch_loss = []

    for iter_num, data in enumerate(dataloader_train):
        try:
            optimizer.zero_grad()

            if torch.cuda.is_available():
                classification_loss, regression_loss = model([data['img'].cuda().float(), data['annot']])
            else:
                classification_loss, regression_loss = model([data['img'].float(), data['annot']])

            classification_loss = classification_loss.mean()
            regression_loss = regression_loss.mean()
        
            loss = classification_loss + regression_loss

            if bool(loss == 0):
                continue

            loss.backward()

            torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)

            optimizer.step()

            classification_loss.detach_()
            regression_loss.detach_()
            loss.detach_()

            if parser.debug != 1:
                writer.add_scalar("ClassLoss/Train", classification_loss.cpu(), (iter_num + len(dataloader_train) * epoch))
                writer.add_scalar("RegLoss/Train", regression_loss.cpu(), (iter_num + len(dataloader_train) * epoch))
                writer.add_scalar("Total/Train", loss.cpu(), (iter_num + len(dataloader_train) * epoch))

            train_hist.append(float(loss.cpu()))

            epoch_loss.append(float(loss.cpu()))

            if iter_num % int(len(dataloader_train)/10) == 0:

                print(
                'Epoch: {:3d}/{:3d} | Iteration: {:4d}/{:4d} | Classification loss: {:1.5f} | Regression loss: {:1.5f} | Epoch loss: {:1.5f} | Average loss: {:1.5f}'.format(
                    epoch, parser.epochs,  iter_num, len(dataloader_train), float(classification_loss.cpu()), float(regression_loss.cpu()), np.mean(epoch_loss), np.mean(train_hist)))

            del classification_loss
            del regression_loss
            del loss
        except Exception as e:
            print(e)
            continue

    if parser.debug != 1:
        writer.add_scalar("EpochLoss/Train", np.mean(epoch_loss), epoch)

    return np.mean(epoch_loss)

 def eval(dataloader_val, model, writer, epoch, val_hist):

    print("Eval")

    model.eval()
    model.module.freeze_bn()
    model.module.set_compute(True)

    epoch_loss = []

    with torch.no_grad():
        for iter_num, data in enumerate(dataloader_val):
            try:
                if torch.cuda.is_available():
                    classification_loss, regression_loss = model([data['img'].cuda().float(), data['annot']])
                else:
                    classification_loss, regression_loss = model([data['img'].float(), data['annot']])

                classification_loss = classification_loss.mean().cpu()
                regression_loss = regression_loss.mean().cpu()

                loss = classification_loss + regression_loss

                if bool(loss == 0):
                    continue

                if parser.debug != 1:
                    writer.add_scalar("ClassLoss/Eval", classification_loss, (iter_num + len(dataloader_val) * epoch))
                    writer.add_scalar("RegLoss/Eval", regression_loss, (iter_num + len(dataloader_val) * epoch))
                    writer.add_scalar("Total/Eval", loss, (iter_num + len(dataloader_val) * epoch))

                val_hist.append(float(loss))
                epoch_loss.append(float(loss))

                if iter_num % int(len(dataloader_val)/10) == 0:

                    print('Epoch: {:3d}/{:3d} | Iteration: {:4d}/{:4d} | Classification loss: {:1.5f} | Regression loss: {:1.5f} | Epoch loss: {:1.5f} | Average loss: {:1.5f}'.format(epoch, parser.epochs,  iter_num, len(dataloader_val), float(classification_loss), float(regression_loss), np.mean(epoch_loss), np.mean(val_hist)))
                
                del classification_loss
                del regression_loss
                del loss

            except Exception as e:
                print(e)
                continue

    if parser.debug != 1:
        writer.add_scalar("EpochLoss/Eval", np.mean(epoch_loss), epoch)

    return np.mean(epoch_loss)

Here is my snippet code.
csv_eval.evaluation(dataset_val, retinanet)

Is just inference to compute the mAP.
What I have observed is that during the first training epoch the nvidia-smi memory usage increases a bit, we have a low value during the eval epoch and a very low during inference (normal behaviour). Second training epoch the memory still increases but what is surprising is during the eval and inference the memory usage stays at the same level as during training.
After 2-3 epochs it stays around the same value whatever the train-eval-inference phase.