Without reading too deeply into your code, I suspect that this reassignment is the problem (just a guess).
What I am trying to say is suppose you have code like
def areTorchModulesEqual(module1, module2):
    for index, (p1, p2) in enumerate(zip(module1.parameters(), module2.parameters())):
        if p1.data.ne(p2.data).sum() > 0:
            return False
    return True

def train_model(model, optimizer, data_iterator, num_epochs, loss_function):
      for epoch in range(num_epochs):
          for batch in data_iterator:
                   training_data = batch[0]
                   targets = batch[1]
                   optimizer.zero_grad()
                   output = model(training_data)
                   loss = loss_function(output, targets)
                   loss.backward()
                   optimizer.step() 
       // here you don't need to return anything, the model is being modified
modelA = getInitialModel()
modelB = getInitialModel()
modelB.load_state_dict(modelA.state_dict())
train_model(modelA, optimizer, data_iterator, num_epochs, loss_function)
print(areTorchModulesEqual(moduleA, moduleB) # should return false

To see where exactly the GPU usage is increasing, in case my theory is wrong, you can use these:
def printGPUInfo(verbose=False):
    print("total memory available: " + str(torch.cuda.get_device_properties(0).total_memory * 1e-9))
    print("Currently reserved: " + str(torch.cuda.memory_reserved(0) * 1e-9))
    print("Currently allocated: " + str(torch.cuda.memory_allocated(0) * 1e-9))
    print("Max reserved " + str(torch.cuda.max_memory_reserved(0) * 1e-9))
    print("Max allocated " + str(torch.cuda.max_memory_reserved(0) * 1e-9))
    print("\n")
    if verbose:
        print(torch.cuda.memory_summary(device=None, abbreviated=False))

Call the above function before and after your call to  self._train_epoches, where I go line by line
in various trouble areas to see where memory is not being released.