My GPU memory keeps on increasing after every iteration. I have 2 losses (h_loss, f_loss) and etas is a list of trainable parameters defined outside the loop. Here is the code snippet inside the training loop.
# define etas (trainable Variables)
while iters < opts['max_iters']:
    data = torch.from_numpy(data)
    input_var = Variable(data)

    optimizer.zero_grad()
	# get h_loss and f_loss
	h_grads = torch.autograd.grad(outputs=h_loss, inputs=cv_params, create_graph=True)
    torch.autograd.backward(cv_params, h_grads)

    for eta_i, param in enumerate(cv_params):
        param.grad = param.grad * etas[eta_i]

    f_grads = torch.autograd.grad(outputs=f_loss, inputs=cv_params, create_graph=True)
    torch.autograd.backward(cv_params, f_grads)

    flat_params = []
    for param in cv_params:
        flat_params.append(param.grad.view(-1))
    flat_params = torch.cat(flat_params, 0)

    var_loss = (flat_params**2).mean()

    var_grads = torch.autograd.grad(outputs=var_loss, inputs=var_params, create_graph=True)
    torch.autograd.backward(var_params, var_grads)

	optimizer.step()
	
	# get next data batch
	iters += 1