Yes, it grows after every SGD iteration and my graphics memory quickly runs out. I have converted my tracking codes to
def graphics_memory():
    xa = torch.cuda.memory_allocated()
    xa = xa / 1024 / 1024
    xb = torch.cuda.memory_cached()
    xb = xb / 1024 / 1024
    return max(xa, xb)

def memory_track():
    for obj in gc.get_objects():
        try:
            if torch.is_tensor(obj) or (hasattr(obj, "data") and torch.is_tensor(obj.data)):
                print(type(obj), obj.size())
            elif isinstance(obj, collections.OrderedDict):
                for key, val in obj.items():
                    if torch.is_tensor(val):
                        print("key {}, val {}".format(key, val.size()))
        except:
            pass
    print("current memory allocated {}".format(graphics_memory()))
    return

It still shows the current memory usage keeps growing despite that the residing tensors or OrderedDicts remain the same.