I cannot reproduce this behavior using this code snippet:
def print_active_bytes():
    stats = torch.cuda.memory_stats()
    current_active_byte =  stats["active_bytes.all.current"]
    print(current_active_byte)


# initial usage
print_active_bytes()
> 0

# vanilla tensor
x = torch.randn(1024, device='cuda')
print_active_bytes()
> 4096

del x
print_active_bytes()
> 0

# requires_grad=True
x = torch.randn(1024, device='cuda', requires_grad=True)
print_active_bytes()
> 4096

del x
print_active_bytes()
> 0