Removing the local reference will not delete the global tensor.
If you cannot free the cache, then a reference is still pointing to the tensor as shown here:
def fun(tensor):
    print(torch.cuda.memory_allocated() / 1024**2)
    # Delete local reference
    del tensor
    print(torch.cuda.memory_allocated() / 1024**2)
    return

# Check that memory is empty
print(torch.cuda.memory_allocated())
> 0
print(torch.cuda.memory_cached())
> 0

# Create tensor
x = torch.randn(1024 * 1024, device='cuda')
print(torch.cuda.memory_allocated() / 1024**2)
> 4.0
print(torch.cuda.memory_cached() / 1024**2)
> 20.0

# Call fun and check, if x is still alive
fun(x)
> 4.0
> 4.0

print(x.device) # still alive
> cuda:0
print(torch.cuda.memory_allocated() / 1024**2)
> 4.0
print(torch.cuda.memory_cached() / 1024**2)
> 20.0

# Delete global tensor
del x
print(torch.cuda.memory_allocated() / 1024**2)
> 0.0
print(torch.cuda.memory_cached() / 1024**2)
> 20.0

# Now empty cache
torch.cuda.empty_cache()
print(torch.cuda.memory_cached() / 1024**2)
> 0.0