Thanks for the answer. 
I tested it by my custom code.
class CustomMM(autograd.Function):
  def forward(ctx, x, y):
    out = torch.mm(x, y)
    return out

  def backward(ctx, grad_out):
    return magic_grad_for_x(grad_out), magic_grad_for_y(grad_out)

print(torch.cuda.memory_allocated())
x = torch.rand(10000, 10, requires_grad=True).cuda()
y = torch.rand(10, 10000).cuda()
z = torch.rand(10000, 100).cuda()
print(torch.cuda.memory_allocated())

output = CustomMM.apply(CustomMM.apply(x, y), z)
print(torch.cuda.memory_allocated())

# output.backward()  # It doesn't work because there is no magic function :)