I see… I made a small demo and yes you’re correct it errors out. Thanks

image.png1039×307 40.2 KB

import torch
from torch.autograd import grad

torch.manual_seed(0)
G = torch.nn.Linear(2,4)
D = torch.nn.Linear(4,1)

real = torch.ones(5,4, requires_grad=False)
z = torch.rand(5,2, requires_grad=False)


D.zero_grad()
y = D(real)
loss = y.mean()
loss.backward()
print(G.weight.grad)
print(D.weight.grad)
print()

fake = G(z)
y = D(fake)
loss = y.mean()
(-loss).backward()
print(G.weight.grad)
print(D.weight.grad)
print()

alpha = torch.rand(5,1)
alpha = alpha.expand(real.size())
interp = alpha*real + (1-alpha)*fake
y = D(interp)
gradients = grad(outputs=y, inputs=interp, grad_outputs=torch.ones(y.size()),
create_graph=False, retain_graph=None, only_inputs=True)[0]
gp = ((gradients.norm(dim=1)-1)**2).mean()
gp.backward()
print(G.weight.grad)
print(D.weight.grad)
print()