Hmm ok, I did have some stray numpy operations in gram_matrix() and a few other places which I’ve gone ahead and translated to torch. I had added the Variable wrap because I was getting ‘RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn’ which I’m still getting now even though I think I’ve replaced all numpy operations and have the pastiche set to require_grad = True.
Can you spot what detaches the computation graph?
def gram_matrix(input):
	a, b, c, d = input.size()
	features = input.view(a * b, c * d)
	G = torch.mm(features, features.t())
	return G.div(a * b * c * d)

Class SelectiveSequential(torch.nn.Module)
        ...
	def forward(self, x, layers_to_return):
		layers = [[] for i in range(len(layers_to_return))]
		for name, module in self._modules.iteritems():
			x = module(x)
			for i,l in enumerate(layers_to_return):
				if l+"_"+name in self._to_select:
					layers[i].append(x)
		return layers if len(layers_to_return) > 1 else layers[0]

style_images = utils.get_style_images(args, scale_current)
style_images = [s.to(device) for s in style_images]

# get style features
gram_style = None
for i, style_image in enumerate(style_images):
	style_features = net(style_image, ['style'])
	target_i = [utils.gram_matrix(y) for y in style_features]
	target_i = [t.div(style_features[n].numel()) for n,t in enumerate(target_i)]
	target_i = [t.mul(style_blend_weights[i]) for t in target_i]
	if i == 0:
		gram_style = target_i
	else:
		gram_style = [sum(x) for x in zip(gram_style, target_i)]

file_name = args.content_filename.format(mod_idx)
content_image = skimage.io.imread(file_name)
content_image = skimage.transform.rescale(content_image, scale_current)
content_image = utils.match_color(content_image, np.moveaxis(style_images[0].cpu().squeeze().numpy(),0,-1)) #TODO hist match multiple styles
content_image = torch.from_numpy(content_image.reshape(1, content_image.shape[2],content_image.shape[0],content_image.shape[1])).float()
content_image = content_image.to(device)

# get content features
content_features = net(content_image, ['content'])

init_image = content_image
pastiche = init_image.to(device)
pastiche.requires_grad = True

optimizer = torch.optim.Adam([pastiche], lr=args.learning_rate)
mse_loss = torch.nn.MSELoss()

# optimize the images
for e in range(args.num_iterations):
	optimizer.zero_grad()
	c_feat, s_feat = net(pastiche, ['content','style'])
	content_loss = 0.
	for m in range(len(c_feat)):
		content_loss += args.content_weight * mse_loss(c_feat[m], content_features[m])

	style_loss = 0.
	for m in range(len(s_feat)):
		gram_y = utils.gram_matrix(s_feat[m])
		gram_s = gram_style[m]
		style_loss += args.style_weight * mse_loss(gram_y, gram_s)

	total_loss = content_loss + style_loss
	total_loss.backward()
	optimizer.step()
	print(total_loss.data.cpu().numpy())

# save frame
img_out = pastiche.clone().squeeze().mul(255).cpu().clamp(0, 255).numpy()
img_out = img_out.transpose(1, 2, 0).astype('uint8')
skimage.io.imsave(args.img_filename.format(run, mod_idx), img_out)
if run == args.passes_per_scale - 1 and mod_idx == 1:
	skimage.io.imsave(args.img_filename.format(max(*img.shape), mod_idx), img_out)

I’m using pytorch==0.4 so I think Variable was just an alias for Tensor, but I’ve removed all Variables anyway. Some other parts of my pipeline rely on pytorch<=0.4 so I’d like to get it working for this version.
Thanks so much for all your help!