Alright that didn’t turn out to be so hard, but I’m still running into the same exact error! The loss is the same every single time. To me it seems like the backward() call just isn’t updating anything in the pastiche image even though I’ve set it to require_grad.
I’ve now changed the building of the network to just insert the modules as read from the vgg19 model without any losses. I’ve wrapped it in a custom Sequential class that returns a list of features to use for calculating the losses when optimizing.


build_net()
content_layers = args.content_layers.split(",")
style_layers = args.style_layers.split(",")

modules_dict = OrderedDict()
feature_list = []

next_content_i, next_style_i = 0, 0

block = 1
conv = 1
for i, layer in enumerate(cnn.features):
    if next_content_i < len(content_layers) or next_style_i < len(style_layers):
        name = 'uhhh'
        if isinstance(layer, torch.nn.Conv2d):
            name = 'conv'+str(block)+'_'+str(conv)
        elif isinstance(layer, torch.nn.ReLU):
            name = 'relu'+str(block)+'_'+str(conv)
            layer = torch.nn.ReLU(inplace=False)
            conv += 1
        elif isinstance(layer, torch.nn.MaxPool2d):
            name = 'pool_'+str(block)
            if args.pooling == 'avg':
                assert(layer.padW == 0 and layer.padH == 0)
                kW, kH = layer.kW, layer.kH
                dW, dH = layer.dW, layer.dH
                avg_pool_layer = torch.nn.SpatialAveragePooling(kW, kH, dW, dH)
                layer = avg_pool_layer
                block += 1
                conv = 1
        else:
            continue

        modules_dict[name] = layer

        if next_content_i < len(content_layers) and name == content_layers[next_content_i]:
            print("Earmarking content loss "+str(next_content_i+1)+": "+name)
            feature_list.append('content_'+name)
            next_content_i = next_content_i + 1

        if next_style_i < len(style_layers) and name == style_layers[next_style_i]:
            print("Earmarking style loss "+str(next_style_i+1)+": "+name)
            feature_list.append('style_'+name)
            next_style_i = next_style_i + 1

return SelectiveSequential(feature_list, modules_dict)

class SelectiveSequential(torch.nn.Module):
	def __init__(self, to_select, modules_dict):
		super(SelectiveSequential, self).__init__()
		for key, module in modules_dict.items():
			self.add_module(key, module)
		self._to_select = to_select

	def forward(self, x, layers_to_return):
		layers = [[] for i in range(len(layers_to_return))]
		for name, module in self._modules.iteritems():
			x = module(x)
			for i,l in enumerate(layers_to_return):
				if l+"_"+name in self._to_select:
					layers[i].append(x)
		return layers if len(layers_to_return) > 1 else layers[0]

def gram_matrix(input):
	a, b, c, d = input.shape
	features = input.contiguous().view(a * b, c * d)
	G = torch.mm(features, features.t()).float()
	return G.div(a * b * c * d)


My optimization now looks like this:
cnn = torchvision.models.vgg19(pretrained=True)
net = utils.build_net(args, cnn)
del cnn
for module in net.children():
    if isinstance(module, torch.nn.Conv2d):
        # remove these, not used, but uses gpu memory
        module.gradWeight = None
        module.gradBias = None
for param in net.parameters():
    param.requires_grad = False

style_images = utils.get_style_images(args, 1)
file_name = args.content_filename.format(mod_idx)
content_image = skimage.io.imread(file_name)
content_image = skimage.transform.rescale(content_image, scale_current)
content_image = utils.match_color(content_image, np.moveaxis(style_images[0].cpu().squeeze().numpy(), 0, -1))
content_image = torch.from_numpy(content_image.reshape(1, content_image.shape[2], content_image.shape[0], content_image.shape[1])).float()
img = content_image.clone()

# get content & flow weighted features
content_features = net(img, ['content'])
f_xc_c = []
for m in range(len(content_features)):
    f_xc_c.append(torch.autograd.Variable(content_features[m].data, requires_grad=False))

# get style features
gram_style = None
for i, style_image in enumerate(style_images):
    style_features = net(style_image, ['style'])
    target_i = [utils.gram_matrix(y) for y in style_features]
    target_i = [t / style_features[n].nelement() for n,t in enumerate(target_i)]
    target_i = [t * style_blend_weights[i] for t in target_i]
    if i == 0:
        gram_style = target_i
    else:
        gram_style = [sum(x) for x in zip(gram_style, target_i)]

# init optimizer
pastiche = torch.autograd.Variable(img.data, requires_grad=True)
optimizer = torch.optim.Adam([pastiche], lr=args.learning_rate)
mse_loss = torch.nn.MSELoss()

# optimize the images
for e in range(args.num_iterations):
    optimizer.zero_grad()
    c_feat, s_feat = net(pastiche, ['content','style'])
    content_loss = 0.
    for m in range(len(c_feat)):
        content_loss += args.content_weight * mse_loss(c_feat[m], f_xc_c[m])

    style_loss = 0.
    for m in range(len(s_feat)):
        gram_y = utils.gram_matrix(s_feat[m])
        gram_s = torch.autograd.Variable(gram_style[m].data, requires_grad=False)
        style_loss += args.style_weight * mse_loss(gram_y, gram_s)

    total_loss = content_loss + style_loss
    loss = torch.autograd.Variable(total_loss.data, requires_grad=True)
    loss.backward()
    optimizer.step()
    print(total_loss.data.cpu().numpy())

How can I get pastiche to be updated? Aren’t the calls to loss.backward() and/or optimizer.step() supposed to handle this?