Thanks for the reply!
I’m building my network up in two steps which loop through the vgg model (although I’d like to be able to do this for many different models eventually) inserting layers and losses. First I insert all the layers from vgg (which are frozen after insertion) and the StyleLosses as these do not change throughout optimization.


First net building loop
content_layers = args.content_layers.split(",")
style_layers = args.style_layers.split(",")

next_content_i, next_style_i, next_temporal_i, current_layer_index = 0, 0, 0, 0

cnn = torchvision.models.vgg19(pretrained=True)
net = torch.nn.Sequential()

block = 1
conv = 1
for i, layer in enumerate(cnn.features):
	if next_content_i < len(content_layers) or next_style_i < len(style_layers):
		name = 'uhhh'
		if isinstance(layer, torch.nn.Conv2d):
			name = 'conv'+str(block)+'_'+str(conv)
		elif isinstance(layer, torch.nn.ReLU):
			name = 'relu'+str(block)+'_'+str(conv)
			conv += 1
			layer = torch.nn.ReLU(inplace=False)
		elif isinstance(layer, torch.nn.MaxPool2d):
			name = 'pool'+str(block)
			if args.pooling == 'avg':
				assert(layer.padW == 0 and layer.padH == 0)
				kW, kH = layer.kW, layer.kH
				dW, dH = layer.dW, layer.dH
				avg_pool_layer = torch.nn.SpatialAveragePooling(kW, kH, dW, dH)
				net.add_module("pooling_"+str(block), avg_pool_layer)
			else:
				net.add_module("pooling_"+str(block), layer)
			block += 1
			conv = 1
		else:
			continue
		net.add_module(name, layer)
		current_layer_index = current_layer_index + 1
		if next_content_i < len(content_layers) and name == content_layers[next_content_i]:
			print("Earmarking content loss "+str(next_content_i+1)+": "+name)
			losses_indices.append(current_layer_index)
			losses_type.append('content_'+name)
			next_content_i = next_content_i + 1
		if next_style_i < len(style_layers) and name == style_layers[next_style_i]:
			print("Setting up style loss "+str(next_style_i+1)+": "+name)
			gram = GramMatrix()
			target = []
			for s in range(len(style_images)):
				target_features = net.forward(style_images[s]).detach()
				target_i = gram.forward(target_features)
				target_i.div(target_features.nelement())
				target_i.mul(style_blend_weights[s])
				if s == 0:
					target = target_i
				else:
					target.append(target_i)
			loss_module = StyleLoss(args.style_weight, target, args.normalize_gradients)
			net.add_module("style_loss_"+name, loss_module)
			current_layer_index = current_layer_index + 1
			style_losses.append(loss_module)
			next_style_i = next_style_i + 1

del cnn
for module in net.children():
    if isinstance(module, torch.nn.Conv2d):
        # remove these, not used, but uses gpu memory
        module.gradWeight = None
        module.gradBias = None

for param in net.parameters():
    param.requires_grad = False


Afterwards I want to loop through many different content images so I insert the content loss into the network, run the optimization, remove it, and repeat for each image.


Content loss insertion
content_losses, prev_plus_flow_losses = [], []
additional_layers = 0

for i in range(len(losses_indices)):
	if losses_type[i].startswith('content'):
		content_loss = get_content_loss_module(net, losses_indices[i] + additional_layers, content_image, args)
		# insert content
		index = losses_indices[i]+additional_layers
		new_modules = list(net.children())[:index] + [content_loss] + list(net.children())[index+1:]
		net = torch.nn.Sequential(*new_modules)

		additional_layers = additional_layers + 1
		content_losses.append(content_loss)

def get_content_loss_module(net, layer_idx, target_img, args):
	tmpNet = torch.nn.Sequential()
	for i, layer in enumerate(net.children()):
		if i == layer_idx: break
		tmpNet.add_module("temp"+str(i), layer)
	target = tmpNet.forward(target_img)
	loss_module = ContentLoss(args.content_weight, target, args.normalize_gradients)
	return loss_module

run_optimization(args, net, content_losses, style_losses, temporal_losses, img, mod_idx, run)

for i,l in enumerate(losses_type):
    if l.startswith('content'):
        additional_layers = additional_layers - 1
        index = losses_indices[i]+additional_layers
        new_modules = list(net.children())[:index] + list(net.children())[index+1:]
        net = torch.nn.Sequential(*new_modules)


Regarding replacing model.forward() calls: I’ve tried this in the optimization part but am still running into the same problem, the printed loss value never changes. Is the model(input) call better in all situations? Should I replace all model.forward() calls throughout? Or only in the optimization closure?
N.B. there are some references to temporal losses throughout the code, these are basically slightly modified content losses but are currently not being inserted yet so temporal_losses is just an empty list. This should eventually be handled in a similar way to the content_losses: being inserted for each individual content image.