Thanks, just tried that out. Unfortunately the loss still increases with each iteration (though strangely accuracy stays at exactly the same number):

image.png947×180 10.3 KB

New code after incorporating your snippet:
import torch
from torch.optim import Optimizer
import torch.nn as nn
import numpy as np
import math

class RandomHillClimb(Optimizer):
    def __init__(self, params, neighborhoodSize = 0.02
                 , loss = nn.CrossEntropyLoss()
                 , model = None
                 , features = None
                 , labels = None): #these represent default values, but can be overridden
        self.loss = loss
        self.model = model
        self.features = features
        self.labels = labels
        self.neighborhoodSize = neighborhoodSize

    def step(self):
        #need to first generate a random new point in the space
        if torch.cuda.is_available():
            oldOutputs = self.model(self.features.cuda())
            oldPerformance = self.loss(oldOutputs, self.labels.type(torch.LongTensor).cuda())
        else:
            oldOutputs = self.model(self.features)
            oldPerformance = self.loss(oldOutputs, self.labels.type(torch.LongTensor))
        # Save init values
        old_state_dict = {}
        for key in self.model.state_dict():
            old_state_dict[key] = self.model.state_dict()[key].clone()

        for name, param in self.model.state_dict().items():
            #generate a matrix of random changes in each param element to be added to each param matrix
            if (len(param.shape) == 2):
                random = torch.Tensor(np.random.uniform(low = self.neighborhoodSize * -1, high = self.neighborhoodSize
                                                        , size = (param.shape[0], param.shape[1])))
            else:
                random = torch.Tensor(np.random.uniform(low = self.neighborhoodSize * -1, high = self.neighborhoodSize
                                                        , size = param.shape[0]))
            #now add random to the params to transform them
            new_param = param + random.cuda()
            self.model.state_dict()[name].copy_(new_param)

        if torch.cuda.is_available():
            newOutputs = self.model(self.features.cuda())
            newWeights = self.model.state_dict().items()
            newPerformance = self.loss(newOutputs, self.labels.type(torch.LongTensor).cuda())
        else:
            newOutputs = self.model(self.features)
            newWeights = self.model.state_dict().items()
            newPerformance = self.loss(newOutputs, self.labels.type(torch.LongTensor))


        if (newPerformance < oldPerformance):
            self.model.load_state_dict(old_state_dict) #put the old weights back in case where we don't jump'