Hey, I’m new to PyTorch and do not understand all the stuff that is happening in the background. For some reason loss.item() becomes “nan” after a couple of iterations, if I activate interStep. Why does this happen?
I’m glad for any help.
class interStep:
    def __init__(self, named_params):
        self.named_params = list(named_params)

        # names to filter out
        to_remove = ['bn'] # Filter out BatchNorm
        fliter_map = lambda x: not any(name in x[0] for name in to_remove)

        # weights and biases
        self.weights = [(n, p) for n, p in self.named_params if 'weight' in n]
        self.weights = list(filter(fliter_map, self.weights))
        self.n_layers = len(self.weights)

    def _get_weight(self, i):
        _, param = self.weights[i]
        return param

    def step(self,st_dev_activations, st_dev_errors):
        for i in range(1, self.n_layers - 1):
            self._get_weight(i - 1).data.mul_(st_dev_errors)
            self._get_weight(i).data.mul_(st_dev_activations)


def train(interStep ):
    for i, (input, target) in enumerate(train_loader):
        output = model(input)
        loss = criterion(output, target)

        if interStep is not None:
            sdactivations = torch.std(output)
            output = torch.div(output, sdactivations)
            sdloss = torch.std(loss)
            loss = torch.div(loss, sdloss)
            interStep.step(sdactivations, sdloss)
 
        print(loss.item())
        loss = torch.mean(loss)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()