It seems that Module.zero_grad() does not like parameters with no grad (source below crashes), but what is the proper way to have model parameters which should not be touched by the backprop, but should benefit from Module’s comfort (cuda() etc.)?
import torch

from torch import Tensor
from torch.nn.parameter import Parameter
from torch.nn import Module

class Blah(Module):
    def __init__(self, dim):
        super(Blah, self).__init__()
        self.s = Parameter(torch.rand(1, dim), requires_grad = False)
        self.t = Parameter(torch.rand(1, dim))

blah = Blah(10)

blah.zero_grad()