Do you mean weight is learnable and calculate y = tanh(weight) * x when learning?
class LeNet(nn.Module):
    def __init__(self):
        super(LeNet, self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv2d(1, 6, 3, 1, 2,bias=False),
            nn.ReLU(),
            nn.MaxPool2d(2, 2)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(6, 16, 5,bias=False),
            nn.ReLU(),
            nn.MaxPool2d(2, 2)
        )
        self.fc1 = nn.Linear(16 * 5 * 5, 120,bias=False)
        self.fc2 = nn.Linear(120, 10,bias=False)

    def forward(self, x):
        with torch.no_grad():
            self.conv1[0].weight[:] = torch.tanh(self.conv1[0].weight[:])
            self.conv2[0].weight[:] = torch.tanh(self.conv2[0].weight[:])
            self.fc1.weight[:] = torch.tanh(self.fc1.weight[:])
            self.fc2.weight[:] = torch.tanh(self.fc2.weight[:])
        x = self.conv1(x)
        x = self.conv2(x)
        x = x.view(x.size()[0], -1)
        x = self.fc1(x)
        x= F.relu(x)
        x = self.fc2(x)
        return x

I am not sure with torch.no_grad is needed or not.