Hi,
Maybe I’ve been a bit negative.
Here is a code sample of how you can do this for linear:
You can use such module as a dropin replacement for Linear in your model and the rest will work as expected! (avoid multithreaded forward in a single copy of this net  )
import torch
from torch import nn


class MyLinear(nn.Linear):
    def __init__(self, in_size, out_size):
        super().__init__(in_size, out_size)
        self.in_size = in_size
        self.out_size = out_size
        # Remove the original weights and bias
        del self.weight
        del self.bias

        # Add our new parameter
        self.M = nn.Parameter(torch.rand(out_size, 1))

    def forward(self, inp):
        # repopulate the original weights with our custom version
        # You can do any differentiable op here
        self.weight = self.M.expand(self.out_size, self.in_size) * 3
        self.weight[1, 1] = 0 # Mask entry 1,1 in in weight to see some difference in the gradients
        self.bias = self.M.squeeze(-1)

        result = super().forward(inp)

        # Avoid side effects in nn.Module
        del self.weight
        del self.bias

        return result


in_size = 10
out_size = 20

my_layer = MyLinear(in_size, out_size)

inp = torch.rand(5, in_size)
out = my_layer(inp)
out.sum().backward()

print(my_layer.M.grad)

Hope this helps !