I found a solution for a problem similar that yours. Check the code below for a convolution block with symmetric coefficients
I initially tried to create the weight matrix as @Naman-ntc but the graph was broken and the gradients did not flow to the scalar variables
For this reason I create the weight matrix by summing the coefficients to a zero matrix. Any other simpler alternatives are welcome
class Conv2d_symmetric(nn.Module):
    def __init__(self):
        super(Conv2d_simple, self).__init__()

        self.a = nn.Parameter(torch.randn(1))
        self.b = nn.Parameter(torch.randn(1))
        self.c = nn.Parameter(torch.randn(1))


        self.bias = None
        self.stride = 2
        self.padding = 1
        self.dilation = 1
        self.groups = 1



    def forward(self, input):

        #in case we use gpu we need to create the weight matrix there
        device = self.a.device

        weight = torch.zeros((1,1,3,3)).to(device)
        weight[0,0,0,0] += self.c[0]
        weight[0,0,0,1] += self.b[0]
        weight[0,0,0,2] += self.c[0]
        weight[0,0,1,0] += self.b[0]
        weight[0,0,1,1] += self.a[0]
        weight[0,0,1,2] += self.b[0]
        weight[0,0,2,0] += self.c[0]
        weight[0,0,2,1] += self.b[0]
        weight[0,0,2,2] += self.c[0]


        # print("weight= ", weight)
        # print("inout = ", input)
        return F.conv2d(input, weight, self.bias, self.stride,
                        self.padding, self.dilation, self.groups)