I’m not sure, how build_model is supposed to work, as build_input_layer does not return anything.
However, if you would just like to pass two matrices, create a single one, and use it as a linear layer, this code should work:
class MyModel(nn.Module):
    def __init__(self, matrix_a, matrix_b):
        super(MyModel, self).__init__()
        self.matrix = nn.Parameter(torch.matmul(matrix_a, matrix_b))
        
    def forward(self, x):
        x = x.matmul(self.matrix)
        return x
        
    
matrix_a = torch.randn(2, 5)
matrix_b = torch.randn(5, 10)
model = MyModel(matrix_a, matrix_b)
x = torch.randn(1, 2)
output = model(x)
loss = output.mean()
loss.backward()