I come acrross with the following:
class BatchedMatMul(nn.Module):
    def __init__(self):
        super().__init__()
        self.quant = torch.quantization.QuantStub()
        self.linear = nn.Linear(3,3, bias=False)
        self.dequant = torch.quantization.DeQuantStub()

    def forward(self, input1, input2):
        y = []
        for b in range(input1.shape[0]):
          print(f"Linear's type: {type(self.linear)}")
          print(f"Linear's weigth type: {type(self.linear.weight)}")
          if isinstance(self.linear.weight, nn.Parameter):
            self.linear.weight.requires_grad = False
            self.linear.weight.copy_ (self.quant(input1[b]))
            y.append(self.linear(self.quant(input2[b])))
          else:
            self.linear.set_weight_bias(self.quant(input1[b]), b=None)
            y.append(self.linear(self.quant(input2[b])))
          
        return self.dequant(torch.stack(y))

self.linear has changeg from  torch.nn.modules.linear.Linear to torch.nn.quantized.modules.linear.Linear so their methods and attributes are different.
Nevertheless, this approach is still throwing an error because the quantized linear layer expects an signed integer as parameter but an unsigned is being given…
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-112-629b4f9ebdc0> in <module>()
     42 print("Quantizing... [OK]")
     43 print("Running INT8 inference...")
---> 44 y = model_int8.forward(inp, inp)
     45 print(y)
     46 print("Running INT8 inference..[OK]")

2 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/quantized/modules/linear.py in set_weight_bias(self, weight, bias)
     21     def set_weight_bias(self, weight: torch.Tensor, bias: Optional[torch.Tensor]) -> None:
     22         if self.dtype == torch.qint8:
---> 23             self._packed_params = torch.ops.quantized.linear_prepack(weight, bias)
     24         elif self.dtype == torch.float16:
     25             self._packed_params = torch.ops.quantized.linear_prepack_fp16(weight, bias)

RuntimeError: expected scalar type QInt8 but found QUInt8