I think somebody have the same error: Issue with Quantization 9
I think i know the answew:
# pytorch 1.4
#save
torch.save(net.state_dict(),'xx') # fp32 model
#load
model.qconfig = torch.quantization.get_default_qat_qconfig("fbgemm")
model.fuse_model()
torch.quantization.prepare_qat(net, inplace=True)
state_dict = torch.load('xx', map_location=torch.device('cpu'))
# remove module.  module
#torch.quantization.convert(net,inplace=True) # convert it to int8 model
x = torch.randn((2,3,300,300))
y = net(x)
print(y)
print('Success')