Hi:
I am trying to run quantization on a model. The model I am using is the pretrained restnet18:
I am using the quantization aware training.  I run into a similar problem like https://discuss.pytorch.org/t/runtimeerror-could-not-run-aten-add-tensor-with-arguments-from-the-quantizedcpu-backend/110039 2. The code is running on GPU first. Before quantization, I transfer it from GPU to CPU and quantize it. It seems like the quantization is working. The problem arises when the quantized model is called later in the code to run the tester.
However, I tried the solution by adding the dequant() and quant() around that ‘aten::add_.Tensor’
as

image704×814 176 KB

I still got errors Could not run ‘aten::add_.Tensor’ with arguments from the ‘QuantizedCPU’ backend

image1702×874 701 KB

I even tried to use, however, I still has this error
out = torch.nn.quantized.modules.FloatFunctional().add(out,identity)
Here is my model:
class Resnet18_ONE(nn.Module):
def __init__(self):

    super(Resnet18_ONE,self).__init__()
    #self.loss = loss
    self.quant = torch.quantization.QuantStub()
    resnet18_tmp = resnet.resnet18(pretrained=True)
    #set_parameter_requires_grad(resnet18,True)
    num_ftrs = resnet18_tmp.fc.in_features
    #num_classes = superclasses
    self.base= nn.Sequential(*list(resnet18_tmp.children())[:-1])
    #print(self.base)
    self.linear_sub = nn.Linear(num_ftrs, superclasses)
    self.linear_bird = nn.Linear(num_ftrs, classes_bird)
    self.linear_boat = nn.Linear(num_ftrs, classes_boat)
    self.linear_car = nn.Linear(num_ftrs, classes_car)
    self.linear_cat = nn.Linear(num_ftrs, classes_cat)
    self.linear_fungus = nn.Linear(num_ftrs, classes_fungus)
    self.linear_insect = nn.Linear(num_ftrs, classes_insect)
    self.linear_monkey = nn.Linear(num_ftrs, classes_monkey)
    self.linear_truck = nn.Linear(num_ftrs, classes_truck)
    self.linear_dog = nn.Linear(num_ftrs, classes_dog)
    self.linear_fruit = nn.Linear(num_ftrs, classes_fruit)
    self.dequant = torch.quantization.DeQuantStub()

def forward(self,x):

    x = self.quant(x)
    x = self.base(x)
    x = self.dequant(x)  
    x = torch.flatten(x, 1)
    if task == 'SUB':
        x = self.linear_sub(x)
    elif task == 'BIRD':
        #print("I am in bird")
        x = self.linear_bird(x)
    elif task == 'BOAT':
        x = self.linear_boat(x)
    elif task == 'CAR':
        x = self.linear_car(x)
    elif task == 'CAT':
        x = self.linear_cat(x)
    elif task == 'FUNGUS':
        x = self.linear_fungus(x)
    elif task == 'INSECT':
        x = self.linear_insect(x)
    elif task == 'MONKEY':
        x = self.linear_monkey(x)
    elif task == 'TRUCK':
        x = self.linear_truck(x)
    elif task == 'DOG':
        x = self.linear_dog(x)
    else:
        #print("enter fruit")
        x = self.linear_fruit(x)
              
    return x

Here is the setting for quantization aware training
model_one.train()
model_one.qconfig = torch.quantization.get_default_qat_qconfig(‘fbgemm’)
model_one_fused = torch.quantization.fuse_modules(model_one,[[‘base.0’,‘base.1’],[‘base.4.0.conv1’,‘base.4.0.bn1’],[‘base.4.0.conv2’,‘base.4.0.bn2’],[‘base.4.1.conv1’,‘base.4.1.bn1’],[‘base.4.1.conv2’,‘base.4.1.bn2’],[‘base.5.0.conv1’,‘base.5.0.bn1’],[‘base.5.0.conv2’,‘base.5.0.bn2’],[‘base.5.1.conv1’,‘base.5.1.bn1’],[‘base.5.1.conv2’,‘base.5.1.bn2’],[‘base.6.0.conv1’,‘base.6.0.bn1’],[‘base.6.0.conv2’,‘base.6.0.bn2’],[‘base.6.1.conv1’,‘base.6.1.bn1’],[‘base.6.1.conv2’,‘base.6.1.bn2’],[‘base.7.0.conv1’,‘base.7.0.bn1’],[‘base.7.0.conv2’,‘base.7.0.bn2’],[‘base.7.1.conv1’,‘base.7.1.bn1’],[‘base.7.1.conv2’,‘base.7.1.bn2’]])
model_one_prepared = torch.quantization.prepare(model_one_fused)
torch.quantization.prepare_qat(model_one_prepared,inplace = True)
Here is where I convert my model and test it:
model_one_prepared.to(‘cpu’)
model_one_int8 = torch.quantization.convert(model_one_prepared.eval(),inplace=False)
acc = test_cpu(testLoader_mix10,model_one_int8)