Yes
 
 The code now 
import torch
from torch import nn, optim
from torch.quantization import QuantStub, DeQuantStub
from copy import deepcopy

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

class Model(nn.Module):
    def __init__(self):
        super().__init__()

        self.backbone0 = nn.Sequential(
            QuantStub(),
            nn.Conv2d(3, 1, 1, bias=False),
            nn.BatchNorm2d(1),
            nn.ReLU(),
            DeQuantStub(),
        )
        self.backbone1 = nn.Sequential(
            nn.Conv2d(1, 2, 3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(2),
            nn.AvgPool2d(14),
            nn.Sigmoid(),
        )

    def forward(self, x):
        x = self.backbone0(x)
        x = self.backbone1(x)
        return x

model = Model()

model.backbone0.qconfig = torch.quantization.get_default_qat_qconfig('fbgemm')
torch.quantization.prepare_qat(model.backbone0, inplace=True)

optimizer = optim.Adam(model.parameters(), lr=1)
model.to(device)
print(model)



criterion = nn.BCELoss()

for epoch in range(10):
    print('EPOCH', epoch)
    model.train()

    inputs = torch.rand(2, 3, 28, 28)
    labels = torch.FloatTensor([[1, 1], [0, 0]])

    inputs = inputs.to(device)
    labels = labels.to(device)
    outputs = model(inputs)
    loss = criterion(outputs.view(2, 2), labels)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if epoch >= 2:
        model.apply(torch.quantization.disable_observer)
        pass

    if epoch >= 3:
        model.apply(torch.nn.intrinsic.qat.freeze_bn_stats)

    quant_model = deepcopy(model)
    quant_model = torch.quantization.convert(quant_model.eval().cpu(), inplace=False)

    with torch.no_grad():
        inp = torch.zeros([1, 3, 28, 28], device='cpu')
        model.eval().cpu()
        quant_model.eval().cpu()
        qout = quant_model.forward(inp)
        out = model.forward(inp)
        print(qout.view(2).tolist())
        print(out.view(2).tolist())

        model.to(device)


I printed the state_dicts of the not converted and converted models if the output from forwarding zeros diverged/stayed the same:

 diverged 
# Forward zeros output
[0.28445518016815186, 0.3933817744255066] #not converted
[0.07719799876213074, 0.6743956208229065] #converted

<================ NON-CONVERTED MODEL ====================>
OrderedDict([('backbone0.0.activation_post_process.scale', tensor([0.0077])), ('backbone0.0.activation_post_process.zero_point', tensor([0])), ('backbone0.0.activation_post_process.activation_post_process.min_val', tensor(0.0006)), ('backbone0.0.activation_post_process.activation_post_process.max_val', tensor(0.9801)), ('backbone0.1.weight', tensor([[[[-4.4932]],

         [[ 3.9661]],

         [[-4.1858]]]])), ('backbone0.1.activation_post_process.scale', tensor([0.0039])), ('backbone0.1.activation_post_process.zero_point', tensor([127])), ('backbone0.1.activation_post_process.activation_post_process.min_val', tensor(-0.4932)), ('backbone0.1.activation_post_process.activation_post_process.max_val', tensor(0.0016)), ('backbone0.1.weight_fake_quant.scale', tensor([0.0028])), ('backbone0.1.weight_fake_quant.zero_point', tensor([0])), ('backbone0.1.weight_fake_quant.activation_post_process.min_vals', tensor([-0.3583])), ('backbone0.1.weight_fake_quant.activation_post_process.max_vals', tensor([0.0458])), ('backbone0.2.weight', tensor([4.2045])), ('backbone0.2.bias', tensor([-0.3767])), ('backbone0.2.running_mean', tensor([-0.1198])), ('backbone0.2.running_var', tensor([0.3677])), ('backbone0.2.num_batches_tracked', tensor(10)), ('backbone0.2.activation_post_process.scale', tensor([0.0341])), ('backbone0.2.activation_post_process.zero_point', tensor([59])), ('backbone0.2.activation_post_process.activation_post_process.min_val', tensor(-2.0061)), ('backbone0.2.activation_post_process.activation_post_process.max_val', tensor(2.3278)), ('backbone0.3.activation_post_process.scale', tensor([0.0184])), ('backbone0.3.activation_post_process.zero_point', tensor([0])), ('backbone0.3.activation_post_process.activation_post_process.min_val', tensor(0.0400)), ('backbone0.3.activation_post_process.activation_post_process.max_val', tensor(2.3427)), ('backbone1.0.weight', tensor([[[[-3.8434, -4.1207,  4.9514],
          [ 5.1071, -4.7582, -3.8411],
          [-3.6406, -4.1518, -1.5902]]],


        [[[-4.2797, -4.4012,  1.1095],
          [-5.2368,  5.8240,  0.5995],
          [-3.5678, -3.8644,  1.4833]]]])), ('backbone1.1.weight', tensor([-2.8446,  2.8394])), ('backbone1.1.bias', tensor([ 0.2494, -0.2510])), ('backbone1.1.running_mean', tensor([-12.0386,  -4.0614])), ('backbone1.1.running_var', tensor([157.3076, 138.7101])), ('backbone1.1.num_batches_tracked', tensor(10))])

<============== QUANT CONVERTED MODEL ===============>
OrderedDict([('backbone0.0.scale', tensor([0.0077])), ('backbone0.0.zero_point', tensor([0])), ('backbone0.1.weight', tensor([[[[-0.3597]],

         [[ 0.3569]],

         [[-0.3597]]]], size=(1, 3, 1, 1), dtype=torch.qint8,
       quantization_scheme=torch.per_channel_affine,
       scale=tensor([0.0028], dtype=torch.float64), zero_point=tensor([0]),
       axis=0)), ('backbone0.1.scale', tensor(0.0039)), ('backbone0.1.zero_point', tensor(127)), ('backbone0.1.bias', None), ('backbone0.2.weight', tensor([1.])), ('backbone0.2.bias', tensor([0.])), ('backbone0.2.running_mean', tensor([0.])), ('backbone0.2.running_var', tensor([1.])), ('backbone0.2.num_batches_tracked', tensor(0)), ('backbone1.0.weight', tensor([[[[-3.8434, -4.1207,  4.9514],
          [ 5.1071, -4.7582, -3.8411],
          [-3.6406, -4.1518, -1.5902]]],


        [[[-4.2797, -4.4012,  1.1095],
          [-5.2368,  5.8240,  0.5995],
          [-3.5678, -3.8644,  1.4833]]]])), ('backbone1.1.weight', tensor([-2.8446,  2.8394])), ('backbone1.1.bias', tensor([ 0.2494, -0.2510])), ('backbone1.1.running_mean', tensor([-12.0386,  -4.0614])), ('backbone1.1.running_var', tensor([157.3076, 138.7101])), ('backbone1.1.num_batches_tracked', tensor(10))])



 still the same 
# Forward zeros output
[0.4619605243206024, 0.3693790137767792] #not converted
[0.4619605243206024, 0.3693790137767792] #converted

<================ NON-CONVERTED MODEL ====================>
OrderedDict([('backbone0.0.activation_post_process.scale', tensor([0.0077])), ('backbone0.0.activation_post_process.zero_point', tensor([0])), ('backbone0.0.activation_post_process.activation_post_process.min_val', tensor(8.7249e-05)), ('backbone0.0.activation_post_process.activation_post_process.max_val', tensor(0.9802)), ('backbone0.1.weight', tensor([[[[4.5793]],

         [[3.9695]],

         [[4.1871]]]])), ('backbone0.1.activation_post_process.scale', tensor([0.0046])), ('backbone0.1.activation_post_process.zero_point', tensor([42])), ('backbone0.1.activation_post_process.activation_post_process.min_val', tensor(-0.1939)), ('backbone0.1.activation_post_process.activation_post_process.max_val', tensor(0.3904)), ('backbone0.1.weight_fake_quant.scale', tensor([0.0035])), ('backbone0.1.weight_fake_quant.zero_point', tensor([0])), ('backbone0.1.weight_fake_quant.activation_post_process.min_vals', tensor([-0.1654])), ('backbone0.1.weight_fake_quant.activation_post_process.max_vals', tensor([0.4444])), ('backbone0.2.weight', tensor([3.7733])), ('backbone0.2.bias', tensor([-3.2874])), ('backbone0.2.running_mean', tensor([0.4043])), ('backbone0.2.running_var', tensor([0.3758])), ('backbone0.2.num_batches_tracked', tensor(10)), ('backbone0.2.activation_post_process.scale', tensor([0.0358])), ('backbone0.2.activation_post_process.zero_point', tensor([65])), ('backbone0.2.activation_post_process.activation_post_process.min_val', tensor(-2.3070)), ('backbone0.2.activation_post_process.activation_post_process.max_val', tensor(2.2333)), ('backbone0.3.activation_post_process.scale', tensor([0.0179])), ('backbone0.3.activation_post_process.zero_point', tensor([0])), ('backbone0.3.activation_post_process.activation_post_process.min_val', tensor(0.)), ('backbone0.3.activation_post_process.activation_post_process.max_val', tensor(2.2680)), ('backbone1.0.weight', tensor([[[[ 4.3533, -3.5816,  5.0651],
          [-4.1010, -3.6161, -4.3417],
          [ 4.0427, -4.3517,  4.1981]]],


        [[[ 2.5564, -3.3695,  4.0380],
          [-3.9976, -7.2543, -4.0428],
          [ 3.7343, -3.5447,  2.7283]]]])), ('backbone1.1.weight', tensor([-0.5695, -2.9817])), ('backbone1.1.bias', tensor([-0.1784, -0.1795])), ('backbone1.1.running_mean', tensor([ 0.1879, -0.5091])), ('backbone1.1.running_var', tensor([16.9781, 18.2449])), ('backbone1.1.num_batches_tracked', tensor(10))])

<============== QUANT CONVERTED MODEL ===============>
OrderedDict([('backbone0.0.scale', tensor([0.0077])), ('backbone0.0.zero_point', tensor([0])), ('backbone0.1.weight', tensor([[[[0.4426]],

         [[0.4426]],

         [[0.4426]]]], size=(1, 3, 1, 1), dtype=torch.qint8,
       quantization_scheme=torch.per_channel_affine,
       scale=tensor([0.0035], dtype=torch.float64), zero_point=tensor([0]),
       axis=0)), ('backbone0.1.scale', tensor(0.0046)), ('backbone0.1.zero_point', tensor(42)), ('backbone0.1.bias', None), ('backbone0.2.weight', tensor([1.])), ('backbone0.2.bias', tensor([0.])), ('backbone0.2.running_mean', tensor([0.])), ('backbone0.2.running_var', tensor([1.])), ('backbone0.2.num_batches_tracked', tensor(0)), ('backbone1.0.weight', tensor([[[[ 4.3533, -3.5816,  5.0651],
          [-4.1010, -3.6161, -4.3417],
          [ 4.0427, -4.3517,  4.1981]]],


        [[[ 2.5564, -3.3695,  4.0380],
          [-3.9976, -7.2543, -4.0428],
          [ 3.7343, -3.5447,  2.7283]]]])), ('backbone1.1.weight', tensor([-0.5695, -2.9817])), ('backbone1.1.bias', tensor([-0.1784, -0.1795])), ('backbone1.1.running_mean', tensor([ 0.1879, -0.5091])), ('backbone1.1.running_var', tensor([16.9781, 18.2449])), ('backbone1.1.num_batches_tracked', tensor(10))])