@wizardk: Thanks for reporting it. It’s an issue in our backend library, FBGEMM, for certain specific depthwise shapes (i_c != o_c ). In your case, depthwise convolution goes through a slower path. https://github.com/pytorch/FBGEMM/issues/347 19 is tracking the progress on improving performance for such cases. If your use case doesn’t need i_c != o_c, please proceed with using i_c == o_c for depthwise convolutions.
Meanwhile I see the following results for your 4 cases, if I make depthwise to have the same i_c and o_c.
Self CPU time total: 64.826ms
Self CPU time total: 31.913ms
Self CPU time total: 50.317ms
Self CPU time total: 17.530ms
The following is the code I used for benchmarking.
import torch
from torch import nn
import torch.quantization as Q

torch.set_num_threads(1)

class TestConv(nn.Module):
    def __init__(self, q, dw, i_c, o_c):
        super(TestConv, self).__init__()
        self.lyr = nn.Sequential(
            nn.Conv2d(in_channels=i_c, out_channels=i_c, kernel_size=1, stride=1, padding=0, dilation=1, groups=1, bias=False),
            nn.BatchNorm2d(num_features=i_c),
            nn.ReLU(inplace=False) if q else nn.ReLU6(inplace=True),
            nn.Conv2d(in_channels=i_c, out_channels=o_c, kernel_size=3, stride=1, padding=1, dilation=1, groups=i_c if dw else 1, bias=False),
            nn.BatchNorm2d(num_features=o_c),
            nn.ReLU(inplace=False) if q else nn.ReLU6(inplace=True),
            nn.Conv2d(in_channels=o_c, out_channels=2*o_c, kernel_size=1, stride=1, padding=0, dilation=1, groups=1, bias=False),
            nn.BatchNorm2d(num_features=2*o_c),
            nn.ReLU(inplace=False) if q else nn.ReLU6(inplace=True),
        )

    def forward(self, x):
        return self.lyr(x)

class TestCNN(nn.Module):
    def __init__(self, q, dw):
        super(TestCNN, self).__init__()
        self.q = q
        self.quant = Q.QuantStub()
        self.dequant = Q.DeQuantStub()
        i_c = 1
        self.cnn = []
        for _ in range(8):
            self.cnn.append(TestConv(q=q, dw=dw, i_c=i_c, o_c=i_c))
            i_c *= 2
        self.cnn = nn.Sequential(*self.cnn)

    def fuse_model(self):
        for m in self.modules():
            if type(m) == TestConv:
                Q.fuse_modules(m.lyr, ['0', '1', '2'], inplace=True)
                Q.fuse_modules(m.lyr, ['3', '4', '5'], inplace=True)
                Q.fuse_modules(m.lyr, ['6', '7', '8'], inplace=True)

    def forward(self, x):
        if self.q:
            x = self.quant(x)
        x = self.cnn(x)
        if self.q:
            x = self.dequant(x)
        return x
def q_test(dw):
    def _eval(m):
        m.eval()
        with torch.no_grad():
            for batch_idx in range(10):
                x = torch.randn(10, 1, 100, 100)
                y = m(x)

    print('\nno quantization\n')
    fm = TestCNN(q=False, dw=dw)
    fm.eval()
    torch.save(fm.state_dict(), 'float.{}.pt'.format('dw' if dw else 'cmn'))
    scriptModel = torch.jit.script(fm)
    x = torch.randn(1, 1, 32, 100)
    with torch.autograd.profiler.profile(record_shapes=True) as prof:
        scriptModel(x)
    print("autograd prof:\n {} \n".format(prof.key_averages(group_by_input_shape=False)))
    #print("autograd prof table:\n {} \n".format(prof.table(row_limit=-1)))

    torch.jit.save(scriptModel, 'jit.f.{}.pt'.format('dw' if dw else 'cmn'))

    print('\npost-training static quantization\n')
    qm = TestCNN(q=True, dw=dw)
    #print(qm)
    qm.load_state_dict(torch.load('float.{}.pt'.format('dw' if dw else 'cmn'), map_location='cpu'))
    qm.eval()
    qm.fuse_model()
    qm.qconfig = Q.get_default_qconfig('fbgemm')
    Q.prepare(qm, inplace=True)
    _eval(qm)  # calibration
    Q.convert(qm, inplace=True)
    qscriptModel = torch.jit.script(qm)
    with torch.autograd.profiler.profile(record_shapes=True) as prof:
        qscriptModel(x)
    print("autograd prof:\n {} \n".format(prof.key_averages(group_by_input_shape=False)))
    #print("autograd prof table:\n {} \n".format(prof.table(row_limit=-1)))
    torch.jit.save(qscriptModel, 'jit.q.{}.pt'.format('dw' if dw else 'cmn'))

q_test(dw=False)  # dump float and quant model without depthwise
q_test(dw=True)  # dump float and quant model with depthwise