Thank you @ptrblck for pointing that out. Actually it happens when I use torch.nn.grad.conv2d_input, torch.nn.grad.conv2d_weight in the backward pass of a custom conv op. with stride=1 number of calls to the mkldnn_convolution operation is normal but backward pass is 100x slow. When stride=2 both number of calls to slow_conv_dilated2d and backward time increases heavily. Hopefully you can reproduce this with the following code. Maybe I am doing something wrong in the custom convolution operation. Thanks.
import torch
import torch.nn as nn
import torch.nn.functional as F

class op_Conv2D(torch.autograd.Function):
  
  @staticmethod
  def forward(ctx, input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1):
    ctx.save_for_backward(input, weight, bias)
    ctx.stride = stride
    ctx.padding = padding
    ctx.groups = groups
    ctx.dilation = dilation
    out = F.conv2d(input, weight, bias, stride, padding, dilation, groups)
    return out
    
  @staticmethod
  def backward(ctx, grad_output):
    input, weight, bias = ctx.saved_tensors
    grad_input = grad_weight= grad_bias = None

    if ctx.needs_input_grad[0]:
      grad_input = torch.nn.grad.conv2d_input(input.shape, weight, grad_output, ctx.stride, ctx.padding, ctx.dilation, ctx.groups)
        
    if ctx.needs_input_grad[1]:
      grad_weight = torch.nn.grad.conv2d_weight(input, weight.shape, grad_output, ctx.stride, ctx.padding, ctx.dilation, ctx.groups)
            
    if bias is not None and ctx.needs_input_grad[2]:
      grad_bias = grad_output.sum((0,2,3))
        
    if bias is not None:
      return grad_input, grad_weight, grad_bias, None, None, None, None
    else:
      return grad_input, grad_weight, None, None, None, None, None

w = nn.Parameter(torch.randn(32, 32, 3, 3, device=torch.device("cpu")))
b = nn.Parameter(torch.zeros(32, device=torch.device("cpu"))) 

data = torch.rand(32, 32, 3, 3)

with torch.autograd.profiler.profile(use_cuda=False) as prof:
  x_act = op_Conv2D.apply(data,w, b, 2, 0, 1, 1)
print("forward pass F.conv2d --- \n", prof.key_averages().table(sort_by="self_cpu_time_total"))

with torch.autograd.profiler.profile(use_cuda=False) as prof:
  x_act.sum().backward()
print("backward pass F.conv2d --- \n", prof.key_averages().table(sort_by="self_cpu_time_total"))


conv_block = nn.Conv2d(32, 32, 3, stride=2).cpu()

with torch.autograd.profiler.profile(use_cuda=False) as prof:
  x_act = conv_block(data)
print("forward pass nn.Conv2d --- \n", prof.key_averages().table(sort_by="self_cpu_time_total"))

with torch.autograd.profiler.profile(use_cuda=False) as prof:
  x_act.sum().backward()
print("backward pass nn.Conv2d --- \n", prof.key_averages().table(sort_by="self_cpu_time_total"))