Hi,
I cannot share the complete (and rather complex) model but I extended the minimal example from above a bit to make the effect more obvious (it’s attached below).
I am not sure about the padding though, I would assume the input shape dimensions need to be divisable by 8, hence I first use a normal conv on the 64x64 input then use padded conv.
But I also tried other combinations…
Running the code in 2D mode I get:
2020-07-27 08:12:32,971 - Executing 2D Test
2020-07-27 08:12:36,583 - FP32 duration: 2.420s
2020-07-27 08:12:37,398 - FP16 duration: 0.814s
2020-07-27 08:12:38,229 - AMP duration: 0.828s

When profiling, I set the rounds to 1 to make the report more readable (and because profiling is slow).
The profiler shows tensor core usage for FP16 and AMP part
Running in 3D mode:
2020-07-27 08:25:13,797 - Executing 3D Test
2020-07-27 08:25:17,523 - FP32 duration: 2.203s
2020-07-27 08:25:21,319 - FP16 duration: 3.790s
2020-07-27 08:25:25,137 - AMP duration: 3.811s

For 3D, the profiler doesn’t show any tensor core usage.
I used the nv-nsight-cu GUI profiler and configured it so that it calls the command line profiler with the following arguments: /opt/nvidia/nsight-compute/2019.5.0/target/linux-desktop-glibc_2_11_3-x64/nv-nsight-cu-cli --export "report" --force-overwrite --target-processes all --kernel-regex-base function --launch-skip-before-match 0 --section LaunchStats --section Occupancy --section SpeedOfLight --sampling-interval auto --sampling-max-passes 5 --sampling-buffer-size 33554432 --profile-from-start 1 --cache-control all --clock-control base --apply-rules --metrics sm__inst_executed_pipe_tensor_op_hmma.sum "conda/envs/pytorchcudnn8/bin/python"  torch-try-tensorcores_minimal2d3d.py
The code:
import torch
from torch import nn
import torch.nn.functional as F
from time import time
import logging
from apex import amp


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv = nn_conv(64, 64, 5)
        self.conv_padded = nn_conv(64, 64, 5, padding=2)
        self.pool = nn_pool(2, 2)

    def forward(self, x):
        x = F.relu(self.conv(x))
        for i in range(5):
            x = F.relu(self.conv_padded(x))
        return x


if __name__ == "__main__":
    logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.DEBUG)
    # === CONFIGURATION ===
    dimensionality = 2  # 2 for 2D, 3 for 3D

    # === TEST CODE ===
    if dimensionality == 2:
        logging.info(f"Executing 2D Test")
        rounds = 500
        nn_conv = nn.Conv2d
        nn_pool = nn.MaxPool2d
        image_dims = (8, 64, 64, 64)
    else:
        logging.info(f"Executing 3D Test")
        rounds = 2
        nn_conv = nn.Conv3d
        nn_pool = nn.MaxPool3d
        image_dims = (8, 64, 64, 64, 64)

    input_image = torch.rand(image_dims, device="cuda")

    # Run in FP32 mode
    net = Net().to("cuda").to(torch.float32)
    start = time()
    for i in range(rounds):
        net(input_image)
    torch.cuda.synchronize()
    logging.info(f"FP32 duration: {time() - start:.03f}s")

    # Run in FP16 mode
    net = Net().to("cuda").to(torch.float32)
    fp16_net = net.to(torch.float16)
    fp16_input = input_image.to(torch.float16)
    start = time()
    for i in range(rounds):
        fp16_net(fp16_input)
    torch.cuda.synchronize()
    logging.info(f"FP16 duration: {time() - start:.03f}s")

    # Run with AMP
    net = Net().to("cuda").to(torch.float32)
    amp_net = amp.initialize(net, opt_level="O1", verbosity=0)
    start = time()
    for i in range(rounds):
        amp_net(input_image)
    torch.cuda.synchronize()
    logging.info(f"AMP duration: {time() - start:.03f}s")