Thanks for checking Patrick, I’m not sure why it didn’t work for me.
However I started from scratch and the below works:
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04

ARG PYTHON_VERSION=3.8
ARG WITH_TORCHVISION=1
ARG pytorch_tag='v1.6.0'
ARG torchvision_tag='v0.7.0'

RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         cmake \
         git \
         curl \
         ca-certificates \
         libjpeg-dev \
         libpng-dev && \
     rm -rf /var/lib/apt/lists/*


RUN curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     chmod +x ~/miniconda.sh && \
     ~/miniconda.sh -b -p /opt/conda && \
     rm ~/miniconda.sh && \
     /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include ninja cython typing && \
     /opt/conda/bin/conda install -y -c pytorch magma-cuda100 && \
     /opt/conda/bin/conda clean -ya

ENV PATH /opt/conda/bin:$PATH

# This must be done before pip so that requirements.txt is available

WORKDIR /opt

RUN git clone https://github.com/pytorch/pytorch.git && cd pytorch && git checkout $pytorch_tag 
WORKDIR /opt/pytorch
RUN git submodule sync && git submodule update --init --recursive
RUN TORCH_CUDA_ARCH_LIST="7.0 7.5" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
    CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \
    pip install -v .

RUN if [ "$WITH_TORCHVISION" = "1" ] ; then git clone https://github.com/pytorch/vision.git && cd vision && git checkout $torchvision_tag && pip install -v . ; else echo "building without torchvision" ; fi

WORKDIR /workspace
RUN chmod -R a+w .

Build: DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --tag XXX/torch16:20201007 --build-arg pytorch_tag='v1.6.0' --build-arg torchvision_tag='v0.7.0' --build-arg PYTHON_VERSION=3.8 .