Hey Patrick, I tried again from scratch - my log 1. My dockerfile:
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
ARG PYTHON_VERSION=3.6

ENV PATH=$PATH:/usr/local/nvidia/bin:/usr/local/cuda/bin
ENV CUDA_HOME=/usr/local/cuda
ENV CUDNN_INSTALL_PATH=/usr/local/cuda
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPT/lib64

RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh  && \
     chmod +x ~/miniconda.sh && \
     ~/miniconda.sh -b -p /opt/conda && \
     rm ~/miniconda.sh

RUN /opt/conda/bin/conda install -y python=$PYTHON_VERSION

# Add conda python to the path
ENV PATH=/opt/conda/bin:$PATH

# Tools to build from source
RUN conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_extensions future six requests dataclasses
RUN conda install -c pytorch magma-cuda100  

# https://github.com/MoonVision/moonbox-docker/blob/master/docker/builders/pytorch/Dockerfile#L31
RUN git clone --recursive https://github.com/pytorch/pytorch \
 && cd pytorch \
 && git submodule sync && git submodule update --init --recursive \
 && export TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" \
 && export TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
 && export CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \
 && python setup.py clean \
 && python setup.py install

RUN git clone --recursive https://github.com/pytorch/vision \
 && cd vision \
 && git submodule sync && git submodule update --init --recursive \
 && export TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" \
 && export TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
 && export CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \
 && python setup.py clean \
 && python setup.py install

Thanks again for the help.
Edit: Just to say if I comment out torchvision; torch does seem to work (at least the stuff without vision):
>>> torch.__version__
'1.8.0a0+f65ab89'
>>> torch.cuda.is_available()
True
>>>
>>> torch.cuda.get_device_name(0)
'GeForce RTX 2080 Ti'
>>> x = torch.randn(16,3,224,224).cuda()
>>> x2 = torch.nn.functional.avg_pool2d(x, (3,3))

My system details
Driver Version: 410.104     
CUDA Version: 10.0