I think I found one way: installing torchvision from pip instead of conda:
# install.sh

PREFIX=/home/gamir/lab/vadim/prefix
mkdir -p $PREFIX

curl -L -so $PREFIX/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh && chmod +x $PREFIX/miniconda.sh && $PREFIX/miniconda.sh -b -p $PREFIX/miniconda && rm $PREFIX/miniconda.sh

export PATH=$PREFIX/miniconda/bin:$PATH

conda install -y opencv pytorch cudatoolkit=11.1 -c pytorch -c conda-forge

python -m pip install torchvision 

# enter.sh

PREFIX=/home/gamir/lab/vadim/prefix

PATH=$PREFIX/miniconda/bin:$PATH
LD_LIBRARY_PATH=$PREFIX/miniconda/lib:$LD_LIBRARY_PATH

PATH="$PATH" LD_LIBRARY_PATH="$LD_LIBRARY_PATH" bash --init-file <(echo "export TERM=xterm-256color; export MPLBACKEND=agg; export PS1='CONDA [\u@\h \W]\$ '")