I’m trying to figure out how to best answer this. What exactly do you mean by model definition? Altogether, faster_rcnn is spread across multiple files, and most of them are in torchvision. At this point, my code makes few modifications to it, but it does some.
To try to aid in sharing the model, I am trying the following, which you should be able to reproduce easily enough. This should also isolate the issue to something in torchvision rather than my code. Here’s my script, simple_train.py:
import torch
import torchvision
import numpy as np
import random
import os
import math
import sys

NCLASS = 3


def reduce_dict(input_dict, average=True):
    """
    Args:
        input_dict (dict): all the values will be reduced
        average (bool): whether to do average or sum
    Reduce the values in the dictionary from all processes so that all processes
    have the averaged results. Returns a dict with the same fields as
    input_dict, after reduction.
    """
    # world_size = get_world_size()
    world_size = 1
    if world_size < 2:
        return input_dict
    with torch.no_grad():
        names = []
        values = []
        # sort the keys so that they are consistent across processes
        for k in sorted(input_dict.keys()):
            names.append(k)
            values.append(input_dict[k])
        values = torch.stack(values, dim=0)
        dist.all_reduce(values)
        if average:
            values /= world_size
        reduced_dict = {k: v for k, v in zip(names, values)}
    return reduced_dict

def generate_target(ind):

    # nboxes = random.randint(1, 3)
    boxes = []
    labels = []
    nboxes = 2
    for i in range(nboxes):
        xtl = 1
        xbr = 2
        ytl = 3
        ybr = 4
        box = [xtl, ytl, xbr, ybr]
        label = 2
        boxes.append(box)
        labels.append(label)

    one_hot = torch.zeros(NCLASS)
    imid = torch.tensor([ind])
    iscrowd = torch.zeros(len(boxes))
    boxes = torch.as_tensor(boxes, dtype=torch.float32)
    labels = torch.as_tensor(labels, dtype=torch.int64)
    area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])

    target = {'one_hot': one_hot, 'id': imid, 'boxes': boxes, 'labels': labels, 'image_id': imid, 'area': area, 'iscrowd': iscrowd}
    return target

seed = 1
'''
torch.use_deterministic_algorithms(True) # , in beta....
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
os.environ['PYTHONHASHSEED'] = str(seed)
'''
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

model = torchvision.models.detection.__dict__['fasterrcnn_resnet50_fpn'](num_classes=NCLASS, pretrained=0)
device = torch.device('cuda')
model.to(device)

'''
# can see all_the_outs[0] and [1] are the same
all_the_outs = []
model.eval()
for h in range(2):
    all_outs = []
    for i in range(10):
        images = torch.full([1, 3, 540, 1920], i/100)
        outputs = model(images)
        all_outs.append(outputs)
    all_the_outs.append(all_outs)
'''
model.train()

params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(
    params, lr=0.00001, momentum=0.9, weight_decay=1e-4)

nsteps = 100
for i in range(nsteps):
    images = torch.full([1, 3, 540, 1920], i/100)
    images = list(image.to(device) for image in images)
    targets = []
    for j in range(len(images)):
        targets.append(generate_target(j))
    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

    loss_dict = model(images, targets)

    losses = sum(loss for loss in loss_dict.values())

    # reduce losses over all GPUs for logging purposes
    # loss_dict_reduced = utils.reduce_dict(loss_dict)
    loss_dict_reduced = reduce_dict(loss_dict)
    losses_reduced = sum(loss for loss in loss_dict_reduced.values())

    loss_value = losses_reduced.item()

    if not math.isfinite(loss_value):
        print("Loss is {}, stopping training".format(loss_value))
        print(loss_dict_reduced)
        sys.exit(1)

    optimizer.zero_grad()
    losses.backward()
    optimizer.step()
    
    '''
    if lr_scheduler is not None:
	lr_scheduler.step()
    '''
print('done')

This works fine and prints done. However, as soon as I uncomment the lines about deterministic behavior I get errors.
Torch nightly error:
$ CUDA_VISIBLE_DEVICES=0 python simple_train.py
/home/austin/.virtualenvs/tchnite/lib/python3.7/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and
subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)
  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)
terminate called after throwing an instance of 'c10::Error'
  what():  linearIndex.numel()*sliceSize*nElemBefore == value.numel()INTERNAL ASSERT FAILED at "/pytorch/aten/src/ATen/native/cuda/Indexing.cu":253, please report a bug to PyTorch. number of flattened indices did not match number of elements in the value tensor8761
Exception raised from index_put_with_sort_kernel at /pytorch/aten/src/ATen/native/cuda/Indexing.cu:253 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7f0dfecd1302 in /home/austin/.virtualenvs/tchnite/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::string const&) + 0x5b (0x7f0dfeccdc9b in /home/austin/.virtualenvs/tchnite/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #2: c10::detail::torchInternalAssertFail(char const*, char const*, unsigned int, char const*, std::string const&) + 0x3e (0x7f0dfecce18e in /home/austin/.virtualenvs/tchnite/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #3: at::native::(anonymous namespace)::index_put_with_sort_kernel(at::Tensor&, c10::List<c10::optional<at::Tensor> > const&, at::Tensor const&, bool, bool) + 0x2218 (0x7f0e00c5c268 in /home/austin/.virtualenvs/tchnite/lib/python3.7/site-packages/torch/lib/libtorch_cuda.so)
frame #4: at::native::_index_put_impl_(at::Tensor&, c10::List<c10::optional<at::Tensor> > const&, at::Tensor const&, bool, bool) + 0x553 (0x7f0e42b6e423 in /home/austin/.virtualenvs/tchnite/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)

Aborted (core dumped)

torch 1.8.1 complains about CUBLAS_WORKSPACE_CONFIG even though it is set
(tch18) $ CUBLAS_WORKSPACE_CONFIG=:16:8
(tch18) $ CUDA_VISIBLE_DEVICES=1 python simple_train.py 
[W Context.cpp:70] Warning: torch.use_deterministic_algorithms is in beta, and its design and functionality may change in the future. (function operator())
Traceback (most recent call last):
  File "simple_train.py", line 107, in <module>
    loss_dict = model(images, targets)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torchvision/models/detection/generalized_rcnn.py", line 98, in forward
    detections, detector_losses = self.roi_heads(features, proposals, images.image_sizes, targets)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torchvision/models/detection/roi_heads.py", line 753, in forward
    box_features = self.box_head(box_features)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torchvision/models/detection/faster_rcnn.py", line 258, in forward
    x = F.relu(self.fc6(x))
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 94, in forward
    return F.linear(input, self.weight, self.bias)
  File "/home/austin/.virtualenvs/tch18/lib/python3.7/site-packages/torch/nn/functional.py", line 1753, in linear
    return torch._C._nn.linear(input, weight, bias)
RuntimeError: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
(tch18) austin@skywalker:/media/ssd1/austin/invert_counter/detection$ echo CUBLAS_WORKSPACE_CONFIG
CUBLAS_WORKSPACE_CONFIG
(tch18) $ echo $CUBLAS_WORKSPACE_CONFIG
:16:8


Let me know if you are able to reproduce these results or if I’m still missing something.