I set up the most basic network and data loader to reproduce the problem (74 lines sorry)
import torch
from torch import nn
import logging

logger = logging.getLogger(__name__)
logging.basicConfig()
logger.setLevel(logging.INFO)
NUM_CLASSES = 10


class Net(nn.Module):
    def __init__(self,):
        super(Net, self).__init__()
        self.network = torch.hub.load(
            "rwightman/pytorch-image-models",
            "efficientnet_b0",
            pretrained=True,
            features_only=True,
            in_chans=3,
        )
        mid_ch = self.network.feature_info[-1]["num_chs"]
        self.classifier = nn.Conv2d(mid_ch, NUM_CLASSES, kernel_size=1)
        self.criterion = nn.CrossEntropyLoss()

    def forward(self, batch):
        batch["image"] = batch["image"].float().cuda()
        batch["gt"] = batch["gt"].cuda()
        x = self.network(batch["image"])[-1]
        out_size = batch["gt"].size()[-2:]
        logits = self.classifier(x)
        logits = nn.functional.interpolate(
            logits,
            size=out_size,
            mode="bilinear",
            align_corners=False,
            recompute_scale_factor=False,
        )
        loss = self.criterion(target=batch["gt"].long().squeeze(1), input=logits)
        return {"loss": loss}


class Dataloader:
    def __init__(self, batch_size, size):
        self.batch_size = batch_size
        self.size = size

    def __iter__(self,):
        for i in range(250):
            img = torch.randn((self.batch_size, 3, self.size, self.size))
            target = torch.randint(NUM_CLASSES, (self.batch_size, 1, self.size, self.size))
            batch = {"image": img, "gt": target}
            yield batch


net = Net()
net.cuda()
optimizer = torch.optim.SGD(
    net.parameters(), lr=1e-2, weight_decay=5e-4, momentum=0.9, nesterov=False,
)
train_data_loader = Dataloader(batch_size=6, size=512)
logger.warning('starting training')
for i, batch in enumerate(train_data_loader):
    try:
        output = net(batch)
        if i % 50 == 0 and i != 0:
            raise ValueError
    except Exception as e:
        logging.error(e, exc_info=True)  # log stack trace
        continue
    logger.info(f'Loss: {output["loss"].item()} | iter: {i}')
    output["loss"].backward()
    optimizer.step()
    optimizer.zero_grad()


What is interesting is that the problem only occurs when the gpu memory consumption is well over 50% (for me that was at batch size 7 at minimum, with a 1070ti w/ ~8Gb memory), but the problem did not occur when using a batch size of 6 for instance. Here is a gpu consumption graph, before the first error, and after 2 errors when the problem does not occur, using a batch size of 6:

(yellow is gpu memory, blue gpu util)
So clearly some extra memory is allocated (to what I don’t know) due to the catched error, but it does not accumulate further beyond that, when another error occurs…
I have also verified that no gpu memory accumulates on catched errors when setting the whole loop in a with torch.no_grad() context, so it must have something to do with autograd right?
For me this issue is resolved in one of two ways:

I am shown a way that I can catch errors without gpu memory consumption building up (i.e. clearing the autograd graph somehow manually?)
or when this is simply not possible currently  although I can’t imagine I’m the only one with this use case though, so then I’ll open a feature request in the pytorch repository.