Not really.
See the modified code here that prints the actual used memory accros iterations:
import torch
from torch import nn
import logging

logger = logging.getLogger(__name__)
logging.basicConfig()
logger.setLevel(logging.INFO)
NUM_CLASSES = 10


class Net(nn.Module):
    def __init__(self,):
        super(Net, self).__init__()
        self.network = torch.hub.load(
            "rwightman/pytorch-image-models",
            "efficientnet_b0",
            pretrained=True,
            features_only=True,
            in_chans=3,
        )
        mid_ch = self.network.feature_info[-1]["num_chs"]
        self.classifier = nn.Conv2d(mid_ch, NUM_CLASSES, kernel_size=1)
        self.criterion = nn.CrossEntropyLoss()

    def forward(self, batch):
        batch["image"] = batch["image"].float().cuda()
        batch["gt"] = batch["gt"].cuda()
        x = self.network(batch["image"])[-1]
        out_size = batch["gt"].size()[-2:]
        logits = self.classifier(x)
        logits = nn.functional.interpolate(
            logits,
            size=out_size,
            mode="bilinear",
            align_corners=False,
            recompute_scale_factor=False,
        )
        loss = self.criterion(target=batch["gt"].long().squeeze(1), input=logits)
        return {"loss": loss}


class Dataloader:
    def __init__(self, batch_size, size):
        self.batch_size = batch_size
        self.size = size

    def __iter__(self,):
        for i in range(10):
            img = torch.randn((self.batch_size, 3, self.size, self.size))
            target = torch.randint(NUM_CLASSES, (self.batch_size, 1, self.size, self.size))
            batch = {"image": img, "gt": target}
            yield batch


net = Net()
net.cuda()
optimizer = torch.optim.SGD(
    net.parameters(), lr=1e-2, weight_decay=5e-4, momentum=0.9, nesterov=False,
)
train_data_loader = Dataloader(batch_size=10, size=40)
logger.warning('starting training')
for i, batch in enumerate(train_data_loader):
    print(torch.cuda.memory_allocated())
    try:
        output = net(batch)
        print(torch.cuda.memory_allocated())
        if i % 3 == 0 and i != 0:
            raise ValueError("Value error")
    except Exception as e:
        # logging.error(e, exc_info=True)  # log stack trace
        print('caugh an error, keep going ' + str(e))
        continue
    print(f'Loss: {output["loss"].item()} | iter: {i}')
    output["loss"].backward()
    optimizer.step()
    optimizer.zero_grad()

Which prints:
WARNING:__main__:starting training

14631936
51151872
Loss: 3.2388575077056885 | iter: 0
43490816
80133120
Loss: 3.1496763229370117 | iter: 1
43490816
80133120
Loss: 3.0223429203033447 | iter: 2
43490816
80133120
caugh an error, keep going Value error
80133120
80010240
Loss: 3.105882167816162 | iter: 4
43490816
80133120
Loss: 3.029092311859131 | iter: 5
43490816
80133120
caugh an error, keep going Value error
80133120
80010240
Loss: 2.8809430599212646 | iter: 7
43490816
80133120
Loss: 2.9971139430999756 | iter: 8
43490816
80133120
caugh an error, keep going Value error


Most iteration, the memory goes down because the call to backward frees the buffers.
When you skip that because of the error, the memory is not freed but that does not change the final usage.