So I came up with this minimal example, consisting of just a Conv Layer and a Linear layer. The code exits after the first iteration.
import torch
import torchvision
import torch.nn as nn
import copy

train_loader = torch.utils.data.DataLoader(
  torchvision.datasets.MNIST("./", train=True, download=True,
                             transform=torchvision.transforms.Compose([
                               torchvision.transforms.ToTensor(),
                               torchvision.transforms.Normalize(
                                 (0.1307,), (0.3081,))
                             ])),
  batch_size=4096, shuffle=True)

class Net(nn.Module):
    
    def __init__(self):
        super().__init__()
        self.encoder = nn.Conv2d(1, 8, 3, stride=1, padding=1)
        self.head = nn.Linear(6272, 10)

    def forward(self, image):
        out = self.encoder(image)
        out = out.flatten(start_dim=1)
        print(out.shape)
        return self.head(out)

torch.manual_seed(42)
model1 = Net()
torch.manual_seed(42)
model2 = Net()
assert all([torch.equal(x[1], y[1]) for x, y in zip(model1.state_dict().items(), model2.state_dict().items())])


optim1 = torch.optim.AdamW(model1.parameters())
optim2 = torch.optim.AdamW(model2.parameters())
loss1 = nn.MSELoss()
loss2 = nn.MSELoss()
dev1 = torch.device("cuda:0")
dev2 = torch.device("cuda:1")
cpu = torch.device("cpu")

model1 = model1.to(dev1)
model2 = model2.to(dev2)

model1.train()
model2.train()

for i, (images, targets) in enumerate(train_loader):
    batch1 = copy.deepcopy(images).to(dev1)
    batch2 = copy.deepcopy(images).to(dev2)

    t1 = copy.deepcopy(targets).to(dev1)
    t2 = copy.deepcopy(targets).to(dev2)
    t1 = nn.functional.one_hot(t1, num_classes=10).float()
    t2 = nn.functional.one_hot(t2, num_classes=10).float()

    optim1.zero_grad()
    result1 = model1.forward(batch1)
    l1 = loss1(result1, t1)
    l1.backward()
    optim1.step()

    optim2.zero_grad()
    result2 = model2.forward(batch2)
    l2 = loss2(result2, t2)
    l2.backward()
    optim2.step()

    if not (model1.to(cpu).encoder.weight.grad == model2.to(cpu).encoder.weight.grad).all():
        print(f"Nope nope nope - Chuck Testa!\n @Iteration {i}")
        break
    else:
        model1 = model1.to(dev1)
        model2 = model2.to(dev2)
        model1.train()
        model2.train()