hi PtrBlck,
I am using a Titan RTX GPU.
It seems changing the learning rate only delays the onset.
the executable code for a vae-GAN is:
            b_size = real_cpu.size(0)
            label_r = torch.full((b_size,), real_label, dtype=torch.float, device=device)
            label_f = torch.full((b_size,), fake_label, dtype=torch.float, device=device)
            # Forward pass real batch through D
            output = netD(real_cpu).view(-1)
            # Calculate loss on all-real batch
            errD_real = criterion(output, label_r)
            # D_x = output.mean().item()


            # label.fill_(fake_label)
            loss_G1, out_G1 = loss_fn_G_I(netG, real_cpu, device)
            output1 = netD(out_G1.detach()).view(-1)
            errD_G_real = criterion(output1, label_f)

            ## Train with all fake based on noise
            noise = torch.randn(b_size, nz, device=device)
            fake = netG.D2_Decoder(noise)
            # label.fill_(fake_label)
            output2 = netD(fake.detach()).view(-1)
            errD_fake = criterion(output2, label_f)


            errD = errD_real + errD_fake + errD_G_real
            # Calculate gradients for D in backward pass
            optimizerD.zero_grad()
            errD.backward(retain_graph=True)
            optimizerD.step()

            label_r2 = torch.full((b_size,), real_label, dtype=torch.float, device=device)
            fake2 = netG.D2_Decoder(noise)

            # with torch.autograd.set_detect_anomaly(True):
            #### now to work on the generator
            #### use just the decoder of the VAE first with fake
           
            optimizerG_D.zero_grad()
            # label.fill_(real_label)
            output4 = netD(fake2).view(-1)
            errG_F = criterion(output4, label_r2)
            output5 = netD(out_G1).view(-1)
            errG_R = criterion(output5, label_r2)
            err_G=errG_F+errG_R
            err_G.backward(retain_graph=True)
            optimizerG_D.step()

            ### now we operate on the encoder part of the VAE
            optimizerG_E.zero_grad()
            loss_G2, out_G2 = loss_fn_G_I(netG, real_cpu, device)
            loss_G2.backward(retain_graph=True)
            optimizerG_E.step()

I am at my wits end as to what is causing this. The data set used is celebrity faces