Following my previous reply:
I’ve created this simplified model that replicates the issue:
import torch
import torch.nn as nn
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

class Model(nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.que = [torch.rand((1,2560,2560), device=device)] * 3 # Init queue with len of 3
        self.fc = nn.Linear(2560, 2560)
        
    def forward(self, x):
        t=0
        while t<=1000:
            print("Itter count: ", t)
            x = self.fc(x)#.detach() # uncomment .detach to fixes the CUDA out of memory
            self.que.pop(0); self.que.append(x) # pop first element from the que and append x to the end of que
            t+=1

        return x

if __name__ == '__main__':
    model = Model().to(device)
    x=torch.rand((1,2560,2560), device=device)

    model(x)

I’ve also noticed that assigning the output of self.fc to a different variable also fixed the issue of Cuda running out of memory:
def forward(self, x):
    t=0
    while t<=1000:
        print("Itter count: ", t)
        y = self.fc(x)
        self.que.pop(0); self.que.append(y) # pop first element from the que and append y to the end of que
        t+=1

    return y