I have got a special deep neural network with special cost function. I have wrote my cost function with this strategy. First i create a batch of data, let say 64 images. I forward this batch through the network and finally calculate the Cost value. for creating the gradient variable for the network, I mean the variable one before my cost layer, I create a variable with same size as output variable of the network. then with for i iterate over the images and fill this newly created tensor. Actually What i return when using backward function is that tesnor.   Here is my code:

for i, (images, labels) in enumerate(trainLoader):
    if s in steps:
        learning_rate = learning_rate * 0.1
        optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=momentum, weight_decay=decay)
    if cuda:
        images = images.cuda()
    images = V(images)
    optimizer.zero_grad()
    output = net(images).cpu()  # batches*95*S*S
    loss = 0
    target, index = jsonToTensor(labels)
    target, index = V(target), V(index)
    cost = criterion(output, target, index)
    loss += cost.data[0]
    cost.backward()
    epochLoss += loss
    optimizer.step()
    print("(%d,%d) -> Current Batch Loss: %f"%(epoch, i, (loss/batches)))
    s = s + 1


And here is my cost:
class Criterion(Function):
def __init__(self, S, B, numClass, coords, l_coord, l_nobj):
    super(Criterion, self).__init__()
    self.S = S # Number of Cell
    self.B = B # Number of Bouning Box
    self.numClass = numClass
    self.coords = coords
    self.l_coord = l_coord
    self.l_nobj = l_nobj

def forward(self, pred_out, real_out, index):
    # pred_out -> 64*95*7*7
    self.save_for_backward(pred_out, real_out ,index)
    cost = 0
    for i in range(pred_out.size()[0]):
        if i==0:
            endInx = int((index[i]).numpy())
            if i==endInx:
                continue
            cost += self.perImageCalc(pred_out[i,:,:,:].unsqueeze(0), real_out[0:endInx,:])
        else:
            startInx = int((index[i-1]).numpy())
            endInx = int((index[i]).numpy())
            if startInx==endInx:
                continue
            cost += self.perImageCalc(pred_out[i,:,:,:].unsqueeze(0), real_out[startInx:endInx,:])
    return cost

def perImageCalc(self, pt, rt):
    # Doing Some stuff to calculate the Cost
    return cost

def backward(self, grad_cost):
    pt, rt, index = self.saved_tensors
    grad_pred_out = torch.zeros(pt.size())
    for i in range(pt.size()[0]):
        if i==0:
            endInx = int((index[i]).numpy())
            grad_pred_out[i,:,:,:] = self.gradPerImageCalc(pt[i,:,:,:].unsqueeze(0), rt[0:endInx,:])
        else:
            startInx = int((index[i-1]).numpy())
            endInx = int((index[i]).numpy())
            grad_pred_out[i,:,:,:] = self.gradPerImageCalc(pt[i,:,:,:].unsqueeze(0), rt[startInx:endInx,:])
    grad_real_out = grad_index = None
    return grad_pred_out, grad_real_out, grad_index

def gradPerImageCalc(self, pt, rt):
    # Doing Some Stuff
    return grad_pred_out
        
def indices(self, a, func):
    return [i for (i, val) in enumerate(a) if func(val)]

The first epoch forwarding action and backward action are good, but when i do the second backward action, i mean in the second stage of the process, I receive this run time error:
`
RuntimeError: Trying to backward through the graph second time, but the buffers have already been freed. Please specify retain_variables=True when calling backward for the first time.

/home/mlcmdeep/Mohammad_Mahdi_Derakhshani/yo/MSCoco/Main_v2.py(182)train()
-> cost.backward()
(Pdb) bt
/home/mlcmdeep/anaconda3/lib/python3.6/bdb.py(431)run()
-> exec(cmd, globals, locals)
(1)()
/home/mlcmdeep/Mohammad_Mahdi_Derakhshani/yo/MSCoco/Main_v2.py(209)()
-> train(epoch)
/home/mlcmdeep/Mohammad_Mahdi_Derakhshani/yo/MSCoco/Main_v2.py(182)train()
-> cost.backward()
/home/mlcmdeep/anaconda3/lib/python3.6/site-packages/torch/autograd/variable.py(146)backward()
-> self._execution_engine.run_backward((self,), (gradient,), retain_variables)
/home/mlcmdeep/Mohammad_Mahdi_Derakhshani/yo/MSCoco/Cost.py(128)backward()
-> pt, rt, index = self.saved_tensors
`

Could you please help me what is the source of error? By the way, here is my network:
class yoloV1CocoTiny(nn.Module):
  def __init__(self, inputChannel, numClass, B, S, ngpu, batchSize):
    super(yoloV1CocoTiny, self).__init__()
    self.ngpu = ngpu
    self.batchSize = batchSize
    self.main = nn.Sequential(
            nn.Conv2d(3,16,3,1,1),
            nn.BatchNorm2d(16),
            nn.LeakyReLU(0.1),
            nn.MaxPool2d(2,2),

            nn.Conv2d(16,32,3,1,1),
            nn.BatchNorm2d(32),
            nn.LeakyReLU(0.1),
            nn.MaxPool2d(2,2),

            nn.Conv2d(32,64,3,1,1),
            nn.BatchNorm2d(64),
            nn.LeakyReLU(0.1),
            nn.MaxPool2d(2,2),

            nn.Conv2d(64,128,3,1,1),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.1),
            nn.MaxPool2d(2,2),

            nn.Conv2d(128,256,3,1,1),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.1),
            nn.MaxPool2d(2,2),

            nn.Conv2d(256,512,3,1,1),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.1),
            nn.MaxPool2d(2,2),

            nn.Conv2d(512,1024,3,1,1),
            nn.BatchNorm2d(1024),
            nn.LeakyReLU(0.1),
            nn.Conv2d(1024,256,3,1,1),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.1),
            nn.Conv2d(256,numClass + B*5,3,1,1),
            nn.ReLU(True)
        )
 def forward(self, input):
    gpu_ids = None
    if isinstance(input.data, torch.cuda.FloatTensor) and self.ngpu>1:
        gpu_ids = range(self.ngpu)
        output = nn.parallel.data_parallel(self.main, input, gpu_ids)
    else:
        output = self.main(input)
    return output