I am running a latent space model on a network, the network itself dosent take up more than 2gb memory when stored in the local ram of the computer.
But when I try to run my pytorch model i get the following error:
RuntimeError: cuda runtime error (2) : out of memory at c:\anaconda2\conda-bld\pytorch_1519501749874\work\torch\lib\thc\generic/THCTensorMathPointwise.cu:343
I am running on a gtx 1080ti which should have more than enough memory to handle this.
def distMatrix(m):
    n = m.size(0)
    d = m.size(1)
    x = m.unsqueeze(1).expand(n, n, d)
    y = m.unsqueeze(0).expand(n, n, d)
    return torch.sqrt(torch.pow(x - y, 2).sum(2) + 1e-4)

def loss(tY):
    d = -distMatrix(tZ)+B
    sigmoidD = torch.sigmoid(d)
    reduce = tY*torch.log(sigmoidD)+(1-tY)*torch.log(1-sigmoidD)
    #remove diagonal
    reduce[torch.eye(n).byte().cuda()] = 0
    return -reduce.sum()

tZ = autograd.Variable(torch.cuda.FloatTensor(Z), requires_grad=True)
B = autograd.Variable(torch.cuda.FloatTensor([0]), requires_grad=True)

tY = autograd.Variable(torch.cuda.FloatTensor(Ytest), requires_grad=False)

losses = []
biases = []
testScore = []

learning_rate = 1e-3
epochs = 10000

sigmoid = np.vectorize(lambda x: math.exp(-np.logaddexp(0, -x)))

percentDone = 0
percent = 2
for i in range(1,epochs):
    count = (float(i)/epochs)*100
    if count % percent == 0:
        print(count)

    l = loss(tY)
    l.backward(retain_graph=True)
    losses.append(float(l))
    biases.append(B.data)
    tZ.data = tZ.data - learning_rate * tZ.grad.data
    B.data = B.data - learning_rate * B.grad.data
    
    tZ.grad.data.zero_()
    B.grad.data.zero_()

Z is 25059 by 2, Ytest is 25059 by 25059
Any hints on how to avoid the memory error would be greatly appreicated.