I tried this:
class MyModel(torch.nn.Module):
def init(self):
super(MyModel, self).init()
self.conv0 = torch.nn.Conv1d(24,512,5).cuda()
self.conv1 = torch.nn.Conv1d(512,512,3,dilation=2).cuda()
self.conv2 = torch.nn.Conv1d(512,512,3,dilation=3).cuda()
self.conv3 = torch.nn.Conv1d(512,512,1).cuda()
self.conv4 = torch.nn.Conv1d(512,1500,1).cuda()
self.lin1 = torch.nn.Linear(3000,512).cuda()
self.lin2 = torch.nn.Linear(512,512).cuda()
self.lin3 = torch.nn.Linear(512,2000).cuda()
def forward(self,x):
    x = torch.sigmoid(self.conv0(x))
    x = torch.sigmoid(self.conv1(x))
    x = torch.sigmoid(self.conv2(x))
    x = torch.sigmoid(self.conv3(x))
    x = torch.sigmoid(self.conv4(x))
    mean = torch.mean(x,dim=2)
    std = torch.std(x,dim=2)
    x = torch.cat([mean,std],dim=1)
    x = torch.sigmoid(self.lin1(x)) 
    x = torch.sigmoid(self.lin2(x))
    x = torch.sigmoid(self.lin3(x))
    x = torch.nn.functional.softmax(x,dim=1)
    return x

def backward(self,x,lable):
    loss = - torch.trace(torch.mm(torch.log(x),torch.t(lable)))
    loss.backward()
    return self.conv0.weight.grad.data

When im calling just forward it works without problems, but when im trying this:
for i in range(10):
x = Net.forward(Variable(torch.FloatTensor(np.random.rand(128,24,500)).cuda(),requires_grad=False) )
print(Net.backward(x,Variable(torch.FloatTensor(np.random.rand(128,2000)).cuda(),requires_grad=False)))
print(get_gpu_memory_map())
it starts to fill gpu like it was before