Ok, so we are hunting it down. 
I’ve created a sample code snippet and tried to simulate your use case:
class SmallModel(nn.Module):
    def __init__(self):
        super(SmallModel, self).__init__()
        self.act = nn.ReLU()
        self.fc1 = nn.Linear(1000, 100)
        self.fc2 = nn.Linear(100, 2)
    
    def forward(self, x):
        x = self.act(self.fc1(x))
        x = F.log_softmax(self.fc2(x), dim=1)
        return x


class NetEnsemble(nn.Module):
    def __init__(self, models):
        super(NetEnsemble, self).__init__()
        self.ensemble_models = nn.ModuleList(models)
        self.fc = nn.Linear(12, 6)
        
        self.mean = torch.randn(1000).cuda()
        self.std = torch.randn(1000).cuda()
        
    def forward(self, x):
        log_softmaxes = []
        
        with torch.no_grad():
            x_normalized = (x - Variable(self.mean)) / Variable(self.std)
        
            for model in self.ensemble_models:
                log_softmax = model(x_normalized)
                log_softmaxes.append(log_softmax)
                
        
        out = torch.cat(log_softmaxes, 1)
        out = out.view(out.size(0), -1)
        out = self.fc(out)
        
        return F.log_softmax(out, dim=1)


y = Variable(torch.LongTensor(10).random_(6).cuda())
x = Variable(torch.randn(10, 1000).cuda())

models = [SmallModel() for i in range(6)]
ens_model = NetEnsemble(models)
ens_model.cuda()

criterion = nn.NLLLoss()
optimizer = optim.Adam(ens_model.fc.parameters())

mem_alloc = torch.cuda.memory_allocated()
mem_cache = torch.cuda.memory_cached()

epochs = 10000
for epoch in range(epochs):
    optimizer.zero_grad()
    output = ens_model(x)

    loss = criterion(output, y) 
    loss.backward()
    
    optimizer.step()

    print('Stage2 grad: {}'.format(ens_model.fc.weight.grad))
    print('Stage1 grad: {}'.format(
        ens_model.ensemble_models[0].fc1.weight.grad))
    print('Epoch {} Loss {}'.format(epoch, loss.data[0]))
    print('New allocated memory {}\tnew cached memory {}'.format(
        torch.cuda.memory_allocated() - mem_alloc,
        torch.cuda.memory_cached() - mem_cache))
    
    mem_alloc = torch.cuda.memory_allocated()
    mem_cache = torch.cuda.memory_cached()

Could you check, if your stage 1 models get somehow gradients?
The volatile flag should take care of this, so I doubt it’s the issue, but somehow we have to get to the leak.
I’ve built PyTorch from source, that’s why I’m using with torch.no_grad() instead of volatile.