I tried to do somehow like this :

def train(epoch):
    for batch_idx, (data, target), (data_2, target_2) in enumerate(train_loader, train_loader_2):
        if use_cuda:
            data, target = data.cuda(), target.cuda()
            data_2,  target_2= data_2.cuda(), target_2.cuda()

        data, target = Variable(data), Variable(target)
        data_2,  target_2 = Variable(data_2), Variable(target_2)

        optimizer.zero_grad()

        data = data.float()
        data_2 = data_2.float()
        output = model(data, data_2 )

        prec1, = accuracy(output.data, target.data)
        loss = criterion(output, torch.max(target, 1)[0])
        loss.backward()
        optimizer.step()

Since both data have same target, the accyracy will remain the same, i kept using two DataLoader for both datasets,  i m wondering if there is anyway to do so ?