Thank you very much for your reply.
Useing multi-gpus by torch.nn.DataParallel.  Input data and models are placed on the master device.  In forward function,   will this result of .cuda(), .cpu() be placed on the corresponding device?  I’m not very sure.  I will use the solution you mentioned above.
And, the incomprehensible thing is that I only get errors after executing several batches. The first few batch programs will execute normally.
This code is open sourced on github. I changed it from distributed to DataParallel.  The URL of code: https://github.com/yaohungt/Gated-Spatio-Temporal-Energy-Graph 1
Major changes：
    def train(self, loader, base_model, logits_model, criterion, base_optimizer, logits_optimizer, epoch, args):
        adjust_learning_rate(args.lr, args.lr_decay_rate, base_optimizer, epoch)
        adjust_learning_rate(args.lr, args.lr_decay_rate, logits_optimizer, epoch)
        batch_time = AverageMeter()
        data_time = AverageMeter()
        losses = AverageMeter()
        s_top1 = AverageMeter()
        s_top5 = AverageMeter()
        o_top1 = AverageMeter()
        o_top5 = AverageMeter()
        v_top1 = AverageMeter()
        v_top5 = AverageMeter()    
        sov_top1 = AverageMeter()

        # switch to train mode
        base_model.train()
        logits_model.train()
        criterion.train()
        base_optimizer.zero_grad()
        logits_optimizer.zero_grad()

        def part(x): return itertools.islice(x, int(len(x)*args.train_size))
        end = time.time()
        for i, (input, s_target, o_target, v_target, meta) in enumerate(part(loader)):
            gc.collect()
            data_time.update(time.time() - end)
            meta['epoch'] = epoch

            print("meta = {}".format(meta))

            # s_target = s_target.long().cuda(async=True)
            # o_target = o_target.long().cuda(async=True)
            # v_target = v_target.long().cuda(async=True)
            # input_var = torch.autograd.Variable(input.cuda())
            s_target = s_target.long().cuda(device=device_ids[0])
            o_target = o_target.long().cuda(device=device_ids[0])
            v_target = v_target.long().cuda(device=device_ids[0])
            input_var = torch.autograd.Variable(input.cuda(device=device_ids[0]))
            s_target_var = torch.autograd.Variable(s_target)
            o_target_var = torch.autograd.Variable(o_target)
            v_target_var = torch.autograd.Variable(v_target)
            
            feat = base_model(input_var)

            feat = feat.cuda(device=device_ids[0])

            s, o, v, so, ov, vs, ss, oo, vv, so_t, ov_t, vs_t, os_t, vo_t, sv_t = logits_model(feat)

            s = s.cuda(device=device_ids[0])
            o = o.cuda(device=device_ids[0])
            v = v.cuda(device=device_ids[0])
            so = so.cuda(device=device_ids[0])
            ov = ov.cuda(device=device_ids[0])
            vs = vs.cuda(device=device_ids[0])
            ss = ss.cuda(device=device_ids[0])
            oo = oo.cuda(device=device_ids[0])
            vv = vv.cuda(device=device_ids[0])
            so_t = so_t.cuda(device=device_ids[0])
            ov_t = ov_t.cuda(device=device_ids[0])
            vs_t = vs_t.cuda(device=device_ids[0])
            os_t = os_t.cuda(device=device_ids[0])
            vo_t = vo_t.cuda(device=device_ids[0])
            sv_t = sv_t.cuda(device=device_ids[0])
            s_target_var = s_target_var.cuda(device=device_ids[0])
            o_target_var = o_target_var.cuda(device=device_ids[0])
            v_target_var = v_target_var.cuda(device=device_ids[0])
            meta['ids'] = meta['ids'].cuda(device=device_ids[0])
            meta['time'] = meta['time'] .cuda(device=device_ids[0])


            
            s_output, o_output, v_output, loss = criterion(*((s, o, v, so, ov, vs, ss, oo, vv, so_t, ov_t, vs_t, os_t, vo_t, sv_t) + (s_target_var, o_target_var, v_target_var, meta)))
            
            s_prec1, s_prec5, s_prec1_output = accuracy_s(s_output.data, s_target, topk=(1, 5))
            o_prec1, o_prec5, o_prec1_output = accuracy(o_output.data, o_target, topk=(1, 5))
            v_prec1, v_prec5, v_prec1_output = accuracy(v_output.data, v_target, topk=(1, 5))

            sov_prec1 = s_prec1_output.cpu() * o_prec1_output * v_prec1_output
            sov_prec1 = sov_prec1.sum(0, keepdim=True)
            sov_prec1 = sov_prec1.mul_(100.0 / input.size(0))