I checked with smaller model and the processor is not full.
cut.png1301×716 268 KB
 The time for parallel model is still longer than sequential model.
Below is my function using thread on single GPU.
def parallel_apply(modules, inputs, kwargs_tup=None, devices=None):
assert len(modules) == len(inputs)
if kwargs_tup is not None:
    assert len(modules) == len(kwargs_tup)
else:
    kwargs_tup = ({},) * len(modules)
if devices is not None:
    assert len(modules) == len(devices)
else:
    devices = [None] * len(modules)

lock = threading.Lock()
results = {}


def _worker(i, module, input, kwargs, lock, device=None):

    if device is None:
        device = get_a_var(input).get_device()
    try:
        with torch.cuda.device(device):
            output = module.cuda(device)(*input,**kwargs);
        with lock:
            results[i] = output
    except Exception as e:
        with lock:
            results[i] = e

if len(modules) > 1:
    threads = [threading.Thread(target=_worker,
                               args=(i, module, input, kwargs, lock, device),)
               for i, (module, input, kwargs, device) in
               enumerate(zip(modules, inputs, kwargs_tup, devices))]

    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()
else:
    _worker(0, modules[0], inputs[0], kwargs_tup[0], devices[0])

outputs = []
for i in range(len(inputs)):
    output = results[i]
    if isinstance(output, Exception):
        raise output
    outputs.append(output)
return outputs