+1 to delegating cpu inference to a subprocess as mentioned by eqy. Additionally you can look into using the RPC framework for this as well (Distributed RPC Framework — PyTorch master documentation 1) which allows you to send tensors between processes easily.
For your use case, here is a basic example of spawning multiple processes programmatically (without command line) and provides a basic example of using RPC to delegate tensors to CPU or GPU model based on the contents of the tensor.
import os
import random
import torch
import torch.multiprocessing as mp
import torch.distributed.rpc as rpc
import torch.nn as nn
import time

os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = "5678"

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.layer = nn.Linear(1, 1)

    def forward(self, x):
        return self.layer(x)

model = Model()
shutdown = 0

def set_shutdown():
    global shutdown
    shutdown = 1

def process_data(input, use_cuda):
    if use_cuda:
        input = input.to("cuda:0")
        model.cuda()
    else:
        model.cpu()
    print(next(model.parameters()).device)

    # return tensor output as cpu
    return model(input).cpu()

def worker_0():
    global shutdown
    rpc.init_rpc("cpu_worker", rank=0, world_size=3)

    # spins there until shutdown is set
    while not shutdown:
        continue
    rpc.shutdown()

def worker_1():
    global shutdown
    rpc.init_rpc("cuda_worker", rank=1, world_size=3)

    # spins there until shutdown is set
    while not shutdown:
        continue
    rpc.shutdown()

def worker_2():
    rpc.init_rpc("controller", rank=2, world_size=3)
    data = torch.rand(10, 1)
    futures = []
    for rand_tensor in data:
        # if random number is > 0.5, then use cpu to process, else use gpu
        if rand_tensor > 0.5:
            fut = rpc.rpc_async("cpu_worker", process_data, args=(rand_tensor, False))
        else:
            fut = rpc.rpc_async("cuda_worker", process_data, args=(rand_tensor, True))
        futures.append(fut)
    for fut in futures:
        result = fut.wait()
        print(result)
    rpc.rpc_sync("cpu_worker", set_shutdown)
    rpc.rpc_sync("cuda_worker", set_shutdown)
    rpc.shutdown()

if __name__ == "__main__":
    p0 = mp.Process(target=worker_0)
    p1 = mp.Process(target=worker_1)
    p2 = mp.Process(target=worker_2)
    p0.start()
    p1.start()
    p2.start()
    p0.join()
    p1.join()
    p2.join()