Thank you @spanev!
In case anyone is wondering, here is how to set process specific env variables:
import torch.multiprocessing as _mp
import torch
import os

mp = _mp.get_context('fork')


class Process(mp.Process):
    def __init__(self):
        super().__init__()
        print("Init Process")
        return

    def run(self):
        print("Hello World!")
        os.environ['CUDA_VISIBLE_DEVICES'] = '1'
        print(torch.cuda.device_count())
        print(torch.cuda.current_device())

if __name__ == "__main__":
    num_processes = 1
    os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
    processes = [Process() for i in range(num_processes)]
    [p.start() for p in processes]
    print("main: " + os.environ['CUDA_VISIBLE_DEVICES'])
    [p.join() for p in processes]

It is important to set it in the run method of the process, as the init method is still called in the main process, therefore setting the env vars of the main process when set there.