import ctypes
import time
import numpy as np
import torch.multiprocessing as mp
import torch
from torch.utils.data import Dataset

class SharedDataset1(Dataset):
    def __init__(self, shared_mem):
        super(SharedDataset1, self).__init__()
        self.shared_mem = shared_mem

    def __len__(self):
        return 10000

    def __getitem__(self, idx):
        return torch.randn(3, 32, 32)

def np_before(gpu, array, num_workers):
    with array.get_lock():
        np_array = np.ctypeslib.as_array(array.get_obj())

    dataset = SharedDataset1(np_array)
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=10, num_workers=num_workers)

    for img in dataloader:
        c = img+0.1
        time.sleep(3)

if __name__ == "__main__":
    mp.set_start_method('spawn')
    array = mp.Array(ctypes.c_byte, 10000*1024*1024, lock=True)
    with array.get_lock():
        np_array = np.ctypeslib.as_array(array.get_obj())
        np_array.fill(100)

    print("allocated")
    mp.spawn(np_before, args=(array, 1), nprocs=2)
    print("started")

    begin = time.time()
    while time.time() - begin < 100:
        a = 1 * 100

This gives the following error

Traceback (most recent call last):
File “mem_data.py”, line 64, in 
mp.spawn(np_before, args=(array, 1), nprocs=2)
File “/home/parawr/.conda/envs/faclab/lib/python3.7/site-packages/torch/multiprocessing/spawn.py”, line 200, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method=‘spawn’)
File “/home/parawr/.conda/envs/faclab/lib/python3.7/site-packages/torch/multiprocessing/spawn.py”, line 158, in start_processes
while not context.join():
File “/home/parawr/.conda/envs/faclab/lib/python3.7/site-packages/torch/multiprocessing/spawn.py”, line 119, in join
raise Exception(msg)