@ptrblck Here is as simple as I could make it.  I also put it into a notebook with 100 256x256 images and the associated CSV on dropbox here if you want:

  
      
      Dropbox
  
  
    

SharedArray.tar.gz 1

Shared with Dropbox


  
  
    
    
  
  


You can see I just do two tests.  One where I set num_workers to 0 and load the cache, then flip use_cache to True, and everything seems to be fine.  But if I have num_workers set to > 0 when I load the cache, then I get an empty image back.
import torch
import torchvision
from torchvision import transforms
from torch.utils.data import Dataset, DataLoader
import numpy as np
import cv2
import matplotlib.pyplot as plt
%matplotlib inline
import ctypes
import multiprocessing as mp
import os
import pandas as pd

class TestDataset(Dataset):
    def __init__(self, imfolder: str, transforms = None):
        
        self.imfolder = imfolder
        self.transforms = transforms
        self.n_images = len([f for f in os.listdir(imfolder) if f.endswith('.jpg')])
        self.df = pd.read_csv(f"{imfolder}/files.csv")
        c=3
        h=256
        w=256
        
        shared_array_base = mp.Array(ctypes.c_uint, self.n_images*c*h*w)
        shared_array = np.ctypeslib.as_array(shared_array_base.get_obj())
        self.shared_array = shared_array.reshape(self.n_images, h, w, c)
        self.shared_array = self.shared_array.astype("uint8")
        self.use_cache = False
        
    def __getitem__(self, index):
        if not self.use_cache:
            im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'])
            x = cv2.imread(im_path)
            x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)
            self.shared_array[index] = x
        x = self.shared_array[index] 
        
        if self.transforms:
            x = self.transforms(x)
            
        return x

    def set_use_cache(self, use_cache):
        self.use_cache = use_cache
    
    def __len__(self):
        return self.n_images
    
    
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])
    ])
pytorch_dataset = TestDataset(imfolder= "./test", transforms=transform)
pytorch_dataloader = DataLoader(dataset=pytorch_dataset, batch_size=10, shuffle=True,  pin_memory=True, num_workers=1)



# Works
pytorch_dataset = TestDataset(imfolder= "./test", transforms=transform)
pytorch_dataloader = DataLoader(dataset=pytorch_dataset)
pytorch_dataloader.dataset.set_use_cache(use_cache=False)
pytorch_dataloader.num_workers = 0

for idx, x in enumerate(pytorch_dataloader):
    print(f"Loading Cache {idx*pytorch_dataloader.batch_size}.\r", end='')
    
pytorch_dataloader.dataset.set_use_cache(use_cache=True)

images = next(iter(pytorch_dataloader))
image = images[0]
plt.imshow(np.transpose(image, (1, 2, 0)))




# Does not work
pytorch_dataset = TestDataset(imfolder= "./test", transforms=transform)
pytorch_dataloader = DataLoader(dataset=pytorch_dataset)
pytorch_dataloader.dataset.set_use_cache(use_cache=False)
pytorch_dataloader.num_workers = 1

for idx, x in enumerate(pytorch_dataloader):
    print(f"Loading Cache {idx*pytorch_dataloader.batch_size}.\r", end='')
    
pytorch_dataloader.dataset.set_use_cache(use_cache=True)

images = next(iter(pytorch_dataloader))
image = images[0]
plt.imshow(np.transpose(image, (1, 2, 0)))