@ptrblck actually your code does show the problem, at least on my system.
The value you got back was “0”…and the next value you will get back is 0 as well, and they will all be 0’s.  During the loading of the cache…whether its num_workers = 0 or num_workers = 1, you will get the correct values back, as your output shows.  But once that iterator completes…the values are hosed in the case of num_workers = 1.  you can verify this by iterating again as you see in my output, I look at index 0 and its value is 0  (using your np.ones code I look at the value of index 1, since index 0 is always “0” anyways).
Here is updated code with just using np.ones() but more verbose output:
import torch
from torch.utils.data import Dataset, DataLoader
import numpy as np
import ctypes
import multiprocessing as mp

class TestDataset(Dataset):
    def __init__(self):
        
        self.n_images = 10

        shared_array_base = mp.Array(ctypes.c_uint, self.n_images*3*256*256)
        shared_array = np.ctypeslib.as_array(shared_array_base.get_obj())
        self.shared_array = shared_array.reshape(self.n_images, 256, 256, 3)
        self.shared_array = self.shared_array.astype("uint8")
        self.use_cache = False
        
    def __getitem__(self, index):
        if not self.use_cache:
            self.shared_array[index] =  np.ones((256, 256, 3)).astype(np.uint8) * index
        x = self.shared_array[index] 
        return x

    def set_use_cache(self, use_cache):
        self.use_cache = use_cache
    
    def __len__(self):
        return self.n_images
    
# Works
print("Works")
pytorch_dataset = TestDataset()
pytorch_dataloader = DataLoader(dataset=pytorch_dataset)
pytorch_dataloader.dataset.set_use_cache(use_cache=False)
pytorch_dataloader.num_workers = 0

for idx, x in enumerate(pytorch_dataloader):
    print(f"Loading Cache {idx*pytorch_dataloader.batch_size}.\r", end='')
    print(f"Index {idx} = {x.sum()}")

pytorch_dataloader.dataset.set_use_cache(use_cache=True)

for idx, x in enumerate(pytorch_dataloader):
    print(f"Read from cache Index {idx} = {x.sum()}")
    
# Does not work
print("Does not work")
pytorch_dataset = TestDataset()
pytorch_dataloader = DataLoader(dataset=pytorch_dataset)
pytorch_dataloader.dataset.set_use_cache(use_cache=False)
pytorch_dataloader.num_workers = 1

for idx, x in enumerate(pytorch_dataloader):
    print(f"Loading Cache {idx*pytorch_dataloader.batch_size}.\r", end='')
    print(f"Index {idx} = {x.sum()}")

pytorch_dataloader.dataset.set_use_cache(use_cache=True)

for idx, x in enumerate(pytorch_dataloader):
    print(f"Read from cache Index {idx} = {x.sum()}")



Produces:
Works
Index 0 = 0he 0.
Index 1 = 196608
Index 2 = 393216
Index 3 = 589824
Index 4 = 786432
Index 5 = 983040
Index 6 = 1179648
Index 7 = 1376256
Index 8 = 1572864
Index 9 = 1769472
Read from cache Index 0 = 0
Read from cache Index 1 = 196608
Read from cache Index 2 = 393216
Read from cache Index 3 = 589824
Read from cache Index 4 = 786432
Read from cache Index 5 = 983040
Read from cache Index 6 = 1179648
Read from cache Index 7 = 1376256
Read from cache Index 8 = 1572864
Read from cache Index 9 = 1769472
Does not work
Index 0 = 0he 0.
Index 1 = 196608
Index 2 = 393216
Index 3 = 589824
Index 4 = 786432
Index 5 = 983040
Index 6 = 1179648
Index 7 = 1376256
Index 8 = 1572864
Index 9 = 1769472
Read from cache Index 0 = 0
Read from cache Index 1 = 0
Read from cache Index 2 = 0
Read from cache Index 3 = 0
Read from cache Index 4 = 0
Read from cache Index 5 = 0
Read from cache Index 6 = 0
Read from cache Index 7 = 0
Read from cache Index 8 = 0
Read from cache Index 9 = 0