@ptrblck more progress.  I eliminated all the transform, cv2, images…but I left pandas.  Problem is still there.  All you need is a dummy CSV file, no actual images.  So you can just save this as files.csv:
image_name,value
ISIC_0030000.jpg,1
ISIC_0030001.jpg,2
ISIC_0030002.jpg,3
ISIC_0030003.jpg,4
ISIC_0030004.jpg,5
ISIC_0030005.jpg,6
ISIC_0030006.jpg,7
ISIC_0030007.jpg,8
ISIC_0030008.jpg,9
ISIC_0030009.jpg,10

This is the updated code:
import torch
from torch.utils.data import Dataset, DataLoader
import numpy as np
import ctypes
import multiprocessing as mp
import pandas as pd

class TestDataset(Dataset):
    def __init__(self, imfolder: str):
        
        self.imfolder = imfolder
        self.df = pd.read_csv(f"{imfolder}/files.csv")
        self.n_images = len(self.df) 

        c=3
        h=256
        w=256
        
        shared_array_base = mp.Array(ctypes.c_uint, self.n_images*c*h*w)
        shared_array = np.ctypeslib.as_array(shared_array_base.get_obj())
        self.shared_array = shared_array.reshape(self.n_images, h, w, c)
        self.shared_array = self.shared_array.astype("uint8")
        self.use_cache = False
        
    def __getitem__(self, index):
        if not self.use_cache:
            self.shared_array[index] = self.df.iloc[index]['value'] # x
        x = self.shared_array[index] 
        print(f" Current index {index}, value of cache index 0 object is {self.shared_array[0].sum()}")
        return x

    def set_use_cache(self, use_cache):
        self.use_cache = use_cache
    
    def __len__(self):
        return self.n_images
    
pytorch_dataset = TestDataset(imfolder= "./test")
pytorch_dataloader = DataLoader(dataset=pytorch_dataset, batch_size=1, shuffle=True,  pin_memory=True, num_workers=1)

# Works
print("Works")
pytorch_dataset = TestDataset(imfolder= "./test")
pytorch_dataloader = DataLoader(dataset=pytorch_dataset)
pytorch_dataloader.dataset.set_use_cache(use_cache=False)
pytorch_dataloader.num_workers = 0

for idx, x in enumerate(pytorch_dataloader):
    next
#     print(f"Loading Cache {idx*pytorch_dataloader.batch_size}.\r", end='')
#     print(x.sum())

pytorch_dataloader.dataset.set_use_cache(use_cache=True)

for idx, x in enumerate(pytorch_dataloader):
    next
#     print(f"x: {mylist[idx]:.5f}   x from cache: {x.sum():.5f}")


# Does not work
print("Does not work")
pytorch_dataset = TestDataset(imfolder= "./test")
pytorch_dataloader = DataLoader(dataset=pytorch_dataset)
pytorch_dataloader.dataset.set_use_cache(use_cache=False)
pytorch_dataloader.num_workers = 1

for idx, x in enumerate(pytorch_dataloader):
    next
#     print(f"Loading Cache {idx*pytorch_dataloader.batch_size}.\r", end='')
#     print(x.sum())

pytorch_dataloader.dataset.set_use_cache(use_cache=True)

for idx, x in enumerate(pytorch_dataloader):
    next
    #     print(f"x: {mylist[idx]:.5f}   x from cache: {x.sum():.5f}")



And here is the output:
Works
 Current index 0, value of cache index 0 object is 196608
 Current index 1, value of cache index 0 object is 196608
 Current index 2, value of cache index 0 object is 196608
 Current index 3, value of cache index 0 object is 196608
 Current index 4, value of cache index 0 object is 196608
 Current index 5, value of cache index 0 object is 196608
 Current index 6, value of cache index 0 object is 196608
 Current index 7, value of cache index 0 object is 196608
 Current index 8, value of cache index 0 object is 196608
 Current index 9, value of cache index 0 object is 196608
 Current index 0, value of cache index 0 object is 196608
 Current index 1, value of cache index 0 object is 196608
 Current index 2, value of cache index 0 object is 196608
 Current index 3, value of cache index 0 object is 196608
 Current index 4, value of cache index 0 object is 196608
 Current index 5, value of cache index 0 object is 196608
 Current index 6, value of cache index 0 object is 196608
 Current index 7, value of cache index 0 object is 196608
 Current index 8, value of cache index 0 object is 196608
 Current index 9, value of cache index 0 object is 196608
Does not work
 Current index 0, value of cache index 0 object is 196608
 Current index 1, value of cache index 0 object is 196608
 Current index 2, value of cache index 0 object is 196608
 Current index 3, value of cache index 0 object is 196608
 Current index 4, value of cache index 0 object is 196608
 Current index 5, value of cache index 0 object is 196608
 Current index 6, value of cache index 0 object is 196608
 Current index 7, value of cache index 0 object is 196608
 Current index 8, value of cache index 0 object is 196608
 Current index 9, value of cache index 0 object is 196608
 Current index 0, value of cache index 0 object is 0
 Current index 1, value of cache index 0 object is 0
 Current index 2, value of cache index 0 object is 0
 Current index 3, value of cache index 0 object is 0
 Current index 4, value of cache index 0 object is 0
 Current index 5, value of cache index 0 object is 0
 Current index 6, value of cache index 0 object is 0
 Current index 7, value of cache index 0 object is 0
 Current index 8, value of cache index 0 object is 0
 Current index 9, value of cache index 0 object is 0