I have been able to test.  What I am finding is that during the loading of the cache, I have to have num_workers set to 0.  If I don’t then I just get no data.  Example:
pytorch_dataloader.num_workers=1

 for idx, (x, y) in enumerate(pytorch_dataloader):
    print(f"Loading Cache {idx*12}.\r", end='')

pytorch_dataloader.dataset.set_use_cache(use_cache=True)
images,meta = next(iter(pytorch_dataloader))
images[0]

tensor([[[-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         ...,
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.]],

        [[-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         ...,
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.]],

        [[-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         ...,
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.],
         [-1., -1., -1.,  ..., -1., -1., -1.]]])


yet this works:
pytorch_dataloader.num_workers=0

 for idx, (x, y) in enumerate(pytorch_dataloader):
    print(f"Loading Cache {idx*12}.\r", end='')

pytorch_dataloader.dataset.set_use_cache(use_cache=True)
images,meta = next(iter(pytorch_dataloader))
images[0]

tensor([[[ 0.7098,  0.5137,  0.5608,  ...,  0.2392,  0.2235, -0.0588],
         [ 0.7176,  0.5137,  0.6078,  ...,  0.0039, -0.0980,  0.3255],
         [ 0.7255,  0.4902,  0.6314,  ..., -0.0275,  0.5294,  0.7647],
         ...,
         [ 0.6235,  0.6392,  0.6627,  ...,  0.7255,  0.7098,  0.6784],
         [ 0.6314,  0.6314,  0.6471,  ...,  0.7020,  0.7020,  0.7098],
         [ 0.6471,  0.6471,  0.6627,  ...,  0.7098,  0.6784,  0.6863]],

        [[ 0.3333,  0.1608,  0.2000,  ..., -0.1137, -0.1373, -0.4196],
         [ 0.3412,  0.1608,  0.2627,  ..., -0.3569, -0.4588, -0.0353],
         [ 0.3647,  0.1294,  0.2863,  ..., -0.3882,  0.1843,  0.4118],
         ...,
         [ 0.3490,  0.3647,  0.3804,  ...,  0.4196,  0.4039,  0.3725],
         [ 0.3647,  0.3647,  0.3647,  ...,  0.3804,  0.3804,  0.3882],
         [ 0.3804,  0.3804,  0.3569,  ...,  0.3725,  0.3412,  0.3490]],

        [[ 0.3020,  0.1216,  0.2000,  ..., -0.2627, -0.2549, -0.5373],
         [ 0.3255,  0.1373,  0.2549,  ..., -0.4824, -0.5608, -0.1373],
         [ 0.3647,  0.1294,  0.2941,  ..., -0.4902,  0.1137,  0.3647],
         ...,
         [ 0.3804,  0.3961,  0.3961,  ...,  0.4431,  0.4275,  0.3961],
         [ 0.3725,  0.3725,  0.3804,  ...,  0.4275,  0.4275,  0.4353],
         [ 0.3882,  0.3882,  0.3804,  ...,  0.4275,  0.3961,  0.4039]]])

I must be doing something wrong that I am not able to have num_workers > 0 while populating the cache, if you have any ideas please let me know, I am troubleshooting.