@peterjc123  I still have the problem, and training my model is very slow. I just tried updating CUDA toolkit to 10.1 now.
class PermDataset(Dataset):
    """Perm dataset."""

    def __init__(self, csv_file, img_dir, Brea, Benth, GH, Leo, Nav, Bn, CL_20, CL_15, CL_10):
        """
        Args:
            csv_file (string): Path to the csv file with labels
            img_dir (string): Directory with all the images.

           return: subvolume (3d numpy array), ID of subvolume, and label

        """
        self.file_name = pd.read_csv(open(csv_file, 'rU'), encoding='utf-8', engine='c')
        self.img_dir = img_dir
        self.Brea = Brea             #3D numpy array
        self.Benth = Benth             #3D numpy array
        self.GH = GH               #3D numpy array
        self.Leo = Leo               #3D numpy array
        self.Nav = Nav                  #3D numpy array
        self.Bn = Bn                 #3D numpy array
        self.CL_20 = CL_20             #3D numpy array
        self.CL_15 = CL_15                   #3D numpy array
        self.CL_10 = CL_10               #3D numpy array


    def __len__(self):
        return len(self.file_name)

    def __getitem__(self, idx):
        """
        1- get the index of training
        2- get the name of rock and id of subvolume
        3- open the image and reterive the subvolume
        4- find the corrsponding permeability value from csv file
            """
        idx_name = self.file_name.iloc[idx, 1]
        idx_label = self.file_name.iloc[idx, 4]

        idx_direction = idx_name[-2]
        name_string = re.search(r'[A-Z]...', idx_name).group(0)
        digits_in_string = re.search(r'[0-9]+', idx_name).group(0)
        s1 = int(''.join(digits_in_string))
        # assign image

        if 'Br' in name_string:
            img = self.Brea
        elif 'Ben' in name_string:
            img = self.Benth
        elif 'BnD' in name_string:
            img = self.Bn
        elif 'ND3' in name_string:
            img = self.Nav
        elif 'Leo' in name_string:
            img = self.Leo
        elif 'GHD' in name_string:
            img = self.GH
        elif 'CL20' in name_string:
            img = self.CL_20
        elif 'CL15' in name_string:
            img = self.CL_15
        elif 'CL10' in name_string:
            img = self.CL_10
        else:
            print('no matching image')

        cube = shift(img, 64, 32, s1, idx_direction) #function for getting a subvolume from the 3d matrix
        cube = np.expand_dims(cube, 0)

        return cube, idx_label, idx_name

my main code structure:
def trainProcess(load_model=False):
    # writer = SummaryWriter()
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    # model = CNN3D().float()
    model = resnet50().float()
    model.to(device)
    criterion = torch.nn.L1Loss()
    optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
    best_val = float(0)
    early_stopping = EarlyStopping(patience=15, verbose=True)
    Brea_image = np.load(os.path.join(img_path, 'BrD2.npy'))
    Benth_image = np.load(os.path.join(img_path, 'BenD2.npy'))
    GH_image = np.load(os.path.join(img_path, 'GHD3.npy'))
    Leo_image = np.load(os.path.join(img_path, 'LeoD3.npy'))
    ND_image = np.load(os.path.join(img_path, 'ND3.npy'))
    Bn_image = np.load(os.path.join(img_path, 'BnD3.npy'))
    CL_20 = np.load(os.path.join(img_path, 'CL_20.npy'))
    CL_15 = np.load(os.path.join(img_path, 'CL_15.npy'))
    CL_10 = np.load(os.path.join(img_path, 'CL_10.npy'))

    # Dataset Split
    jobid = r"C:\Users\drn-4\Desktop\3D\logs_loss\Regresion{}".format(time.strftime("%Y%m%d-%H%M%S"))
    dataset = PermDataset(labels_file, img_path, Brea_image, Benth_image, GH_image, Leo_image, ND_image, Bn_image,
                          CL_20, CL_15, CL_10)
    batch_size = 3
    validation_split = .25
    testing_split = 0.5
    shuffle_dataset = True
    random_seed = 4
    validate_every = 1

    # Creating data indices for training and validation splits:
    dataset_size = len(dataset)
    indices = list(range(dataset_size))
    split = int(np.floor(validation_split * dataset_size))

    if shuffle_dataset:
        np.random.seed(random_seed)
        np.random.shuffle(indices)
    train_indices, val_test_indices = indices[split:], indices[:split]

    split1 = int(np.floor(testing_split * len(val_test_indices)))

    val_indices, test_indices = val_test_indices[split1:], val_test_indices[:split1]

    # Creating PT data samplers and loaders:
    train_dataset1 = torch.utils.data.Subset(dataset, train_indices)
    valid_dataset1 = torch.utils.data.Subset(dataset, val_indices)
    test_dataset1 = torch.utils.data.Subset(dataset, test_indices)

    train_loader = torch.utils.data.DataLoader(train_dataset1, batch_size=batch_size,
                                               num_workers=4, shuffle=True, pin_memory=False)
    validation_loader = torch.utils.data.DataLoader(valid_dataset1, batch_size=batch_size,
                                                    num_workers=0, shuffle=True, pin_memory=False)

    test_loader = torch.utils.data.DataLoader(test_dataset1, batch_size=batch_size,
                                              num_workers=0, shuffle=True, pin_memory=False)
    win = viz.scatter(
        X=np.array([[1, 2], [3, 4]]),
        opts=dict(
            markersize=10,
            legend=['Samples']
        ),
    )

    # Now, let's start the training process!
    print('Training...')
    for epoch in range(100):
        # Compute a training epoch
        loss = trainEpoch(train_loader, model, criterion, optimizer, epoch, device)

        # Compute a validation epoch
        if epoch % validate_every == 0:
            val_loss = valEpoch(device, validation_loader, model, criterion, epoch, win)

        # Print validation accuracy and best validation accuracy
        is_best = bool(val_loss > best_val)
        best_val = max(val_loss, best_val)
        print('** Validation: %f (best) - %f (current)' % (best_val, val_loss))

        early_stopping(val_loss, model, epoch, optimizer)

        if early_stopping.early_stop:
            print("Early stopping")
            break

        if epoch == 0:
            torch.save({
                'train_indices': train_indices, 'validation_indices': val_indices, 'test_indices': test_indices
            }, r'C:\Users\drn-4\Desktop\3D\checkdir\train_val_test_indices.pth'.format(time.strftime("%Y%m%d-%H%M%S"),
                                                                                       epoch))


if __name__ == "__main__":
    global writer
    writer = SummaryWriter()
    DEFAULT_PORT = 8097
    DEFAULT_HOSTNAME = "http://localhost"
    viz = Visdom(port=DEFAULT_PORT, server=DEFAULT_HOSTNAME, base_url='/', username='',
                 password='',
                 use_incoming_socket=True,
                 env=str('ResNext50_{}'.format(time.strftime("%Y%m%d-%H%M%S"))))

    trainProcess()
    writer.close()