You can use Subset to create different folds for cross-validation by providing data/train/validation indices.
class Subset(Dataset):
    """
    Subset of a dataset at specified indices.
    Arguments:
        dataset (Dataset): The whole Dataset
        indices (sequence): Indices in the whole set selected for subset
    """
    def __init__(self, dataset, indices):
        self.dataset = dataset
        self.indices = indices

    def __len__(self):
        if self.indices.shape == ():
            print('this happens: Subset')
            return 1
        else:
            return len(self.indices)

    def __getitem__(self, idx):
        return self.dataset[self.indices[idx]]

for i in range(k):
    print('Processing fold: ', i + 1)
    """%%%% Initiate new model %%%%""" #in every fold
    valid_idx = np.arange(len(dataset))[i * num_val_samples:(i + 1) * num_val_samples]
    train_idx = np.concatenate([np.arange(len(dataset))[:i * num_val_samples], np.arange(len(dataset))[(i + 1) * num_val_samples:]], axis=0)
    train_dataset = Subset(dataset, train_idx)
    valid_dataset = Subset(dataset, valid_idx)
    train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True, num_workers=1)
    valid_loader = DataLoader(valid_dataset, batch_size=1, shuffle=False, num_workers=1)

Though I don’t know your data structure, the codes look ok to me.