Actually, if your memory is large enough, you can try this
dataset = torch.empty_like(data_loader.dataset)
cur_ind = 0
for batch_idx, input in enumerate(data_loader):
    dataset[cur_ind:cur_ind+len(input)] = input
    cur_ind += len(input)
new_data_loader = DataLoader(dataset[:cur_ind, ...], ...)