never mind. I missed to convert one of the variables. I added     landmarks_batch = landmarks_batch.cpu() and now it works.
# Helper function to show a batch
def show_landmarks_batch(sample_batched):
    """Show image with landmarks for a batch of samples."""
    images_batch, landmarks_batch = \
            sample_batched['image'], sample_batched['landmarks']
    batch_size = len(images_batch)
    im_size = images_batch.size(2)

    grid = utils.make_grid(images_batch)
    grid = grid.cpu()
    plt.imshow(grid.numpy().transpose((1, 2, 0)))
    #sample_batched = sample_batched.to('cpu')
    cpu_sample_batched = {}
    ##for k, v in sample_batched.items():
    ##    cpu_sample_batched[k] = v.cpu()
    
    landmarks_batch = landmarks_batch.cpu()

    for i in range(batch_size):
        plt.scatter(landmarks_batch[i, :, 0].numpy() + i * im_size,
                    landmarks_batch[i, :, 1].numpy(),
                    s=10, marker='.', c='r')

        plt.title('Batch from dataloader')