To distribute your computation, you will need to use strategy.run, here is the snippet(from your colab notebook) that shows the same. This will use both the GPUs to run the computation.
def _inference_step(data):
    images, labels = data
    with tf.GradientTape() as tape:
        tape.watch(pre_images)
        outputs = model(pre_images, training=False)
        grads = tape.gradient(outputs, pre_images)
    return grads

@tf.function
def _distributed_inference_step(data):
    images, labels = data
    grads = strategy.run(_inference_step, args=((data,)))
    # gather values from all replicas
    return strategy.gather(grads, axis=0)

for pre_images, true_label in eval_dataset:
    grads = _distributed_inference_step((pre_images, true_label))