input_feature_key=input_feature_key)

if optional_task_prefixes:

ds = prepend_prompt(

ds,
output_features,
prompt_mode=optional_task_prefixes[i],
mode=optional_task_prefixes[i])

processed_ds.append(ds)

if use_prefix_lm_task:

ds = prepend_prompt(

ds,
output_features,
prompt_mode=optional_task_prefixes[-1],
mode=optional_task_prefixes[-1])

processed_ds.append(ds)

ds = ds_shards[-1]
ds = t5.data.preprocessors.prefix_lm(ds, sequence_lengths, output_features)
if optional_task_prefixes:

ds = tf.data.experimental.sample_from_datasets(processed_ds, rates, seed)
return ds

9.3 Details of Supervised Finetuning SOTA runs
Most of our supervised ﬁnetuning runs were ﬁnetuned as single tasks. The only exception was that:

• We ﬁnetuned GLUE as a single mixture with proportionate sampling. This has become standard and

defacto setup (Raﬀel et al., 2019; He et al., 2022; Tay et al., 2020, 2021b).
UL2- Unifying Language Learning Paradigms