import{S as Gp,i as Vp,s as Yp,e as i,t as a,k as f,w as v,c as p,a as c,h as n,d as s,m as u,x as $,g as h,G as t,y as j,q as b,o as g,B as x,l as Hp,M as Jp,b as T,p as qt,v as Kp,n as Dt}from"../../chunks/vendor-hf-doc-builder.js";import{T as ti}from"../../chunks/Tip-hf-doc-builder.js";import{Y as va}from"../../chunks/Youtube-hf-doc-builder.js";import{I as In}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Up}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as Zp}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function Qp(E){let o,d;return o=new Up({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter3/section2_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter3/section2_tf.ipynb"}]}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function Xp(E){let o,d;return o=new Up({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter3/section2_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter3/section2_pt.ipynb"}]}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function ec(E){let o,d,l,w,y,m,_,z;return _=new C({props:{code:`import tensorflow as tf
import numpy as np
from transformers import AutoTokenizer, TFAutoModelForSequenceClassification

# Same as before
checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    "I've been waiting for a HuggingFace course my whole life.",
    "This course is amazing!",
]
batch = dict(tokenizer(sequences, padding=True, truncation=True, return_tensors="tf"))

# This is new
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy")
labels = tf.convert_to_tensor([1, 1])
model.train_on_batch(batch, labels)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

<span class="hljs-comment"># Same as before</span>
checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;This course is amazing!&quot;</span>,
]
batch = <span class="hljs-built_in">dict</span>(tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>))

<span class="hljs-comment"># This is new</span>
model.<span class="hljs-built_in">compile</span>(optimizer=<span class="hljs-string">&quot;adam&quot;</span>, loss=<span class="hljs-string">&quot;sparse_categorical_crossentropy&quot;</span>)
labels = tf.convert_to_tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>])
model.train_on_batch(batch, labels)`}}),{c(){o=i("p"),d=a("Continuing with the example from the "),l=i("a"),w=a("previous chapter"),y=a(", here is how we would train a sequence classifier on one batch in TensorFlow:"),m=f(),v(_.$$.fragment),this.h()},l(k){o=p(k,"P",{});var q=c(o);d=n(q,"Continuing with the example from the "),l=p(q,"A",{href:!0});var L=c(l);w=n(L,"previous chapter"),L.forEach(s),y=n(q,", here is how we would train a sequence classifier on one batch in TensorFlow:"),q.forEach(s),m=u(k),$(_.$$.fragment,k),this.h()},h(){T(l,"href","/course/chapter2")},m(k,q){h(k,o,q),t(o,d),t(o,l),t(l,w),t(o,y),h(k,m,q),j(_,k,q),z=!0},i(k){z||(b(_.$$.fragment,k),z=!0)},o(k){g(_.$$.fragment,k),z=!1},d(k){k&&s(o),k&&s(m),x(_,k)}}}function tc(E){let o,d,l,w,y,m,_,z;return _=new C({props:{code:`import torch
from transformers import AdamW, AutoTokenizer, AutoModelForSequenceClassification

# Same as before
checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    "I've been waiting for a HuggingFace course my whole life.",
    "This course is amazing!",
]
batch = tokenizer(sequences, padding=True, truncation=True, return_tensors="pt")

# This is new
batch["labels"] = torch.tensor([1, 1])

optimizer = AdamW(model.parameters())
loss = model(**batch).loss
loss.backward()
optimizer.step()`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AdamW, AutoTokenizer, AutoModelForSequenceClassification

<span class="hljs-comment"># Same as before</span>
checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;This course is amazing!&quot;</span>,
]
batch = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># This is new</span>
batch[<span class="hljs-string">&quot;labels&quot;</span>] = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>])

optimizer = AdamW(model.parameters())
loss = model(**batch).loss
loss.backward()
optimizer.step()`}}),{c(){o=i("p"),d=a("Continuing with the example from the "),l=i("a"),w=a("previous chapter"),y=a(", here is how we would train a sequence classifier on one batch in PyTorch:"),m=f(),v(_.$$.fragment),this.h()},l(k){o=p(k,"P",{});var q=c(o);d=n(q,"Continuing with the example from the "),l=p(q,"A",{href:!0});var L=c(l);w=n(L,"previous chapter"),L.forEach(s),y=n(q,", here is how we would train a sequence classifier on one batch in PyTorch:"),q.forEach(s),m=u(k),$(_.$$.fragment,k),this.h()},h(){T(l,"href","/course/chapter2")},m(k,q){h(k,o,q),t(o,d),t(o,l),t(l,w),t(o,y),h(k,m,q),j(_,k,q),z=!0},i(k){z||(b(_.$$.fragment,k),z=!0)},o(k){g(_.$$.fragment,k),z=!1},d(k){k&&s(o),k&&s(m),x(_,k)}}}function sc(E){let o,d;return o=new va({props:{id:"W_gMJF0xomE"}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function ac(E){let o,d;return o=new va({props:{id:"_BZearw7f0w"}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function nc(E){let o,d,l,w,y;return{c(){o=i("p"),d=a("\u270F\uFE0F "),l=i("strong"),w=a("Try it out!"),y=a(" Look at element 15 of the training set and element 87 of the validation set. What are their labels?")},l(m){o=p(m,"P",{});var _=c(o);d=n(_,"\u270F\uFE0F "),l=p(_,"STRONG",{});var z=c(l);w=n(z,"Try it out!"),z.forEach(s),y=n(_," Look at element 15 of the training set and element 87 of the validation set. What are their labels?"),_.forEach(s)},m(m,_){h(m,o,_),t(o,d),t(o,l),t(l,w),t(o,y)},d(m){m&&s(o)}}}function oc(E){let o,d;return o=new va({props:{id:"P-rZWqcB6CE"}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function lc(E){let o,d;return o=new va({props:{id:"0u3ioSwev3s"}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function rc(E){let o,d,l,w,y;return{c(){o=i("p"),d=a("\u270F\uFE0F "),l=i("strong"),w=a("Try it out!"),y=a(" Take element 15 of the training set and tokenize the two sentences separately and as a pair. What\u2019s the difference between the two results?")},l(m){o=p(m,"P",{});var _=c(o);d=n(_,"\u270F\uFE0F "),l=p(_,"STRONG",{});var z=c(l);w=n(z,"Try it out!"),z.forEach(s),y=n(_," Take element 15 of the training set and tokenize the two sentences separately and as a pair. What\u2019s the difference between the two results?"),_.forEach(s)},m(m,_){h(m,o,_),t(o,d),t(o,l),t(l,w),t(o,y)},d(m){m&&s(o)}}}function ic(E){let o,d,l,w,y;return{c(){o=i("p"),d=a("The function that is responsible for putting together samples inside a batch is called a "),l=i("em"),w=a("collate function"),y=a(". The default collator is a function that will just convert your samples to tf.Tensor and concatenate them (recursively if your elements are lists, tuples, or dictionaries). This won\u2019t be possible in our case since the inputs we have won\u2019t all be of the same size. We have deliberately postponed the padding, to only apply it as necessary on each batch and avoid having over-long inputs with a lot of padding. This will speed up training by quite a bit, but note that if you\u2019re training on a TPU it can cause problems \u2014 TPUs prefer fixed shapes, even when that requires extra padding.")},l(m){o=p(m,"P",{});var _=c(o);d=n(_,"The function that is responsible for putting together samples inside a batch is called a "),l=p(_,"EM",{});var z=c(l);w=n(z,"collate function"),z.forEach(s),y=n(_,". The default collator is a function that will just convert your samples to tf.Tensor and concatenate them (recursively if your elements are lists, tuples, or dictionaries). This won\u2019t be possible in our case since the inputs we have won\u2019t all be of the same size. We have deliberately postponed the padding, to only apply it as necessary on each batch and avoid having over-long inputs with a lot of padding. This will speed up training by quite a bit, but note that if you\u2019re training on a TPU it can cause problems \u2014 TPUs prefer fixed shapes, even when that requires extra padding."),_.forEach(s)},m(m,_){h(m,o,_),t(o,d),t(o,l),t(l,w),t(o,y)},d(m){m&&s(o)}}}function pc(E){let o,d,l,w,y,m,_,z;return{c(){o=i("p"),d=a("The function that is responsible for putting together samples inside a batch is called a "),l=i("em"),w=a("collate function"),y=a(". It\u2019s an argument you can pass when you build a "),m=i("code"),_=a("DataLoader"),z=a(", the default being a function that will just convert your samples to PyTorch tensors and concatenate them (recursively if your elements are lists, tuples, or dictionaries). This won\u2019t be possible in our case since the inputs we have won\u2019t all be of the same size. We have deliberately postponed the padding, to only apply it as necessary on each batch and avoid having over-long inputs with a lot of padding. This will speed up training by quite a bit, but note that if you\u2019re training on a TPU it can cause problems \u2014 TPUs prefer fixed shapes, even when that requires extra padding.")},l(k){o=p(k,"P",{});var q=c(o);d=n(q,"The function that is responsible for putting together samples inside a batch is called a "),l=p(q,"EM",{});var L=c(l);w=n(L,"collate function"),L.forEach(s),y=n(q,". It\u2019s an argument you can pass when you build a "),m=p(q,"CODE",{});var Y=c(m);_=n(Y,"DataLoader"),Y.forEach(s),z=n(q,", the default being a function that will just convert your samples to PyTorch tensors and concatenate them (recursively if your elements are lists, tuples, or dictionaries). This won\u2019t be possible in our case since the inputs we have won\u2019t all be of the same size. We have deliberately postponed the padding, to only apply it as necessary on each batch and avoid having over-long inputs with a lot of padding. This will speed up training by quite a bit, but note that if you\u2019re training on a TPU it can cause problems \u2014 TPUs prefer fixed shapes, even when that requires extra padding."),q.forEach(s)},m(k,q){h(k,o,q),t(o,d),t(o,l),t(l,w),t(o,y),t(o,m),t(m,_),t(o,z)},d(k){k&&s(o)}}}function cc(E){let o,d;return o=new C({props:{code:`from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function hc(E){let o,d;return o=new C({props:{code:`from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)`}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function dc(E){let o,d,l,w,y;return o=new C({props:{code:`{'attention_mask': torch.Size([8, 67]),
 'input_ids': torch.Size([8, 67]),
 'token_type_ids': torch.Size([8, 67]),
 'labels': torch.Size([8])}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;labels&#x27;</span>: torch.Size([<span class="hljs-number">8</span>])}`}}),{c(){v(o.$$.fragment),d=f(),l=i("p"),w=a("Looking good! Now that we\u2019ve gone from raw text to batches our model can deal with, we\u2019re ready to fine-tune it!")},l(m){$(o.$$.fragment,m),d=u(m),l=p(m,"P",{});var _=c(l);w=n(_,"Looking good! Now that we\u2019ve gone from raw text to batches our model can deal with, we\u2019re ready to fine-tune it!"),_.forEach(s)},m(m,_){j(o,m,_),h(m,d,_),h(m,l,_),t(l,w),y=!0},i(m){y||(b(o.$$.fragment,m),y=!0)},o(m){g(o.$$.fragment,m),y=!1},d(m){x(o,m),m&&s(d),m&&s(l)}}}function fc(E){let o,d;return o=new C({props:{code:`{'attention_mask': TensorShape([8, 67]),
 'input_ids': TensorShape([8, 67]),
 'token_type_ids': TensorShape([8, 67]),
 'labels': TensorShape([8])}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;labels&#x27;</span>: TensorShape([<span class="hljs-number">8</span>])}`}}),{c(){v(o.$$.fragment)},l(l){$(o.$$.fragment,l)},m(l,w){j(o,l,w),d=!0},i(l){d||(b(o.$$.fragment,l),d=!0)},o(l){g(o.$$.fragment,l),d=!1},d(l){x(o,l)}}}function uc(E){let o,d,l,w,y;return{c(){o=i("p"),d=a("\u270F\uFE0F "),l=i("strong"),w=a("Try it out!"),y=a(" Replicate the preprocessing on the GLUE SST-2 dataset. It\u2019s a little bit different since it\u2019s composed of single sentences instead of pairs, but the rest of what we did should look the same. For a harder challenge, try to write a preprocessing function that works on any of the GLUE tasks.")},l(m){o=p(m,"P",{});var _=c(o);d=n(_,"\u270F\uFE0F "),l=p(_,"STRONG",{});var z=c(l);w=n(z,"Try it out!"),z.forEach(s),y=n(_," Replicate the preprocessing on the GLUE SST-2 dataset. It\u2019s a little bit different since it\u2019s composed of single sentences instead of pairs, but the rest of what we did should look the same. For a harder challenge, try to write a preprocessing function that works on any of the GLUE tasks."),_.forEach(s)},m(m,_){h(m,o,_),t(o,d),t(o,l),t(l,w),t(o,y)},d(m){m&&s(o)}}}function Bp(E){let o,d,l,w,y,m,_,z,k,q,L,Y,H,B,ee,W,F,J,pe,Ee;return W=new C({props:{code:`tf_train_dataset = tokenized_datasets["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=8,
)

tf_validation_dataset = tokenized_datasets["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=False,
    collate_fn=data_collator,
    batch_size=8,
)`,highlighted:`tf_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)

tf_validation_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">False</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)`}}),{c(){o=i("p"),d=a("Now that we have our dataset and a data collator, we need to put them together. We could manually load batches and collate them, but that\u2019s a lot of work, and probably not very performant either. Instead, there\u2019s a simple method that offers a performant solution to this problem: "),l=i("code"),w=a("to_tf_dataset()"),y=a(". This will wrap a "),m=i("code"),_=a("tf.data.Dataset"),z=a(" around your dataset, with an optional collation function. "),k=i("code"),q=a("tf.data.Dataset"),L=a(" is a native TensorFlow format that Keras can use for "),Y=i("code"),H=a("model.fit()"),B=a(", so this one method immediately converts a \u{1F917} Dataset to a format that\u2019s ready for training. Let\u2019s see it in action with our dataset!"),ee=f(),v(W.$$.fragment),F=f(),J=i("p"),pe=a("And that\u2019s it! We can take those datasets forward into the next lecture, where training will be pleasantly straightforward after all the hard work of data preprocessing.")},l(D){o=p(D,"P",{});var P=c(o);d=n(P,"Now that we have our dataset and a data collator, we need to put them together. We could manually load batches and collate them, but that\u2019s a lot of work, and probably not very performant either. Instead, there\u2019s a simple method that offers a performant solution to this problem: "),l=p(P,"CODE",{});var Ct=c(l);w=n(Ct,"to_tf_dataset()"),Ct.forEach(s),y=n(P,". This will wrap a "),m=p(P,"CODE",{});var ce=c(m);_=n(ce,"tf.data.Dataset"),ce.forEach(s),z=n(P," around your dataset, with an optional collation function. "),k=p(P,"CODE",{});var Pt=c(k);q=n(Pt,"tf.data.Dataset"),Pt.forEach(s),L=n(P," is a native TensorFlow format that Keras can use for "),Y=p(P,"CODE",{});var St=c(Y);H=n(St,"model.fit()"),St.forEach(s),B=n(P,", so this one method immediately converts a \u{1F917} Dataset to a format that\u2019s ready for training. Let\u2019s see it in action with our dataset!"),P.forEach(s),ee=u(D),$(W.$$.fragment,D),F=u(D),J=p(D,"P",{});var Ge=c(J);pe=n(Ge,"And that\u2019s it! We can take those datasets forward into the next lecture, where training will be pleasantly straightforward after all the hard work of data preprocessing."),Ge.forEach(s)},m(D,P){h(D,o,P),t(o,d),t(o,l),t(l,w),t(o,y),t(o,m),t(m,_),t(o,z),t(o,k),t(k,q),t(o,L),t(o,Y),t(Y,H),t(o,B),h(D,ee,P),j(W,D,P),h(D,F,P),h(D,J,P),t(J,pe),Ee=!0},i(D){Ee||(b(W.$$.fragment,D),Ee=!0)},o(D){g(W.$$.fragment,D),Ee=!1},d(D){D&&s(o),D&&s(ee),x(W,D),D&&s(F),D&&s(J)}}}function mc(E){let o,d,l,w,y,m,_,z,k,q,L,Y,H,B,ee,W,F,J,pe,Ee,D,P,Ct,ce,Pt,St,Ge,ve,Te,is,Ve,Mn,ps,Rn,$a,te,se,At,K,Hn,Ye,Bn,Un,Je,Gn,Vn,Ke,Yn,Jn,ja,Ot,Kn,xa,Ze,Ea,Qe,Ta,I,Zn,cs,Qn,Xn,hs,eo,to,ds,so,ao,fs,no,oo,us,lo,ro,za,he,io,ms,po,co,_s,ho,fo,qa,ze,uo,ws,mo,_o,Da,Xe,Ca,et,Pa,de,wo,bs,bo,go,gs,ko,yo,Sa,tt,Aa,st,Oa,A,vo,ks,$o,jo,ys,xo,Eo,vs,To,zo,$s,qo,Do,js,Co,Po,xs,So,Ao,Es,Oo,Wo,Wa,qe,Na,$e,De,Ts,at,No,zs,Lo,La,ae,ne,Wt,Ce,Fo,Nt,Io,Mo,Fa,nt,Ia,Lt,Ro,Ma,ot,Ra,lt,Ha,U,Ho,qs,Bo,Uo,Ds,Go,Vo,Ft,Yo,Jo,Cs,Ko,Zo,Ba,Pe,Ua,Se,Qo,Ps,Xo,el,Ga,rt,Va,It,tl,Ya,it,Ja,fe,sl,Ss,al,nl,As,ol,ll,Ka,pt,Za,G,rl,Os,il,pl,Ws,cl,hl,Ns,dl,fl,Ls,ul,ml,Qa,Ae,_l,Fs,wl,bl,Xa,ue,gl,Mt,kl,yl,Is,vl,$l,en,Rt,jl,tn,Oe,xl,Ms,El,Tl,sn,me,zl,Ht,ql,Dl,Bt,Cl,Pl,an,ct,nn,V,Sl,Rs,Al,Ol,Hs,Wl,Nl,Bs,Ll,Fl,ht,Il,Ml,on,_e,Rl,dt,Us,Hl,Bl,Gs,Ul,Gl,ln,ft,rn,S,Vl,Vs,Yl,Jl,Ys,Kl,Zl,Js,Ql,Xl,Ks,er,tr,Zs,sr,ar,Qs,nr,or,Xs,lr,rr,ea,ir,pr,ut,cr,hr,pn,We,dr,ta,fr,ur,cn,we,mr,sa,_r,wr,aa,br,gr,hn,mt,dn,Ut,kr,fn,_t,un,be,yr,na,vr,$r,oa,jr,xr,mn,M,Er,la,Tr,zr,ra,qr,Dr,ia,Cr,Pr,pa,Sr,Ar,ca,Or,Wr,_n,Ne,Nr,ha,Lr,Fr,wn,je,Le,da,wt,Ir,fa,Mr,bn,bt,gn,Gt,Fe,Rr,ua,Hr,Br,kn,oe,le,Vt,Z,Ur,ma,Gr,Vr,_a,Yr,Jr,wa,Kr,Zr,yn,gt,vn,kt,$n,Ie,Qr,ba,Xr,ei,jn,yt,xn,re,ie,Yt,Me,En,Jt,Tn;l=new Zp({props:{fw:E[0]}}),z=new In({});const si=[Xp,Qp],vt=[];function ai(e,r){return e[0]==="pt"?0:1}H=ai(E),B=vt[H]=si[H](E);const ni=[tc,ec],$t=[];function oi(e,r){return e[0]==="pt"?0:1}W=oi(E),F=$t[W]=ni[W](E),Ve=new In({});const li=[ac,sc],jt=[];function ri(e,r){return e[0]==="pt"?0:1}te=ri(E),se=jt[te]=li[te](E),Ze=new C({props:{code:`from datasets import load_dataset

raw_datasets = load_dataset("glue", "mrpc")
raw_datasets`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

raw_datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>)
raw_datasets`}}),Qe=new C({props:{code:`DatasetDict({
    train: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 408
    })
    test: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 1725
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">3668</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">408</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">1725</span>
    })
})`}}),Xe=new C({props:{code:`raw_train_dataset = raw_datasets["train"]
raw_train_dataset[0]`,highlighted:`raw_train_dataset = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]
raw_train_dataset[<span class="hljs-number">0</span>]`}}),et=new C({props:{code:`{'idx': 0,
 'label': 1,
 'sentence1': 'Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
 'sentence2': 'Referring to him as only " the witness " , Amrozi accused his brother of deliberately distorting his evidence .'}`,highlighted:`{<span class="hljs-string">&#x27;idx&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">1</span>,
 <span class="hljs-string">&#x27;sentence1&#x27;</span>: <span class="hljs-string">&#x27;Amrozi accused his brother , whom he called &quot; the witness &quot; , of deliberately distorting his evidence .&#x27;</span>,
 <span class="hljs-string">&#x27;sentence2&#x27;</span>: <span class="hljs-string">&#x27;Referring to him as only &quot; the witness &quot; , Amrozi accused his brother of deliberately distorting his evidence .&#x27;</span>}`}}),tt=new C({props:{code:"raw_train_dataset.features",highlighted:"raw_train_dataset.features"}}),st=new C({props:{code:`{'sentence1': Value(dtype='string', id=None),
 'sentence2': Value(dtype='string', id=None),
 'label': ClassLabel(num_classes=2, names=['not_equivalent', 'equivalent'], names_file=None, id=None),
 'idx': Value(dtype='int32', id=None)}`,highlighted:`{<span class="hljs-string">&#x27;sentence1&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;sentence2&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;label&#x27;</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">&#x27;not_equivalent&#x27;</span>, <span class="hljs-string">&#x27;equivalent&#x27;</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;idx&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`}}),qe=new ti({props:{$$slots:{default:[nc]},$$scope:{ctx:E}}}),at=new In({});const ii=[lc,oc],xt=[];function pi(e,r){return e[0]==="pt"?0:1}ae=pi(E),ne=xt[ae]=ii[ae](E),nt=new C({props:{code:`from transformers import AutoTokenizer

checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
tokenized_sentences_1 = tokenizer(raw_datasets["train"]["sentence1"])
tokenized_sentences_2 = tokenizer(raw_datasets["train"]["sentence2"])`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
tokenized_sentences_1 = tokenizer(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence1&quot;</span>])
tokenized_sentences_2 = tokenizer(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence2&quot;</span>])`}}),ot=new C({props:{code:`inputs = tokenizer("This is the first sentence.", "This is the second one.")
inputs`,highlighted:`inputs = tokenizer(<span class="hljs-string">&quot;This is the first sentence.&quot;</span>, <span class="hljs-string">&quot;This is the second one.&quot;</span>)
inputs`}}),lt=new C({props:{code:`{ 
  'input_ids': [101, 2023, 2003, 1996, 2034, 6251, 1012, 102, 2023, 2003, 1996, 2117, 2028, 1012, 102],
  'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
  'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
}`,highlighted:`{ 
  <span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2023</span>, <span class="hljs-number">2003</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">2034</span>, <span class="hljs-number">6251</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>, <span class="hljs-number">2023</span>, <span class="hljs-number">2003</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">2117</span>, <span class="hljs-number">2028</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>],
  <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
  <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]
}`}}),Pe=new ti({props:{$$slots:{default:[rc]},$$scope:{ctx:E}}}),rt=new C({props:{code:'tokenizer.convert_ids_to_tokens(inputs["input_ids"])',highlighted:'tokenizer.convert_ids_to_tokens(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])'}}),it=new C({props:{code:"['[CLS]', 'this', 'is', 'the', 'first', 'sentence', '.', '[SEP]', 'this', 'is', 'the', 'second', 'one', '.', '[SEP]']",highlighted:'[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;first&#x27;</span>, <span class="hljs-string">&#x27;sentence&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;second&#x27;</span>, <span class="hljs-string">&#x27;one&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]'}}),pt=new C({props:{code:`['[CLS]', 'this', 'is', 'the', 'first', 'sentence', '.', '[SEP]', 'this', 'is', 'the', 'second', 'one', '.', '[SEP]']
[      0,      0,    0,     0,       0,          0,   0,       0,      1,    1,     1,        1,     1,   1,       1]`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;first&#x27;</span>, <span class="hljs-string">&#x27;sentence&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;second&#x27;</span>, <span class="hljs-string">&#x27;one&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]
[      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,       <span class="hljs-number">0</span>,          <span class="hljs-number">0</span>,   <span class="hljs-number">0</span>,       <span class="hljs-number">0</span>,      <span class="hljs-number">1</span>,    <span class="hljs-number">1</span>,     <span class="hljs-number">1</span>,        <span class="hljs-number">1</span>,     <span class="hljs-number">1</span>,   <span class="hljs-number">1</span>,       <span class="hljs-number">1</span>]`}}),ct=new C({props:{code:`tokenized_dataset = tokenizer(
    raw_datasets["train"]["sentence1"],
    raw_datasets["train"]["sentence2"],
    padding=True,
    truncation=True,
)`,highlighted:`tokenized_dataset = tokenizer(
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence1&quot;</span>],
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence2&quot;</span>],
    padding=<span class="hljs-literal">True</span>,
    truncation=<span class="hljs-literal">True</span>,
)`}}),ft=new C({props:{code:`def tokenize_function(example):
    return tokenizer(example["sentence1"], example["sentence2"], truncation=True)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">example</span>):
    <span class="hljs-keyword">return</span> tokenizer(example[<span class="hljs-string">&quot;sentence1&quot;</span>], example[<span class="hljs-string">&quot;sentence2&quot;</span>], truncation=<span class="hljs-literal">True</span>)`}}),mt=new C({props:{code:`tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)
tokenized_datasets`,highlighted:`tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>(tokenize_function, batched=<span class="hljs-literal">True</span>)
tokenized_datasets`}}),_t=new C({props:{code:`DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 408
    })
    test: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 1725
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">3668</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">408</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">1725</span>
    })
})`}}),wt=new In({}),bt=new va({props:{id:"7q5NyFT8REg"}});function ci(e,r){return e[0]==="pt"?pc:ic}let zn=ci(E),xe=zn(E);const hi=[hc,cc],Et=[];function di(e,r){return e[0]==="pt"?0:1}oe=di(E),le=Et[oe]=hi[oe](E),gt=new C({props:{code:`samples = tokenized_datasets["train"][:8]
samples = {k: v for k, v in samples.items() if k not in ["idx", "sentence1", "sentence2"]}
[len(x) for x in samples["input_ids"]]`,highlighted:`samples = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">8</span>]
samples = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> samples.items() <span class="hljs-keyword">if</span> k <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;idx&quot;</span>, <span class="hljs-string">&quot;sentence1&quot;</span>, <span class="hljs-string">&quot;sentence2&quot;</span>]}
[<span class="hljs-built_in">len</span>(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> samples[<span class="hljs-string">&quot;input_ids&quot;</span>]]`}}),kt=new C({props:{code:"[50, 59, 47, 67, 59, 50, 62, 32]",highlighted:'[<span class="hljs-number">50</span>, <span class="hljs-number">59</span>, <span class="hljs-number">47</span>, <span class="hljs-number">67</span>, <span class="hljs-number">59</span>, <span class="hljs-number">50</span>, <span class="hljs-number">62</span>, <span class="hljs-number">32</span>]'}}),yt=new C({props:{code:`batch = data_collator(samples)
{k: v.shape for k, v in batch.items()}`,highlighted:`batch = data_collator(samples)
{k: v.shape <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}`}});const fi=[fc,dc],Tt=[];function ui(e,r){return e[0]==="tf"?0:1}re=ui(E),ie=Tt[re]=fi[re](E),Me=new ti({props:{$$slots:{default:[uc]},$$scope:{ctx:E}}});let N=E[0]==="tf"&&Bp();return{c(){o=i("meta"),d=f(),v(l.$$.fragment),w=f(),y=i("h1"),m=i("a"),_=i("span"),v(z.$$.fragment),k=f(),q=i("span"),L=a("Processing the data"),Y=f(),B.c(),ee=f(),F.c(),J=f(),pe=i("p"),Ee=a("Of course, just training the model on two sentences is not going to yield very good results. To get better results, you will need to prepare a bigger dataset."),D=f(),P=i("p"),Ct=a("In this section we will use as an example the MRPC (Microsoft Research Paraphrase Corpus) dataset, introduced in a "),ce=i("a"),Pt=a("paper"),St=a(" by William B. Dolan and Chris Brockett. The dataset consists of 5,801 pairs of sentences, with a label indicating if they are paraphrases or not (i.e., if both sentences mean the same thing). We\u2019ve selected it for this chapter because it\u2019s a small dataset, so it\u2019s easy to experiment with training on it."),Ge=f(),ve=i("h3"),Te=i("a"),is=i("span"),v(Ve.$$.fragment),Mn=f(),ps=i("span"),Rn=a("Loading a dataset from the Hub"),$a=f(),se.c(),At=f(),K=i("p"),Hn=a("The Hub doesn\u2019t just contain models; it also has multiple datasets in lots of different languages. You can browse the datasets "),Ye=i("a"),Bn=a("here"),Un=a(", and we recommend you try to load and process a new dataset once you have gone through this section (see the general documentation "),Je=i("a"),Gn=a("here"),Vn=a("). But for now, let\u2019s focus on the MRPC dataset! This is one of the 10 datasets composing the "),Ke=i("a"),Yn=a("GLUE benchmark"),Jn=a(", which is an academic benchmark that is used to measure the performance of ML models across 10 different text classification tasks."),ja=f(),Ot=i("p"),Kn=a("The \u{1F917} Datasets library provides a very simple command to download and cache a dataset on the Hub. We can download the MRPC dataset like this:"),xa=f(),v(Ze.$$.fragment),Ea=f(),v(Qe.$$.fragment),Ta=f(),I=i("p"),Zn=a("As you can see, we get a "),cs=i("code"),Qn=a("DatasetDict"),Xn=a(" object which contains the training set, the validation set, and the test set. Each of those contains several columns ("),hs=i("code"),eo=a("sentence1"),to=a(", "),ds=i("code"),so=a("sentence2"),ao=a(", "),fs=i("code"),no=a("label"),oo=a(", and "),us=i("code"),lo=a("idx"),ro=a(") and a variable number of rows, which are the number of elements in each set (so, there are 3,668 pairs of sentences in the training set, 408 in the validation set, and 1,725 in the test set)."),za=f(),he=i("p"),io=a("This command downloads and caches the dataset, by default in "),ms=i("em"),po=a("~/.cache/huggingface/datasets"),co=a(". Recall from Chapter 2 that you can customize your cache folder by setting the "),_s=i("code"),ho=a("HF_HOME"),fo=a(" environment variable."),qa=f(),ze=i("p"),uo=a("We can access each pair of sentences in our "),ws=i("code"),mo=a("raw_datasets"),_o=a(" object by indexing, like with a dictionary:"),Da=f(),v(Xe.$$.fragment),Ca=f(),v(et.$$.fragment),Pa=f(),de=i("p"),wo=a("We can see the labels are already integers, so we won\u2019t have to do any preprocessing there. To know which integer corresponds to which label, we can inspect the "),bs=i("code"),bo=a("features"),go=a(" of our "),gs=i("code"),ko=a("raw_train_dataset"),yo=a(". This will tell us the type of each column:"),Sa=f(),v(tt.$$.fragment),Aa=f(),v(st.$$.fragment),Oa=f(),A=i("p"),vo=a("Behind the scenes, "),ks=i("code"),$o=a("label"),jo=a(" is of type "),ys=i("code"),xo=a("ClassLabel"),Eo=a(", and the mapping of integers to label name is stored in the "),vs=i("em"),To=a("names"),zo=a(" folder. "),$s=i("code"),qo=a("0"),Do=a(" corresponds to "),js=i("code"),Co=a("not_equivalent"),Po=a(", and "),xs=i("code"),So=a("1"),Ao=a(" corresponds to "),Es=i("code"),Oo=a("equivalent"),Wo=a("."),Wa=f(),v(qe.$$.fragment),Na=f(),$e=i("h3"),De=i("a"),Ts=i("span"),v(at.$$.fragment),No=f(),zs=i("span"),Lo=a("Preprocessing a dataset"),La=f(),ne.c(),Wt=f(),Ce=i("p"),Fo=a("To preprocess the dataset, we need to convert the text to numbers the model can make sense of. As you saw in the "),Nt=i("a"),Io=a("previous chapter"),Mo=a(", this is done with a tokenizer. We can feed the tokenizer one sentence or a list of sentences, so we can directly tokenize all the first sentences and all the second sentences of each pair like this:"),Fa=f(),v(nt.$$.fragment),Ia=f(),Lt=i("p"),Ro=a("However, we can\u2019t just pass two sequences to the model and get a prediction of whether the two sentences are paraphrases or not. We need to handle the two sequences as a pair, and apply the appropriate preprocessing. Fortunately, the tokenizer can also take a pair of sequences and prepare it the way our BERT model expects:"),Ma=f(),v(ot.$$.fragment),Ra=f(),v(lt.$$.fragment),Ha=f(),U=i("p"),Ho=a("We discussed the "),qs=i("code"),Bo=a("input_ids"),Uo=a(" and "),Ds=i("code"),Go=a("attention_mask"),Vo=a(" keys in "),Ft=i("a"),Yo=a("Chapter 2"),Jo=a(", but we put off talking about "),Cs=i("code"),Ko=a("token_type_ids"),Zo=a(". In this example, this is what tells the model which part of the input is the first sentence and which is the second sentence."),Ba=f(),v(Pe.$$.fragment),Ua=f(),Se=i("p"),Qo=a("If we decode the IDs inside "),Ps=i("code"),Xo=a("input_ids"),el=a(" back to words:"),Ga=f(),v(rt.$$.fragment),Va=f(),It=i("p"),tl=a("we will get:"),Ya=f(),v(it.$$.fragment),Ja=f(),fe=i("p"),sl=a("So we see the model expects the inputs to be of the form "),Ss=i("code"),al=a("[CLS] sentence1 [SEP] sentence2 [SEP]"),nl=a(" when there are two sentences. Aligning this with the "),As=i("code"),ol=a("token_type_ids"),ll=a(" gives us:"),Ka=f(),v(pt.$$.fragment),Za=f(),G=i("p"),rl=a("As you can see, the parts of the input corresponding to "),Os=i("code"),il=a("[CLS] sentence1 [SEP]"),pl=a(" all have a token type ID of "),Ws=i("code"),cl=a("0"),hl=a(", while the other parts, corresponding to "),Ns=i("code"),dl=a("sentence2 [SEP]"),fl=a(", all have a token type ID of "),Ls=i("code"),ul=a("1"),ml=a("."),Qa=f(),Ae=i("p"),_l=a("Note that if you select a different checkpoint, you won\u2019t necessarily have the "),Fs=i("code"),wl=a("token_type_ids"),bl=a(" in your tokenized inputs (for instance, they\u2019re not returned if you use a DistilBERT model). They are only returned when the model will know what to do with them, because it has seen them during its pretraining."),Xa=f(),ue=i("p"),gl=a("Here, BERT is pretrained with token type IDs, and on top of the masked language modeling objective we talked about in "),Mt=i("a"),kl=a("Chapter 1"),yl=a(", it has an additional objective called "),Is=i("em"),vl=a("next sentence prediction"),$l=a(". The goal with this task is to model the relationship between pairs of sentences."),en=f(),Rt=i("p"),jl=a("With next sentence prediction, the model is provided pairs of sentences (with randomly masked tokens) and asked to predict whether the second sentence follows the first. To make the task non-trivial, half of the time the sentences follow each other in the original document they were extracted from, and the other half of the time the two sentences come from two different documents."),tn=f(),Oe=i("p"),xl=a("In general, you don\u2019t need to worry about whether or not there are "),Ms=i("code"),El=a("token_type_ids"),Tl=a(" in your tokenized inputs: as long as you use the same checkpoint for the tokenizer and the model, everything will be fine as the tokenizer knows what to provide to its model."),sn=f(),me=i("p"),zl=a("Now that we have seen how our tokenizer can deal with one pair of sentences, we can use it to tokenize our whole dataset: like in the "),Ht=i("a"),ql=a("previous chapter"),Dl=a(", we can feed the tokenizer a list of pairs of sentences by giving it the list of first sentences, then the list of second sentences. This is also compatible with the padding and truncation options we saw in "),Bt=i("a"),Cl=a("Chapter 2"),Pl=a(". So, one way to preprocess the training dataset is:"),an=f(),v(ct.$$.fragment),nn=f(),V=i("p"),Sl=a("This works well, but it has the disadvantage of returning a dictionary (with our keys, "),Rs=i("code"),Al=a("input_ids"),Ol=a(", "),Hs=i("code"),Wl=a("attention_mask"),Nl=a(", and "),Bs=i("code"),Ll=a("token_type_ids"),Fl=a(", and values that are lists of lists). It will also only work if you have enough RAM to store your whole dataset during the tokenization (whereas the datasets from the \u{1F917} Datasets library are "),ht=i("a"),Il=a("Apache Arrow"),Ml=a(" files stored on the disk, so you only keep the samples you ask for loaded in memory)."),on=f(),_e=i("p"),Rl=a("To keep the data as a dataset, we will use the "),dt=i("a"),Us=i("code"),Hl=a("Dataset.map()"),Bl=a(" method. This also allows us some extra flexibility, if we need more preprocessing done than just tokenization. The "),Gs=i("code"),Ul=a("map()"),Gl=a(" method works by applying a function on each element of the dataset, so let\u2019s define a function that tokenizes our inputs:"),ln=f(),v(ft.$$.fragment),rn=f(),S=i("p"),Vl=a("This function takes a dictionary (like the items of our dataset) and returns a new dictionary with the keys "),Vs=i("code"),Yl=a("input_ids"),Jl=a(", "),Ys=i("code"),Kl=a("attention_mask"),Zl=a(", and "),Js=i("code"),Ql=a("token_type_ids"),Xl=a(". Note that it also works if the "),Ks=i("code"),er=a("example"),tr=a(" dictionary contains several samples (each key as a list of sentences) since the "),Zs=i("code"),sr=a("tokenizer"),ar=a(" works on lists of pairs of sentences, as seen before. This will allow us to use the option "),Qs=i("code"),nr=a("batched=True"),or=a(" in our call to "),Xs=i("code"),lr=a("map()"),rr=a(", which will greatly speed up the tokenization. The "),ea=i("code"),ir=a("tokenizer"),pr=a(" is backed by a tokenizer written in Rust from the "),ut=i("a"),cr=a("\u{1F917} Tokenizers"),hr=a(" library. This tokenizer can be very fast, but only if we give it lots of inputs at once."),pn=f(),We=i("p"),dr=a("Note that we\u2019ve left the "),ta=i("code"),fr=a("padding"),ur=a(" argument out in our tokenization function for now. This is because padding all the samples to the maximum length is not efficient: it\u2019s better to pad the samples when we\u2019re building a batch, as then we only need to pad to the maximum length in that batch, and not the maximum length in the entire dataset. This can save a lot of time and processing power when the inputs have very variable lengths!"),cn=f(),we=i("p"),mr=a("Here is how we apply the tokenization function on all our datasets at once. We\u2019re using "),sa=i("code"),_r=a("batched=True"),wr=a(" in our call to "),aa=i("code"),br=a("map"),gr=a(" so the function is applied to multiple elements of our dataset at once, and not on each element separately. This allows for faster preprocessing."),hn=f(),v(mt.$$.fragment),dn=f(),Ut=i("p"),kr=a("The way the \u{1F917} Datasets library applies this processing is by adding new fields to the datasets, one for each key in the dictionary returned by the preprocessing function:"),fn=f(),v(_t.$$.fragment),un=f(),be=i("p"),yr=a("You can even use multiprocessing when applying your preprocessing function with "),na=i("code"),vr=a("map()"),$r=a(" by passing along a "),oa=i("code"),jr=a("num_proc"),xr=a(" argument. We didn\u2019t do this here because the \u{1F917} Tokenizers library already uses multiple threads to tokenize our samples faster, but if you are not using a fast tokenizer backed by this library, this could speed up your preprocessing."),mn=f(),M=i("p"),Er=a("Our "),la=i("code"),Tr=a("tokenize_function"),zr=a(" returns a dictionary with the keys "),ra=i("code"),qr=a("input_ids"),Dr=a(", "),ia=i("code"),Cr=a("attention_mask"),Pr=a(", and "),pa=i("code"),Sr=a("token_type_ids"),Ar=a(", so those three fields are added to all splits of our dataset. Note that we could also have changed existing fields if our preprocessing function returned a new value for an existing key in the dataset to which we applied "),ca=i("code"),Or=a("map()"),Wr=a("."),_n=f(),Ne=i("p"),Nr=a("The last thing we will need to do is pad all the examples to the length of the longest element when we batch elements together \u2014 a technique we refer to as "),ha=i("em"),Lr=a("dynamic padding"),Fr=a("."),wn=f(),je=i("h3"),Le=i("a"),da=i("span"),v(wt.$$.fragment),Ir=f(),fa=i("span"),Mr=a("Dynamic padding"),bn=f(),v(bt.$$.fragment),gn=f(),xe.c(),Gt=f(),Fe=i("p"),Rr=a("To do this in practice, we have to define a collate function that will apply the correct amount of padding to the items of the dataset we want to batch together. Fortunately, the \u{1F917} Transformers library provides us with such a function via "),ua=i("code"),Hr=a("DataCollatorWithPadding"),Br=a(". It takes a tokenizer when you instantiate it (to know which padding token to use, and whether the model expects padding to be on the left or on the right of the inputs) and will do everything you need:"),kn=f(),le.c(),Vt=f(),Z=i("p"),Ur=a("To test this new toy, let\u2019s grab a few samples from our training set that we would like to batch together. Here, we remove the columns "),ma=i("code"),Gr=a("idx"),Vr=a(", "),_a=i("code"),Yr=a("sentence1"),Jr=a(", and "),wa=i("code"),Kr=a("sentence2"),Zr=a(" as they won\u2019t be needed and contain strings (and we can\u2019t create tensors with strings) and have a look at the lengths of each entry in the batch:"),yn=f(),v(gt.$$.fragment),vn=f(),v(kt.$$.fragment),$n=f(),Ie=i("p"),Qr=a("No surprise, we get samples of varying length, from 32 to 67. Dynamic padding means the samples in this batch should all be padded to a length of 67, the maximum length inside the batch. Without dynamic padding, all of the samples would have to be padded to the maximum length in the whole dataset, or the maximum length the model can accept. Let\u2019s double-check that our "),ba=i("code"),Xr=a("data_collator"),ei=a(" is dynamically padding the batch properly:"),jn=f(),v(yt.$$.fragment),xn=f(),ie.c(),Yt=f(),v(Me.$$.fragment),En=f(),N&&N.c(),Jt=Hp(),this.h()},l(e){const r=Jp('[data-svelte="svelte-1phssyn"]',document.head);o=p(r,"META",{name:!0,content:!0}),r.forEach(s),d=u(e),$(l.$$.fragment,e),w=u(e),y=p(e,"H1",{class:!0});var zt=c(y);m=p(zt,"A",{id:!0,class:!0,href:!0});var Kt=c(m);_=p(Kt,"SPAN",{});var Zt=c(_);$(z.$$.fragment,Zt),Zt.forEach(s),Kt.forEach(s),k=u(zt),q=p(zt,"SPAN",{});var Qt=c(q);L=n(Qt,"Processing the data"),Qt.forEach(s),zt.forEach(s),Y=u(e),B.l(e),ee=u(e),F.l(e),J=u(e),pe=p(e,"P",{});var ga=c(pe);Ee=n(ga,"Of course, just training the model on two sentences is not going to yield very good results. To get better results, you will need to prepare a bigger dataset."),ga.forEach(s),D=u(e),P=p(e,"P",{});var Re=c(P);Ct=n(Re,"In this section we will use as an example the MRPC (Microsoft Research Paraphrase Corpus) dataset, introduced in a "),ce=p(Re,"A",{href:!0,rel:!0});var ka=c(ce);Pt=n(ka,"paper"),ka.forEach(s),St=n(Re," by William B. Dolan and Chris Brockett. The dataset consists of 5,801 pairs of sentences, with a label indicating if they are paraphrases or not (i.e., if both sentences mean the same thing). We\u2019ve selected it for this chapter because it\u2019s a small dataset, so it\u2019s easy to experiment with training on it."),Re.forEach(s),Ge=u(e),ve=p(e,"H3",{class:!0});var He=c(ve);Te=p(He,"A",{id:!0,class:!0,href:!0});var Xt=c(Te);is=p(Xt,"SPAN",{});var ya=c(is);$(Ve.$$.fragment,ya),ya.forEach(s),Xt.forEach(s),Mn=u(He),ps=p(He,"SPAN",{});var mi=c(ps);Rn=n(mi,"Loading a dataset from the Hub"),mi.forEach(s),He.forEach(s),$a=u(e),se.l(e),At=u(e),K=p(e,"P",{});var Be=c(K);Hn=n(Be,"The Hub doesn\u2019t just contain models; it also has multiple datasets in lots of different languages. You can browse the datasets "),Ye=p(Be,"A",{href:!0,rel:!0});var _i=c(Ye);Bn=n(_i,"here"),_i.forEach(s),Un=n(Be,", and we recommend you try to load and process a new dataset once you have gone through this section (see the general documentation "),Je=p(Be,"A",{href:!0,rel:!0});var wi=c(Je);Gn=n(wi,"here"),wi.forEach(s),Vn=n(Be,"). But for now, let\u2019s focus on the MRPC dataset! This is one of the 10 datasets composing the "),Ke=p(Be,"A",{href:!0,rel:!0});var bi=c(Ke);Yn=n(bi,"GLUE benchmark"),bi.forEach(s),Jn=n(Be,", which is an academic benchmark that is used to measure the performance of ML models across 10 different text classification tasks."),Be.forEach(s),ja=u(e),Ot=p(e,"P",{});var gi=c(Ot);Kn=n(gi,"The \u{1F917} Datasets library provides a very simple command to download and cache a dataset on the Hub. We can download the MRPC dataset like this:"),gi.forEach(s),xa=u(e),$(Ze.$$.fragment,e),Ea=u(e),$(Qe.$$.fragment,e),Ta=u(e),I=p(e,"P",{});var Q=c(I);Zn=n(Q,"As you can see, we get a "),cs=p(Q,"CODE",{});var ki=c(cs);Qn=n(ki,"DatasetDict"),ki.forEach(s),Xn=n(Q," object which contains the training set, the validation set, and the test set. Each of those contains several columns ("),hs=p(Q,"CODE",{});var yi=c(hs);eo=n(yi,"sentence1"),yi.forEach(s),to=n(Q,", "),ds=p(Q,"CODE",{});var vi=c(ds);so=n(vi,"sentence2"),vi.forEach(s),ao=n(Q,", "),fs=p(Q,"CODE",{});var $i=c(fs);no=n($i,"label"),$i.forEach(s),oo=n(Q,", and "),us=p(Q,"CODE",{});var ji=c(us);lo=n(ji,"idx"),ji.forEach(s),ro=n(Q,") and a variable number of rows, which are the number of elements in each set (so, there are 3,668 pairs of sentences in the training set, 408 in the validation set, and 1,725 in the test set)."),Q.forEach(s),za=u(e),he=p(e,"P",{});var es=c(he);io=n(es,"This command downloads and caches the dataset, by default in "),ms=p(es,"EM",{});var xi=c(ms);po=n(xi,"~/.cache/huggingface/datasets"),xi.forEach(s),co=n(es,". Recall from Chapter 2 that you can customize your cache folder by setting the "),_s=p(es,"CODE",{});var Ei=c(_s);ho=n(Ei,"HF_HOME"),Ei.forEach(s),fo=n(es," environment variable."),es.forEach(s),qa=u(e),ze=p(e,"P",{});var qn=c(ze);uo=n(qn,"We can access each pair of sentences in our "),ws=p(qn,"CODE",{});var Ti=c(ws);mo=n(Ti,"raw_datasets"),Ti.forEach(s),_o=n(qn," object by indexing, like with a dictionary:"),qn.forEach(s),Da=u(e),$(Xe.$$.fragment,e),Ca=u(e),$(et.$$.fragment,e),Pa=u(e),de=p(e,"P",{});var ts=c(de);wo=n(ts,"We can see the labels are already integers, so we won\u2019t have to do any preprocessing there. To know which integer corresponds to which label, we can inspect the "),bs=p(ts,"CODE",{});var zi=c(bs);bo=n(zi,"features"),zi.forEach(s),go=n(ts," of our "),gs=p(ts,"CODE",{});var qi=c(gs);ko=n(qi,"raw_train_dataset"),qi.forEach(s),yo=n(ts,". This will tell us the type of each column:"),ts.forEach(s),Sa=u(e),$(tt.$$.fragment,e),Aa=u(e),$(st.$$.fragment,e),Oa=u(e),A=p(e,"P",{});var R=c(A);vo=n(R,"Behind the scenes, "),ks=p(R,"CODE",{});var Di=c(ks);$o=n(Di,"label"),Di.forEach(s),jo=n(R," is of type "),ys=p(R,"CODE",{});var Ci=c(ys);xo=n(Ci,"ClassLabel"),Ci.forEach(s),Eo=n(R,", and the mapping of integers to label name is stored in the "),vs=p(R,"EM",{});var Pi=c(vs);To=n(Pi,"names"),Pi.forEach(s),zo=n(R," folder. "),$s=p(R,"CODE",{});var Si=c($s);qo=n(Si,"0"),Si.forEach(s),Do=n(R," corresponds to "),js=p(R,"CODE",{});var Ai=c(js);Co=n(Ai,"not_equivalent"),Ai.forEach(s),Po=n(R,", and "),xs=p(R,"CODE",{});var Oi=c(xs);So=n(Oi,"1"),Oi.forEach(s),Ao=n(R," corresponds to "),Es=p(R,"CODE",{});var Wi=c(Es);Oo=n(Wi,"equivalent"),Wi.forEach(s),Wo=n(R,"."),R.forEach(s),Wa=u(e),$(qe.$$.fragment,e),Na=u(e),$e=p(e,"H3",{class:!0});var Dn=c($e);De=p(Dn,"A",{id:!0,class:!0,href:!0});var Ni=c(De);Ts=p(Ni,"SPAN",{});var Li=c(Ts);$(at.$$.fragment,Li),Li.forEach(s),Ni.forEach(s),No=u(Dn),zs=p(Dn,"SPAN",{});var Fi=c(zs);Lo=n(Fi,"Preprocessing a dataset"),Fi.forEach(s),Dn.forEach(s),La=u(e),ne.l(e),Wt=u(e),Ce=p(e,"P",{});var Cn=c(Ce);Fo=n(Cn,"To preprocess the dataset, we need to convert the text to numbers the model can make sense of. As you saw in the "),Nt=p(Cn,"A",{href:!0});var Ii=c(Nt);Io=n(Ii,"previous chapter"),Ii.forEach(s),Mo=n(Cn,", this is done with a tokenizer. We can feed the tokenizer one sentence or a list of sentences, so we can directly tokenize all the first sentences and all the second sentences of each pair like this:"),Cn.forEach(s),Fa=u(e),$(nt.$$.fragment,e),Ia=u(e),Lt=p(e,"P",{});var Mi=c(Lt);Ro=n(Mi,"However, we can\u2019t just pass two sequences to the model and get a prediction of whether the two sentences are paraphrases or not. We need to handle the two sequences as a pair, and apply the appropriate preprocessing. Fortunately, the tokenizer can also take a pair of sequences and prepare it the way our BERT model expects:"),Mi.forEach(s),Ma=u(e),$(ot.$$.fragment,e),Ra=u(e),$(lt.$$.fragment,e),Ha=u(e),U=p(e,"P",{});var ge=c(U);Ho=n(ge,"We discussed the "),qs=p(ge,"CODE",{});var Ri=c(qs);Bo=n(Ri,"input_ids"),Ri.forEach(s),Uo=n(ge," and "),Ds=p(ge,"CODE",{});var Hi=c(Ds);Go=n(Hi,"attention_mask"),Hi.forEach(s),Vo=n(ge," keys in "),Ft=p(ge,"A",{href:!0});var Bi=c(Ft);Yo=n(Bi,"Chapter 2"),Bi.forEach(s),Jo=n(ge,", but we put off talking about "),Cs=p(ge,"CODE",{});var Ui=c(Cs);Ko=n(Ui,"token_type_ids"),Ui.forEach(s),Zo=n(ge,". In this example, this is what tells the model which part of the input is the first sentence and which is the second sentence."),ge.forEach(s),Ba=u(e),$(Pe.$$.fragment,e),Ua=u(e),Se=p(e,"P",{});var Pn=c(Se);Qo=n(Pn,"If we decode the IDs inside "),Ps=p(Pn,"CODE",{});var Gi=c(Ps);Xo=n(Gi,"input_ids"),Gi.forEach(s),el=n(Pn," back to words:"),Pn.forEach(s),Ga=u(e),$(rt.$$.fragment,e),Va=u(e),It=p(e,"P",{});var Vi=c(It);tl=n(Vi,"we will get:"),Vi.forEach(s),Ya=u(e),$(it.$$.fragment,e),Ja=u(e),fe=p(e,"P",{});var ss=c(fe);sl=n(ss,"So we see the model expects the inputs to be of the form "),Ss=p(ss,"CODE",{});var Yi=c(Ss);al=n(Yi,"[CLS] sentence1 [SEP] sentence2 [SEP]"),Yi.forEach(s),nl=n(ss," when there are two sentences. Aligning this with the "),As=p(ss,"CODE",{});var Ji=c(As);ol=n(Ji,"token_type_ids"),Ji.forEach(s),ll=n(ss," gives us:"),ss.forEach(s),Ka=u(e),$(pt.$$.fragment,e),Za=u(e),G=p(e,"P",{});var ke=c(G);rl=n(ke,"As you can see, the parts of the input corresponding to "),Os=p(ke,"CODE",{});var Ki=c(Os);il=n(Ki,"[CLS] sentence1 [SEP]"),Ki.forEach(s),pl=n(ke," all have a token type ID of "),Ws=p(ke,"CODE",{});var Zi=c(Ws);cl=n(Zi,"0"),Zi.forEach(s),hl=n(ke,", while the other parts, corresponding to "),Ns=p(ke,"CODE",{});var Qi=c(Ns);dl=n(Qi,"sentence2 [SEP]"),Qi.forEach(s),fl=n(ke,", all have a token type ID of "),Ls=p(ke,"CODE",{});var Xi=c(Ls);ul=n(Xi,"1"),Xi.forEach(s),ml=n(ke,"."),ke.forEach(s),Qa=u(e),Ae=p(e,"P",{});var Sn=c(Ae);_l=n(Sn,"Note that if you select a different checkpoint, you won\u2019t necessarily have the "),Fs=p(Sn,"CODE",{});var ep=c(Fs);wl=n(ep,"token_type_ids"),ep.forEach(s),bl=n(Sn," in your tokenized inputs (for instance, they\u2019re not returned if you use a DistilBERT model). They are only returned when the model will know what to do with them, because it has seen them during its pretraining."),Sn.forEach(s),Xa=u(e),ue=p(e,"P",{});var as=c(ue);gl=n(as,"Here, BERT is pretrained with token type IDs, and on top of the masked language modeling objective we talked about in "),Mt=p(as,"A",{href:!0});var tp=c(Mt);kl=n(tp,"Chapter 1"),tp.forEach(s),yl=n(as,", it has an additional objective called "),Is=p(as,"EM",{});var sp=c(Is);vl=n(sp,"next sentence prediction"),sp.forEach(s),$l=n(as,". The goal with this task is to model the relationship between pairs of sentences."),as.forEach(s),en=u(e),Rt=p(e,"P",{});var ap=c(Rt);jl=n(ap,"With next sentence prediction, the model is provided pairs of sentences (with randomly masked tokens) and asked to predict whether the second sentence follows the first. To make the task non-trivial, half of the time the sentences follow each other in the original document they were extracted from, and the other half of the time the two sentences come from two different documents."),ap.forEach(s),tn=u(e),Oe=p(e,"P",{});var An=c(Oe);xl=n(An,"In general, you don\u2019t need to worry about whether or not there are "),Ms=p(An,"CODE",{});var np=c(Ms);El=n(np,"token_type_ids"),np.forEach(s),Tl=n(An," in your tokenized inputs: as long as you use the same checkpoint for the tokenizer and the model, everything will be fine as the tokenizer knows what to provide to its model."),An.forEach(s),sn=u(e),me=p(e,"P",{});var ns=c(me);zl=n(ns,"Now that we have seen how our tokenizer can deal with one pair of sentences, we can use it to tokenize our whole dataset: like in the "),Ht=p(ns,"A",{href:!0});var op=c(Ht);ql=n(op,"previous chapter"),op.forEach(s),Dl=n(ns,", we can feed the tokenizer a list of pairs of sentences by giving it the list of first sentences, then the list of second sentences. This is also compatible with the padding and truncation options we saw in "),Bt=p(ns,"A",{href:!0});var lp=c(Bt);Cl=n(lp,"Chapter 2"),lp.forEach(s),Pl=n(ns,". So, one way to preprocess the training dataset is:"),ns.forEach(s),an=u(e),$(ct.$$.fragment,e),nn=u(e),V=p(e,"P",{});var ye=c(V);Sl=n(ye,"This works well, but it has the disadvantage of returning a dictionary (with our keys, "),Rs=p(ye,"CODE",{});var rp=c(Rs);Al=n(rp,"input_ids"),rp.forEach(s),Ol=n(ye,", "),Hs=p(ye,"CODE",{});var ip=c(Hs);Wl=n(ip,"attention_mask"),ip.forEach(s),Nl=n(ye,", and "),Bs=p(ye,"CODE",{});var pp=c(Bs);Ll=n(pp,"token_type_ids"),pp.forEach(s),Fl=n(ye,", and values that are lists of lists). It will also only work if you have enough RAM to store your whole dataset during the tokenization (whereas the datasets from the \u{1F917} Datasets library are "),ht=p(ye,"A",{href:!0,rel:!0});var cp=c(ht);Il=n(cp,"Apache Arrow"),cp.forEach(s),Ml=n(ye," files stored on the disk, so you only keep the samples you ask for loaded in memory)."),ye.forEach(s),on=u(e),_e=p(e,"P",{});var os=c(_e);Rl=n(os,"To keep the data as a dataset, we will use the "),dt=p(os,"A",{href:!0,rel:!0});var hp=c(dt);Us=p(hp,"CODE",{});var dp=c(Us);Hl=n(dp,"Dataset.map()"),dp.forEach(s),hp.forEach(s),Bl=n(os," method. This also allows us some extra flexibility, if we need more preprocessing done than just tokenization. The "),Gs=p(os,"CODE",{});var fp=c(Gs);Ul=n(fp,"map()"),fp.forEach(s),Gl=n(os," method works by applying a function on each element of the dataset, so let\u2019s define a function that tokenizes our inputs:"),os.forEach(s),ln=u(e),$(ft.$$.fragment,e),rn=u(e),S=p(e,"P",{});var O=c(S);Vl=n(O,"This function takes a dictionary (like the items of our dataset) and returns a new dictionary with the keys "),Vs=p(O,"CODE",{});var up=c(Vs);Yl=n(up,"input_ids"),up.forEach(s),Jl=n(O,", "),Ys=p(O,"CODE",{});var mp=c(Ys);Kl=n(mp,"attention_mask"),mp.forEach(s),Zl=n(O,", and "),Js=p(O,"CODE",{});var _p=c(Js);Ql=n(_p,"token_type_ids"),_p.forEach(s),Xl=n(O,". Note that it also works if the "),Ks=p(O,"CODE",{});var wp=c(Ks);er=n(wp,"example"),wp.forEach(s),tr=n(O," dictionary contains several samples (each key as a list of sentences) since the "),Zs=p(O,"CODE",{});var bp=c(Zs);sr=n(bp,"tokenizer"),bp.forEach(s),ar=n(O," works on lists of pairs of sentences, as seen before. This will allow us to use the option "),Qs=p(O,"CODE",{});var gp=c(Qs);nr=n(gp,"batched=True"),gp.forEach(s),or=n(O," in our call to "),Xs=p(O,"CODE",{});var kp=c(Xs);lr=n(kp,"map()"),kp.forEach(s),rr=n(O,", which will greatly speed up the tokenization. The "),ea=p(O,"CODE",{});var yp=c(ea);ir=n(yp,"tokenizer"),yp.forEach(s),pr=n(O," is backed by a tokenizer written in Rust from the "),ut=p(O,"A",{href:!0,rel:!0});var vp=c(ut);cr=n(vp,"\u{1F917} Tokenizers"),vp.forEach(s),hr=n(O," library. This tokenizer can be very fast, but only if we give it lots of inputs at once."),O.forEach(s),pn=u(e),We=p(e,"P",{});var On=c(We);dr=n(On,"Note that we\u2019ve left the "),ta=p(On,"CODE",{});var $p=c(ta);fr=n($p,"padding"),$p.forEach(s),ur=n(On," argument out in our tokenization function for now. This is because padding all the samples to the maximum length is not efficient: it\u2019s better to pad the samples when we\u2019re building a batch, as then we only need to pad to the maximum length in that batch, and not the maximum length in the entire dataset. This can save a lot of time and processing power when the inputs have very variable lengths!"),On.forEach(s),cn=u(e),we=p(e,"P",{});var ls=c(we);mr=n(ls,"Here is how we apply the tokenization function on all our datasets at once. We\u2019re using "),sa=p(ls,"CODE",{});var jp=c(sa);_r=n(jp,"batched=True"),jp.forEach(s),wr=n(ls," in our call to "),aa=p(ls,"CODE",{});var xp=c(aa);br=n(xp,"map"),xp.forEach(s),gr=n(ls," so the function is applied to multiple elements of our dataset at once, and not on each element separately. This allows for faster preprocessing."),ls.forEach(s),hn=u(e),$(mt.$$.fragment,e),dn=u(e),Ut=p(e,"P",{});var Ep=c(Ut);kr=n(Ep,"The way the \u{1F917} Datasets library applies this processing is by adding new fields to the datasets, one for each key in the dictionary returned by the preprocessing function:"),Ep.forEach(s),fn=u(e),$(_t.$$.fragment,e),un=u(e),be=p(e,"P",{});var rs=c(be);yr=n(rs,"You can even use multiprocessing when applying your preprocessing function with "),na=p(rs,"CODE",{});var Tp=c(na);vr=n(Tp,"map()"),Tp.forEach(s),$r=n(rs," by passing along a "),oa=p(rs,"CODE",{});var zp=c(oa);jr=n(zp,"num_proc"),zp.forEach(s),xr=n(rs," argument. We didn\u2019t do this here because the \u{1F917} Tokenizers library already uses multiple threads to tokenize our samples faster, but if you are not using a fast tokenizer backed by this library, this could speed up your preprocessing."),rs.forEach(s),mn=u(e),M=p(e,"P",{});var X=c(M);Er=n(X,"Our "),la=p(X,"CODE",{});var qp=c(la);Tr=n(qp,"tokenize_function"),qp.forEach(s),zr=n(X," returns a dictionary with the keys "),ra=p(X,"CODE",{});var Dp=c(ra);qr=n(Dp,"input_ids"),Dp.forEach(s),Dr=n(X,", "),ia=p(X,"CODE",{});var Cp=c(ia);Cr=n(Cp,"attention_mask"),Cp.forEach(s),Pr=n(X,", and "),pa=p(X,"CODE",{});var Pp=c(pa);Sr=n(Pp,"token_type_ids"),Pp.forEach(s),Ar=n(X,", so those three fields are added to all splits of our dataset. Note that we could also have changed existing fields if our preprocessing function returned a new value for an existing key in the dataset to which we applied "),ca=p(X,"CODE",{});var Sp=c(ca);Or=n(Sp,"map()"),Sp.forEach(s),Wr=n(X,"."),X.forEach(s),_n=u(e),Ne=p(e,"P",{});var Wn=c(Ne);Nr=n(Wn,"The last thing we will need to do is pad all the examples to the length of the longest element when we batch elements together \u2014 a technique we refer to as "),ha=p(Wn,"EM",{});var Ap=c(ha);Lr=n(Ap,"dynamic padding"),Ap.forEach(s),Fr=n(Wn,"."),Wn.forEach(s),wn=u(e),je=p(e,"H3",{class:!0});var Nn=c(je);Le=p(Nn,"A",{id:!0,class:!0,href:!0});var Op=c(Le);da=p(Op,"SPAN",{});var Wp=c(da);$(wt.$$.fragment,Wp),Wp.forEach(s),Op.forEach(s),Ir=u(Nn),fa=p(Nn,"SPAN",{});var Np=c(fa);Mr=n(Np,"Dynamic padding"),Np.forEach(s),Nn.forEach(s),bn=u(e),$(bt.$$.fragment,e),gn=u(e),xe.l(e),Gt=u(e),Fe=p(e,"P",{});var Ln=c(Fe);Rr=n(Ln,"To do this in practice, we have to define a collate function that will apply the correct amount of padding to the items of the dataset we want to batch together. Fortunately, the \u{1F917} Transformers library provides us with such a function via "),ua=p(Ln,"CODE",{});var Lp=c(ua);Hr=n(Lp,"DataCollatorWithPadding"),Lp.forEach(s),Br=n(Ln,". It takes a tokenizer when you instantiate it (to know which padding token to use, and whether the model expects padding to be on the left or on the right of the inputs) and will do everything you need:"),Ln.forEach(s),kn=u(e),le.l(e),Vt=u(e),Z=p(e,"P",{});var Ue=c(Z);Ur=n(Ue,"To test this new toy, let\u2019s grab a few samples from our training set that we would like to batch together. Here, we remove the columns "),ma=p(Ue,"CODE",{});var Fp=c(ma);Gr=n(Fp,"idx"),Fp.forEach(s),Vr=n(Ue,", "),_a=p(Ue,"CODE",{});var Ip=c(_a);Yr=n(Ip,"sentence1"),Ip.forEach(s),Jr=n(Ue,", and "),wa=p(Ue,"CODE",{});var Mp=c(wa);Kr=n(Mp,"sentence2"),Mp.forEach(s),Zr=n(Ue," as they won\u2019t be needed and contain strings (and we can\u2019t create tensors with strings) and have a look at the lengths of each entry in the batch:"),Ue.forEach(s),yn=u(e),$(gt.$$.fragment,e),vn=u(e),$(kt.$$.fragment,e),$n=u(e),Ie=p(e,"P",{});var Fn=c(Ie);Qr=n(Fn,"No surprise, we get samples of varying length, from 32 to 67. Dynamic padding means the samples in this batch should all be padded to a length of 67, the maximum length inside the batch. Without dynamic padding, all of the samples would have to be padded to the maximum length in the whole dataset, or the maximum length the model can accept. Let\u2019s double-check that our "),ba=p(Fn,"CODE",{});var Rp=c(ba);Xr=n(Rp,"data_collator"),Rp.forEach(s),ei=n(Fn," is dynamically padding the batch properly:"),Fn.forEach(s),jn=u(e),$(yt.$$.fragment,e),xn=u(e),ie.l(e),Yt=u(e),$(Me.$$.fragment,e),En=u(e),N&&N.l(e),Jt=Hp(),this.h()},h(){T(o,"name","hf:doc:metadata"),T(o,"content",JSON.stringify(_c)),T(m,"id","processing-the-data"),T(m,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(m,"href","#processing-the-data"),T(y,"class","relative group"),T(ce,"href","https://www.aclweb.org/anthology/I05-5002.pdf"),T(ce,"rel","nofollow"),T(Te,"id","loading-a-dataset-from-the-hub"),T(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(Te,"href","#loading-a-dataset-from-the-hub"),T(ve,"class","relative group"),T(Ye,"href","https://huggingface.co/datasets"),T(Ye,"rel","nofollow"),T(Je,"href","https://huggingface.co/docs/datasets/loading_datasets.html#from-the-huggingface-hub"),T(Je,"rel","nofollow"),T(Ke,"href","https://gluebenchmark.com/"),T(Ke,"rel","nofollow"),T(De,"id","preprocessing-a-dataset"),T(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(De,"href","#preprocessing-a-dataset"),T($e,"class","relative group"),T(Nt,"href","/course/chapter2"),T(Ft,"href","/course/chapter2"),T(Mt,"href","/course/chapter1"),T(Ht,"href","/course/chapter2"),T(Bt,"href","/course/chapter2"),T(ht,"href","https://arrow.apache.org/"),T(ht,"rel","nofollow"),T(dt,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),T(dt,"rel","nofollow"),T(ut,"href","https://github.com/huggingface/tokenizers"),T(ut,"rel","nofollow"),T(Le,"id","dynamic-padding"),T(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(Le,"href","#dynamic-padding"),T(je,"class","relative group")},m(e,r){t(document.head,o),h(e,d,r),j(l,e,r),h(e,w,r),h(e,y,r),t(y,m),t(m,_),j(z,_,null),t(y,k),t(y,q),t(q,L),h(e,Y,r),vt[H].m(e,r),h(e,ee,r),$t[W].m(e,r),h(e,J,r),h(e,pe,r),t(pe,Ee),h(e,D,r),h(e,P,r),t(P,Ct),t(P,ce),t(ce,Pt),t(P,St),h(e,Ge,r),h(e,ve,r),t(ve,Te),t(Te,is),j(Ve,is,null),t(ve,Mn),t(ve,ps),t(ps,Rn),h(e,$a,r),jt[te].m(e,r),h(e,At,r),h(e,K,r),t(K,Hn),t(K,Ye),t(Ye,Bn),t(K,Un),t(K,Je),t(Je,Gn),t(K,Vn),t(K,Ke),t(Ke,Yn),t(K,Jn),h(e,ja,r),h(e,Ot,r),t(Ot,Kn),h(e,xa,r),j(Ze,e,r),h(e,Ea,r),j(Qe,e,r),h(e,Ta,r),h(e,I,r),t(I,Zn),t(I,cs),t(cs,Qn),t(I,Xn),t(I,hs),t(hs,eo),t(I,to),t(I,ds),t(ds,so),t(I,ao),t(I,fs),t(fs,no),t(I,oo),t(I,us),t(us,lo),t(I,ro),h(e,za,r),h(e,he,r),t(he,io),t(he,ms),t(ms,po),t(he,co),t(he,_s),t(_s,ho),t(he,fo),h(e,qa,r),h(e,ze,r),t(ze,uo),t(ze,ws),t(ws,mo),t(ze,_o),h(e,Da,r),j(Xe,e,r),h(e,Ca,r),j(et,e,r),h(e,Pa,r),h(e,de,r),t(de,wo),t(de,bs),t(bs,bo),t(de,go),t(de,gs),t(gs,ko),t(de,yo),h(e,Sa,r),j(tt,e,r),h(e,Aa,r),j(st,e,r),h(e,Oa,r),h(e,A,r),t(A,vo),t(A,ks),t(ks,$o),t(A,jo),t(A,ys),t(ys,xo),t(A,Eo),t(A,vs),t(vs,To),t(A,zo),t(A,$s),t($s,qo),t(A,Do),t(A,js),t(js,Co),t(A,Po),t(A,xs),t(xs,So),t(A,Ao),t(A,Es),t(Es,Oo),t(A,Wo),h(e,Wa,r),j(qe,e,r),h(e,Na,r),h(e,$e,r),t($e,De),t(De,Ts),j(at,Ts,null),t($e,No),t($e,zs),t(zs,Lo),h(e,La,r),xt[ae].m(e,r),h(e,Wt,r),h(e,Ce,r),t(Ce,Fo),t(Ce,Nt),t(Nt,Io),t(Ce,Mo),h(e,Fa,r),j(nt,e,r),h(e,Ia,r),h(e,Lt,r),t(Lt,Ro),h(e,Ma,r),j(ot,e,r),h(e,Ra,r),j(lt,e,r),h(e,Ha,r),h(e,U,r),t(U,Ho),t(U,qs),t(qs,Bo),t(U,Uo),t(U,Ds),t(Ds,Go),t(U,Vo),t(U,Ft),t(Ft,Yo),t(U,Jo),t(U,Cs),t(Cs,Ko),t(U,Zo),h(e,Ba,r),j(Pe,e,r),h(e,Ua,r),h(e,Se,r),t(Se,Qo),t(Se,Ps),t(Ps,Xo),t(Se,el),h(e,Ga,r),j(rt,e,r),h(e,Va,r),h(e,It,r),t(It,tl),h(e,Ya,r),j(it,e,r),h(e,Ja,r),h(e,fe,r),t(fe,sl),t(fe,Ss),t(Ss,al),t(fe,nl),t(fe,As),t(As,ol),t(fe,ll),h(e,Ka,r),j(pt,e,r),h(e,Za,r),h(e,G,r),t(G,rl),t(G,Os),t(Os,il),t(G,pl),t(G,Ws),t(Ws,cl),t(G,hl),t(G,Ns),t(Ns,dl),t(G,fl),t(G,Ls),t(Ls,ul),t(G,ml),h(e,Qa,r),h(e,Ae,r),t(Ae,_l),t(Ae,Fs),t(Fs,wl),t(Ae,bl),h(e,Xa,r),h(e,ue,r),t(ue,gl),t(ue,Mt),t(Mt,kl),t(ue,yl),t(ue,Is),t(Is,vl),t(ue,$l),h(e,en,r),h(e,Rt,r),t(Rt,jl),h(e,tn,r),h(e,Oe,r),t(Oe,xl),t(Oe,Ms),t(Ms,El),t(Oe,Tl),h(e,sn,r),h(e,me,r),t(me,zl),t(me,Ht),t(Ht,ql),t(me,Dl),t(me,Bt),t(Bt,Cl),t(me,Pl),h(e,an,r),j(ct,e,r),h(e,nn,r),h(e,V,r),t(V,Sl),t(V,Rs),t(Rs,Al),t(V,Ol),t(V,Hs),t(Hs,Wl),t(V,Nl),t(V,Bs),t(Bs,Ll),t(V,Fl),t(V,ht),t(ht,Il),t(V,Ml),h(e,on,r),h(e,_e,r),t(_e,Rl),t(_e,dt),t(dt,Us),t(Us,Hl),t(_e,Bl),t(_e,Gs),t(Gs,Ul),t(_e,Gl),h(e,ln,r),j(ft,e,r),h(e,rn,r),h(e,S,r),t(S,Vl),t(S,Vs),t(Vs,Yl),t(S,Jl),t(S,Ys),t(Ys,Kl),t(S,Zl),t(S,Js),t(Js,Ql),t(S,Xl),t(S,Ks),t(Ks,er),t(S,tr),t(S,Zs),t(Zs,sr),t(S,ar),t(S,Qs),t(Qs,nr),t(S,or),t(S,Xs),t(Xs,lr),t(S,rr),t(S,ea),t(ea,ir),t(S,pr),t(S,ut),t(ut,cr),t(S,hr),h(e,pn,r),h(e,We,r),t(We,dr),t(We,ta),t(ta,fr),t(We,ur),h(e,cn,r),h(e,we,r),t(we,mr),t(we,sa),t(sa,_r),t(we,wr),t(we,aa),t(aa,br),t(we,gr),h(e,hn,r),j(mt,e,r),h(e,dn,r),h(e,Ut,r),t(Ut,kr),h(e,fn,r),j(_t,e,r),h(e,un,r),h(e,be,r),t(be,yr),t(be,na),t(na,vr),t(be,$r),t(be,oa),t(oa,jr),t(be,xr),h(e,mn,r),h(e,M,r),t(M,Er),t(M,la),t(la,Tr),t(M,zr),t(M,ra),t(ra,qr),t(M,Dr),t(M,ia),t(ia,Cr),t(M,Pr),t(M,pa),t(pa,Sr),t(M,Ar),t(M,ca),t(ca,Or),t(M,Wr),h(e,_n,r),h(e,Ne,r),t(Ne,Nr),t(Ne,ha),t(ha,Lr),t(Ne,Fr),h(e,wn,r),h(e,je,r),t(je,Le),t(Le,da),j(wt,da,null),t(je,Ir),t(je,fa),t(fa,Mr),h(e,bn,r),j(bt,e,r),h(e,gn,r),xe.m(e,r),h(e,Gt,r),h(e,Fe,r),t(Fe,Rr),t(Fe,ua),t(ua,Hr),t(Fe,Br),h(e,kn,r),Et[oe].m(e,r),h(e,Vt,r),h(e,Z,r),t(Z,Ur),t(Z,ma),t(ma,Gr),t(Z,Vr),t(Z,_a),t(_a,Yr),t(Z,Jr),t(Z,wa),t(wa,Kr),t(Z,Zr),h(e,yn,r),j(gt,e,r),h(e,vn,r),j(kt,e,r),h(e,$n,r),h(e,Ie,r),t(Ie,Qr),t(Ie,ba),t(ba,Xr),t(Ie,ei),h(e,jn,r),j(yt,e,r),h(e,xn,r),Tt[re].m(e,r),h(e,Yt,r),j(Me,e,r),h(e,En,r),N&&N.m(e,r),h(e,Jt,r),Tn=!0},p(e,[r]){const zt={};r&1&&(zt.fw=e[0]),l.$set(zt);let Kt=H;H=ai(e),H!==Kt&&(Dt(),g(vt[Kt],1,1,()=>{vt[Kt]=null}),qt(),B=vt[H],B||(B=vt[H]=si[H](e),B.c()),b(B,1),B.m(ee.parentNode,ee));let Zt=W;W=oi(e),W!==Zt&&(Dt(),g($t[Zt],1,1,()=>{$t[Zt]=null}),qt(),F=$t[W],F||(F=$t[W]=ni[W](e),F.c()),b(F,1),F.m(J.parentNode,J));let Qt=te;te=ri(e),te!==Qt&&(Dt(),g(jt[Qt],1,1,()=>{jt[Qt]=null}),qt(),se=jt[te],se||(se=jt[te]=li[te](e),se.c()),b(se,1),se.m(At.parentNode,At));const ga={};r&2&&(ga.$$scope={dirty:r,ctx:e}),qe.$set(ga);let Re=ae;ae=pi(e),ae!==Re&&(Dt(),g(xt[Re],1,1,()=>{xt[Re]=null}),qt(),ne=xt[ae],ne||(ne=xt[ae]=ii[ae](e),ne.c()),b(ne,1),ne.m(Wt.parentNode,Wt));const ka={};r&2&&(ka.$$scope={dirty:r,ctx:e}),Pe.$set(ka),zn!==(zn=ci(e))&&(xe.d(1),xe=zn(e),xe&&(xe.c(),xe.m(Gt.parentNode,Gt)));let He=oe;oe=di(e),oe!==He&&(Dt(),g(Et[He],1,1,()=>{Et[He]=null}),qt(),le=Et[oe],le||(le=Et[oe]=hi[oe](e),le.c()),b(le,1),le.m(Vt.parentNode,Vt));let Xt=re;re=ui(e),re!==Xt&&(Dt(),g(Tt[Xt],1,1,()=>{Tt[Xt]=null}),qt(),ie=Tt[re],ie||(ie=Tt[re]=fi[re](e),ie.c()),b(ie,1),ie.m(Yt.parentNode,Yt));const ya={};r&2&&(ya.$$scope={dirty:r,ctx:e}),Me.$set(ya),e[0]==="tf"?N?r&1&&b(N,1):(N=Bp(),N.c(),b(N,1),N.m(Jt.parentNode,Jt)):N&&(Dt(),g(N,1,1,()=>{N=null}),qt())},i(e){Tn||(b(l.$$.fragment,e),b(z.$$.fragment,e),b(B),b(F),b(Ve.$$.fragment,e),b(se),b(Ze.$$.fragment,e),b(Qe.$$.fragment,e),b(Xe.$$.fragment,e),b(et.$$.fragment,e),b(tt.$$.fragment,e),b(st.$$.fragment,e),b(qe.$$.fragment,e),b(at.$$.fragment,e),b(ne),b(nt.$$.fragment,e),b(ot.$$.fragment,e),b(lt.$$.fragment,e),b(Pe.$$.fragment,e),b(rt.$$.fragment,e),b(it.$$.fragment,e),b(pt.$$.fragment,e),b(ct.$$.fragment,e),b(ft.$$.fragment,e),b(mt.$$.fragment,e),b(_t.$$.fragment,e),b(wt.$$.fragment,e),b(bt.$$.fragment,e),b(le),b(gt.$$.fragment,e),b(kt.$$.fragment,e),b(yt.$$.fragment,e),b(ie),b(Me.$$.fragment,e),b(N),Tn=!0)},o(e){g(l.$$.fragment,e),g(z.$$.fragment,e),g(B),g(F),g(Ve.$$.fragment,e),g(se),g(Ze.$$.fragment,e),g(Qe.$$.fragment,e),g(Xe.$$.fragment,e),g(et.$$.fragment,e),g(tt.$$.fragment,e),g(st.$$.fragment,e),g(qe.$$.fragment,e),g(at.$$.fragment,e),g(ne),g(nt.$$.fragment,e),g(ot.$$.fragment,e),g(lt.$$.fragment,e),g(Pe.$$.fragment,e),g(rt.$$.fragment,e),g(it.$$.fragment,e),g(pt.$$.fragment,e),g(ct.$$.fragment,e),g(ft.$$.fragment,e),g(mt.$$.fragment,e),g(_t.$$.fragment,e),g(wt.$$.fragment,e),g(bt.$$.fragment,e),g(le),g(gt.$$.fragment,e),g(kt.$$.fragment,e),g(yt.$$.fragment,e),g(ie),g(Me.$$.fragment,e),g(N),Tn=!1},d(e){s(o),e&&s(d),x(l,e),e&&s(w),e&&s(y),x(z),e&&s(Y),vt[H].d(e),e&&s(ee),$t[W].d(e),e&&s(J),e&&s(pe),e&&s(D),e&&s(P),e&&s(Ge),e&&s(ve),x(Ve),e&&s($a),jt[te].d(e),e&&s(At),e&&s(K),e&&s(ja),e&&s(Ot),e&&s(xa),x(Ze,e),e&&s(Ea),x(Qe,e),e&&s(Ta),e&&s(I),e&&s(za),e&&s(he),e&&s(qa),e&&s(ze),e&&s(Da),x(Xe,e),e&&s(Ca),x(et,e),e&&s(Pa),e&&s(de),e&&s(Sa),x(tt,e),e&&s(Aa),x(st,e),e&&s(Oa),e&&s(A),e&&s(Wa),x(qe,e),e&&s(Na),e&&s($e),x(at),e&&s(La),xt[ae].d(e),e&&s(Wt),e&&s(Ce),e&&s(Fa),x(nt,e),e&&s(Ia),e&&s(Lt),e&&s(Ma),x(ot,e),e&&s(Ra),x(lt,e),e&&s(Ha),e&&s(U),e&&s(Ba),x(Pe,e),e&&s(Ua),e&&s(Se),e&&s(Ga),x(rt,e),e&&s(Va),e&&s(It),e&&s(Ya),x(it,e),e&&s(Ja),e&&s(fe),e&&s(Ka),x(pt,e),e&&s(Za),e&&s(G),e&&s(Qa),e&&s(Ae),e&&s(Xa),e&&s(ue),e&&s(en),e&&s(Rt),e&&s(tn),e&&s(Oe),e&&s(sn),e&&s(me),e&&s(an),x(ct,e),e&&s(nn),e&&s(V),e&&s(on),e&&s(_e),e&&s(ln),x(ft,e),e&&s(rn),e&&s(S),e&&s(pn),e&&s(We),e&&s(cn),e&&s(we),e&&s(hn),x(mt,e),e&&s(dn),e&&s(Ut),e&&s(fn),x(_t,e),e&&s(un),e&&s(be),e&&s(mn),e&&s(M),e&&s(_n),e&&s(Ne),e&&s(wn),e&&s(je),x(wt),e&&s(bn),x(bt,e),e&&s(gn),xe.d(e),e&&s(Gt),e&&s(Fe),e&&s(kn),Et[oe].d(e),e&&s(Vt),e&&s(Z),e&&s(yn),x(gt,e),e&&s(vn),x(kt,e),e&&s($n),e&&s(Ie),e&&s(jn),x(yt,e),e&&s(xn),Tt[re].d(e),e&&s(Yt),x(Me,e),e&&s(En),N&&N.d(e),e&&s(Jt)}}}const _c={local:"processing-the-data",sections:[{local:"loading-a-dataset-from-the-hub",title:"Loading a dataset from the Hub"},{local:"preprocessing-a-dataset",title:"Preprocessing a dataset"},{local:"dynamic-padding",title:"Dynamic padding"}],title:"Processing the data"};function wc(E,o,d){let l="pt";return Kp(()=>{const w=new URLSearchParams(window.location.search);d(0,l=w.get("fw")||"pt")}),[l]}class xc extends Gp{constructor(o){super();Vp(this,o,wc,mc,Yp,{})}}export{xc as default,_c as metadata};
