import{S as jt,i as zt,s as Et,e as r,k as h,w as g,t as i,l as _t,M as It,c as p,d as s,m as f,x as k,a as c,h as u,b as q,G as n,g as a,y as b,o as d,p as vt,q as w,B as $,v as Pt,n as qt}from"../../chunks/vendor-hf-doc-builder.js";import{I as Js}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as y}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as yt}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as Tt}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function xt(j){let l,m;return l=new yt({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter2/section6_tf.ipynb"}]}}),{c(){g(l.$$.fragment)},l(o){k(l.$$.fragment,o)},m(o,_){b(l,o,_),m=!0},i(o){m||(w(l.$$.fragment,o),m=!0)},o(o){d(l.$$.fragment,o),m=!1},d(o){$(l,o)}}}function St(j){let l,m;return l=new yt({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter2/section6_pt.ipynb"}]}}),{c(){g(l.$$.fragment)},l(o){k(l.$$.fragment,o)},m(o,_){b(l,o,_),m=!0},i(o){m||(w(l.$$.fragment,o),m=!0)},o(o){d(l.$$.fragment,o),m=!1},d(o){$(l,o)}}}function At(j){let l,m;return l=new y({props:{code:`
`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`}}),{c(){g(l.$$.fragment)},l(o){k(l.$$.fragment,o)},m(o,_){b(l,o,_),m=!0},i(o){m||(w(l.$$.fragment,o),m=!0)},o(o){d(l.$$.fragment,o),m=!1},d(o){$(l,o)}}}function Ft(j){let l,m;return l=new y({props:{code:`
`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`}}),{c(){g(l.$$.fragment)},l(o){k(l.$$.fragment,o)},m(o,_){b(l,o,_),m=!0},i(o){m||(w(l.$$.fragment,o),m=!0)},o(o){d(l.$$.fragment,o),m=!1},d(o){$(l,o)}}}function Dt(j){let l,m,o,_,A,C,we,R,is,ge,ps,Se,z,E,ne,oe,us,Ae,N,cs,ke,hs,fs,Fe,B,De,T,ms,be,ds,ws,$e,gs,ks,Ce,ae,bs,Ne,L,He,le,$s,Oe,M,We,re,_s,Re,G,Be,ie,vs,Le,J,Me,v,qs,_e,ys,js,ve,zs,Es,qe,Is,Ps,ye,Ts,xs,Ge,U,Je,F,H,je,K,Ss,ze,As,Ue,pe,Fs,Ke,Q,Qe,V,Ve,ue,Ds,Xe,X,Ye,Y,Ze,x,Cs,Ee,Ns,Hs,Ie,Os,Ws,es,D,O,Pe,Z,Rs,Te,Bs,ss,W,Ls,xe,Ms,Gs,ts,I,P,ce,ns;o=new Tt({props:{fw:j[0]}}),R=new Js({});const Us=[St,xt],ee=[];function Ks(e,t){return e[0]==="pt"?0:1}z=Ks(j),E=ee[z]=Us[z](j),B=new y({props:{code:`

`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`}}),L=new y({props:{code:"",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`}}),M=new y({props:{code:"",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`}}),G=new y({props:{code:`
`,highlighted:`<span class="hljs-comment"># Will pad the sequences up to the maximum sequence length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the specified max length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`}}),J=new y({props:{code:`
`,highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Will truncate the sequences that are longer than the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Will truncate the sequences that are longer than the specified max length</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`}}),U=new y({props:{code:`

`,highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Returns PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Returns TensorFlow tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Returns NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`}}),K=new Js({}),Q=new y({props:{code:`
`,highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`}}),V=new y({props:{code:`[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102]
[1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012]`,highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`}}),X=new y({props:{code:`print(tokenizer.decode(model_inputs["input_ids"]))
print(tokenizer.decode(ids))`,highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`}}),Y=new y({props:{code:`"[CLS] i've been waiting for a huggingface course my whole life. [SEP]"
"i've been waiting for a huggingface course my whole life."`,highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`}}),Z=new Js({});const Qs=[Ft,At],se=[];function Vs(e,t){return e[0]==="pt"?0:1}return I=Vs(j),P=se[I]=Qs[I](j),{c(){l=r("meta"),m=h(),g(o.$$.fragment),_=h(),A=r("h1"),C=r("a"),we=r("span"),g(R.$$.fragment),is=h(),ge=r("span"),ps=i("Putting it all together"),Se=h(),E.c(),ne=h(),oe=r("p"),us=i("In the last few sections, we\u2019ve been trying our best to do most of the work by hand. We\u2019ve explored how tokenizers work and looked at tokenization, conversion to input IDs, padding, truncation, and attention masks."),Ae=h(),N=r("p"),cs=i("However, as we saw in section 2, the \u{1F917} Transformers API can handle all of this for us with a high-level function that we\u2019ll dive into here. When you call your "),ke=r("code"),hs=i("tokenizer"),fs=i(" directly on the sentence, you get back inputs that are ready to pass through your model:"),Fe=h(),g(B.$$.fragment),De=h(),T=r("p"),ms=i("Here, the "),be=r("code"),ds=i("model_inputs"),ws=i(" variable contains everything that\u2019s necessary for a model to operate well. For DistilBERT, that includes the input IDs as well as the attention mask. Other models that accept additional inputs will also have those output by the "),$e=r("code"),gs=i("tokenizer"),ks=i(" object."),Ce=h(),ae=r("p"),bs=i("As we\u2019ll see in some examples below, this method is very powerful. First, it can tokenize a single sequence:"),Ne=h(),g(L.$$.fragment),He=h(),le=r("p"),$s=i("It also handles multiple sequences at a time, with no change in the API:"),Oe=h(),g(M.$$.fragment),We=h(),re=r("p"),_s=i("It can pad according to several objectives:"),Re=h(),g(G.$$.fragment),Be=h(),ie=r("p"),vs=i("It can also truncate sequences:"),Le=h(),g(J.$$.fragment),Me=h(),v=r("p"),qs=i("The "),_e=r("code"),ys=i("tokenizer"),js=i(" object can handle the conversion to specific framework tensors, which can then be directly sent to the model. For example, in the following code sample we are prompting the tokenizer to return tensors from the different frameworks \u2014 "),ve=r("code"),zs=i('"pt"'),Es=i(" returns PyTorch tensors, "),qe=r("code"),Is=i('"tf"'),Ps=i(" returns TensorFlow tensors, and "),ye=r("code"),Ts=i('"np"'),xs=i(" returns NumPy arrays:"),Ge=h(),g(U.$$.fragment),Je=h(),F=r("h2"),H=r("a"),je=r("span"),g(K.$$.fragment),Ss=h(),ze=r("span"),As=i("Special tokens"),Ue=h(),pe=r("p"),Fs=i("If we take a look at the input IDs returned by the tokenizer, we will see they are a tiny bit different from what we had earlier:"),Ke=h(),g(Q.$$.fragment),Qe=h(),g(V.$$.fragment),Ve=h(),ue=r("p"),Ds=i("One token ID was added at the beginning, and one at the end. Let\u2019s decode the two sequences of IDs above to see what this is about:"),Xe=h(),g(X.$$.fragment),Ye=h(),g(Y.$$.fragment),Ze=h(),x=r("p"),Cs=i("The tokenizer added the special word "),Ee=r("code"),Ns=i("[CLS]"),Hs=i(" at the beginning and the special word "),Ie=r("code"),Os=i("[SEP]"),Ws=i(" at the end. This is because the model was pretrained with those, so to get the same results for inference we need to add them as well. Note that some models don\u2019t add special words, or add different ones; models may also add these special words only at the beginning, or only at the end. In any case, the tokenizer knows which ones are expected and will deal with this for you."),es=h(),D=r("h2"),O=r("a"),Pe=r("span"),g(Z.$$.fragment),Rs=h(),Te=r("span"),Bs=i("Wrapping up: From tokenizer to model"),ss=h(),W=r("p"),Ls=i("Now that we\u2019ve seen all the individual steps the "),xe=r("code"),Ms=i("tokenizer"),Gs=i(" object uses when applied on texts, let\u2019s see one final time how it can handle multiple sequences (padding!), very long sequences (truncation!), and multiple types of tensors with its main API:"),ts=h(),P.c(),ce=_t(),this.h()},l(e){const t=It('[data-svelte="svelte-1phssyn"]',document.head);l=p(t,"META",{name:!0,content:!0}),t.forEach(s),m=f(e),k(o.$$.fragment,e),_=f(e),A=p(e,"H1",{class:!0});var te=c(A);C=p(te,"A",{id:!0,class:!0,href:!0});var he=c(C);we=p(he,"SPAN",{});var fe=c(we);k(R.$$.fragment,fe),fe.forEach(s),he.forEach(s),is=f(te),ge=p(te,"SPAN",{});var Xs=c(ge);ps=u(Xs,"Putting it all together"),Xs.forEach(s),te.forEach(s),Se=f(e),E.l(e),ne=f(e),oe=p(e,"P",{});var Ys=c(oe);us=u(Ys,"In the last few sections, we\u2019ve been trying our best to do most of the work by hand. We\u2019ve explored how tokenizers work and looked at tokenization, conversion to input IDs, padding, truncation, and attention masks."),Ys.forEach(s),Ae=f(e),N=p(e,"P",{});var os=c(N);cs=u(os,"However, as we saw in section 2, the \u{1F917} Transformers API can handle all of this for us with a high-level function that we\u2019ll dive into here. When you call your "),ke=p(os,"CODE",{});var Zs=c(ke);hs=u(Zs,"tokenizer"),Zs.forEach(s),fs=u(os," directly on the sentence, you get back inputs that are ready to pass through your model:"),os.forEach(s),Fe=f(e),k(B.$$.fragment,e),De=f(e),T=p(e,"P",{});var me=c(T);ms=u(me,"Here, the "),be=p(me,"CODE",{});var et=c(be);ds=u(et,"model_inputs"),et.forEach(s),ws=u(me," variable contains everything that\u2019s necessary for a model to operate well. For DistilBERT, that includes the input IDs as well as the attention mask. Other models that accept additional inputs will also have those output by the "),$e=p(me,"CODE",{});var st=c($e);gs=u(st,"tokenizer"),st.forEach(s),ks=u(me," object."),me.forEach(s),Ce=f(e),ae=p(e,"P",{});var tt=c(ae);bs=u(tt,"As we\u2019ll see in some examples below, this method is very powerful. First, it can tokenize a single sequence:"),tt.forEach(s),Ne=f(e),k(L.$$.fragment,e),He=f(e),le=p(e,"P",{});var nt=c(le);$s=u(nt,"It also handles multiple sequences at a time, with no change in the API:"),nt.forEach(s),Oe=f(e),k(M.$$.fragment,e),We=f(e),re=p(e,"P",{});var ot=c(re);_s=u(ot,"It can pad according to several objectives:"),ot.forEach(s),Re=f(e),k(G.$$.fragment,e),Be=f(e),ie=p(e,"P",{});var at=c(ie);vs=u(at,"It can also truncate sequences:"),at.forEach(s),Le=f(e),k(J.$$.fragment,e),Me=f(e),v=p(e,"P",{});var S=c(v);qs=u(S,"The "),_e=p(S,"CODE",{});var lt=c(_e);ys=u(lt,"tokenizer"),lt.forEach(s),js=u(S," object can handle the conversion to specific framework tensors, which can then be directly sent to the model. For example, in the following code sample we are prompting the tokenizer to return tensors from the different frameworks \u2014 "),ve=p(S,"CODE",{});var rt=c(ve);zs=u(rt,'"pt"'),rt.forEach(s),Es=u(S," returns PyTorch tensors, "),qe=p(S,"CODE",{});var it=c(qe);Is=u(it,'"tf"'),it.forEach(s),Ps=u(S," returns TensorFlow tensors, and "),ye=p(S,"CODE",{});var pt=c(ye);Ts=u(pt,'"np"'),pt.forEach(s),xs=u(S," returns NumPy arrays:"),S.forEach(s),Ge=f(e),k(U.$$.fragment,e),Je=f(e),F=p(e,"H2",{class:!0});var as=c(F);H=p(as,"A",{id:!0,class:!0,href:!0});var ut=c(H);je=p(ut,"SPAN",{});var ct=c(je);k(K.$$.fragment,ct),ct.forEach(s),ut.forEach(s),Ss=f(as),ze=p(as,"SPAN",{});var ht=c(ze);As=u(ht,"Special tokens"),ht.forEach(s),as.forEach(s),Ue=f(e),pe=p(e,"P",{});var ft=c(pe);Fs=u(ft,"If we take a look at the input IDs returned by the tokenizer, we will see they are a tiny bit different from what we had earlier:"),ft.forEach(s),Ke=f(e),k(Q.$$.fragment,e),Qe=f(e),k(V.$$.fragment,e),Ve=f(e),ue=p(e,"P",{});var mt=c(ue);Ds=u(mt,"One token ID was added at the beginning, and one at the end. Let\u2019s decode the two sequences of IDs above to see what this is about:"),mt.forEach(s),Xe=f(e),k(X.$$.fragment,e),Ye=f(e),k(Y.$$.fragment,e),Ze=f(e),x=p(e,"P",{});var de=c(x);Cs=u(de,"The tokenizer added the special word "),Ee=p(de,"CODE",{});var dt=c(Ee);Ns=u(dt,"[CLS]"),dt.forEach(s),Hs=u(de," at the beginning and the special word "),Ie=p(de,"CODE",{});var wt=c(Ie);Os=u(wt,"[SEP]"),wt.forEach(s),Ws=u(de," at the end. This is because the model was pretrained with those, so to get the same results for inference we need to add them as well. Note that some models don\u2019t add special words, or add different ones; models may also add these special words only at the beginning, or only at the end. In any case, the tokenizer knows which ones are expected and will deal with this for you."),de.forEach(s),es=f(e),D=p(e,"H2",{class:!0});var ls=c(D);O=p(ls,"A",{id:!0,class:!0,href:!0});var gt=c(O);Pe=p(gt,"SPAN",{});var kt=c(Pe);k(Z.$$.fragment,kt),kt.forEach(s),gt.forEach(s),Rs=f(ls),Te=p(ls,"SPAN",{});var bt=c(Te);Bs=u(bt,"Wrapping up: From tokenizer to model"),bt.forEach(s),ls.forEach(s),ss=f(e),W=p(e,"P",{});var rs=c(W);Ls=u(rs,"Now that we\u2019ve seen all the individual steps the "),xe=p(rs,"CODE",{});var $t=c(xe);Ms=u($t,"tokenizer"),$t.forEach(s),Gs=u(rs," object uses when applied on texts, let\u2019s see one final time how it can handle multiple sequences (padding!), very long sequences (truncation!), and multiple types of tensors with its main API:"),rs.forEach(s),ts=f(e),P.l(e),ce=_t(),this.h()},h(){q(l,"name","hf:doc:metadata"),q(l,"content",JSON.stringify(Ct)),q(C,"id","putting-it-all-together"),q(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),q(C,"href","#putting-it-all-together"),q(A,"class","relative group"),q(H,"id","special-tokens"),q(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),q(H,"href","#special-tokens"),q(F,"class","relative group"),q(O,"id","wrapping-up-from-tokenizer-to-model"),q(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),q(O,"href","#wrapping-up-from-tokenizer-to-model"),q(D,"class","relative group")},m(e,t){n(document.head,l),a(e,m,t),b(o,e,t),a(e,_,t),a(e,A,t),n(A,C),n(C,we),b(R,we,null),n(A,is),n(A,ge),n(ge,ps),a(e,Se,t),ee[z].m(e,t),a(e,ne,t),a(e,oe,t),n(oe,us),a(e,Ae,t),a(e,N,t),n(N,cs),n(N,ke),n(ke,hs),n(N,fs),a(e,Fe,t),b(B,e,t),a(e,De,t),a(e,T,t),n(T,ms),n(T,be),n(be,ds),n(T,ws),n(T,$e),n($e,gs),n(T,ks),a(e,Ce,t),a(e,ae,t),n(ae,bs),a(e,Ne,t),b(L,e,t),a(e,He,t),a(e,le,t),n(le,$s),a(e,Oe,t),b(M,e,t),a(e,We,t),a(e,re,t),n(re,_s),a(e,Re,t),b(G,e,t),a(e,Be,t),a(e,ie,t),n(ie,vs),a(e,Le,t),b(J,e,t),a(e,Me,t),a(e,v,t),n(v,qs),n(v,_e),n(_e,ys),n(v,js),n(v,ve),n(ve,zs),n(v,Es),n(v,qe),n(qe,Is),n(v,Ps),n(v,ye),n(ye,Ts),n(v,xs),a(e,Ge,t),b(U,e,t),a(e,Je,t),a(e,F,t),n(F,H),n(H,je),b(K,je,null),n(F,Ss),n(F,ze),n(ze,As),a(e,Ue,t),a(e,pe,t),n(pe,Fs),a(e,Ke,t),b(Q,e,t),a(e,Qe,t),b(V,e,t),a(e,Ve,t),a(e,ue,t),n(ue,Ds),a(e,Xe,t),b(X,e,t),a(e,Ye,t),b(Y,e,t),a(e,Ze,t),a(e,x,t),n(x,Cs),n(x,Ee),n(Ee,Ns),n(x,Hs),n(x,Ie),n(Ie,Os),n(x,Ws),a(e,es,t),a(e,D,t),n(D,O),n(O,Pe),b(Z,Pe,null),n(D,Rs),n(D,Te),n(Te,Bs),a(e,ss,t),a(e,W,t),n(W,Ls),n(W,xe),n(xe,Ms),n(W,Gs),a(e,ts,t),se[I].m(e,t),a(e,ce,t),ns=!0},p(e,[t]){const te={};t&1&&(te.fw=e[0]),o.$set(te);let he=z;z=Ks(e),z!==he&&(qt(),d(ee[he],1,1,()=>{ee[he]=null}),vt(),E=ee[z],E||(E=ee[z]=Us[z](e),E.c()),w(E,1),E.m(ne.parentNode,ne));let fe=I;I=Vs(e),I!==fe&&(qt(),d(se[fe],1,1,()=>{se[fe]=null}),vt(),P=se[I],P||(P=se[I]=Qs[I](e),P.c()),w(P,1),P.m(ce.parentNode,ce))},i(e){ns||(w(o.$$.fragment,e),w(R.$$.fragment,e),w(E),w(B.$$.fragment,e),w(L.$$.fragment,e),w(M.$$.fragment,e),w(G.$$.fragment,e),w(J.$$.fragment,e),w(U.$$.fragment,e),w(K.$$.fragment,e),w(Q.$$.fragment,e),w(V.$$.fragment,e),w(X.$$.fragment,e),w(Y.$$.fragment,e),w(Z.$$.fragment,e),w(P),ns=!0)},o(e){d(o.$$.fragment,e),d(R.$$.fragment,e),d(E),d(B.$$.fragment,e),d(L.$$.fragment,e),d(M.$$.fragment,e),d(G.$$.fragment,e),d(J.$$.fragment,e),d(U.$$.fragment,e),d(K.$$.fragment,e),d(Q.$$.fragment,e),d(V.$$.fragment,e),d(X.$$.fragment,e),d(Y.$$.fragment,e),d(Z.$$.fragment,e),d(P),ns=!1},d(e){s(l),e&&s(m),$(o,e),e&&s(_),e&&s(A),$(R),e&&s(Se),ee[z].d(e),e&&s(ne),e&&s(oe),e&&s(Ae),e&&s(N),e&&s(Fe),$(B,e),e&&s(De),e&&s(T),e&&s(Ce),e&&s(ae),e&&s(Ne),$(L,e),e&&s(He),e&&s(le),e&&s(Oe),$(M,e),e&&s(We),e&&s(re),e&&s(Re),$(G,e),e&&s(Be),e&&s(ie),e&&s(Le),$(J,e),e&&s(Me),e&&s(v),e&&s(Ge),$(U,e),e&&s(Je),e&&s(F),$(K),e&&s(Ue),e&&s(pe),e&&s(Ke),$(Q,e),e&&s(Qe),$(V,e),e&&s(Ve),e&&s(ue),e&&s(Xe),$(X,e),e&&s(Ye),$(Y,e),e&&s(Ze),e&&s(x),e&&s(es),e&&s(D),$(Z),e&&s(ss),e&&s(W),e&&s(ts),se[I].d(e),e&&s(ce)}}}const Ct={local:"putting-it-all-together",sections:[{local:"special-tokens",title:"Special tokens"},{local:"wrapping-up-from-tokenizer-to-model",title:"Wrapping up: From tokenizer to model"}],title:"Putting it all together"};function Nt(j,l,m){let o="pt";return Pt(()=>{const _=new URLSearchParams(window.location.search);m(0,o=_.get("fw")||"pt")}),[o]}class Lt extends jt{constructor(l){super();zt(this,l,Nt,Dt,Et,{})}}export{Lt as default,Ct as metadata};
