import{S as yn,i as En,s as Tn,e as l,k as m,w as b,t as i,l as vn,M as Pn,c as u,d as s,m as d,x as k,a as c,h as p,b as j,G as o,g as t,y as _,o as h,p as jn,q as g,B as q,v as Sn,n as wn}from"../../chunks/vendor-hf-doc-builder.js";import{I as Js}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as w}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as zn}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as In}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function An(z){let r,f;return r=new zn({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"}]}}),{c(){b(r.$$.fragment)},l(a){k(r.$$.fragment,a)},m(a,$){_(r,a,$),f=!0},i(a){f||(g(r.$$.fragment,a),f=!0)},o(a){h(r.$$.fragment,a),f=!1},d(a){q(r,a)}}}function xn(z){let r,f;return r=new zn({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"}]}}),{c(){b(r.$$.fragment)},l(a){k(r.$$.fragment,a)},m(a,$){_(r,a,$),f=!0},i(a){f||(g(r.$$.fragment,a),f=!0)},o(a){h(r.$$.fragment,a),f=!1},d(a){q(r,a)}}}function Cn(z){let r,f;return r=new w({props:{code:`import tensorflow as tf
from transformers import AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

tokens = tokenizer(sequences, padding=True, truncation=True, return_tensors="tf")
output = model(**tokens)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`}}),{c(){b(r.$$.fragment)},l(a){k(r.$$.fragment,a)},m(a,$){_(r,a,$),f=!0},i(a){f||(g(r.$$.fragment,a),f=!0)},o(a){h(r.$$.fragment,a),f=!1},d(a){q(r,a)}}}function Fn(z){let r,f;return r=new w({props:{code:`import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification

checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

tokens = tokenizer(sequences, padding=True, truncation=True, return_tensors="pt")
output = model(**tokens)`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`}}),{c(){b(r.$$.fragment)},l(a){k(r.$$.fragment,a)},m(a,$){_(r,a,$),f=!0},i(a){f||(g(r.$$.fragment,a),f=!0)},o(a){h(r.$$.fragment,a),f=!1},d(a){q(r,a)}}}function Dn(z){let r,f,a,$,x,D,ge,O,is,be,us,Ae,y,E,oe,ae,ps,xe,H,cs,ke,ms,ds,Ce,M,Fe,S,fs,_e,hs,gs,qe,bs,ks,De,te,_s,He,W,Ne,re,qs,Re,L,Be,le,$s,Oe,G,Me,ie,vs,We,J,Le,v,js,$e,ws,zs,ve,ys,Es,je,Ts,Ps,we,Ss,Is,Ge,U,Je,C,N,ze,K,As,ye,xs,Ue,ue,Cs,Ke,Q,Qe,V,Ve,pe,Fs,Xe,X,Ye,Y,Ze,I,Ds,Ee,Hs,Ns,Te,Rs,Bs,es,F,R,Pe,Z,Os,Se,Ms,ss,B,Ws,Ie,Ls,Gs,ns,T,P,ce,os;a=new In({props:{fw:z[0]}}),O=new Js({});const Us=[xn,An],ee=[];function Ks(e,n){return e[0]==="pt"?0:1}y=Ks(z),E=ee[y]=Us[y](z),M=new w({props:{code:`from transformers import AutoTokenizer

checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = "I've been waiting for a HuggingFace course my whole life."

model_inputs = tokenizer(sequence)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`}}),W=new w({props:{code:`sequence = "I've been waiting for a HuggingFace course my whole life."

model_inputs = tokenizer(sequence)`,highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`}}),L=new w({props:{code:`sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

model_inputs = tokenizer(sequences)`,highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`}}),G=new w({props:{code:`# Will pad the sequences up to the maximum sequence length
model_inputs = tokenizer(sequences, padding="longest")

# Will pad the sequences up to the model max length
# (512 for BERT or DistilBERT)
model_inputs = tokenizer(sequences, padding="max_length")

# Will pad the sequences up to the specified max length
model_inputs = tokenizer(sequences, padding="max_length", max_length=8)`,highlighted:`<span class="hljs-comment"># Will pad the sequences up to the maximum sequence length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the specified max length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`}}),J=new w({props:{code:`sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

# Will truncate the sequences that are longer than the model max length
# (512 for BERT or DistilBERT)
model_inputs = tokenizer(sequences, truncation=True)

# Will truncate the sequences that are longer than the specified max length
model_inputs = tokenizer(sequences, max_length=8, truncation=True)`,highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Will truncate the sequences that are longer than the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Will truncate the sequences that are longer than the specified max length</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`}}),U=new w({props:{code:`sequences = ["I've been waiting for a HuggingFace course my whole life.", "So have I!"]

# Returns PyTorch tensors
model_inputs = tokenizer(sequences, padding=True, return_tensors="pt")

# Returns TensorFlow tensors
model_inputs = tokenizer(sequences, padding=True, return_tensors="tf")

# Returns NumPy arrays
model_inputs = tokenizer(sequences, padding=True, return_tensors="np")`,highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Returns PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Returns TensorFlow tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Returns NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`}}),K=new Js({}),Q=new w({props:{code:`sequence = "I've been waiting for a HuggingFace course my whole life."

model_inputs = tokenizer(sequence)
print(model_inputs["input_ids"])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
print(ids)`,highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`}}),V=new w({props:{code:`[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102]
[1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012]`,highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`}}),X=new w({props:{code:`print(tokenizer.decode(model_inputs["input_ids"]))
print(tokenizer.decode(ids))`,highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`}}),Y=new w({props:{code:`"[CLS] i've been waiting for a huggingface course my whole life. [SEP]"
"i've been waiting for a huggingface course my whole life."`,highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`}}),Z=new Js({});const Qs=[Fn,Cn],se=[];function Vs(e,n){return e[0]==="pt"?0:1}return T=Vs(z),P=se[T]=Qs[T](z),{c(){r=l("meta"),f=m(),b(a.$$.fragment),$=m(),x=l("h1"),D=l("a"),ge=l("span"),b(O.$$.fragment),is=m(),be=l("span"),us=i("Poniendo todo junto"),Ae=m(),E.c(),oe=m(),ae=l("p"),ps=i("En las \xFAltimas secciones, hemos hecho nuestro mejor esfuerzo para realizar la mayor parte del trabajo a mano. Exploramos como funcionan los tokenizadores y vimos la tokenizaci\xF3n, conversi\xF3n a IDs de entrada, relleno, truncado, y m\xE1scaras de atenci\xF3n."),xe=m(),H=l("p"),cs=i("Sin embargo, como vimos en la secci\xF3n 3, la API de transformadores \u{1F917} puede manejar todo esto por nosotros con una funci\xF3n de alto nivel la cual trataremos aqu\xED. Cuando llamas a tu "),ke=l("code"),ms=i("tokenizer"),ds=i(" directamente en una sentencia, obtienes entradas que est\xE1n lista para pasar a tu modelo:"),Ce=m(),b(M.$$.fragment),Fe=m(),S=l("p"),fs=i("Aqu\xED la varibale "),_e=l("code"),hs=i("model_inputs"),gs=i(" contiene todo lo necesario para que un modelo opere bien. Para DistilBERT, que incluye los IDs de entrada tambi\xE9n como la m\xE1scara de atenci\xF3n. Otros modelos que aceptan entradas adicionales tambi\xE9n tendr\xE1n las salidas del objeto "),qe=l("code"),bs=i("tokenizer"),ks=i("."),De=m(),te=l("p"),_s=i("Como veremos en los ejemplos de abajo, este m\xE9todo es muy poderoso. Primero, puede tokenizar una sola secuencia:"),He=m(),b(W.$$.fragment),Ne=m(),re=l("p"),qs=i("Tambi\xE9n maneja m\xFAltiples secuencias a la vez, sin cambios en la API:"),Re=m(),b(L.$$.fragment),Be=m(),le=l("p"),$s=i("Puede rellenar de acuerdo a varios objetivos:"),Oe=m(),b(G.$$.fragment),Me=m(),ie=l("p"),vs=i("Tambi\xE9n puede truncar secuencias:"),We=m(),b(J.$$.fragment),Le=m(),v=l("p"),js=i("El objeto "),$e=l("code"),ws=i("tokenizer"),zs=i(" puede manejar la conversi\xF3n a tensores de frameworks espec\xEDficos, los cuales pueden ser enviados directametne al modelo. Por ejemplo, en el siguiente c\xF3digo de ejemplo estamos solicitando al tokenizer que regrese los tensores de los distintos frameworks \u2014 "),ve=l("code"),ys=i('"pt"'),Es=i(" regresa tensores de PyTorch, "),je=l("code"),Ts=i('"tf"'),Ps=i(" regresa tensores de TensorFlow, y "),we=l("code"),Ss=i('"np"'),Is=i(" regresa arreglos de NumPy:"),Ge=m(),b(U.$$.fragment),Je=m(),C=l("h2"),N=l("a"),ze=l("span"),b(K.$$.fragment),As=m(),ye=l("span"),xs=i("Tokens especiales"),Ue=m(),ue=l("p"),Cs=i("Si damos un vistazo a los IDs de entrada retornados por el tokenizer, veremos que son un poquito diferentes a lo que ten\xEDamos anteriormente:"),Ke=m(),b(Q.$$.fragment),Qe=m(),b(V.$$.fragment),Ve=m(),pe=l("p"),Fs=i("Se agreg\xF3 un ID de token al principio, y uno al final. Decodifiquemos las dos secuencias de IDs de arriba para ver de que se trata:"),Xe=m(),b(X.$$.fragment),Ye=m(),b(Y.$$.fragment),Ze=m(),I=l("p"),Ds=i("El tokenizador agreg\xF3 la palabra especial "),Ee=l("code"),Hs=i("[CLS]"),Ns=i(" al principio y la palabra especial "),Te=l("code"),Rs=i("[SEP]"),Bs=i(" al final. Esto se debe a que el modelo fue preentrenado con esos, as\xED para obtener los mismos resultados por inferencia necesitamos agregarlos tambi\xE9n. Nota que algunos modelos no agregan palabras especiales, o agregan unas distintas;  los modelos tambi\xE9n pueden agregar estas palabras especiales s\xF3lo al principio, o s\xF3lo al final. En cualquier caso, el tokenizador sabe cu\xE1les son las esperadas y se encargar\xE1 de ello por t\xED."),es=m(),F=l("h2"),R=l("a"),Pe=l("span"),b(Z.$$.fragment),Os=m(),Se=l("span"),Ms=i("Conclusi\xF3n: Del tokenizador al moelo"),ss=m(),B=l("p"),Ws=i("Ahora que hemos visto todos los pasos individuales que el objeto "),Ie=l("code"),Ls=i("tokenizer"),Gs=i(" usa cuando se aplica a textos, veamos una \xFAltima vez c\xF3mo maneja varias secuencias (\xA1relleno!), secuencias muy largas (\xA1truncado!), y m\xFAltiples tipos de tensores con su API principal:"),ns=m(),P.c(),ce=vn(),this.h()},l(e){const n=Pn('[data-svelte="svelte-1phssyn"]',document.head);r=u(n,"META",{name:!0,content:!0}),n.forEach(s),f=d(e),k(a.$$.fragment,e),$=d(e),x=u(e,"H1",{class:!0});var ne=c(x);D=u(ne,"A",{id:!0,class:!0,href:!0});var me=c(D);ge=u(me,"SPAN",{});var de=c(ge);k(O.$$.fragment,de),de.forEach(s),me.forEach(s),is=d(ne),be=u(ne,"SPAN",{});var Xs=c(be);us=p(Xs,"Poniendo todo junto"),Xs.forEach(s),ne.forEach(s),Ae=d(e),E.l(e),oe=d(e),ae=u(e,"P",{});var Ys=c(ae);ps=p(Ys,"En las \xFAltimas secciones, hemos hecho nuestro mejor esfuerzo para realizar la mayor parte del trabajo a mano. Exploramos como funcionan los tokenizadores y vimos la tokenizaci\xF3n, conversi\xF3n a IDs de entrada, relleno, truncado, y m\xE1scaras de atenci\xF3n."),Ys.forEach(s),xe=d(e),H=u(e,"P",{});var as=c(H);cs=p(as,"Sin embargo, como vimos en la secci\xF3n 3, la API de transformadores \u{1F917} puede manejar todo esto por nosotros con una funci\xF3n de alto nivel la cual trataremos aqu\xED. Cuando llamas a tu "),ke=u(as,"CODE",{});var Zs=c(ke);ms=p(Zs,"tokenizer"),Zs.forEach(s),ds=p(as," directamente en una sentencia, obtienes entradas que est\xE1n lista para pasar a tu modelo:"),as.forEach(s),Ce=d(e),k(M.$$.fragment,e),Fe=d(e),S=u(e,"P",{});var fe=c(S);fs=p(fe,"Aqu\xED la varibale "),_e=u(fe,"CODE",{});var en=c(_e);hs=p(en,"model_inputs"),en.forEach(s),gs=p(fe," contiene todo lo necesario para que un modelo opere bien. Para DistilBERT, que incluye los IDs de entrada tambi\xE9n como la m\xE1scara de atenci\xF3n. Otros modelos que aceptan entradas adicionales tambi\xE9n tendr\xE1n las salidas del objeto "),qe=u(fe,"CODE",{});var sn=c(qe);bs=p(sn,"tokenizer"),sn.forEach(s),ks=p(fe,"."),fe.forEach(s),De=d(e),te=u(e,"P",{});var nn=c(te);_s=p(nn,"Como veremos en los ejemplos de abajo, este m\xE9todo es muy poderoso. Primero, puede tokenizar una sola secuencia:"),nn.forEach(s),He=d(e),k(W.$$.fragment,e),Ne=d(e),re=u(e,"P",{});var on=c(re);qs=p(on,"Tambi\xE9n maneja m\xFAltiples secuencias a la vez, sin cambios en la API:"),on.forEach(s),Re=d(e),k(L.$$.fragment,e),Be=d(e),le=u(e,"P",{});var an=c(le);$s=p(an,"Puede rellenar de acuerdo a varios objetivos:"),an.forEach(s),Oe=d(e),k(G.$$.fragment,e),Me=d(e),ie=u(e,"P",{});var tn=c(ie);vs=p(tn,"Tambi\xE9n puede truncar secuencias:"),tn.forEach(s),We=d(e),k(J.$$.fragment,e),Le=d(e),v=u(e,"P",{});var A=c(v);js=p(A,"El objeto "),$e=u(A,"CODE",{});var rn=c($e);ws=p(rn,"tokenizer"),rn.forEach(s),zs=p(A," puede manejar la conversi\xF3n a tensores de frameworks espec\xEDficos, los cuales pueden ser enviados directametne al modelo. Por ejemplo, en el siguiente c\xF3digo de ejemplo estamos solicitando al tokenizer que regrese los tensores de los distintos frameworks \u2014 "),ve=u(A,"CODE",{});var ln=c(ve);ys=p(ln,'"pt"'),ln.forEach(s),Es=p(A," regresa tensores de PyTorch, "),je=u(A,"CODE",{});var un=c(je);Ts=p(un,'"tf"'),un.forEach(s),Ps=p(A," regresa tensores de TensorFlow, y "),we=u(A,"CODE",{});var pn=c(we);Ss=p(pn,'"np"'),pn.forEach(s),Is=p(A," regresa arreglos de NumPy:"),A.forEach(s),Ge=d(e),k(U.$$.fragment,e),Je=d(e),C=u(e,"H2",{class:!0});var ts=c(C);N=u(ts,"A",{id:!0,class:!0,href:!0});var cn=c(N);ze=u(cn,"SPAN",{});var mn=c(ze);k(K.$$.fragment,mn),mn.forEach(s),cn.forEach(s),As=d(ts),ye=u(ts,"SPAN",{});var dn=c(ye);xs=p(dn,"Tokens especiales"),dn.forEach(s),ts.forEach(s),Ue=d(e),ue=u(e,"P",{});var fn=c(ue);Cs=p(fn,"Si damos un vistazo a los IDs de entrada retornados por el tokenizer, veremos que son un poquito diferentes a lo que ten\xEDamos anteriormente:"),fn.forEach(s),Ke=d(e),k(Q.$$.fragment,e),Qe=d(e),k(V.$$.fragment,e),Ve=d(e),pe=u(e,"P",{});var hn=c(pe);Fs=p(hn,"Se agreg\xF3 un ID de token al principio, y uno al final. Decodifiquemos las dos secuencias de IDs de arriba para ver de que se trata:"),hn.forEach(s),Xe=d(e),k(X.$$.fragment,e),Ye=d(e),k(Y.$$.fragment,e),Ze=d(e),I=u(e,"P",{});var he=c(I);Ds=p(he,"El tokenizador agreg\xF3 la palabra especial "),Ee=u(he,"CODE",{});var gn=c(Ee);Hs=p(gn,"[CLS]"),gn.forEach(s),Ns=p(he," al principio y la palabra especial "),Te=u(he,"CODE",{});var bn=c(Te);Rs=p(bn,"[SEP]"),bn.forEach(s),Bs=p(he," al final. Esto se debe a que el modelo fue preentrenado con esos, as\xED para obtener los mismos resultados por inferencia necesitamos agregarlos tambi\xE9n. Nota que algunos modelos no agregan palabras especiales, o agregan unas distintas;  los modelos tambi\xE9n pueden agregar estas palabras especiales s\xF3lo al principio, o s\xF3lo al final. En cualquier caso, el tokenizador sabe cu\xE1les son las esperadas y se encargar\xE1 de ello por t\xED."),he.forEach(s),es=d(e),F=u(e,"H2",{class:!0});var rs=c(F);R=u(rs,"A",{id:!0,class:!0,href:!0});var kn=c(R);Pe=u(kn,"SPAN",{});var _n=c(Pe);k(Z.$$.fragment,_n),_n.forEach(s),kn.forEach(s),Os=d(rs),Se=u(rs,"SPAN",{});var qn=c(Se);Ms=p(qn,"Conclusi\xF3n: Del tokenizador al moelo"),qn.forEach(s),rs.forEach(s),ss=d(e),B=u(e,"P",{});var ls=c(B);Ws=p(ls,"Ahora que hemos visto todos los pasos individuales que el objeto "),Ie=u(ls,"CODE",{});var $n=c(Ie);Ls=p($n,"tokenizer"),$n.forEach(s),Gs=p(ls," usa cuando se aplica a textos, veamos una \xFAltima vez c\xF3mo maneja varias secuencias (\xA1relleno!), secuencias muy largas (\xA1truncado!), y m\xFAltiples tipos de tensores con su API principal:"),ls.forEach(s),ns=d(e),P.l(e),ce=vn(),this.h()},h(){j(r,"name","hf:doc:metadata"),j(r,"content",JSON.stringify(Hn)),j(D,"id","poniendo-todo-junto"),j(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(D,"href","#poniendo-todo-junto"),j(x,"class","relative group"),j(N,"id","tokens-especiales"),j(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(N,"href","#tokens-especiales"),j(C,"class","relative group"),j(R,"id","conclusin-del-tokenizador-al-moelo"),j(R,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(R,"href","#conclusin-del-tokenizador-al-moelo"),j(F,"class","relative group")},m(e,n){o(document.head,r),t(e,f,n),_(a,e,n),t(e,$,n),t(e,x,n),o(x,D),o(D,ge),_(O,ge,null),o(x,is),o(x,be),o(be,us),t(e,Ae,n),ee[y].m(e,n),t(e,oe,n),t(e,ae,n),o(ae,ps),t(e,xe,n),t(e,H,n),o(H,cs),o(H,ke),o(ke,ms),o(H,ds),t(e,Ce,n),_(M,e,n),t(e,Fe,n),t(e,S,n),o(S,fs),o(S,_e),o(_e,hs),o(S,gs),o(S,qe),o(qe,bs),o(S,ks),t(e,De,n),t(e,te,n),o(te,_s),t(e,He,n),_(W,e,n),t(e,Ne,n),t(e,re,n),o(re,qs),t(e,Re,n),_(L,e,n),t(e,Be,n),t(e,le,n),o(le,$s),t(e,Oe,n),_(G,e,n),t(e,Me,n),t(e,ie,n),o(ie,vs),t(e,We,n),_(J,e,n),t(e,Le,n),t(e,v,n),o(v,js),o(v,$e),o($e,ws),o(v,zs),o(v,ve),o(ve,ys),o(v,Es),o(v,je),o(je,Ts),o(v,Ps),o(v,we),o(we,Ss),o(v,Is),t(e,Ge,n),_(U,e,n),t(e,Je,n),t(e,C,n),o(C,N),o(N,ze),_(K,ze,null),o(C,As),o(C,ye),o(ye,xs),t(e,Ue,n),t(e,ue,n),o(ue,Cs),t(e,Ke,n),_(Q,e,n),t(e,Qe,n),_(V,e,n),t(e,Ve,n),t(e,pe,n),o(pe,Fs),t(e,Xe,n),_(X,e,n),t(e,Ye,n),_(Y,e,n),t(e,Ze,n),t(e,I,n),o(I,Ds),o(I,Ee),o(Ee,Hs),o(I,Ns),o(I,Te),o(Te,Rs),o(I,Bs),t(e,es,n),t(e,F,n),o(F,R),o(R,Pe),_(Z,Pe,null),o(F,Os),o(F,Se),o(Se,Ms),t(e,ss,n),t(e,B,n),o(B,Ws),o(B,Ie),o(Ie,Ls),o(B,Gs),t(e,ns,n),se[T].m(e,n),t(e,ce,n),os=!0},p(e,[n]){const ne={};n&1&&(ne.fw=e[0]),a.$set(ne);let me=y;y=Ks(e),y!==me&&(wn(),h(ee[me],1,1,()=>{ee[me]=null}),jn(),E=ee[y],E||(E=ee[y]=Us[y](e),E.c()),g(E,1),E.m(oe.parentNode,oe));let de=T;T=Vs(e),T!==de&&(wn(),h(se[de],1,1,()=>{se[de]=null}),jn(),P=se[T],P||(P=se[T]=Qs[T](e),P.c()),g(P,1),P.m(ce.parentNode,ce))},i(e){os||(g(a.$$.fragment,e),g(O.$$.fragment,e),g(E),g(M.$$.fragment,e),g(W.$$.fragment,e),g(L.$$.fragment,e),g(G.$$.fragment,e),g(J.$$.fragment,e),g(U.$$.fragment,e),g(K.$$.fragment,e),g(Q.$$.fragment,e),g(V.$$.fragment,e),g(X.$$.fragment,e),g(Y.$$.fragment,e),g(Z.$$.fragment,e),g(P),os=!0)},o(e){h(a.$$.fragment,e),h(O.$$.fragment,e),h(E),h(M.$$.fragment,e),h(W.$$.fragment,e),h(L.$$.fragment,e),h(G.$$.fragment,e),h(J.$$.fragment,e),h(U.$$.fragment,e),h(K.$$.fragment,e),h(Q.$$.fragment,e),h(V.$$.fragment,e),h(X.$$.fragment,e),h(Y.$$.fragment,e),h(Z.$$.fragment,e),h(P),os=!1},d(e){s(r),e&&s(f),q(a,e),e&&s($),e&&s(x),q(O),e&&s(Ae),ee[y].d(e),e&&s(oe),e&&s(ae),e&&s(xe),e&&s(H),e&&s(Ce),q(M,e),e&&s(Fe),e&&s(S),e&&s(De),e&&s(te),e&&s(He),q(W,e),e&&s(Ne),e&&s(re),e&&s(Re),q(L,e),e&&s(Be),e&&s(le),e&&s(Oe),q(G,e),e&&s(Me),e&&s(ie),e&&s(We),q(J,e),e&&s(Le),e&&s(v),e&&s(Ge),q(U,e),e&&s(Je),e&&s(C),q(K),e&&s(Ue),e&&s(ue),e&&s(Ke),q(Q,e),e&&s(Qe),q(V,e),e&&s(Ve),e&&s(pe),e&&s(Xe),q(X,e),e&&s(Ye),q(Y,e),e&&s(Ze),e&&s(I),e&&s(es),e&&s(F),q(Z),e&&s(ss),e&&s(B),e&&s(ns),se[T].d(e),e&&s(ce)}}}const Hn={local:"poniendo-todo-junto",sections:[{local:"tokens-especiales",title:"Tokens especiales"},{local:"conclusin-del-tokenizador-al-moelo",title:"Conclusi\xF3n: Del tokenizador al moelo"}],title:"Poniendo todo junto"};function Nn(z,r,f){let a="pt";return Sn(()=>{const $=new URLSearchParams(window.location.search);f(0,a=$.get("fw")||"pt")}),[a]}class Ln extends yn{constructor(r){super();En(this,r,Nn,Dn,Tn,{})}}export{Ln as default,Hn as metadata};
