import{S as zl,i as Tl,s as xl,e as n,k as h,w as k,t as l,M as jl,c as r,d as t,m as f,x as b,a as i,h as p,b as u,N as Lt,F as s,g as a,y as v,o as m,p as Pl,q as w,B as g,v as Al,n as Il}from"../../chunks/vendor-1e8b365d.js";import{T as Dl}from"../../chunks/Tip-62b14c6e.js";import{Y as Ns}from"../../chunks/Youtube-c2a8cc39.js";import{I as O}from"../../chunks/IconCopyLink-483c28ba.js";import{C as I}from"../../chunks/CodeBlock-e5764662.js";import{D as El}from"../../chunks/DocNotebookDropdown-37d928d3.js";import{F as Sl}from"../../chunks/FrameworkSwitchCourse-7f8f0f31.js";function ql(D){let d,y;return d=new El({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter2/section4_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter2/section4_tf.ipynb"}]}}),{c(){k(d.$$.fragment)},l(c){b(d.$$.fragment,c)},m(c,E){v(d,c,E),y=!0},i(c){y||(w(d.$$.fragment,c),y=!0)},o(c){m(d.$$.fragment,c),y=!1},d(c){g(d,c)}}}function Nl(D){let d,y;return d=new El({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter2/section4_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter2/section4_pt.ipynb"}]}}),{c(){k(d.$$.fragment)},l(c){b(d.$$.fragment,c)},m(c,E){v(d,c,E),y=!0},i(c){y||(w(d.$$.fragment,c),y=!0)},o(c){m(d.$$.fragment,c),y=!1},d(c){g(d,c)}}}function Cl(D){let d,y,c,E,z,$,T,x;return{c(){d=n("p"),y=l("Similar to "),c=n("code"),E=l("TFAutoModel"),z=l(", the "),$=n("code"),T=l("AutoTokenizer"),x=l(" class will grab the proper tokenizer class in the library based on the checkpoint name, and can be used directly with any checkpoint:")},l(j){d=r(j,"P",{});var _=i(d);y=p(_,"Similar to "),c=r(_,"CODE",{});var B=i(c);E=p(B,"TFAutoModel"),B.forEach(t),z=p(_,", the "),$=r(_,"CODE",{});var C=i($);T=p(C,"AutoTokenizer"),C.forEach(t),x=p(_," class will grab the proper tokenizer class in the library based on the checkpoint name, and can be used directly with any checkpoint:"),_.forEach(t)},m(j,_){a(j,d,_),s(d,y),s(d,c),s(c,E),s(d,z),s(d,$),s($,T),s(d,x)},d(j){j&&t(d)}}}function Hl(D){let d,y,c,E,z,$,T,x;return{c(){d=n("p"),y=l("Similar to "),c=n("code"),E=l("AutoModel"),z=l(", the "),$=n("code"),T=l("AutoTokenizer"),x=l(" class will grab the proper tokenizer class in the library based on the checkpoint name, and can be used directly with any checkpoint:")},l(j){d=r(j,"P",{});var _=i(d);y=p(_,"Similar to "),c=r(_,"CODE",{});var B=i(c);E=p(B,"AutoModel"),B.forEach(t),z=p(_,", the "),$=r(_,"CODE",{});var C=i($);T=p(C,"AutoTokenizer"),C.forEach(t),x=p(_," class will grab the proper tokenizer class in the library based on the checkpoint name, and can be used directly with any checkpoint:"),_.forEach(t)},m(j,_){a(j,d,_),s(d,y),s(d,c),s(c,E),s(d,z),s(d,$),s($,T),s(d,x)},d(j){j&&t(d)}}}function Ll(D){let d,y,c,E,z;return{c(){d=n("p"),y=l("\u270F\uFE0F "),c=n("strong"),E=l("Try it out!"),z=l(" Replicate the two last steps (tokenization and conversion to input IDs) on the input sentences we used in section 2 (\u201CI\u2019ve been waiting for a HuggingFace course my whole life.\u201D and \u201CI hate this so much!\u201D). Check that you get the same input IDs we got earlier!")},l($){d=r($,"P",{});var T=i(d);y=p(T,"\u270F\uFE0F "),c=r(T,"STRONG",{});var x=i(c);E=p(x,"Try it out!"),x.forEach(t),z=p(T," Replicate the two last steps (tokenization and conversion to input IDs) on the input sentences we used in section 2 (\u201CI\u2019ve been waiting for a HuggingFace course my whole life.\u201D and \u201CI hate this so much!\u201D). Check that you get the same input IDs we got earlier!"),T.forEach(t)},m($,T){a($,d,T),s(d,y),s(d,c),s(c,E),s(d,z)},d($){$&&t(d)}}}function Fl(D){let d,y,c,E,z,$,T,x,j,_,B,C,q,N,lt,ze,Cs,pt,xa,Hs,ht,ja,Ls,Te,Fs,ft,Pa,Ms,ut,Aa,Os,U,oe,Ft,xe,Ia,Mt,Da,Bs,je,Us,ae,Sa,Ot,qa,Na,Gs,G,Pe,Mr,Ca,Ae,Or,Ws,ne,Ha,Bt,La,Fa,Rs,Ie,Js,De,Ys,dt,Ma,Ks,ct,Oa,Vs,mt,Ba,Qs,Se,Ua,Ut,Ga,Xs,re,Wa,Gt,Ra,Ja,Zs,W,ie,Wt,qe,Ya,Rt,Ka,eo,Ne,to,wt,Va,so,le,Jt,Qa,Xa,Yt,Za,oo,kt,en,ao,R,Ce,Br,tn,He,Ur,no,bt,sn,ro,vt,on,io,pe,an,Kt,nn,rn,lo,J,he,Vt,Le,ln,Qt,pn,po,Fe,ho,gt,hn,fo,yt,fn,uo,$t,un,co,Y,Me,Gr,dn,Oe,Wr,mo,_t,cn,wo,Et,mn,ko,K,fe,Xt,Be,wn,Zt,kn,bo,zt,bn,vo,H,es,vn,gn,ts,yn,$n,ss,_n,go,Tt,En,yo,V,ue,os,Ue,zn,as,Tn,$o,P,xn,ns,jn,Pn,rs,An,In,is,Dn,Sn,ls,qn,Nn,_o,de,Cn,ps,Hn,Ln,Eo,Ge,zo,xt,We,To,jt,Fn,xo,Re,jo,Je,Po,Pt,Mn,Ao,Ye,Io,A,On,hs,Bn,Un,At,Gn,Wn,fs,Rn,Jn,us,Yn,Kn,Do,Q,ce,ds,Ke,Vn,cs,Qn,So,Ve,qo,me,Xn,ms,Zn,er,No,we,tr,ws,sr,or,Co,L,ar,ks,nr,rr,bs,ir,lr,Ho,It,pr,Lo,X,ke,vs,Qe,hr,gs,fr,Fo,be,ur,ys,dr,cr,Mo,Xe,Oo,Dt,mr,Bo,Ze,Uo,S,wr,$s,kr,br,_s,vr,gr,Es,yr,$r,Go,Z,ve,zs,et,_r,Ts,Er,Wo,ge,zr,xs,Tr,xr,Ro,tt,Jo,st,Yo,St,jr,Ko,ye,Vo,ee,$e,js,ot,Pr,Ps,Ar,Qo,te,As,Ir,Dr,Is,Sr,qr,Xo,at,Zo,nt,ea,_e,Nr,Ds,Cr,Hr,ta,qt,Lr,sa;c=new Sl({props:{fw:D[0]}}),x=new O({});const Rr=[Nl,ql],rt=[];function Jr(e,o){return e[0]==="pt"?0:1}q=Jr(D),N=rt[q]=Rr[q](D),ze=new Ns({props:{id:"VFp38yj8h3A"}}),Te=new I({props:{code:"Jim Henson was a puppeteer",highlighted:'<span class="hljs-comment">Jim Henson was a puppeteer</span>'}}),xe=new O({}),je=new Ns({props:{id:"nhJxYji1aho"}}),Ie=new I({props:{code:`tokenized_text = "Jim Henson was a puppeteer".split()
print(tokenized_text)`,highlighted:`tokenized_text = <span class="hljs-string">&quot;Jim Henson was a puppeteer&quot;</span>.split()
<span class="hljs-built_in">print</span>(tokenized_text)`}}),De=new I({props:{code:"['Jim', 'Henson', 'was', 'a', 'puppeteer']",highlighted:'[<span class="hljs-string">&#x27;Jim&#x27;</span>, <span class="hljs-string">&#x27;Henson&#x27;</span>, <span class="hljs-string">&#x27;was&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;puppeteer&#x27;</span>]'}}),qe=new O({}),Ne=new Ns({props:{id:"ssLq_EK2jLE"}}),Le=new O({}),Fe=new Ns({props:{id:"zHvTiHr506c"}}),Be=new O({}),Ue=new O({}),Ge=new I({props:{code:`from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertTokenizer

tokenizer = BertTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}});function Yr(e,o){return e[0]==="pt"?Hl:Cl}let oa=Yr(D),se=oa(D);return We=new I({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Re=new I({props:{code:'tokenizer("Using a Transformer network is simple")',highlighted:'tokenizer(<span class="hljs-string">&quot;Using a Transformer network is simple&quot;</span>)'}}),Je=new I({props:{code:`{'input_ids': [101, 7993, 170, 11303, 1200, 2443, 1110, 3014, 102],
 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0],
 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}`,highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>, <span class="hljs-number">102</span>],
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Ye=new I({props:{code:'tokenizer.save_pretrained("directory_on_my_computer")',highlighted:'tokenizer.save_pretrained(<span class="hljs-string">&quot;directory_on_my_computer&quot;</span>)'}}),Ke=new O({}),Ve=new Ns({props:{id:"Yffk5aydLzg"}}),Qe=new O({}),Xe=new I({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

sequence = "Using a Transformer network is simple"
tokens = tokenizer.tokenize(sequence)

print(tokens)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)

sequence = <span class="hljs-string">&quot;Using a Transformer network is simple&quot;</span>
tokens = tokenizer.tokenize(sequence)

<span class="hljs-built_in">print</span>(tokens)`}}),Ze=new I({props:{code:"['Using', 'a', 'transform', '##er', 'network', 'is', 'simple']",highlighted:'[<span class="hljs-string">&#x27;Using&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;transform&#x27;</span>, <span class="hljs-string">&#x27;##er&#x27;</span>, <span class="hljs-string">&#x27;network&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;simple&#x27;</span>]'}}),et=new O({}),tt=new I({props:{code:`ids = tokenizer.convert_tokens_to_ids(tokens)

print(ids)`,highlighted:`ids = tokenizer.convert_tokens_to_ids(tokens)

<span class="hljs-built_in">print</span>(ids)`}}),st=new I({props:{code:"[7993, 170, 11303, 1200, 2443, 1110, 3014]",highlighted:'[<span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>]'}}),ye=new Dl({props:{$$slots:{default:[Ll]},$$scope:{ctx:D}}}),ot=new O({}),at=new I({props:{code:`decoded_string = tokenizer.decode([7993, 170, 11303, 1200, 2443, 1110, 3014])
print(decoded_string)`,highlighted:`decoded_string = tokenizer.decode([<span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>])
<span class="hljs-built_in">print</span>(decoded_string)`}}),nt=new I({props:{code:"'Using a Transformer network is simple'",highlighted:'<span class="hljs-string">&#x27;Using a Transformer network is simple&#x27;</span>'}}),{c(){d=n("meta"),y=h(),k(c.$$.fragment),E=h(),z=n("h1"),$=n("a"),T=n("span"),k(x.$$.fragment),j=h(),_=n("span"),B=l("Tokenizers"),C=h(),N.c(),lt=h(),k(ze.$$.fragment),Cs=h(),pt=n("p"),xa=l("Tokenizers are one of the core components of the NLP pipeline. They serve one purpose: to translate text into data that can be processed by the model. Models can only process numbers, so tokenizers need to convert our text inputs to numerical data. In this section, we\u2019ll explore exactly what happens in the tokenization pipeline."),Hs=h(),ht=n("p"),ja=l("In NLP tasks, the data that is generally processed is raw text. Here\u2019s an example of such text:"),Ls=h(),k(Te.$$.fragment),Fs=h(),ft=n("p"),Pa=l("However, models can only process numbers, so we need to find a way to convert the raw text to numbers. That\u2019s what the tokenizers do, and there are a lot of ways to go about this. The goal is to find the most meaningful representation \u2014 that is, the one that makes the most sense to the model \u2014 and, if possible, the smallest representation."),Ms=h(),ut=n("p"),Aa=l("Let\u2019s take a look at some examples of tokenization algorithms, and try to answer some of the questions you may have about tokenization."),Os=h(),U=n("h2"),oe=n("a"),Ft=n("span"),k(xe.$$.fragment),Ia=h(),Mt=n("span"),Da=l("Word-based"),Bs=h(),k(je.$$.fragment),Us=h(),ae=n("p"),Sa=l("The first type of tokenizer that comes to mind is "),Ot=n("em"),qa=l("word-based"),Na=l(". It\u2019s generally very easy to set up and use with only a few rules, and it often yields decent results. For example, in the image below, the goal is to split the raw text into words and find a numerical representation for each of them:"),Gs=h(),G=n("div"),Pe=n("img"),Ca=h(),Ae=n("img"),Ws=h(),ne=n("p"),Ha=l("There are different ways to split the text. For example, we could could use whitespace to tokenize the text into words by applying Python\u2019s "),Bt=n("code"),La=l("split()"),Fa=l(" function:"),Rs=h(),k(Ie.$$.fragment),Js=h(),k(De.$$.fragment),Ys=h(),dt=n("p"),Ma=l("There are also variations of word tokenizers that have extra rules for punctuation. With this kind of tokenizer, we can end up with some pretty large \u201Cvocabularies,\u201D where a vocabulary is defined by the total number of independent tokens that we have in our corpus."),Ks=h(),ct=n("p"),Oa=l("Each word gets assigned an ID, starting from 0 and going up to the size of the vocabulary. The model uses these IDs to identify each word."),Vs=h(),mt=n("p"),Ba=l("If we want to completely cover a language with a word-based tokenizer, we\u2019ll need to have an identifier for each word in the language, which will generate a huge amount of tokens. For example, there are over 500,000 words in the English language, so to build a map from each word to an input ID we\u2019d need to keep track of that many IDs. Furthermore, words like \u201Cdog\u201D are represented differently from words like \u201Cdogs\u201D, and the model will initially have no way of knowing that \u201Cdog\u201D and \u201Cdogs\u201D are similar: it will identify the two words as unrelated. The same applies to other similar words, like \u201Crun\u201D and \u201Crunning\u201D, which the model will not see as being similar initially."),Qs=h(),Se=n("p"),Ua=l("Finally, we need a custom token to represent words that are not in our vocabulary. This is known as the \u201Cunknown\u201D token, often represented as \u201D[UNK]\u201D or \u201D"),Ut=n("unk"),Ga=l("\u201D. It\u2019s generally a bad sign if you see that the tokenizer is producing a lot of these tokens, as it wasn\u2019t able to retrieve a sensible representation of a word and you\u2019re losing information along the way. The goal when crafting the vocabulary is to do it in such a way that the tokenizer tokenizes as few words as possible into the unknown token."),Xs=h(),re=n("p"),Wa=l("One way to reduce the amount of unknown tokens is to go one level deeper, using a "),Gt=n("em"),Ra=l("character-based"),Ja=l(" tokenizer."),Zs=h(),W=n("h2"),ie=n("a"),Wt=n("span"),k(qe.$$.fragment),Ya=h(),Rt=n("span"),Ka=l("Character-based"),eo=h(),k(Ne.$$.fragment),to=h(),wt=n("p"),Va=l("Character-based tokenizers split the text into characters, rather than words. This has two primary benefits:"),so=h(),le=n("ul"),Jt=n("li"),Qa=l("The vocabulary is much smaller."),Xa=h(),Yt=n("li"),Za=l("There are much fewer out-of-vocabulary (unknown) tokens, since every word can be built from characters."),oo=h(),kt=n("p"),en=l("But here too some questions arise concerning spaces and punctuation:"),ao=h(),R=n("div"),Ce=n("img"),tn=h(),He=n("img"),no=h(),bt=n("p"),sn=l("This approach isn\u2019t perfect either. Since the representation is now based on characters rather than words, one could argue that, intuitively, it\u2019s less meaningful: each character doesn\u2019t mean a lot on its own, whereas that is the case with words. However, this again differs according to the language; in Chinese, for example, each character carries more information than a character in a Latin language."),ro=h(),vt=n("p"),on=l("Another thing to consider is that we\u2019ll end up with a very large amount of tokens to be processed by our model: whereas a word would only be a single token with a word-based tokenizer, it can easily turn into 10 or more tokens when converted into characters."),io=h(),pe=n("p"),an=l("To get the best of both worlds, we can use a third technique that combines the two approaches: "),Kt=n("em"),nn=l("subword tokenization"),rn=l("."),lo=h(),J=n("h2"),he=n("a"),Vt=n("span"),k(Le.$$.fragment),ln=h(),Qt=n("span"),pn=l("Subword tokenization"),po=h(),k(Fe.$$.fragment),ho=h(),gt=n("p"),hn=l("Subword tokenization algorithms rely on the principle that frequently used words should not be split into smaller subwords, but rare words should be decomposed into meaningful subwords."),fo=h(),yt=n("p"),fn=l("For instance, \u201Cannoyingly\u201D might be considered a rare word and could be decomposed into \u201Cannoying\u201D and \u201Cly\u201D. These are both likely to appear more frequently as standalone subwords, while at the same time the meaning of \u201Cannoyingly\u201D is kept by the composite meaning of \u201Cannoying\u201D and \u201Cly\u201D."),uo=h(),$t=n("p"),un=l("Here is an example showing how a subword tokenization algorithm would tokenize the sequence \u201CLet\u2019s do tokenization!\u201C:"),co=h(),Y=n("div"),Me=n("img"),dn=h(),Oe=n("img"),mo=h(),_t=n("p"),cn=l("These subwords end up providing a lot of semantic meaning: for instance, in the example above \u201Ctokenization\u201D was split into \u201Ctoken\u201D and \u201Cization\u201D, two tokens that have a semantic meaning while being space-efficient (only two tokens are needed to represent a long word). This allows us to have relatively good coverage with small vocabularies, and close to no unknown tokens."),wo=h(),Et=n("p"),mn=l("This approach is especially useful in agglutinative languages such as Turkish, where you can form (almost) arbitrarily long complex words by stringing together subwords."),ko=h(),K=n("h3"),fe=n("a"),Xt=n("span"),k(Be.$$.fragment),wn=h(),Zt=n("span"),kn=l("And more!"),bo=h(),zt=n("p"),bn=l("Unsurprisingly, there are many more techniques out there. To name a few:"),vo=h(),H=n("ul"),es=n("li"),vn=l("Byte-level BPE, as used in GPT-2"),gn=h(),ts=n("li"),yn=l("WordPiece, as used in BERT"),$n=h(),ss=n("li"),_n=l("SentencePiece or Unigram, as used in several multilingual models"),go=h(),Tt=n("p"),En=l("You should now have sufficient knowledge of how tokenizers work to get started with the API."),yo=h(),V=n("h2"),ue=n("a"),os=n("span"),k(Ue.$$.fragment),zn=h(),as=n("span"),Tn=l("Loading and saving"),$o=h(),P=n("p"),xn=l("Loading and saving tokenizers is as simple as it is with models. Actually, it\u2019s based on the same two methods: "),ns=n("code"),jn=l("from_pretrained()"),Pn=l(" and "),rs=n("code"),An=l("save_pretrained()"),In=l(". These methods will load or save the algorithm used by the tokenizer (a bit like the "),is=n("em"),Dn=l("architecture"),Sn=l(" of the model) as well as its vocabulary (a bit like the "),ls=n("em"),qn=l("weights"),Nn=l(" of the model)."),_o=h(),de=n("p"),Cn=l("Loading the BERT tokenizer trained with the same checkpoint as BERT is done the same way as loading the model, except we use the "),ps=n("code"),Hn=l("BertTokenizer"),Ln=l(" class:"),Eo=h(),k(Ge.$$.fragment),zo=h(),se.c(),xt=h(),k(We.$$.fragment),To=h(),jt=n("p"),Fn=l("We can now use the tokenizer as shown in the previous section:"),xo=h(),k(Re.$$.fragment),jo=h(),k(Je.$$.fragment),Po=h(),Pt=n("p"),Mn=l("Saving a tokenizer is identical to saving a model:"),Ao=h(),k(Ye.$$.fragment),Io=h(),A=n("p"),On=l("We\u2019ll talk more about "),hs=n("code"),Bn=l("token_type_ids"),Un=l(" in "),At=n("a"),Gn=l("Chapter 3"),Wn=l(", and we\u2019ll explain the "),fs=n("code"),Rn=l("attention_mask"),Jn=l(" key a little later. First, let\u2019s see how the "),us=n("code"),Yn=l("input_ids"),Kn=l(" are generated. To do this, we\u2019ll need to look at the intermediate methods of the tokenizer."),Do=h(),Q=n("h2"),ce=n("a"),ds=n("span"),k(Ke.$$.fragment),Vn=h(),cs=n("span"),Qn=l("Encoding"),So=h(),k(Ve.$$.fragment),qo=h(),me=n("p"),Xn=l("Translating text to numbers is known as "),ms=n("em"),Zn=l("encoding"),er=l(". Encoding is done in a two-step process: the tokenization, followed by the conversion to input IDs."),No=h(),we=n("p"),tr=l("As we\u2019ve seen, the first step is to split the text into words (or parts of words, punctuation symbols, etc.), usually called "),ws=n("em"),sr=l("tokens"),or=l(". There are multiple rules that can govern that process, which is why we need to instantiate the tokenizer using the name of the model, to make sure we use the same rules that were used when the model was pretrained."),Co=h(),L=n("p"),ar=l("The second step is to convert those tokens into numbers, so we can build a tensor out of them and feed them to the model. To do this, the tokenizer has a "),ks=n("em"),nr=l("vocabulary"),rr=l(", which is the part we download when we instantiate it with the "),bs=n("code"),ir=l("from_pretrained()"),lr=l(" method. Again, we need to use the same vocabulary used when the model was pretrained."),Ho=h(),It=n("p"),pr=l("To get a better understanding of the two steps, we\u2019ll explore them separately. Note that we will use some methods that perform parts of the tokenization pipeline separately to show you the intermediate results of those steps, but in practice, you should call the tokenizer directly on your inputs (as shown in the section 2)."),Lo=h(),X=n("h3"),ke=n("a"),vs=n("span"),k(Qe.$$.fragment),hr=h(),gs=n("span"),fr=l("Tokenization"),Fo=h(),be=n("p"),ur=l("The tokenization process is done by the "),ys=n("code"),dr=l("tokenize()"),cr=l(" method of the tokenizer:"),Mo=h(),k(Xe.$$.fragment),Oo=h(),Dt=n("p"),mr=l("The output of this method is a list of strings, or tokens:"),Bo=h(),k(Ze.$$.fragment),Uo=h(),S=n("p"),wr=l("This tokenizer is a subword tokenizer: it splits the words until it obtains tokens that can be represented by its vocabulary. That\u2019s the case here with "),$s=n("code"),kr=l("transformer"),br=l(", which is split into two tokens: "),_s=n("code"),vr=l("transform"),gr=l(" and "),Es=n("code"),yr=l("##er"),$r=l("."),Go=h(),Z=n("h3"),ve=n("a"),zs=n("span"),k(et.$$.fragment),_r=h(),Ts=n("span"),Er=l("From tokens to input IDs"),Wo=h(),ge=n("p"),zr=l("The conversion to input IDs is handled by the "),xs=n("code"),Tr=l("convert_tokens_to_ids()"),xr=l(" tokenizer method:"),Ro=h(),k(tt.$$.fragment),Jo=h(),k(st.$$.fragment),Yo=h(),St=n("p"),jr=l("These outputs, once converted to the appropriate framework tensor, can then be used as inputs to a model as seen earlier in this chapter."),Ko=h(),k(ye.$$.fragment),Vo=h(),ee=n("h2"),$e=n("a"),js=n("span"),k(ot.$$.fragment),Pr=h(),Ps=n("span"),Ar=l("Decoding"),Qo=h(),te=n("p"),As=n("em"),Ir=l("Decoding"),Dr=l(" is going the other way around: from vocabulary indices, we want to get a string. This can be done with the "),Is=n("code"),Sr=l("decode()"),qr=l(" method as follows:"),Xo=h(),k(at.$$.fragment),Zo=h(),k(nt.$$.fragment),ea=h(),_e=n("p"),Nr=l("Note that the "),Ds=n("code"),Cr=l("decode"),Hr=l(" method not only converts the indices back to tokens, but also groups together the tokens that were part of the same words to produce a readable sentence. This behavior will be extremely useful when we use models that predict new text (either text generated from a prompt, or for sequence-to-sequence problems like translation or summarization)."),ta=h(),qt=n("p"),Lr=l("By now you should understand the atomic operations a tokenizer can handle: tokenization, conversion to IDs, and converting IDs back to a string. However, we\u2019ve just scraped the tip of the iceberg. In the following section, we\u2019ll take our approach to its limits and take a look at how to overcome them."),this.h()},l(e){const o=jl('[data-svelte="svelte-1phssyn"]',document.head);d=r(o,"META",{name:!0,content:!0}),o.forEach(t),y=f(e),b(c.$$.fragment,e),E=f(e),z=r(e,"H1",{class:!0});var it=i(z);$=r(it,"A",{id:!0,class:!0,href:!0});var Nt=i($);T=r(Nt,"SPAN",{});var Ss=i(T);b(x.$$.fragment,Ss),Ss.forEach(t),Nt.forEach(t),j=f(it),_=r(it,"SPAN",{});var Kr=i(_);B=p(Kr,"Tokenizers"),Kr.forEach(t),it.forEach(t),C=f(e),N.l(e),lt=f(e),b(ze.$$.fragment,e),Cs=f(e),pt=r(e,"P",{});var Vr=i(pt);xa=p(Vr,"Tokenizers are one of the core components of the NLP pipeline. They serve one purpose: to translate text into data that can be processed by the model. Models can only process numbers, so tokenizers need to convert our text inputs to numerical data. In this section, we\u2019ll explore exactly what happens in the tokenization pipeline."),Vr.forEach(t),Hs=f(e),ht=r(e,"P",{});var Qr=i(ht);ja=p(Qr,"In NLP tasks, the data that is generally processed is raw text. Here\u2019s an example of such text:"),Qr.forEach(t),Ls=f(e),b(Te.$$.fragment,e),Fs=f(e),ft=r(e,"P",{});var Xr=i(ft);Pa=p(Xr,"However, models can only process numbers, so we need to find a way to convert the raw text to numbers. That\u2019s what the tokenizers do, and there are a lot of ways to go about this. The goal is to find the most meaningful representation \u2014 that is, the one that makes the most sense to the model \u2014 and, if possible, the smallest representation."),Xr.forEach(t),Ms=f(e),ut=r(e,"P",{});var Zr=i(ut);Aa=p(Zr,"Let\u2019s take a look at some examples of tokenization algorithms, and try to answer some of the questions you may have about tokenization."),Zr.forEach(t),Os=f(e),U=r(e,"H2",{class:!0});var aa=i(U);oe=r(aa,"A",{id:!0,class:!0,href:!0});var ei=i(oe);Ft=r(ei,"SPAN",{});var ti=i(Ft);b(xe.$$.fragment,ti),ti.forEach(t),ei.forEach(t),Ia=f(aa),Mt=r(aa,"SPAN",{});var si=i(Mt);Da=p(si,"Word-based"),si.forEach(t),aa.forEach(t),Bs=f(e),b(je.$$.fragment,e),Us=f(e),ae=r(e,"P",{});var na=i(ae);Sa=p(na,"The first type of tokenizer that comes to mind is "),Ot=r(na,"EM",{});var oi=i(Ot);qa=p(oi,"word-based"),oi.forEach(t),Na=p(na,". It\u2019s generally very easy to set up and use with only a few rules, and it often yields decent results. For example, in the image below, the goal is to split the raw text into words and find a numerical representation for each of them:"),na.forEach(t),Gs=f(e),G=r(e,"DIV",{class:!0});var ra=i(G);Pe=r(ra,"IMG",{class:!0,src:!0,alt:!0}),Ca=f(ra),Ae=r(ra,"IMG",{class:!0,src:!0,alt:!0}),ra.forEach(t),Ws=f(e),ne=r(e,"P",{});var ia=i(ne);Ha=p(ia,"There are different ways to split the text. For example, we could could use whitespace to tokenize the text into words by applying Python\u2019s "),Bt=r(ia,"CODE",{});var ai=i(Bt);La=p(ai,"split()"),ai.forEach(t),Fa=p(ia," function:"),ia.forEach(t),Rs=f(e),b(Ie.$$.fragment,e),Js=f(e),b(De.$$.fragment,e),Ys=f(e),dt=r(e,"P",{});var ni=i(dt);Ma=p(ni,"There are also variations of word tokenizers that have extra rules for punctuation. With this kind of tokenizer, we can end up with some pretty large \u201Cvocabularies,\u201D where a vocabulary is defined by the total number of independent tokens that we have in our corpus."),ni.forEach(t),Ks=f(e),ct=r(e,"P",{});var ri=i(ct);Oa=p(ri,"Each word gets assigned an ID, starting from 0 and going up to the size of the vocabulary. The model uses these IDs to identify each word."),ri.forEach(t),Vs=f(e),mt=r(e,"P",{});var ii=i(mt);Ba=p(ii,"If we want to completely cover a language with a word-based tokenizer, we\u2019ll need to have an identifier for each word in the language, which will generate a huge amount of tokens. For example, there are over 500,000 words in the English language, so to build a map from each word to an input ID we\u2019d need to keep track of that many IDs. Furthermore, words like \u201Cdog\u201D are represented differently from words like \u201Cdogs\u201D, and the model will initially have no way of knowing that \u201Cdog\u201D and \u201Cdogs\u201D are similar: it will identify the two words as unrelated. The same applies to other similar words, like \u201Crun\u201D and \u201Crunning\u201D, which the model will not see as being similar initially."),ii.forEach(t),Qs=f(e),Se=r(e,"P",{});var Fr=i(Se);Ua=p(Fr,"Finally, we need a custom token to represent words that are not in our vocabulary. This is known as the \u201Cunknown\u201D token, often represented as \u201D[UNK]\u201D or \u201D"),Ut=r(Fr,"UNK",{});var li=i(Ut);Ga=p(li,"\u201D. It\u2019s generally a bad sign if you see that the tokenizer is producing a lot of these tokens, as it wasn\u2019t able to retrieve a sensible representation of a word and you\u2019re losing information along the way. The goal when crafting the vocabulary is to do it in such a way that the tokenizer tokenizes as few words as possible into the unknown token."),li.forEach(t),Fr.forEach(t),Xs=f(e),re=r(e,"P",{});var la=i(re);Wa=p(la,"One way to reduce the amount of unknown tokens is to go one level deeper, using a "),Gt=r(la,"EM",{});var pi=i(Gt);Ra=p(pi,"character-based"),pi.forEach(t),Ja=p(la," tokenizer."),la.forEach(t),Zs=f(e),W=r(e,"H2",{class:!0});var pa=i(W);ie=r(pa,"A",{id:!0,class:!0,href:!0});var hi=i(ie);Wt=r(hi,"SPAN",{});var fi=i(Wt);b(qe.$$.fragment,fi),fi.forEach(t),hi.forEach(t),Ya=f(pa),Rt=r(pa,"SPAN",{});var ui=i(Rt);Ka=p(ui,"Character-based"),ui.forEach(t),pa.forEach(t),eo=f(e),b(Ne.$$.fragment,e),to=f(e),wt=r(e,"P",{});var di=i(wt);Va=p(di,"Character-based tokenizers split the text into characters, rather than words. This has two primary benefits:"),di.forEach(t),so=f(e),le=r(e,"UL",{});var ha=i(le);Jt=r(ha,"LI",{});var ci=i(Jt);Qa=p(ci,"The vocabulary is much smaller."),ci.forEach(t),Xa=f(ha),Yt=r(ha,"LI",{});var mi=i(Yt);Za=p(mi,"There are much fewer out-of-vocabulary (unknown) tokens, since every word can be built from characters."),mi.forEach(t),ha.forEach(t),oo=f(e),kt=r(e,"P",{});var wi=i(kt);en=p(wi,"But here too some questions arise concerning spaces and punctuation:"),wi.forEach(t),ao=f(e),R=r(e,"DIV",{class:!0});var fa=i(R);Ce=r(fa,"IMG",{class:!0,src:!0,alt:!0}),tn=f(fa),He=r(fa,"IMG",{class:!0,src:!0,alt:!0}),fa.forEach(t),no=f(e),bt=r(e,"P",{});var ki=i(bt);sn=p(ki,"This approach isn\u2019t perfect either. Since the representation is now based on characters rather than words, one could argue that, intuitively, it\u2019s less meaningful: each character doesn\u2019t mean a lot on its own, whereas that is the case with words. However, this again differs according to the language; in Chinese, for example, each character carries more information than a character in a Latin language."),ki.forEach(t),ro=f(e),vt=r(e,"P",{});var bi=i(vt);on=p(bi,"Another thing to consider is that we\u2019ll end up with a very large amount of tokens to be processed by our model: whereas a word would only be a single token with a word-based tokenizer, it can easily turn into 10 or more tokens when converted into characters."),bi.forEach(t),io=f(e),pe=r(e,"P",{});var ua=i(pe);an=p(ua,"To get the best of both worlds, we can use a third technique that combines the two approaches: "),Kt=r(ua,"EM",{});var vi=i(Kt);nn=p(vi,"subword tokenization"),vi.forEach(t),rn=p(ua,"."),ua.forEach(t),lo=f(e),J=r(e,"H2",{class:!0});var da=i(J);he=r(da,"A",{id:!0,class:!0,href:!0});var gi=i(he);Vt=r(gi,"SPAN",{});var yi=i(Vt);b(Le.$$.fragment,yi),yi.forEach(t),gi.forEach(t),ln=f(da),Qt=r(da,"SPAN",{});var $i=i(Qt);pn=p($i,"Subword tokenization"),$i.forEach(t),da.forEach(t),po=f(e),b(Fe.$$.fragment,e),ho=f(e),gt=r(e,"P",{});var _i=i(gt);hn=p(_i,"Subword tokenization algorithms rely on the principle that frequently used words should not be split into smaller subwords, but rare words should be decomposed into meaningful subwords."),_i.forEach(t),fo=f(e),yt=r(e,"P",{});var Ei=i(yt);fn=p(Ei,"For instance, \u201Cannoyingly\u201D might be considered a rare word and could be decomposed into \u201Cannoying\u201D and \u201Cly\u201D. These are both likely to appear more frequently as standalone subwords, while at the same time the meaning of \u201Cannoyingly\u201D is kept by the composite meaning of \u201Cannoying\u201D and \u201Cly\u201D."),Ei.forEach(t),uo=f(e),$t=r(e,"P",{});var zi=i($t);un=p(zi,"Here is an example showing how a subword tokenization algorithm would tokenize the sequence \u201CLet\u2019s do tokenization!\u201C:"),zi.forEach(t),co=f(e),Y=r(e,"DIV",{class:!0});var ca=i(Y);Me=r(ca,"IMG",{class:!0,src:!0,alt:!0}),dn=f(ca),Oe=r(ca,"IMG",{class:!0,src:!0,alt:!0}),ca.forEach(t),mo=f(e),_t=r(e,"P",{});var Ti=i(_t);cn=p(Ti,"These subwords end up providing a lot of semantic meaning: for instance, in the example above \u201Ctokenization\u201D was split into \u201Ctoken\u201D and \u201Cization\u201D, two tokens that have a semantic meaning while being space-efficient (only two tokens are needed to represent a long word). This allows us to have relatively good coverage with small vocabularies, and close to no unknown tokens."),Ti.forEach(t),wo=f(e),Et=r(e,"P",{});var xi=i(Et);mn=p(xi,"This approach is especially useful in agglutinative languages such as Turkish, where you can form (almost) arbitrarily long complex words by stringing together subwords."),xi.forEach(t),ko=f(e),K=r(e,"H3",{class:!0});var ma=i(K);fe=r(ma,"A",{id:!0,class:!0,href:!0});var ji=i(fe);Xt=r(ji,"SPAN",{});var Pi=i(Xt);b(Be.$$.fragment,Pi),Pi.forEach(t),ji.forEach(t),wn=f(ma),Zt=r(ma,"SPAN",{});var Ai=i(Zt);kn=p(Ai,"And more!"),Ai.forEach(t),ma.forEach(t),bo=f(e),zt=r(e,"P",{});var Ii=i(zt);bn=p(Ii,"Unsurprisingly, there are many more techniques out there. To name a few:"),Ii.forEach(t),vo=f(e),H=r(e,"UL",{});var Ct=i(H);es=r(Ct,"LI",{});var Di=i(es);vn=p(Di,"Byte-level BPE, as used in GPT-2"),Di.forEach(t),gn=f(Ct),ts=r(Ct,"LI",{});var Si=i(ts);yn=p(Si,"WordPiece, as used in BERT"),Si.forEach(t),$n=f(Ct),ss=r(Ct,"LI",{});var qi=i(ss);_n=p(qi,"SentencePiece or Unigram, as used in several multilingual models"),qi.forEach(t),Ct.forEach(t),go=f(e),Tt=r(e,"P",{});var Ni=i(Tt);En=p(Ni,"You should now have sufficient knowledge of how tokenizers work to get started with the API."),Ni.forEach(t),yo=f(e),V=r(e,"H2",{class:!0});var wa=i(V);ue=r(wa,"A",{id:!0,class:!0,href:!0});var Ci=i(ue);os=r(Ci,"SPAN",{});var Hi=i(os);b(Ue.$$.fragment,Hi),Hi.forEach(t),Ci.forEach(t),zn=f(wa),as=r(wa,"SPAN",{});var Li=i(as);Tn=p(Li,"Loading and saving"),Li.forEach(t),wa.forEach(t),$o=f(e),P=r(e,"P",{});var F=i(P);xn=p(F,"Loading and saving tokenizers is as simple as it is with models. Actually, it\u2019s based on the same two methods: "),ns=r(F,"CODE",{});var Fi=i(ns);jn=p(Fi,"from_pretrained()"),Fi.forEach(t),Pn=p(F," and "),rs=r(F,"CODE",{});var Mi=i(rs);An=p(Mi,"save_pretrained()"),Mi.forEach(t),In=p(F,". These methods will load or save the algorithm used by the tokenizer (a bit like the "),is=r(F,"EM",{});var Oi=i(is);Dn=p(Oi,"architecture"),Oi.forEach(t),Sn=p(F," of the model) as well as its vocabulary (a bit like the "),ls=r(F,"EM",{});var Bi=i(ls);qn=p(Bi,"weights"),Bi.forEach(t),Nn=p(F," of the model)."),F.forEach(t),_o=f(e),de=r(e,"P",{});var ka=i(de);Cn=p(ka,"Loading the BERT tokenizer trained with the same checkpoint as BERT is done the same way as loading the model, except we use the "),ps=r(ka,"CODE",{});var Ui=i(ps);Hn=p(Ui,"BertTokenizer"),Ui.forEach(t),Ln=p(ka," class:"),ka.forEach(t),Eo=f(e),b(Ge.$$.fragment,e),zo=f(e),se.l(e),xt=f(e),b(We.$$.fragment,e),To=f(e),jt=r(e,"P",{});var Gi=i(jt);Fn=p(Gi,"We can now use the tokenizer as shown in the previous section:"),Gi.forEach(t),xo=f(e),b(Re.$$.fragment,e),jo=f(e),b(Je.$$.fragment,e),Po=f(e),Pt=r(e,"P",{});var Wi=i(Pt);Mn=p(Wi,"Saving a tokenizer is identical to saving a model:"),Wi.forEach(t),Ao=f(e),b(Ye.$$.fragment,e),Io=f(e),A=r(e,"P",{});var M=i(A);On=p(M,"We\u2019ll talk more about "),hs=r(M,"CODE",{});var Ri=i(hs);Bn=p(Ri,"token_type_ids"),Ri.forEach(t),Un=p(M," in "),At=r(M,"A",{href:!0});var Ji=i(At);Gn=p(Ji,"Chapter 3"),Ji.forEach(t),Wn=p(M,", and we\u2019ll explain the "),fs=r(M,"CODE",{});var Yi=i(fs);Rn=p(Yi,"attention_mask"),Yi.forEach(t),Jn=p(M," key a little later. First, let\u2019s see how the "),us=r(M,"CODE",{});var Ki=i(us);Yn=p(Ki,"input_ids"),Ki.forEach(t),Kn=p(M," are generated. To do this, we\u2019ll need to look at the intermediate methods of the tokenizer."),M.forEach(t),Do=f(e),Q=r(e,"H2",{class:!0});var ba=i(Q);ce=r(ba,"A",{id:!0,class:!0,href:!0});var Vi=i(ce);ds=r(Vi,"SPAN",{});var Qi=i(ds);b(Ke.$$.fragment,Qi),Qi.forEach(t),Vi.forEach(t),Vn=f(ba),cs=r(ba,"SPAN",{});var Xi=i(cs);Qn=p(Xi,"Encoding"),Xi.forEach(t),ba.forEach(t),So=f(e),b(Ve.$$.fragment,e),qo=f(e),me=r(e,"P",{});var va=i(me);Xn=p(va,"Translating text to numbers is known as "),ms=r(va,"EM",{});var Zi=i(ms);Zn=p(Zi,"encoding"),Zi.forEach(t),er=p(va,". Encoding is done in a two-step process: the tokenization, followed by the conversion to input IDs."),va.forEach(t),No=f(e),we=r(e,"P",{});var ga=i(we);tr=p(ga,"As we\u2019ve seen, the first step is to split the text into words (or parts of words, punctuation symbols, etc.), usually called "),ws=r(ga,"EM",{});var el=i(ws);sr=p(el,"tokens"),el.forEach(t),or=p(ga,". There are multiple rules that can govern that process, which is why we need to instantiate the tokenizer using the name of the model, to make sure we use the same rules that were used when the model was pretrained."),ga.forEach(t),Co=f(e),L=r(e,"P",{});var Ht=i(L);ar=p(Ht,"The second step is to convert those tokens into numbers, so we can build a tensor out of them and feed them to the model. To do this, the tokenizer has a "),ks=r(Ht,"EM",{});var tl=i(ks);nr=p(tl,"vocabulary"),tl.forEach(t),rr=p(Ht,", which is the part we download when we instantiate it with the "),bs=r(Ht,"CODE",{});var sl=i(bs);ir=p(sl,"from_pretrained()"),sl.forEach(t),lr=p(Ht," method. Again, we need to use the same vocabulary used when the model was pretrained."),Ht.forEach(t),Ho=f(e),It=r(e,"P",{});var ol=i(It);pr=p(ol,"To get a better understanding of the two steps, we\u2019ll explore them separately. Note that we will use some methods that perform parts of the tokenization pipeline separately to show you the intermediate results of those steps, but in practice, you should call the tokenizer directly on your inputs (as shown in the section 2)."),ol.forEach(t),Lo=f(e),X=r(e,"H3",{class:!0});var ya=i(X);ke=r(ya,"A",{id:!0,class:!0,href:!0});var al=i(ke);vs=r(al,"SPAN",{});var nl=i(vs);b(Qe.$$.fragment,nl),nl.forEach(t),al.forEach(t),hr=f(ya),gs=r(ya,"SPAN",{});var rl=i(gs);fr=p(rl,"Tokenization"),rl.forEach(t),ya.forEach(t),Fo=f(e),be=r(e,"P",{});var $a=i(be);ur=p($a,"The tokenization process is done by the "),ys=r($a,"CODE",{});var il=i(ys);dr=p(il,"tokenize()"),il.forEach(t),cr=p($a," method of the tokenizer:"),$a.forEach(t),Mo=f(e),b(Xe.$$.fragment,e),Oo=f(e),Dt=r(e,"P",{});var ll=i(Dt);mr=p(ll,"The output of this method is a list of strings, or tokens:"),ll.forEach(t),Bo=f(e),b(Ze.$$.fragment,e),Uo=f(e),S=r(e,"P",{});var Ee=i(S);wr=p(Ee,"This tokenizer is a subword tokenizer: it splits the words until it obtains tokens that can be represented by its vocabulary. That\u2019s the case here with "),$s=r(Ee,"CODE",{});var pl=i($s);kr=p(pl,"transformer"),pl.forEach(t),br=p(Ee,", which is split into two tokens: "),_s=r(Ee,"CODE",{});var hl=i(_s);vr=p(hl,"transform"),hl.forEach(t),gr=p(Ee," and "),Es=r(Ee,"CODE",{});var fl=i(Es);yr=p(fl,"##er"),fl.forEach(t),$r=p(Ee,"."),Ee.forEach(t),Go=f(e),Z=r(e,"H3",{class:!0});var _a=i(Z);ve=r(_a,"A",{id:!0,class:!0,href:!0});var ul=i(ve);zs=r(ul,"SPAN",{});var dl=i(zs);b(et.$$.fragment,dl),dl.forEach(t),ul.forEach(t),_r=f(_a),Ts=r(_a,"SPAN",{});var cl=i(Ts);Er=p(cl,"From tokens to input IDs"),cl.forEach(t),_a.forEach(t),Wo=f(e),ge=r(e,"P",{});var Ea=i(ge);zr=p(Ea,"The conversion to input IDs is handled by the "),xs=r(Ea,"CODE",{});var ml=i(xs);Tr=p(ml,"convert_tokens_to_ids()"),ml.forEach(t),xr=p(Ea," tokenizer method:"),Ea.forEach(t),Ro=f(e),b(tt.$$.fragment,e),Jo=f(e),b(st.$$.fragment,e),Yo=f(e),St=r(e,"P",{});var wl=i(St);jr=p(wl,"These outputs, once converted to the appropriate framework tensor, can then be used as inputs to a model as seen earlier in this chapter."),wl.forEach(t),Ko=f(e),b(ye.$$.fragment,e),Vo=f(e),ee=r(e,"H2",{class:!0});var za=i(ee);$e=r(za,"A",{id:!0,class:!0,href:!0});var kl=i($e);js=r(kl,"SPAN",{});var bl=i(js);b(ot.$$.fragment,bl),bl.forEach(t),kl.forEach(t),Pr=f(za),Ps=r(za,"SPAN",{});var vl=i(Ps);Ar=p(vl,"Decoding"),vl.forEach(t),za.forEach(t),Qo=f(e),te=r(e,"P",{});var qs=i(te);As=r(qs,"EM",{});var gl=i(As);Ir=p(gl,"Decoding"),gl.forEach(t),Dr=p(qs," is going the other way around: from vocabulary indices, we want to get a string. This can be done with the "),Is=r(qs,"CODE",{});var yl=i(Is);Sr=p(yl,"decode()"),yl.forEach(t),qr=p(qs," method as follows:"),qs.forEach(t),Xo=f(e),b(at.$$.fragment,e),Zo=f(e),b(nt.$$.fragment,e),ea=f(e),_e=r(e,"P",{});var Ta=i(_e);Nr=p(Ta,"Note that the "),Ds=r(Ta,"CODE",{});var $l=i(Ds);Cr=p($l,"decode"),$l.forEach(t),Hr=p(Ta," method not only converts the indices back to tokens, but also groups together the tokens that were part of the same words to produce a readable sentence. This behavior will be extremely useful when we use models that predict new text (either text generated from a prompt, or for sequence-to-sequence problems like translation or summarization)."),Ta.forEach(t),ta=f(e),qt=r(e,"P",{});var _l=i(qt);Lr=p(_l,"By now you should understand the atomic operations a tokenizer can handle: tokenization, conversion to IDs, and converting IDs back to a string. However, we\u2019ve just scraped the tip of the iceberg. In the following section, we\u2019ll take our approach to its limits and take a look at how to overcome them."),_l.forEach(t),this.h()},h(){u(d,"name","hf:doc:metadata"),u(d,"content",JSON.stringify(Ml)),u($,"id","tokenizers"),u($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($,"href","#tokenizers"),u(z,"class","relative group"),u(oe,"id","wordbased"),u(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(oe,"href","#wordbased"),u(U,"class","relative group"),u(Pe,"class","block dark:hidden"),Lt(Pe.src,Mr="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/word_based_tokenization.svg")||u(Pe,"src",Mr),u(Pe,"alt","An example of word-based tokenization."),u(Ae,"class","hidden dark:block"),Lt(Ae.src,Or="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/word_based_tokenization-dark.svg")||u(Ae,"src",Or),u(Ae,"alt","An example of word-based tokenization."),u(G,"class","flex justify-center"),u(ie,"id","characterbased"),u(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ie,"href","#characterbased"),u(W,"class","relative group"),u(Ce,"class","block dark:hidden"),Lt(Ce.src,Br="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/character_based_tokenization.svg")||u(Ce,"src",Br),u(Ce,"alt","An example of character-based tokenization."),u(He,"class","hidden dark:block"),Lt(He.src,Ur="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/character_based_tokenization-dark.svg")||u(He,"src",Ur),u(He,"alt","An example of character-based tokenization."),u(R,"class","flex justify-center"),u(he,"id","subword-tokenization"),u(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(he,"href","#subword-tokenization"),u(J,"class","relative group"),u(Me,"class","block dark:hidden"),Lt(Me.src,Gr="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/bpe_subword.svg")||u(Me,"src",Gr),u(Me,"alt","A subword tokenization algorithm."),u(Oe,"class","hidden dark:block"),Lt(Oe.src,Wr="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/bpe_subword-dark.svg")||u(Oe,"src",Wr),u(Oe,"alt","A subword tokenization algorithm."),u(Y,"class","flex justify-center"),u(fe,"id","and-more"),u(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(fe,"href","#and-more"),u(K,"class","relative group"),u(ue,"id","loading-and-saving"),u(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ue,"href","#loading-and-saving"),u(V,"class","relative group"),u(At,"href","/course/chapter3"),u(ce,"id","encoding"),u(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ce,"href","#encoding"),u(Q,"class","relative group"),u(ke,"id","tokenization"),u(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ke,"href","#tokenization"),u(X,"class","relative group"),u(ve,"id","from-tokens-to-input-ids"),u(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ve,"href","#from-tokens-to-input-ids"),u(Z,"class","relative group"),u($e,"id","decoding"),u($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($e,"href","#decoding"),u(ee,"class","relative group")},m(e,o){s(document.head,d),a(e,y,o),v(c,e,o),a(e,E,o),a(e,z,o),s(z,$),s($,T),v(x,T,null),s(z,j),s(z,_),s(_,B),a(e,C,o),rt[q].m(e,o),a(e,lt,o),v(ze,e,o),a(e,Cs,o),a(e,pt,o),s(pt,xa),a(e,Hs,o),a(e,ht,o),s(ht,ja),a(e,Ls,o),v(Te,e,o),a(e,Fs,o),a(e,ft,o),s(ft,Pa),a(e,Ms,o),a(e,ut,o),s(ut,Aa),a(e,Os,o),a(e,U,o),s(U,oe),s(oe,Ft),v(xe,Ft,null),s(U,Ia),s(U,Mt),s(Mt,Da),a(e,Bs,o),v(je,e,o),a(e,Us,o),a(e,ae,o),s(ae,Sa),s(ae,Ot),s(Ot,qa),s(ae,Na),a(e,Gs,o),a(e,G,o),s(G,Pe),s(G,Ca),s(G,Ae),a(e,Ws,o),a(e,ne,o),s(ne,Ha),s(ne,Bt),s(Bt,La),s(ne,Fa),a(e,Rs,o),v(Ie,e,o),a(e,Js,o),v(De,e,o),a(e,Ys,o),a(e,dt,o),s(dt,Ma),a(e,Ks,o),a(e,ct,o),s(ct,Oa),a(e,Vs,o),a(e,mt,o),s(mt,Ba),a(e,Qs,o),a(e,Se,o),s(Se,Ua),s(Se,Ut),s(Ut,Ga),a(e,Xs,o),a(e,re,o),s(re,Wa),s(re,Gt),s(Gt,Ra),s(re,Ja),a(e,Zs,o),a(e,W,o),s(W,ie),s(ie,Wt),v(qe,Wt,null),s(W,Ya),s(W,Rt),s(Rt,Ka),a(e,eo,o),v(Ne,e,o),a(e,to,o),a(e,wt,o),s(wt,Va),a(e,so,o),a(e,le,o),s(le,Jt),s(Jt,Qa),s(le,Xa),s(le,Yt),s(Yt,Za),a(e,oo,o),a(e,kt,o),s(kt,en),a(e,ao,o),a(e,R,o),s(R,Ce),s(R,tn),s(R,He),a(e,no,o),a(e,bt,o),s(bt,sn),a(e,ro,o),a(e,vt,o),s(vt,on),a(e,io,o),a(e,pe,o),s(pe,an),s(pe,Kt),s(Kt,nn),s(pe,rn),a(e,lo,o),a(e,J,o),s(J,he),s(he,Vt),v(Le,Vt,null),s(J,ln),s(J,Qt),s(Qt,pn),a(e,po,o),v(Fe,e,o),a(e,ho,o),a(e,gt,o),s(gt,hn),a(e,fo,o),a(e,yt,o),s(yt,fn),a(e,uo,o),a(e,$t,o),s($t,un),a(e,co,o),a(e,Y,o),s(Y,Me),s(Y,dn),s(Y,Oe),a(e,mo,o),a(e,_t,o),s(_t,cn),a(e,wo,o),a(e,Et,o),s(Et,mn),a(e,ko,o),a(e,K,o),s(K,fe),s(fe,Xt),v(Be,Xt,null),s(K,wn),s(K,Zt),s(Zt,kn),a(e,bo,o),a(e,zt,o),s(zt,bn),a(e,vo,o),a(e,H,o),s(H,es),s(es,vn),s(H,gn),s(H,ts),s(ts,yn),s(H,$n),s(H,ss),s(ss,_n),a(e,go,o),a(e,Tt,o),s(Tt,En),a(e,yo,o),a(e,V,o),s(V,ue),s(ue,os),v(Ue,os,null),s(V,zn),s(V,as),s(as,Tn),a(e,$o,o),a(e,P,o),s(P,xn),s(P,ns),s(ns,jn),s(P,Pn),s(P,rs),s(rs,An),s(P,In),s(P,is),s(is,Dn),s(P,Sn),s(P,ls),s(ls,qn),s(P,Nn),a(e,_o,o),a(e,de,o),s(de,Cn),s(de,ps),s(ps,Hn),s(de,Ln),a(e,Eo,o),v(Ge,e,o),a(e,zo,o),se.m(e,o),a(e,xt,o),v(We,e,o),a(e,To,o),a(e,jt,o),s(jt,Fn),a(e,xo,o),v(Re,e,o),a(e,jo,o),v(Je,e,o),a(e,Po,o),a(e,Pt,o),s(Pt,Mn),a(e,Ao,o),v(Ye,e,o),a(e,Io,o),a(e,A,o),s(A,On),s(A,hs),s(hs,Bn),s(A,Un),s(A,At),s(At,Gn),s(A,Wn),s(A,fs),s(fs,Rn),s(A,Jn),s(A,us),s(us,Yn),s(A,Kn),a(e,Do,o),a(e,Q,o),s(Q,ce),s(ce,ds),v(Ke,ds,null),s(Q,Vn),s(Q,cs),s(cs,Qn),a(e,So,o),v(Ve,e,o),a(e,qo,o),a(e,me,o),s(me,Xn),s(me,ms),s(ms,Zn),s(me,er),a(e,No,o),a(e,we,o),s(we,tr),s(we,ws),s(ws,sr),s(we,or),a(e,Co,o),a(e,L,o),s(L,ar),s(L,ks),s(ks,nr),s(L,rr),s(L,bs),s(bs,ir),s(L,lr),a(e,Ho,o),a(e,It,o),s(It,pr),a(e,Lo,o),a(e,X,o),s(X,ke),s(ke,vs),v(Qe,vs,null),s(X,hr),s(X,gs),s(gs,fr),a(e,Fo,o),a(e,be,o),s(be,ur),s(be,ys),s(ys,dr),s(be,cr),a(e,Mo,o),v(Xe,e,o),a(e,Oo,o),a(e,Dt,o),s(Dt,mr),a(e,Bo,o),v(Ze,e,o),a(e,Uo,o),a(e,S,o),s(S,wr),s(S,$s),s($s,kr),s(S,br),s(S,_s),s(_s,vr),s(S,gr),s(S,Es),s(Es,yr),s(S,$r),a(e,Go,o),a(e,Z,o),s(Z,ve),s(ve,zs),v(et,zs,null),s(Z,_r),s(Z,Ts),s(Ts,Er),a(e,Wo,o),a(e,ge,o),s(ge,zr),s(ge,xs),s(xs,Tr),s(ge,xr),a(e,Ro,o),v(tt,e,o),a(e,Jo,o),v(st,e,o),a(e,Yo,o),a(e,St,o),s(St,jr),a(e,Ko,o),v(ye,e,o),a(e,Vo,o),a(e,ee,o),s(ee,$e),s($e,js),v(ot,js,null),s(ee,Pr),s(ee,Ps),s(Ps,Ar),a(e,Qo,o),a(e,te,o),s(te,As),s(As,Ir),s(te,Dr),s(te,Is),s(Is,Sr),s(te,qr),a(e,Xo,o),v(at,e,o),a(e,Zo,o),v(nt,e,o),a(e,ea,o),a(e,_e,o),s(_e,Nr),s(_e,Ds),s(Ds,Cr),s(_e,Hr),a(e,ta,o),a(e,qt,o),s(qt,Lr),sa=!0},p(e,[o]){const it={};o&1&&(it.fw=e[0]),c.$set(it);let Nt=q;q=Jr(e),q!==Nt&&(Il(),m(rt[Nt],1,1,()=>{rt[Nt]=null}),Pl(),N=rt[q],N||(N=rt[q]=Rr[q](e),N.c()),w(N,1),N.m(lt.parentNode,lt)),oa!==(oa=Yr(e))&&(se.d(1),se=oa(e),se&&(se.c(),se.m(xt.parentNode,xt)));const Ss={};o&2&&(Ss.$$scope={dirty:o,ctx:e}),ye.$set(Ss)},i(e){sa||(w(c.$$.fragment,e),w(x.$$.fragment,e),w(N),w(ze.$$.fragment,e),w(Te.$$.fragment,e),w(xe.$$.fragment,e),w(je.$$.fragment,e),w(Ie.$$.fragment,e),w(De.$$.fragment,e),w(qe.$$.fragment,e),w(Ne.$$.fragment,e),w(Le.$$.fragment,e),w(Fe.$$.fragment,e),w(Be.$$.fragment,e),w(Ue.$$.fragment,e),w(Ge.$$.fragment,e),w(We.$$.fragment,e),w(Re.$$.fragment,e),w(Je.$$.fragment,e),w(Ye.$$.fragment,e),w(Ke.$$.fragment,e),w(Ve.$$.fragment,e),w(Qe.$$.fragment,e),w(Xe.$$.fragment,e),w(Ze.$$.fragment,e),w(et.$$.fragment,e),w(tt.$$.fragment,e),w(st.$$.fragment,e),w(ye.$$.fragment,e),w(ot.$$.fragment,e),w(at.$$.fragment,e),w(nt.$$.fragment,e),sa=!0)},o(e){m(c.$$.fragment,e),m(x.$$.fragment,e),m(N),m(ze.$$.fragment,e),m(Te.$$.fragment,e),m(xe.$$.fragment,e),m(je.$$.fragment,e),m(Ie.$$.fragment,e),m(De.$$.fragment,e),m(qe.$$.fragment,e),m(Ne.$$.fragment,e),m(Le.$$.fragment,e),m(Fe.$$.fragment,e),m(Be.$$.fragment,e),m(Ue.$$.fragment,e),m(Ge.$$.fragment,e),m(We.$$.fragment,e),m(Re.$$.fragment,e),m(Je.$$.fragment,e),m(Ye.$$.fragment,e),m(Ke.$$.fragment,e),m(Ve.$$.fragment,e),m(Qe.$$.fragment,e),m(Xe.$$.fragment,e),m(Ze.$$.fragment,e),m(et.$$.fragment,e),m(tt.$$.fragment,e),m(st.$$.fragment,e),m(ye.$$.fragment,e),m(ot.$$.fragment,e),m(at.$$.fragment,e),m(nt.$$.fragment,e),sa=!1},d(e){t(d),e&&t(y),g(c,e),e&&t(E),e&&t(z),g(x),e&&t(C),rt[q].d(e),e&&t(lt),g(ze,e),e&&t(Cs),e&&t(pt),e&&t(Hs),e&&t(ht),e&&t(Ls),g(Te,e),e&&t(Fs),e&&t(ft),e&&t(Ms),e&&t(ut),e&&t(Os),e&&t(U),g(xe),e&&t(Bs),g(je,e),e&&t(Us),e&&t(ae),e&&t(Gs),e&&t(G),e&&t(Ws),e&&t(ne),e&&t(Rs),g(Ie,e),e&&t(Js),g(De,e),e&&t(Ys),e&&t(dt),e&&t(Ks),e&&t(ct),e&&t(Vs),e&&t(mt),e&&t(Qs),e&&t(Se),e&&t(Xs),e&&t(re),e&&t(Zs),e&&t(W),g(qe),e&&t(eo),g(Ne,e),e&&t(to),e&&t(wt),e&&t(so),e&&t(le),e&&t(oo),e&&t(kt),e&&t(ao),e&&t(R),e&&t(no),e&&t(bt),e&&t(ro),e&&t(vt),e&&t(io),e&&t(pe),e&&t(lo),e&&t(J),g(Le),e&&t(po),g(Fe,e),e&&t(ho),e&&t(gt),e&&t(fo),e&&t(yt),e&&t(uo),e&&t($t),e&&t(co),e&&t(Y),e&&t(mo),e&&t(_t),e&&t(wo),e&&t(Et),e&&t(ko),e&&t(K),g(Be),e&&t(bo),e&&t(zt),e&&t(vo),e&&t(H),e&&t(go),e&&t(Tt),e&&t(yo),e&&t(V),g(Ue),e&&t($o),e&&t(P),e&&t(_o),e&&t(de),e&&t(Eo),g(Ge,e),e&&t(zo),se.d(e),e&&t(xt),g(We,e),e&&t(To),e&&t(jt),e&&t(xo),g(Re,e),e&&t(jo),g(Je,e),e&&t(Po),e&&t(Pt),e&&t(Ao),g(Ye,e),e&&t(Io),e&&t(A),e&&t(Do),e&&t(Q),g(Ke),e&&t(So),g(Ve,e),e&&t(qo),e&&t(me),e&&t(No),e&&t(we),e&&t(Co),e&&t(L),e&&t(Ho),e&&t(It),e&&t(Lo),e&&t(X),g(Qe),e&&t(Fo),e&&t(be),e&&t(Mo),g(Xe,e),e&&t(Oo),e&&t(Dt),e&&t(Bo),g(Ze,e),e&&t(Uo),e&&t(S),e&&t(Go),e&&t(Z),g(et),e&&t(Wo),e&&t(ge),e&&t(Ro),g(tt,e),e&&t(Jo),g(st,e),e&&t(Yo),e&&t(St),e&&t(Ko),g(ye,e),e&&t(Vo),e&&t(ee),g(ot),e&&t(Qo),e&&t(te),e&&t(Xo),g(at,e),e&&t(Zo),g(nt,e),e&&t(ea),e&&t(_e),e&&t(ta),e&&t(qt)}}}const Ml={local:"tokenizers",sections:[{local:"wordbased",title:"Word-based"},{local:"characterbased",title:"Character-based"},{local:"subword-tokenization",sections:[{local:"and-more",title:"And more!"}],title:"Subword tokenization"},{local:"loading-and-saving",title:"Loading and saving"},{local:"encoding",sections:[{local:"tokenization",title:"Tokenization"},{local:"from-tokens-to-input-ids",title:"From tokens to input IDs"}],title:"Encoding"},{local:"decoding",title:"Decoding"}],title:"Tokenizers"};function Ol(D,d,y){let c="pt";return Al(()=>{const E=new URLSearchParams(window.location.search);y(0,c=E.get("fw")||"pt")}),[c]}class Kl extends zl{constructor(d){super();Tl(this,d,Ol,Fl,xl,{})}}export{Kl as default,Ml as metadata};
