import{S as wr,i as vr,s as gr,e as o,k as p,w as u,t as n,M as br,c as a,d as s,m as c,a as i,x as m,h as r,b as f,Z as mr,G as t,g as h,y as k,q as w,o as v,B as g,v as zr}from"../../chunks/vendor-hf-doc-builder.js";import{T as _r}from"../../chunks/Tip-hf-doc-builder.js";import{Y as kr}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Jt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as R}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as $r}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Er(Yt){let d,K,b,E,M,_,ue,U;return{c(){d=o("p"),K=n("\u270F\uFE0F "),b=o("strong"),E=n("Try it out!"),M=n(" Load a tokenizer from the "),_=o("code"),ue=n("bert-base-cased"),U=n(" checkpoint and pass the same example to it. What are the main differences you can see between the cased and uncased versions of the tokenizer?")},l(F){d=a(F,"P",{});var $=i(d);K=r($,"\u270F\uFE0F "),b=a($,"STRONG",{});var q=i(b);E=r(q,"Try it out!"),q.forEach(s),M=r($," Load a tokenizer from the "),_=a($,"CODE",{});var me=i(_);ue=r(me,"bert-base-cased"),me.forEach(s),U=r($," checkpoint and pass the same example to it. What are the main differences you can see between the cased and uncased versions of the tokenizer?"),$.forEach(s)},m(F,$){h(F,d,$),t(d,K),t(d,b),t(b,E),t(d,M),t(d,_),t(_,ue),t(d,U)},d(F){F&&s(d)}}}function yr(Yt){let d,K,b,E,M,_,ue,U,F,$,q,me,Oe,Fs,Kt,G,ke,Za,Gs,we,Qa,Vt,H,Ls,wt,Ws,Js,vt,Ys,Ks,Xt,L,V,gt,ve,Vs,bt,Xs,Zt,ge,Qt,X,Zs,be,Qs,eo,es,N,to,zt,so,oo,_t,ao,no,ts,ze,ss,_e,os,y,ro,$t,io,lo,Et,ho,po,yt,co,fo,as,$e,ns,Ee,rs,Z,uo,xt,mo,ko,is,Q,ls,W,ee,Tt,ye,wo,jt,vo,hs,xe,ps,te,go,Ie,bo,zo,cs,x,_o,Pt,$o,Eo,At,yo,xo,Dt,To,jo,fs,Te,ds,je,us,B,Po,St,Ao,Do,Ct,So,Co,ms,Re,qo,ks,Pe,ws,se,Ho,qt,No,Bo,vs,Ae,gs,Me,Oo,bs,Ue,Io,zs,De,_s,Se,$s,z,Ro,Ht,Mo,Uo,Nt,Fo,Go,Bt,Lo,Wo,Ot,Jo,Yo,Es,Fe,Ko,ys,J,oe,It,Ce,Vo,Rt,Xo,xs,T,qe,Zo,Qo,Mt,ea,ta,Ge,sa,oa,Ts,O,aa,Ut,na,ra,Ft,ia,la,js,Y,ae,Gt,He,ha,Lt,pa,Ps,Le,ca,As,ne,Wt,j,We,fa,da,Je,ua,ma,Ye,ka,wa,Ke,va,ga,P,A,Ve,ba,za,Xe,_a,$a,Ze,Ea,ya,Qe,xa,Ta,D,et,ja,Pa,tt,Aa,Da,st,Sa,Ca,ot,qa,Ha,S,at,Na,Ba,nt,Oa,Ia,rt,Ra,Ma,it,Ua,Fa,C,lt,Ga,La,ht,Wa,Ja,pt,Ya,Ka,ct,Va,Ds,ft,Xa,Ss;return _=new Jt({}),q=new $r({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section4.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section4.ipynb"}]}}),ve=new Jt({}),ge=new kr({props:{id:"4IIC2jI9CaU"}}),ze=new R({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
print(type(tokenizer.backend_tokenizer))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(tokenizer.backend_tokenizer))`}}),_e=new R({props:{code:"<class 'tokenizers.Tokenizer'>",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;tokenizers.Tokenizer&#x27;</span>&gt;'}}),$e=new R({props:{code:'print(tokenizer.backend_tokenizer.normalizer.normalize_str("H\xE9ll\xF2 h\xF4w are \xFC?"))',highlighted:'<span class="hljs-built_in">print</span>(tokenizer.backend_tokenizer.normalizer.normalize_str(<span class="hljs-string">&quot;H\xE9ll\xF2 h\xF4w are \xFC?&quot;</span>))'}}),Ee=new R({props:{code:"'hello how are u?'",highlighted:'<span class="hljs-string">&#x27;hello how are u?&#x27;</span>'}}),Q=new _r({props:{$$slots:{default:[Er]},$$scope:{ctx:Yt}}}),ye=new Jt({}),xe=new kr({props:{id:"grlLV8AIXug"}}),Te=new R({props:{code:'tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")',highlighted:'tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)'}}),je=new R({props:{code:"[('Hello', (0, 5)), (',', (5, 6)), ('how', (7, 10)), ('are', (11, 14)), ('you', (16, 19)), ('?', (19, 20))]",highlighted:'[(<span class="hljs-string">&#x27;Hello&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;,&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;how&#x27;</span>, (<span class="hljs-number">7</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;are&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;you&#x27;</span>, (<span class="hljs-number">16</span>, <span class="hljs-number">19</span>)), (<span class="hljs-string">&#x27;?&#x27;</span>, (<span class="hljs-number">19</span>, <span class="hljs-number">20</span>))]'}}),Pe=new R({props:{code:`tokenizer = AutoTokenizer.from_pretrained("gpt2")
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")`,highlighted:`tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)`}}),Ae=new R({props:{code:`[('Hello', (0, 5)), (',', (5, 6)), ('\u0120how', (6, 10)), ('\u0120are', (10, 14)), ('\u0120', (14, 15)), ('\u0120you', (15, 19)),
 ('?', (19, 20))]`,highlighted:`[(<span class="hljs-string">&#x27;Hello&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;,&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;\u0120how&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u0120are&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u0120&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">15</span>)), (<span class="hljs-string">&#x27;\u0120you&#x27;</span>, (<span class="hljs-number">15</span>, <span class="hljs-number">19</span>)),
 (<span class="hljs-string">&#x27;?&#x27;</span>, (<span class="hljs-number">19</span>, <span class="hljs-number">20</span>))]`}}),De=new R({props:{code:`tokenizer = AutoTokenizer.from_pretrained("t5-small")
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")`,highlighted:`tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;t5-small&quot;</span>)
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)`}}),Se=new R({props:{code:"[('\u2581Hello,', (0, 6)), ('\u2581how', (7, 10)), ('\u2581are', (11, 14)), ('\u2581you?', (16, 20))]",highlighted:'[(<span class="hljs-string">&#x27;\u2581Hello,&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;\u2581how&#x27;</span>, (<span class="hljs-number">7</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u2581are&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u2581you?&#x27;</span>, (<span class="hljs-number">16</span>, <span class="hljs-number">20</span>))]'}}),Ce=new Jt({}),He=new Jt({}),{c(){d=o("meta"),K=p(),b=o("h1"),E=o("a"),M=o("span"),u(_.$$.fragment),ue=p(),U=o("span"),F=n("Normalization and pre-tokenization"),$=p(),u(q.$$.fragment),me=p(),Oe=o("p"),Fs=n("Before we dive more deeply into the three most common subword tokenization algorithms used with Transformer models (Byte-Pair Encoding [BPE], WordPiece, and Unigram), we\u2019ll first take a look at the preprocessing that each tokenizer applies to text. Here\u2019s a high-level overview of the steps in the tokenization pipeline:"),Kt=p(),G=o("div"),ke=o("img"),Gs=p(),we=o("img"),Vt=p(),H=o("p"),Ls=n("Before splitting a text into subtokens (according to its model), the tokenizer performs two steps: "),wt=o("em"),Ws=n("normalization"),Js=n(" and "),vt=o("em"),Ys=n("pre-tokenization"),Ks=n("."),Xt=p(),L=o("h2"),V=o("a"),gt=o("span"),u(ve.$$.fragment),Vs=p(),bt=o("span"),Xs=n("Normalization"),Zt=p(),u(ge.$$.fragment),Qt=p(),X=o("p"),Zs=n("The normalization step involves some general cleanup, such as removing needless whitespace, lowercasing, and/or removing accents. If you\u2019re familiar with "),be=o("a"),Qs=n("Unicode normalization"),eo=n(" (such as NFC or NFKC), this is also something the tokenizer may apply."),es=p(),N=o("p"),to=n("The \u{1F917} Transformers "),zt=o("code"),so=n("tokenizer"),oo=n(" has an attribute called "),_t=o("code"),ao=n("backend_tokenizer"),no=n(" that provides access to the underlying tokenizer from the \u{1F917} Tokenizers library:"),ts=p(),u(ze.$$.fragment),ss=p(),u(_e.$$.fragment),os=p(),y=o("p"),ro=n("The "),$t=o("code"),io=n("normalizer"),lo=n(" attribute of the "),Et=o("code"),ho=n("tokenizer"),po=n(" object has a "),yt=o("code"),co=n("normalize_str()"),fo=n(" method that we can use to see how the normalization is performed:"),as=p(),u($e.$$.fragment),ns=p(),u(Ee.$$.fragment),rs=p(),Z=o("p"),uo=n("In this example, since we picked the "),xt=o("code"),mo=n("bert-base-uncased"),ko=n(" checkpoint, the normalization applied lowercasing and removed the accents."),is=p(),u(Q.$$.fragment),ls=p(),W=o("h2"),ee=o("a"),Tt=o("span"),u(ye.$$.fragment),wo=p(),jt=o("span"),vo=n("Pre-tokenization"),hs=p(),u(xe.$$.fragment),ps=p(),te=o("p"),go=n("As we will see in the next sections, a tokenizer cannot be trained on raw text alone. Instead, we first need to split the texts into small entities, like words. That\u2019s where the pre-tokenization step comes in. As we saw in "),Ie=o("a"),bo=n("Chapter 2"),zo=n(", a word-based tokenizer can simply split a raw text into words on whitespace and punctuation. Those words will be the boundaries of the subtokens the tokenizer can learn during its training."),cs=p(),x=o("p"),_o=n("To see how a fast tokenizer performs pre-tokenization, we can use the "),Pt=o("code"),$o=n("pre_tokenize_str()"),Eo=n(" method of the "),At=o("code"),yo=n("pre_tokenizer"),xo=n(" attribute of the "),Dt=o("code"),To=n("tokenizer"),jo=n(" object:"),fs=p(),u(Te.$$.fragment),ds=p(),u(je.$$.fragment),us=p(),B=o("p"),Po=n("Notice how the tokenizer is already keeping track of the offsets, which is how it can give us the offset mapping we used in the previous section. Here the tokenizer ignores the two spaces and replaces them with just one, but the offset jumps between "),St=o("code"),Ao=n("are"),Do=n(" and "),Ct=o("code"),So=n("you"),Co=n(" to account for that."),ms=p(),Re=o("p"),qo=n("Since we\u2019re using a BERT tokenizer, the pre-tokenization involves splitting on whitespace and punctuation. Other tokenizers can have different rules for this step. For example, if we use the GPT-2 tokenizer:"),ks=p(),u(Pe.$$.fragment),ws=p(),se=o("p"),Ho=n("it will split on whitespace and punctuation as well, but it will keep the spaces and replace them with a "),qt=o("code"),No=n("\u0120"),Bo=n(" symbol, enabling it to recover the original spaces if we decode the tokens:"),vs=p(),u(Ae.$$.fragment),gs=p(),Me=o("p"),Oo=n("Also note that unlike the BERT tokenizer, this tokenizer does not ignore the double space."),bs=p(),Ue=o("p"),Io=n("For a last example, let\u2019s have a look at the T5 tokenizer, which is based on the SentencePiece algorithm:"),zs=p(),u(De.$$.fragment),_s=p(),u(Se.$$.fragment),$s=p(),z=o("p"),Ro=n("Like the GPT-2 tokenizer, this one keeps spaces and replaces them with a specific token ("),Ht=o("code"),Mo=n("_"),Uo=n("), but the T5 tokenizer only splits on whitespace, not punctuation. Also note that it added a space by default at the beginning of the sentence (before "),Nt=o("code"),Fo=n("Hello"),Go=n(") and ignored the double space between "),Bt=o("code"),Lo=n("are"),Wo=n(" and "),Ot=o("code"),Jo=n("you"),Yo=n("."),Es=p(),Fe=o("p"),Ko=n("Now that we\u2019ve seen a little of how some different tokenizers process text, we can start to explore the underlying algorithms themselves. We\u2019ll begin with a quick look at the broadly widely applicable SentencePiece; then, over the next three sections, we\u2019ll examine how the three main algorithms used for subword tokenization work."),ys=p(),J=o("h2"),oe=o("a"),It=o("span"),u(Ce.$$.fragment),Vo=p(),Rt=o("span"),Xo=n("SentencePiece"),xs=p(),T=o("p"),qe=o("a"),Zo=n("SentencePiece"),Qo=n(" is a tokenization algorithm for the preprocessing of text that you can use with any of the models we will see in the next three sections. It considers the text as a sequence of Unicode characters, and replaces spaces with a special character, "),Mt=o("code"),ea=n("\u2581"),ta=n(". Used in conjunction with the Unigram algorithm (see "),Ge=o("a"),sa=n("section 7"),oa=n("), it doesn\u2019t even require a pre-tokenization step, which is very useful for languages where the space character is not used (like Chinese or Japanese)."),Ts=p(),O=o("p"),aa=n("The other main feature of SentencePiece is "),Ut=o("em"),na=n("reversible tokenization"),ra=n(": since there is no special treatment of spaces, decoding the tokens is done simply by concatenating them and replacing the "),Ft=o("code"),ia=n("_"),la=n("s with spaces \u2014 this results in the normalized text. As we saw earlier, the BERT tokenizer removes repeating spaces, so its tokenization is not reversible."),js=p(),Y=o("h2"),ae=o("a"),Gt=o("span"),u(He.$$.fragment),ha=p(),Lt=o("span"),pa=n("Algorithm overview"),Ps=p(),Le=o("p"),ca=n("In the following sections, we\u2019ll dive into the three main subword tokenization algorithms: BPE (used by GPT-2 and others), WordPiece (used for example by BERT), and Unigram (used by T5 and others). Before we get started, here\u2019s a quick overview of how they each work. Don\u2019t hesitate to come back to this table after reading each of the next sections if it doesn\u2019t make sense to you yet."),As=p(),ne=o("table"),Wt=o("thead"),j=o("tr"),We=o("th"),fa=n("Model"),da=p(),Je=o("th"),ua=n("BPE"),ma=p(),Ye=o("th"),ka=n("WordPiece"),wa=p(),Ke=o("th"),va=n("Unigram"),ga=p(),P=o("tbody"),A=o("tr"),Ve=o("td"),ba=n("Training"),za=p(),Xe=o("td"),_a=n("Starts from a small vocabulary and learns rules to merge tokens"),$a=p(),Ze=o("td"),Ea=n("Starts from a small vocabulary and learns rules to merge tokens"),ya=p(),Qe=o("td"),xa=n("Starts from a large vocabulary and learns rules to remove tokens"),Ta=p(),D=o("tr"),et=o("td"),ja=n("Training step"),Pa=p(),tt=o("td"),Aa=n("Merges the tokens corresponding to the most common pair"),Da=p(),st=o("td"),Sa=n("Merges the tokens corresponding to the pair with the best score based on the frequency of the pair, privileging pairs where each individual token is less frequent"),Ca=p(),ot=o("td"),qa=n("Removes all the tokens in the vocabulary that will minimize the loss computed on the whole corpus"),Ha=p(),S=o("tr"),at=o("td"),Na=n("Learns"),Ba=p(),nt=o("td"),Oa=n("Merge rules and a vocabulary"),Ia=p(),rt=o("td"),Ra=n("Just a vocabulary"),Ma=p(),it=o("td"),Ua=n("A vocabulary with a score for each token"),Fa=p(),C=o("tr"),lt=o("td"),Ga=n("Encoding"),La=p(),ht=o("td"),Wa=n("Splits a word into characters and applies the merges learned during training"),Ja=p(),pt=o("td"),Ya=n("Finds the longest subword starting from the beginning that is in the vocabulary, then does the same for the rest of the word"),Ka=p(),ct=o("td"),Va=n("Finds the most likely split into tokens, using the scores learned during training"),Ds=p(),ft=o("p"),Xa=n("Now let\u2019s dive into BPE!"),this.h()},l(e){const l=br('[data-svelte="svelte-1phssyn"]',document.head);d=a(l,"META",{name:!0,content:!0}),l.forEach(s),K=c(e),b=a(e,"H1",{class:!0});var Ne=i(b);E=a(Ne,"A",{id:!0,class:!0,href:!0});var en=i(E);M=a(en,"SPAN",{});var tn=i(M);m(_.$$.fragment,tn),tn.forEach(s),en.forEach(s),ue=c(Ne),U=a(Ne,"SPAN",{});var sn=i(U);F=r(sn,"Normalization and pre-tokenization"),sn.forEach(s),Ne.forEach(s),$=c(e),m(q.$$.fragment,e),me=c(e),Oe=a(e,"P",{});var on=i(Oe);Fs=r(on,"Before we dive more deeply into the three most common subword tokenization algorithms used with Transformer models (Byte-Pair Encoding [BPE], WordPiece, and Unigram), we\u2019ll first take a look at the preprocessing that each tokenizer applies to text. Here\u2019s a high-level overview of the steps in the tokenization pipeline:"),on.forEach(s),Kt=c(e),G=a(e,"DIV",{class:!0});var Cs=i(G);ke=a(Cs,"IMG",{class:!0,src:!0,alt:!0}),Gs=c(Cs),we=a(Cs,"IMG",{class:!0,src:!0,alt:!0}),Cs.forEach(s),Vt=c(e),H=a(e,"P",{});var dt=i(H);Ls=r(dt,"Before splitting a text into subtokens (according to its model), the tokenizer performs two steps: "),wt=a(dt,"EM",{});var an=i(wt);Ws=r(an,"normalization"),an.forEach(s),Js=r(dt," and "),vt=a(dt,"EM",{});var nn=i(vt);Ys=r(nn,"pre-tokenization"),nn.forEach(s),Ks=r(dt,"."),dt.forEach(s),Xt=c(e),L=a(e,"H2",{class:!0});var qs=i(L);V=a(qs,"A",{id:!0,class:!0,href:!0});var rn=i(V);gt=a(rn,"SPAN",{});var ln=i(gt);m(ve.$$.fragment,ln),ln.forEach(s),rn.forEach(s),Vs=c(qs),bt=a(qs,"SPAN",{});var hn=i(bt);Xs=r(hn,"Normalization"),hn.forEach(s),qs.forEach(s),Zt=c(e),m(ge.$$.fragment,e),Qt=c(e),X=a(e,"P",{});var Hs=i(X);Zs=r(Hs,"The normalization step involves some general cleanup, such as removing needless whitespace, lowercasing, and/or removing accents. If you\u2019re familiar with "),be=a(Hs,"A",{href:!0,rel:!0});var pn=i(be);Qs=r(pn,"Unicode normalization"),pn.forEach(s),eo=r(Hs," (such as NFC or NFKC), this is also something the tokenizer may apply."),Hs.forEach(s),es=c(e),N=a(e,"P",{});var ut=i(N);to=r(ut,"The \u{1F917} Transformers "),zt=a(ut,"CODE",{});var cn=i(zt);so=r(cn,"tokenizer"),cn.forEach(s),oo=r(ut," has an attribute called "),_t=a(ut,"CODE",{});var fn=i(_t);ao=r(fn,"backend_tokenizer"),fn.forEach(s),no=r(ut," that provides access to the underlying tokenizer from the \u{1F917} Tokenizers library:"),ut.forEach(s),ts=c(e),m(ze.$$.fragment,e),ss=c(e),m(_e.$$.fragment,e),os=c(e),y=a(e,"P",{});var re=i(y);ro=r(re,"The "),$t=a(re,"CODE",{});var dn=i($t);io=r(dn,"normalizer"),dn.forEach(s),lo=r(re," attribute of the "),Et=a(re,"CODE",{});var un=i(Et);ho=r(un,"tokenizer"),un.forEach(s),po=r(re," object has a "),yt=a(re,"CODE",{});var mn=i(yt);co=r(mn,"normalize_str()"),mn.forEach(s),fo=r(re," method that we can use to see how the normalization is performed:"),re.forEach(s),as=c(e),m($e.$$.fragment,e),ns=c(e),m(Ee.$$.fragment,e),rs=c(e),Z=a(e,"P",{});var Ns=i(Z);uo=r(Ns,"In this example, since we picked the "),xt=a(Ns,"CODE",{});var kn=i(xt);mo=r(kn,"bert-base-uncased"),kn.forEach(s),ko=r(Ns," checkpoint, the normalization applied lowercasing and removed the accents."),Ns.forEach(s),is=c(e),m(Q.$$.fragment,e),ls=c(e),W=a(e,"H2",{class:!0});var Bs=i(W);ee=a(Bs,"A",{id:!0,class:!0,href:!0});var wn=i(ee);Tt=a(wn,"SPAN",{});var vn=i(Tt);m(ye.$$.fragment,vn),vn.forEach(s),wn.forEach(s),wo=c(Bs),jt=a(Bs,"SPAN",{});var gn=i(jt);vo=r(gn,"Pre-tokenization"),gn.forEach(s),Bs.forEach(s),hs=c(e),m(xe.$$.fragment,e),ps=c(e),te=a(e,"P",{});var Os=i(te);go=r(Os,"As we will see in the next sections, a tokenizer cannot be trained on raw text alone. Instead, we first need to split the texts into small entities, like words. That\u2019s where the pre-tokenization step comes in. As we saw in "),Ie=a(Os,"A",{href:!0});var bn=i(Ie);bo=r(bn,"Chapter 2"),bn.forEach(s),zo=r(Os,", a word-based tokenizer can simply split a raw text into words on whitespace and punctuation. Those words will be the boundaries of the subtokens the tokenizer can learn during its training."),Os.forEach(s),cs=c(e),x=a(e,"P",{});var ie=i(x);_o=r(ie,"To see how a fast tokenizer performs pre-tokenization, we can use the "),Pt=a(ie,"CODE",{});var zn=i(Pt);$o=r(zn,"pre_tokenize_str()"),zn.forEach(s),Eo=r(ie," method of the "),At=a(ie,"CODE",{});var _n=i(At);yo=r(_n,"pre_tokenizer"),_n.forEach(s),xo=r(ie," attribute of the "),Dt=a(ie,"CODE",{});var $n=i(Dt);To=r($n,"tokenizer"),$n.forEach(s),jo=r(ie," object:"),ie.forEach(s),fs=c(e),m(Te.$$.fragment,e),ds=c(e),m(je.$$.fragment,e),us=c(e),B=a(e,"P",{});var mt=i(B);Po=r(mt,"Notice how the tokenizer is already keeping track of the offsets, which is how it can give us the offset mapping we used in the previous section. Here the tokenizer ignores the two spaces and replaces them with just one, but the offset jumps between "),St=a(mt,"CODE",{});var En=i(St);Ao=r(En,"are"),En.forEach(s),Do=r(mt," and "),Ct=a(mt,"CODE",{});var yn=i(Ct);So=r(yn,"you"),yn.forEach(s),Co=r(mt," to account for that."),mt.forEach(s),ms=c(e),Re=a(e,"P",{});var xn=i(Re);qo=r(xn,"Since we\u2019re using a BERT tokenizer, the pre-tokenization involves splitting on whitespace and punctuation. Other tokenizers can have different rules for this step. For example, if we use the GPT-2 tokenizer:"),xn.forEach(s),ks=c(e),m(Pe.$$.fragment,e),ws=c(e),se=a(e,"P",{});var Is=i(se);Ho=r(Is,"it will split on whitespace and punctuation as well, but it will keep the spaces and replace them with a "),qt=a(Is,"CODE",{});var Tn=i(qt);No=r(Tn,"\u0120"),Tn.forEach(s),Bo=r(Is," symbol, enabling it to recover the original spaces if we decode the tokens:"),Is.forEach(s),vs=c(e),m(Ae.$$.fragment,e),gs=c(e),Me=a(e,"P",{});var jn=i(Me);Oo=r(jn,"Also note that unlike the BERT tokenizer, this tokenizer does not ignore the double space."),jn.forEach(s),bs=c(e),Ue=a(e,"P",{});var Pn=i(Ue);Io=r(Pn,"For a last example, let\u2019s have a look at the T5 tokenizer, which is based on the SentencePiece algorithm:"),Pn.forEach(s),zs=c(e),m(De.$$.fragment,e),_s=c(e),m(Se.$$.fragment,e),$s=c(e),z=a(e,"P",{});var I=i(z);Ro=r(I,"Like the GPT-2 tokenizer, this one keeps spaces and replaces them with a specific token ("),Ht=a(I,"CODE",{});var An=i(Ht);Mo=r(An,"_"),An.forEach(s),Uo=r(I,"), but the T5 tokenizer only splits on whitespace, not punctuation. Also note that it added a space by default at the beginning of the sentence (before "),Nt=a(I,"CODE",{});var Dn=i(Nt);Fo=r(Dn,"Hello"),Dn.forEach(s),Go=r(I,") and ignored the double space between "),Bt=a(I,"CODE",{});var Sn=i(Bt);Lo=r(Sn,"are"),Sn.forEach(s),Wo=r(I," and "),Ot=a(I,"CODE",{});var Cn=i(Ot);Jo=r(Cn,"you"),Cn.forEach(s),Yo=r(I,"."),I.forEach(s),Es=c(e),Fe=a(e,"P",{});var qn=i(Fe);Ko=r(qn,"Now that we\u2019ve seen a little of how some different tokenizers process text, we can start to explore the underlying algorithms themselves. We\u2019ll begin with a quick look at the broadly widely applicable SentencePiece; then, over the next three sections, we\u2019ll examine how the three main algorithms used for subword tokenization work."),qn.forEach(s),ys=c(e),J=a(e,"H2",{class:!0});var Rs=i(J);oe=a(Rs,"A",{id:!0,class:!0,href:!0});var Hn=i(oe);It=a(Hn,"SPAN",{});var Nn=i(It);m(Ce.$$.fragment,Nn),Nn.forEach(s),Hn.forEach(s),Vo=c(Rs),Rt=a(Rs,"SPAN",{});var Bn=i(Rt);Xo=r(Bn,"SentencePiece"),Bn.forEach(s),Rs.forEach(s),xs=c(e),T=a(e,"P",{});var Be=i(T);qe=a(Be,"A",{href:!0,rel:!0});var On=i(qe);Zo=r(On,"SentencePiece"),On.forEach(s),Qo=r(Be," is a tokenization algorithm for the preprocessing of text that you can use with any of the models we will see in the next three sections. It considers the text as a sequence of Unicode characters, and replaces spaces with a special character, "),Mt=a(Be,"CODE",{});var In=i(Mt);ea=r(In,"\u2581"),In.forEach(s),ta=r(Be,". Used in conjunction with the Unigram algorithm (see "),Ge=a(Be,"A",{href:!0});var Rn=i(Ge);sa=r(Rn,"section 7"),Rn.forEach(s),oa=r(Be,"), it doesn\u2019t even require a pre-tokenization step, which is very useful for languages where the space character is not used (like Chinese or Japanese)."),Be.forEach(s),Ts=c(e),O=a(e,"P",{});var kt=i(O);aa=r(kt,"The other main feature of SentencePiece is "),Ut=a(kt,"EM",{});var Mn=i(Ut);na=r(Mn,"reversible tokenization"),Mn.forEach(s),ra=r(kt,": since there is no special treatment of spaces, decoding the tokens is done simply by concatenating them and replacing the "),Ft=a(kt,"CODE",{});var Un=i(Ft);ia=r(Un,"_"),Un.forEach(s),la=r(kt,"s with spaces \u2014 this results in the normalized text. As we saw earlier, the BERT tokenizer removes repeating spaces, so its tokenization is not reversible."),kt.forEach(s),js=c(e),Y=a(e,"H2",{class:!0});var Ms=i(Y);ae=a(Ms,"A",{id:!0,class:!0,href:!0});var Fn=i(ae);Gt=a(Fn,"SPAN",{});var Gn=i(Gt);m(He.$$.fragment,Gn),Gn.forEach(s),Fn.forEach(s),ha=c(Ms),Lt=a(Ms,"SPAN",{});var Ln=i(Lt);pa=r(Ln,"Algorithm overview"),Ln.forEach(s),Ms.forEach(s),Ps=c(e),Le=a(e,"P",{});var Wn=i(Le);ca=r(Wn,"In the following sections, we\u2019ll dive into the three main subword tokenization algorithms: BPE (used by GPT-2 and others), WordPiece (used for example by BERT), and Unigram (used by T5 and others). Before we get started, here\u2019s a quick overview of how they each work. Don\u2019t hesitate to come back to this table after reading each of the next sections if it doesn\u2019t make sense to you yet."),Wn.forEach(s),As=c(e),ne=a(e,"TABLE",{});var Us=i(ne);Wt=a(Us,"THEAD",{});var Jn=i(Wt);j=a(Jn,"TR",{});var le=i(j);We=a(le,"TH",{align:!0});var Yn=i(We);fa=r(Yn,"Model"),Yn.forEach(s),da=c(le),Je=a(le,"TH",{align:!0});var Kn=i(Je);ua=r(Kn,"BPE"),Kn.forEach(s),ma=c(le),Ye=a(le,"TH",{align:!0});var Vn=i(Ye);ka=r(Vn,"WordPiece"),Vn.forEach(s),wa=c(le),Ke=a(le,"TH",{align:!0});var Xn=i(Ke);va=r(Xn,"Unigram"),Xn.forEach(s),le.forEach(s),Jn.forEach(s),ga=c(Us),P=a(Us,"TBODY",{});var he=i(P);A=a(he,"TR",{});var pe=i(A);Ve=a(pe,"TD",{align:!0});var Zn=i(Ve);ba=r(Zn,"Training"),Zn.forEach(s),za=c(pe),Xe=a(pe,"TD",{align:!0});var Qn=i(Xe);_a=r(Qn,"Starts from a small vocabulary and learns rules to merge tokens"),Qn.forEach(s),$a=c(pe),Ze=a(pe,"TD",{align:!0});var er=i(Ze);Ea=r(er,"Starts from a small vocabulary and learns rules to merge tokens"),er.forEach(s),ya=c(pe),Qe=a(pe,"TD",{align:!0});var tr=i(Qe);xa=r(tr,"Starts from a large vocabulary and learns rules to remove tokens"),tr.forEach(s),pe.forEach(s),Ta=c(he),D=a(he,"TR",{});var ce=i(D);et=a(ce,"TD",{align:!0});var sr=i(et);ja=r(sr,"Training step"),sr.forEach(s),Pa=c(ce),tt=a(ce,"TD",{align:!0});var or=i(tt);Aa=r(or,"Merges the tokens corresponding to the most common pair"),or.forEach(s),Da=c(ce),st=a(ce,"TD",{align:!0});var ar=i(st);Sa=r(ar,"Merges the tokens corresponding to the pair with the best score based on the frequency of the pair, privileging pairs where each individual token is less frequent"),ar.forEach(s),Ca=c(ce),ot=a(ce,"TD",{align:!0});var nr=i(ot);qa=r(nr,"Removes all the tokens in the vocabulary that will minimize the loss computed on the whole corpus"),nr.forEach(s),ce.forEach(s),Ha=c(he),S=a(he,"TR",{});var fe=i(S);at=a(fe,"TD",{align:!0});var rr=i(at);Na=r(rr,"Learns"),rr.forEach(s),Ba=c(fe),nt=a(fe,"TD",{align:!0});var ir=i(nt);Oa=r(ir,"Merge rules and a vocabulary"),ir.forEach(s),Ia=c(fe),rt=a(fe,"TD",{align:!0});var lr=i(rt);Ra=r(lr,"Just a vocabulary"),lr.forEach(s),Ma=c(fe),it=a(fe,"TD",{align:!0});var hr=i(it);Ua=r(hr,"A vocabulary with a score for each token"),hr.forEach(s),fe.forEach(s),Fa=c(he),C=a(he,"TR",{});var de=i(C);lt=a(de,"TD",{align:!0});var pr=i(lt);Ga=r(pr,"Encoding"),pr.forEach(s),La=c(de),ht=a(de,"TD",{align:!0});var cr=i(ht);Wa=r(cr,"Splits a word into characters and applies the merges learned during training"),cr.forEach(s),Ja=c(de),pt=a(de,"TD",{align:!0});var fr=i(pt);Ya=r(fr,"Finds the longest subword starting from the beginning that is in the vocabulary, then does the same for the rest of the word"),fr.forEach(s),Ka=c(de),ct=a(de,"TD",{align:!0});var dr=i(ct);Va=r(dr,"Finds the most likely split into tokens, using the scores learned during training"),dr.forEach(s),de.forEach(s),he.forEach(s),Us.forEach(s),Ds=c(e),ft=a(e,"P",{});var ur=i(ft);Xa=r(ur,"Now let\u2019s dive into BPE!"),ur.forEach(s),this.h()},h(){f(d,"name","hf:doc:metadata"),f(d,"content",JSON.stringify(xr)),f(E,"id","normalization-and-pretokenization"),f(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(E,"href","#normalization-and-pretokenization"),f(b,"class","relative group"),f(ke,"class","block dark:hidden"),mr(ke.src,Za="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline.svg")||f(ke,"src",Za),f(ke,"alt","The tokenization pipeline."),f(we,"class","hidden dark:block"),mr(we.src,Qa="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline-dark.svg")||f(we,"src",Qa),f(we,"alt","The tokenization pipeline."),f(G,"class","flex justify-center"),f(V,"id","normalization"),f(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(V,"href","#normalization"),f(L,"class","relative group"),f(be,"href","http://www.unicode.org/reports/tr15/"),f(be,"rel","nofollow"),f(ee,"id","pretokenization"),f(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ee,"href","#pretokenization"),f(W,"class","relative group"),f(Ie,"href","/course/chapter2"),f(oe,"id","sentencepiece"),f(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(oe,"href","#sentencepiece"),f(J,"class","relative group"),f(qe,"href","https://github.com/google/sentencepiece"),f(qe,"rel","nofollow"),f(Ge,"href","/course/chapter7/7"),f(ae,"id","algorithm-overview"),f(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ae,"href","#algorithm-overview"),f(Y,"class","relative group"),f(We,"align","center"),f(Je,"align","center"),f(Ye,"align","center"),f(Ke,"align","center"),f(Ve,"align","center"),f(Xe,"align","center"),f(Ze,"align","center"),f(Qe,"align","center"),f(et,"align","center"),f(tt,"align","center"),f(st,"align","center"),f(ot,"align","center"),f(at,"align","center"),f(nt,"align","center"),f(rt,"align","center"),f(it,"align","center"),f(lt,"align","center"),f(ht,"align","center"),f(pt,"align","center"),f(ct,"align","center")},m(e,l){t(document.head,d),h(e,K,l),h(e,b,l),t(b,E),t(E,M),k(_,M,null),t(b,ue),t(b,U),t(U,F),h(e,$,l),k(q,e,l),h(e,me,l),h(e,Oe,l),t(Oe,Fs),h(e,Kt,l),h(e,G,l),t(G,ke),t(G,Gs),t(G,we),h(e,Vt,l),h(e,H,l),t(H,Ls),t(H,wt),t(wt,Ws),t(H,Js),t(H,vt),t(vt,Ys),t(H,Ks),h(e,Xt,l),h(e,L,l),t(L,V),t(V,gt),k(ve,gt,null),t(L,Vs),t(L,bt),t(bt,Xs),h(e,Zt,l),k(ge,e,l),h(e,Qt,l),h(e,X,l),t(X,Zs),t(X,be),t(be,Qs),t(X,eo),h(e,es,l),h(e,N,l),t(N,to),t(N,zt),t(zt,so),t(N,oo),t(N,_t),t(_t,ao),t(N,no),h(e,ts,l),k(ze,e,l),h(e,ss,l),k(_e,e,l),h(e,os,l),h(e,y,l),t(y,ro),t(y,$t),t($t,io),t(y,lo),t(y,Et),t(Et,ho),t(y,po),t(y,yt),t(yt,co),t(y,fo),h(e,as,l),k($e,e,l),h(e,ns,l),k(Ee,e,l),h(e,rs,l),h(e,Z,l),t(Z,uo),t(Z,xt),t(xt,mo),t(Z,ko),h(e,is,l),k(Q,e,l),h(e,ls,l),h(e,W,l),t(W,ee),t(ee,Tt),k(ye,Tt,null),t(W,wo),t(W,jt),t(jt,vo),h(e,hs,l),k(xe,e,l),h(e,ps,l),h(e,te,l),t(te,go),t(te,Ie),t(Ie,bo),t(te,zo),h(e,cs,l),h(e,x,l),t(x,_o),t(x,Pt),t(Pt,$o),t(x,Eo),t(x,At),t(At,yo),t(x,xo),t(x,Dt),t(Dt,To),t(x,jo),h(e,fs,l),k(Te,e,l),h(e,ds,l),k(je,e,l),h(e,us,l),h(e,B,l),t(B,Po),t(B,St),t(St,Ao),t(B,Do),t(B,Ct),t(Ct,So),t(B,Co),h(e,ms,l),h(e,Re,l),t(Re,qo),h(e,ks,l),k(Pe,e,l),h(e,ws,l),h(e,se,l),t(se,Ho),t(se,qt),t(qt,No),t(se,Bo),h(e,vs,l),k(Ae,e,l),h(e,gs,l),h(e,Me,l),t(Me,Oo),h(e,bs,l),h(e,Ue,l),t(Ue,Io),h(e,zs,l),k(De,e,l),h(e,_s,l),k(Se,e,l),h(e,$s,l),h(e,z,l),t(z,Ro),t(z,Ht),t(Ht,Mo),t(z,Uo),t(z,Nt),t(Nt,Fo),t(z,Go),t(z,Bt),t(Bt,Lo),t(z,Wo),t(z,Ot),t(Ot,Jo),t(z,Yo),h(e,Es,l),h(e,Fe,l),t(Fe,Ko),h(e,ys,l),h(e,J,l),t(J,oe),t(oe,It),k(Ce,It,null),t(J,Vo),t(J,Rt),t(Rt,Xo),h(e,xs,l),h(e,T,l),t(T,qe),t(qe,Zo),t(T,Qo),t(T,Mt),t(Mt,ea),t(T,ta),t(T,Ge),t(Ge,sa),t(T,oa),h(e,Ts,l),h(e,O,l),t(O,aa),t(O,Ut),t(Ut,na),t(O,ra),t(O,Ft),t(Ft,ia),t(O,la),h(e,js,l),h(e,Y,l),t(Y,ae),t(ae,Gt),k(He,Gt,null),t(Y,ha),t(Y,Lt),t(Lt,pa),h(e,Ps,l),h(e,Le,l),t(Le,ca),h(e,As,l),h(e,ne,l),t(ne,Wt),t(Wt,j),t(j,We),t(We,fa),t(j,da),t(j,Je),t(Je,ua),t(j,ma),t(j,Ye),t(Ye,ka),t(j,wa),t(j,Ke),t(Ke,va),t(ne,ga),t(ne,P),t(P,A),t(A,Ve),t(Ve,ba),t(A,za),t(A,Xe),t(Xe,_a),t(A,$a),t(A,Ze),t(Ze,Ea),t(A,ya),t(A,Qe),t(Qe,xa),t(P,Ta),t(P,D),t(D,et),t(et,ja),t(D,Pa),t(D,tt),t(tt,Aa),t(D,Da),t(D,st),t(st,Sa),t(D,Ca),t(D,ot),t(ot,qa),t(P,Ha),t(P,S),t(S,at),t(at,Na),t(S,Ba),t(S,nt),t(nt,Oa),t(S,Ia),t(S,rt),t(rt,Ra),t(S,Ma),t(S,it),t(it,Ua),t(P,Fa),t(P,C),t(C,lt),t(lt,Ga),t(C,La),t(C,ht),t(ht,Wa),t(C,Ja),t(C,pt),t(pt,Ya),t(C,Ka),t(C,ct),t(ct,Va),h(e,Ds,l),h(e,ft,l),t(ft,Xa),Ss=!0},p(e,[l]){const Ne={};l&2&&(Ne.$$scope={dirty:l,ctx:e}),Q.$set(Ne)},i(e){Ss||(w(_.$$.fragment,e),w(q.$$.fragment,e),w(ve.$$.fragment,e),w(ge.$$.fragment,e),w(ze.$$.fragment,e),w(_e.$$.fragment,e),w($e.$$.fragment,e),w(Ee.$$.fragment,e),w(Q.$$.fragment,e),w(ye.$$.fragment,e),w(xe.$$.fragment,e),w(Te.$$.fragment,e),w(je.$$.fragment,e),w(Pe.$$.fragment,e),w(Ae.$$.fragment,e),w(De.$$.fragment,e),w(Se.$$.fragment,e),w(Ce.$$.fragment,e),w(He.$$.fragment,e),Ss=!0)},o(e){v(_.$$.fragment,e),v(q.$$.fragment,e),v(ve.$$.fragment,e),v(ge.$$.fragment,e),v(ze.$$.fragment,e),v(_e.$$.fragment,e),v($e.$$.fragment,e),v(Ee.$$.fragment,e),v(Q.$$.fragment,e),v(ye.$$.fragment,e),v(xe.$$.fragment,e),v(Te.$$.fragment,e),v(je.$$.fragment,e),v(Pe.$$.fragment,e),v(Ae.$$.fragment,e),v(De.$$.fragment,e),v(Se.$$.fragment,e),v(Ce.$$.fragment,e),v(He.$$.fragment,e),Ss=!1},d(e){s(d),e&&s(K),e&&s(b),g(_),e&&s($),g(q,e),e&&s(me),e&&s(Oe),e&&s(Kt),e&&s(G),e&&s(Vt),e&&s(H),e&&s(Xt),e&&s(L),g(ve),e&&s(Zt),g(ge,e),e&&s(Qt),e&&s(X),e&&s(es),e&&s(N),e&&s(ts),g(ze,e),e&&s(ss),g(_e,e),e&&s(os),e&&s(y),e&&s(as),g($e,e),e&&s(ns),g(Ee,e),e&&s(rs),e&&s(Z),e&&s(is),g(Q,e),e&&s(ls),e&&s(W),g(ye),e&&s(hs),g(xe,e),e&&s(ps),e&&s(te),e&&s(cs),e&&s(x),e&&s(fs),g(Te,e),e&&s(ds),g(je,e),e&&s(us),e&&s(B),e&&s(ms),e&&s(Re),e&&s(ks),g(Pe,e),e&&s(ws),e&&s(se),e&&s(vs),g(Ae,e),e&&s(gs),e&&s(Me),e&&s(bs),e&&s(Ue),e&&s(zs),g(De,e),e&&s(_s),g(Se,e),e&&s($s),e&&s(z),e&&s(Es),e&&s(Fe),e&&s(ys),e&&s(J),g(Ce),e&&s(xs),e&&s(T),e&&s(Ts),e&&s(O),e&&s(js),e&&s(Y),g(He),e&&s(Ps),e&&s(Le),e&&s(As),e&&s(ne),e&&s(Ds),e&&s(ft)}}}const xr={local:"normalization-and-pretokenization",sections:[{local:"normalization",title:"Normalization"},{local:"pretokenization",title:"Pre-tokenization"},{local:"sentencepiece",title:"SentencePiece"},{local:"algorithm-overview",title:"Algorithm overview"}],title:"Normalization and pre-tokenization"};function Tr(Yt){return zr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qr extends wr{constructor(d){super();vr(this,d,Tr,yr,gr,{})}}export{qr as default,xr as metadata};
