import{S as B_,i as A_,s as N_,e as a,k as c,w as f,t as n,M as W_,c as i,d as s,m as h,a as l,x as d,h as o,b as $,N as S_,G as t,g as p,y as u,q as m,o as k,B as _,v as F_}from"../../chunks/vendor-hf-doc-builder.js";import{T as I_}from"../../chunks/Tip-hf-doc-builder.js";import{Y as R_}from"../../chunks/Youtube-hf-doc-builder.js";import{I as na}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as w}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as U_}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function G_(oa){let z,ae,A,F,Q,ee,ct,te,ht,qe,N,Pe,ie,ve,Ce,y,se,ft,dt,ne,ut,mt,oe,kt,_t;return{c(){z=a("p"),ae=a("strong"),A=n("To go further"),F=n(" If you test the two versions of the previous normalizers on a string containing the unicode character "),Q=a("code"),ee=n('u"\\u0085"'),ct=n(` you will surely notice that these two normalizers are not exactly equivalent.
To not over-complicate the version with `),te=a("code"),ht=n("normalizers.Sequence"),qe=n(" too much , we haven\u2019t included the Regex replacements that the "),N=a("code"),Pe=n("BertNormalizer"),ie=n(" requires when the "),ve=a("code"),Ce=n("clean_text"),y=n(" argument is set to "),se=a("code"),ft=n("True"),dt=n(" - which is the default behavior. But don\u2019t worry: it is possible to get exactly the same normalization without using the handy "),ne=a("code"),ut=n("BertNormalizer"),mt=n(" by adding two "),oe=a("code"),kt=n("normalizers.Replace"),_t=n("\u2019s to the normalizers sequence.")},l(re){z=i(re,"P",{});var g=l(z);ae=i(g,"STRONG",{});var wt=l(ae);A=o(wt,"To go further"),wt.forEach(s),F=o(g," If you test the two versions of the previous normalizers on a string containing the unicode character "),Q=i(g,"CODE",{});var Ee=l(Q);ee=o(Ee,'u"\\u0085"'),Ee.forEach(s),ct=o(g,` you will surely notice that these two normalizers are not exactly equivalent.
To not over-complicate the version with `),te=i(g,"CODE",{});var Qs=l(te);ht=o(Qs,"normalizers.Sequence"),Qs.forEach(s),qe=o(g," too much , we haven\u2019t included the Regex replacements that the "),N=i(g,"CODE",{});var $t=l(N);Pe=o($t,"BertNormalizer"),$t.forEach(s),ie=o(g," requires when the "),ve=i(g,"CODE",{});var I=l(ve);Ce=o(I,"clean_text"),I.forEach(s),y=o(g," argument is set to "),se=i(g,"CODE",{});var le=l(se);ft=o(le,"True"),le.forEach(s),dt=o(g," - which is the default behavior. But don\u2019t worry: it is possible to get exactly the same normalization without using the handy "),ne=i(g,"CODE",{});var Rn=l(ne);ut=o(Rn,"BertNormalizer"),Rn.forEach(s),mt=o(g," by adding two "),oe=i(g,"CODE",{});var en=l(oe);kt=o(en,"normalizers.Replace"),en.forEach(s),_t=o(g,"\u2019s to the normalizers sequence."),g.forEach(s)},m(re,g){p(re,z,g),t(z,ae),t(ae,A),t(z,F),t(z,Q),t(Q,ee),t(z,ct),t(z,te),t(te,ht),t(z,qe),t(z,N),t(N,Pe),t(z,ie),t(z,ve),t(ve,Ce),t(z,y),t(z,se),t(se,ft),t(z,dt),t(z,ne),t(ne,ut),t(z,mt),t(z,oe),t(oe,kt),t(z,_t)},d(re){re&&s(z)}}}function M_(oa){let z,ae,A,F,Q,ee,ct,te,ht,qe,N,Pe,ie,ve,Ce,y,se,ft,dt,ne,ut,mt,oe,kt,_t,re,g,wt,Ee,Qs,$t,I,le,Rn,en,zt,gm,ra,De,up,tn,mp,kp,aa,bt,ia,Oe,_p,Un,wp,$p,la,T,pe,Gn,zp,bp,Mn,gp,vp,gt,Ep,jp,xp,ce,Xn,yp,Tp,Kn,qp,Pp,vt,Cp,Dp,Op,q,Yn,Lp,Sp,Hn,Bp,Ap,Jn,Np,Wp,Vn,Fp,Ip,Zn,Rp,Up,Et,Gp,Mp,Xp,he,Qn,Kp,Yp,eo,Hp,Jp,jt,Vp,Zp,Qp,fe,to,ec,tc,so,sc,nc,xt,oc,rc,ac,de,no,ic,lc,oo,pc,cc,yt,hc,fc,pa,Le,dc,Tt,uc,mc,ca,je,Se,ro,qt,kc,ao,_c,ha,ue,wc,sn,$c,zc,Pt,bc,gc,fa,Ct,da,Be,vc,io,Ec,jc,ua,nn,xc,ma,Dt,ka,on,yc,_a,xe,Ae,lo,Ot,Tc,po,qc,wa,E,Pc,co,Cc,Dc,ho,Oc,Lc,fo,Sc,Bc,uo,Ac,Nc,mo,Wc,Fc,ko,Ic,Rc,$a,Ne,Uc,_o,Gc,Mc,za,Lt,ba,R,Xc,wo,Kc,Yc,$o,Hc,Jc,zo,Vc,Zc,ga,j,Qc,bo,eh,th,go,sh,nh,vo,oh,rh,Eo,ah,ih,jo,lh,ph,xo,ch,hh,va,St,Ea,U,fh,yo,dh,uh,To,mh,kh,qo,_h,wh,ja,Bt,xa,me,$h,Po,zh,bh,Co,gh,vh,ya,ke,Eh,Do,jh,xh,Oo,yh,Th,Ta,At,qa,Nt,Pa,We,Ca,Fe,qh,Lo,Ph,Ch,Da,Wt,Oa,rn,Dh,La,Ft,Sa,Ie,Oh,So,Lh,Sh,Ba,It,Aa,Rt,Na,Re,Bh,Bo,Ah,Nh,Wa,Ut,Fa,Gt,Ia,Ue,Wh,Ao,Fh,Ih,Ra,Mt,Ua,Xt,Ga,Ge,Rh,No,Uh,Gh,Ma,Kt,Xa,P,Mh,Wo,Xh,Kh,Fo,Yh,Hh,Io,Jh,Vh,Ro,Zh,Qh,Uo,ef,tf,Ka,an,sf,Ya,Yt,Ha,Me,nf,Go,of,rf,Ja,Ht,Va,Xe,af,Mo,lf,pf,Za,Jt,Qa,Vt,ei,b,cf,Xo,hf,ff,Ko,df,uf,Yo,mf,kf,Ho,_f,wf,Jo,$f,zf,Vo,bf,gf,Zo,vf,Ef,Qo,jf,xf,er,yf,Tf,ti,C,qf,tr,Pf,Cf,sr,Df,Of,nr,Lf,Sf,or,Bf,Af,rr,Nf,Wf,si,Zt,ni,Qt,oi,G,Ff,ar,If,Rf,ir,Uf,Gf,lr,Mf,Xf,ri,ln,Kf,ai,es,ii,pn,Yf,li,cn,Hf,pi,ts,ci,ss,hi,hn,Jf,fi,ns,di,os,ui,fn,Vf,mi,rs,ki,Ke,Zf,pr,Qf,ed,_i,as,wi,is,$i,dn,td,zi,ls,bi,_e,sd,cr,nd,od,hr,rd,ad,gi,ps,vi,we,id,fr,ld,pd,dr,cd,hd,Ei,D,fd,ur,dd,ud,mr,md,kd,kr,_d,wd,_r,$d,zd,wr,bd,gd,ji,cs,xi,Ye,vd,$r,Ed,jd,yi,hs,Ti,$e,xd,zr,yd,Td,br,qd,Pd,qi,un,Cd,Pi,ye,He,gr,fs,Dd,vr,Od,Ci,Je,Ld,Er,Sd,Bd,Di,ds,Oi,M,Ad,jr,Nd,Wd,xr,Fd,Id,yr,Rd,Ud,Li,mn,Gd,Si,us,Bi,Ve,Md,Tr,Xd,Kd,Ai,ms,Ni,ks,Wi,kn,Yd,Fi,_s,Ii,x,Hd,qr,Jd,Vd,Pr,Zd,Qd,Cr,eu,tu,Dr,su,nu,Or,ou,ru,Lr,au,iu,Ri,_n,lu,Ui,ws,Gi,wn,pu,Mi,$s,Xi,zs,Ki,$n,cu,Yi,bs,Hi,ze,hu,Sr,fu,du,Br,uu,mu,Ji,gs,Vi,vs,Zi,zn,ku,Qi,Es,el,bn,_u,tl,js,sl,xs,nl,be,wu,Ar,$u,zu,Nr,bu,gu,ol,ys,rl,gn,vu,al,Ts,il,vn,Eu,ll,Te,Ze,Wr,qs,ju,Fr,xu,pl,Qe,yu,Ir,Tu,qu,cl,Ps,hl,En,Pu,fl,jn,Cu,dl,Cs,ul,X,Du,Rr,Ou,Lu,Ur,Su,Bu,Gr,Au,Nu,ml,et,Wu,Mr,Fu,Iu,kl,Ds,_l,xn,Ru,wl,Os,$l,Ls,zl,yn,Uu,bl,Ss,gl,O,Gu,Xr,Mu,Xu,Kr,Ku,Yu,Yr,Hu,Ju,Hr,Vu,Zu,vl,Tn,Qu,El,Bs,jl,qn,em,xl,As,yl,Ns,Tl,K,tm,Jr,sm,nm,Vr,om,rm,Zr,am,im,ql,Ws,Pl,Fs,Cl,Pn,lm,Dl,Is,Ol,Cn,pm,Ll,Rs,Sl,Us,Bl,tt,cm,Qr,hm,fm,Al,Gs,Nl,Y,dm,ea,um,mm,ta,km,_m,sa,wm,$m,Wl,Ms,Fl,Dn,zm,Il,Xs,Rl,On,bm,Ul;return ee=new na({}),N=new U_({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section8.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section8.ipynb"}]}}),bt=new R_({props:{id:"MR8tZm5ViWU"}}),qt=new na({}),Ct=new w({props:{code:`from datasets import load_dataset

dataset = load_dataset("wikitext", name="wikitext-2-raw-v1", split="train")


def get_training_corpus():
    for i in range(0, len(dataset), 1000):
        yield dataset[i : i + 1000]["text"]`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;wikitext&quot;</span>, name=<span class="hljs-string">&quot;wikitext-2-raw-v1&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(dataset), <span class="hljs-number">1000</span>):
        <span class="hljs-keyword">yield</span> dataset[i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;text&quot;</span>]`}}),Dt=new w({props:{code:`with open("wikitext-2.txt", "w", encoding="utf-8") as f:
    for i in range(len(dataset)):
        f.write(dataset[i]["text"] + "\\n")`,highlighted:`<span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>, encoding=<span class="hljs-string">&quot;utf-8&quot;</span>) <span class="hljs-keyword">as</span> f:
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(dataset)):
        f.write(dataset[i][<span class="hljs-string">&quot;text&quot;</span>] + <span class="hljs-string">&quot;\\n&quot;</span>)`}}),Ot=new na({}),Lt=new w({props:{code:`from tokenizers import (
    decoders,
    models,
    normalizers,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)

tokenizer = Tokenizer(models.WordPiece(unk_token="[UNK]"))`,highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> (
    decoders,
    models,
    normalizers,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)

tokenizer = Tokenizer(models.WordPiece(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))`}}),St=new w({props:{code:"tokenizer.normalizer = normalizers.BertNormalizer(lowercase=True)",highlighted:'tokenizer.normalizer = normalizers.BertNormalizer(lowercase=<span class="hljs-literal">True</span>)'}}),Bt=new w({props:{code:`tokenizer.normalizer = normalizers.Sequence(
    [normalizers.NFD(), normalizers.Lowercase(), normalizers.StripAccents()]
)`,highlighted:`tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>(
    [normalizers.NFD(), normalizers.Lowercase(), normalizers.StripAccents()]
)`}}),At=new w({props:{code:'print(tokenizer.normalizer.normalize_str("H\xE9ll\xF2 h\xF4w are \xFC?"))',highlighted:'<span class="hljs-built_in">print</span>(tokenizer.normalizer.normalize_str(<span class="hljs-string">&quot;H\xE9ll\xF2 h\xF4w are \xFC?&quot;</span>))'}}),Nt=new w({props:{code:"hello how are u?",highlighted:"hello how are u?"}}),We=new I_({props:{$$slots:{default:[G_]},$$scope:{ctx:oa}}}),Wt=new w({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()"}}),Ft=new w({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()"}}),It=new w({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)'}}),Rt=new w({props:{code:`[('Let', (0, 3)), ("'", (3, 4)), ('s', (4, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre', (14, 17)),
 ('-', (17, 18)), ('tokenizer', (18, 27)), ('.', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">&#x27;s&#x27;</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)),
 (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">&#x27;tokenizer&#x27;</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;.&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),Ut=new w({props:{code:`pre_tokenizer = pre_tokenizers.WhitespaceSplit()
pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:`pre_tokenizer = pre_tokenizers.WhitespaceSplit()
pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)`}}),Gt=new w({props:{code:`[("Let's", (0, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre-tokenizer.', (14, 28))]`,highlighted:'[(<span class="hljs-string">&quot;Let&#x27;s&quot;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre-tokenizer.&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">28</span>))]'}}),Mt=new w({props:{code:`pre_tokenizer = pre_tokenizers.Sequence(
    [pre_tokenizers.WhitespaceSplit(), pre_tokenizers.Punctuation()]
)
pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:`pre_tokenizer = pre_tokenizers.<span class="hljs-type">Sequence</span>(
    [pre_tokenizers.WhitespaceSplit(), pre_tokenizers.Punctuation()]
)
pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)`}}),Xt=new w({props:{code:`[('Let', (0, 3)), ("'", (3, 4)), ('s', (4, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre', (14, 17)),
 ('-', (17, 18)), ('tokenizer', (18, 27)), ('.', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">&#x27;s&#x27;</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)),
 (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">&#x27;tokenizer&#x27;</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;.&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),Kt=new w({props:{code:`special_tokens = ["[UNK]", "[PAD]", "[CLS]", "[SEP]", "[MASK]"]
trainer = trainers.WordPieceTrainer(vocab_size=25000, special_tokens=special_tokens)`,highlighted:`special_tokens = [<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>]
trainer = trainers.WordPieceTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens)`}}),Yt=new w({props:{code:"tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)",highlighted:"tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)"}}),Ht=new w({props:{code:`tokenizer.model = models.WordPiece(unk_token="[UNK]")
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.WordPiece(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>)
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),Jt=new w({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),Vt=new w({props:{code:`['let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '.']`,highlighted:'[<span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),Zt=new w({props:{code:`cls_token_id = tokenizer.token_to_id("[CLS]")
sep_token_id = tokenizer.token_to_id("[SEP]")
print(cls_token_id, sep_token_id)`,highlighted:`cls_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;[CLS]&quot;</span>)
sep_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;[SEP]&quot;</span>)
<span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)`}}),Qt=new w({props:{code:"(2, 3)",highlighted:'(<span class="hljs-number">2</span>, <span class="hljs-number">3</span>)'}}),es=new w({props:{code:`tokenizer.post_processor = processors.TemplateProcessing(
    single=f"[CLS]:0 $A:0 [SEP]:0",
    pair=f"[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1",
    special_tokens=[("[CLS]", cls_token_id), ("[SEP]", sep_token_id)],
)`,highlighted:`tokenizer.post_processor = processors.TemplateProcessing(
    single=<span class="hljs-string">f&quot;[CLS]:0 $A:0 [SEP]:0&quot;</span>,
    pair=<span class="hljs-string">f&quot;[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1&quot;</span>,
    special_tokens=[(<span class="hljs-string">&quot;[CLS]&quot;</span>, cls_token_id), (<span class="hljs-string">&quot;[SEP]&quot;</span>, sep_token_id)],
)`}}),ts=new w({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),ss=new w({props:{code:`['[CLS]', 'let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '.', '[SEP]']`,highlighted:'[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]'}}),ns=new w({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer...", "on a pair of sentences.")
print(encoding.tokens)
print(encoding.type_ids)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer...&quot;</span>, <span class="hljs-string">&quot;on a pair of sentences.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)
<span class="hljs-built_in">print</span>(encoding.type_ids)`}}),os=new w({props:{code:`['[CLS]', 'let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '...', '[SEP]', 'on', 'a', 'pair', 'of', 'sentences', '.', '[SEP]']
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;...&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;on&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;pair&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;sentences&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]
[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]`}}),rs=new w({props:{code:'tokenizer.decoder = decoders.WordPiece(prefix="##")',highlighted:'tokenizer.decoder = decoders.WordPiece(prefix=<span class="hljs-string">&quot;##&quot;</span>)'}}),as=new w({props:{code:"tokenizer.decode(encoding.ids)",highlighted:"tokenizer.decode(encoding.ids)"}}),is=new w({props:{code:`"let's test this tokenizer... on a pair of sentences."`,highlighted:'<span class="hljs-string">&quot;let&#x27;s test this tokenizer... on a pair of sentences.&quot;</span>'}}),ls=new w({props:{code:'tokenizer.save("tokenizer.json")',highlighted:'tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),ps=new w({props:{code:'new_tokenizer = Tokenizer.from_file("tokenizer.json")',highlighted:'new_tokenizer = Tokenizer.from_file(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),cs=new w({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    # tokenizer_file="tokenizer.json", # You can load from the tokenizer file, alternatively
    unk_token="[UNK]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    sep_token="[SEP]",
    mask_token="[MASK]",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    <span class="hljs-comment"># tokenizer_file=&quot;tokenizer.json&quot;, # You can load from the tokenizer file, alternatively</span>
    unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>,
    pad_token=<span class="hljs-string">&quot;[PAD]&quot;</span>,
    cls_token=<span class="hljs-string">&quot;[CLS]&quot;</span>,
    sep_token=<span class="hljs-string">&quot;[SEP]&quot;</span>,
    mask_token=<span class="hljs-string">&quot;[MASK]&quot;</span>,
)`}}),hs=new w({props:{code:`from transformers import BertTokenizerFast

wrapped_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertTokenizerFast

wrapped_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)`}}),fs=new na({}),ds=new w({props:{code:"tokenizer = Tokenizer(models.BPE())",highlighted:"tokenizer = Tokenizer(models.BPE())"}}),us=new w({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)",highlighted:'tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=<span class="hljs-literal">False</span>)'}}),ms=new w({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test pre-tokenization!")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test pre-tokenization!&quot;</span>)'}}),ks=new w({props:{code:`[('Let', (0, 3)), ("'s", (3, 5)), ('\u0120test', (5, 10)), ('\u0120pre', (10, 14)), ('-', (14, 15)),
 ('tokenization', (15, 27)), ('!', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;s&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;\u0120test&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u0120pre&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">15</span>)),
 (<span class="hljs-string">&#x27;tokenization&#x27;</span>, (<span class="hljs-number">15</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;!&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),_s=new w({props:{code:`trainer = trainers.BpeTrainer(vocab_size=25000, special_tokens=["<|endoftext|>"])
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`,highlighted:`trainer = trainers.BpeTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=[<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>])
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`}}),ws=new w({props:{code:`tokenizer.model = models.BPE()
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.BPE()
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),$s=new w({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),zs=new w({props:{code:`['L', 'et', "'", 's', '\u0120test', '\u0120this', '\u0120to', 'ken', 'izer', '.']`,highlighted:'[<span class="hljs-string">&#x27;L&#x27;</span>, <span class="hljs-string">&#x27;et&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u0120test&#x27;</span>, <span class="hljs-string">&#x27;\u0120this&#x27;</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),bs=new w({props:{code:"tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)",highlighted:'tokenizer.post_processor = processors.ByteLevel(trim_offsets=<span class="hljs-literal">False</span>)'}}),gs=new w({props:{code:`sentence = "Let's test this tokenizer."
encoding = tokenizer.encode(sentence)
start, end = encoding.offsets[4]
sentence[start:end]`,highlighted:`sentence = <span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>
encoding = tokenizer.encode(sentence)
start, end = encoding.offsets[<span class="hljs-number">4</span>]
sentence[start:end]`}}),vs=new w({props:{code:"' test'",highlighted:'<span class="hljs-string">&#x27; test&#x27;</span>'}}),Es=new w({props:{code:"tokenizer.decoder = decoders.ByteLevel()",highlighted:"tokenizer.decoder = decoders.ByteLevel()"}}),js=new w({props:{code:"tokenizer.decode(encoding.ids)",highlighted:"tokenizer.decode(encoding.ids)"}}),xs=new w({props:{code:`"Let's test this tokenizer."`,highlighted:'<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>'}}),ys=new w({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token=<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>,
    eos_token=<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>,
)`}}),Ts=new w({props:{code:`from transformers import GPT2TokenizerFast

wrapped_tokenizer = GPT2TokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPT2TokenizerFast

wrapped_tokenizer = GPT2TokenizerFast(tokenizer_object=tokenizer)`}}),qs=new na({}),Ps=new w({props:{code:"tokenizer = Tokenizer(models.Unigram())",highlighted:"tokenizer = Tokenizer(models.Unigram())"}}),Cs=new w({props:{code:`from tokenizers import Regex

tokenizer.normalizer = normalizers.Sequence(
    [
        normalizers.Replace("\`\`", '"'),
        normalizers.Replace("''", '"'),
        normalizers.NFKD(),
        normalizers.StripAccents(),
        normalizers.Replace(Regex(" {2,}"), " "),
    ]
)`,highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Regex

tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>(
    [
        normalizers.Replace(<span class="hljs-string">&quot;\`\`&quot;</span>, <span class="hljs-string">&#x27;&quot;&#x27;</span>),
        normalizers.Replace(<span class="hljs-string">&quot;&#x27;&#x27;&quot;</span>, <span class="hljs-string">&#x27;&quot;&#x27;</span>),
        normalizers.NFKD(),
        normalizers.StripAccents(),
        normalizers.Replace(Regex(<span class="hljs-string">&quot; {2,}&quot;</span>), <span class="hljs-string">&quot; &quot;</span>),
    ]
)`}}),Ds=new w({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.Metaspace()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.Metaspace()"}}),Os=new w({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test the pre-tokenizer!")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test the pre-tokenizer!&quot;</span>)'}}),Ls=new w({props:{code:`[("\u2581Let's", (0, 5)), ('\u2581test', (5, 10)), ('\u2581the', (10, 14)), ('\u2581pre-tokenizer!', (14, 29))]`,highlighted:'[(<span class="hljs-string">&quot;\u2581Let&#x27;s&quot;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;\u2581test&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u2581the&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u2581pre-tokenizer!&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">29</span>))]'}}),Ss=new w({props:{code:`special_tokens = ["<cls>", "<sep>", "<unk>", "<pad>", "<mask>", "<s>", "</s>"]
trainer = trainers.UnigramTrainer(
    vocab_size=25000, special_tokens=special_tokens, unk_token="<unk>"
)
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`,highlighted:`special_tokens = [<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;pad&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;mask&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;s&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>]
trainer = trainers.UnigramTrainer(
    vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens, unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>
)
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`}}),Bs=new w({props:{code:`tokenizer.model = models.Unigram()
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.Unigram()
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),As=new w({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),Ns=new w({props:{code:`['\u2581Let', "'", 's', '\u2581test', '\u2581this', '\u2581to', 'ken', 'izer', '.']`,highlighted:'[<span class="hljs-string">&#x27;\u2581Let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581test&#x27;</span>, <span class="hljs-string">&#x27;\u2581this&#x27;</span>, <span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),Ws=new w({props:{code:`cls_token_id = tokenizer.token_to_id("<cls>")
sep_token_id = tokenizer.token_to_id("<sep>")
print(cls_token_id, sep_token_id)`,highlighted:`cls_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>)
sep_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>)
<span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)`}}),Fs=new w({props:{code:"0 1",highlighted:'<span class="hljs-number">0</span> <span class="hljs-number">1</span>'}}),Is=new w({props:{code:`tokenizer.post_processor = processors.TemplateProcessing(
    single="$A:0 <sep>:0 <cls>:2",
    pair="$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2",
    special_tokens=[("<sep>", sep_token_id), ("<cls>", cls_token_id)],
)`,highlighted:`tokenizer.post_processor = processors.TemplateProcessing(
    single=<span class="hljs-string">&quot;$A:0 &lt;sep&gt;:0 &lt;cls&gt;:2&quot;</span>,
    pair=<span class="hljs-string">&quot;$A:0 &lt;sep&gt;:0 $B:1 &lt;sep&gt;:1 &lt;cls&gt;:2&quot;</span>,
    special_tokens=[(<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>, sep_token_id), (<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>, cls_token_id)],
)`}}),Rs=new w({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer...", "on a pair of sentences!")
print(encoding.tokens)
print(encoding.type_ids)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer...&quot;</span>, <span class="hljs-string">&quot;on a pair of sentences!&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)
<span class="hljs-built_in">print</span>(encoding.type_ids)`}}),Us=new w({props:{code:`['\u2581Let', "'", 's', '\u2581test', '\u2581this', '\u2581to', 'ken', 'izer', '.', '.', '.', '<sep>', '\u2581', 'on', '\u2581', 'a', '\u2581pair', 
  '\u2581of', '\u2581sentence', 's', '!', '<sep>', '<cls>']
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2]`,highlighted:`[<span class="hljs-string">&#x27;\u2581Let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581test&#x27;</span>, <span class="hljs-string">&#x27;\u2581this&#x27;</span>, <span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;&lt;sep&gt;&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;on&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;\u2581pair&#x27;</span>, 
  <span class="hljs-string">&#x27;\u2581of&#x27;</span>, <span class="hljs-string">&#x27;\u2581sentence&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;!&#x27;</span>, <span class="hljs-string">&#x27;&lt;sep&gt;&#x27;</span>, <span class="hljs-string">&#x27;&lt;cls&gt;&#x27;</span>]
[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>]`}}),Gs=new w({props:{code:"tokenizer.decoder = decoders.Metaspace()",highlighted:"tokenizer.decoder = decoders.Metaspace()"}}),Ms=new w({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<s>",
    eos_token="</s>",
    unk_token="<unk>",
    pad_token="<pad>",
    cls_token="<cls>",
    sep_token="<sep>",
    mask_token="<mask>",
    padding_side="left",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token=<span class="hljs-string">&quot;&lt;s&gt;&quot;</span>,
    eos_token=<span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>,
    unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>,
    pad_token=<span class="hljs-string">&quot;&lt;pad&gt;&quot;</span>,
    cls_token=<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>,
    sep_token=<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>,
    mask_token=<span class="hljs-string">&quot;&lt;mask&gt;&quot;</span>,
    padding_side=<span class="hljs-string">&quot;left&quot;</span>,
)`}}),Xs=new w({props:{code:`from transformers import XLNetTokenizerFast

wrapped_tokenizer = XLNetTokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> XLNetTokenizerFast

wrapped_tokenizer = XLNetTokenizerFast(tokenizer_object=tokenizer)`}}),{c(){z=a("meta"),ae=c(),A=a("h1"),F=a("a"),Q=a("span"),f(ee.$$.fragment),ct=c(),te=a("span"),ht=n("Building a tokenizer, block by block"),qe=c(),f(N.$$.fragment),Pe=c(),ie=a("p"),ve=n("As we\u2019ve seen in the previous sections, tokenization comprises several steps:"),Ce=c(),y=a("ul"),se=a("li"),ft=n("Normalization (any cleanup of the text that is deemed necessary, such as removing spaces or accents, Unicode normalization, etc.)"),dt=c(),ne=a("li"),ut=n("Pre-tokenization (splitting the input into words)"),mt=c(),oe=a("li"),kt=n("Running the input through the model (using the pre-tokenized words to produce a sequence of tokens)"),_t=c(),re=a("li"),g=n("Post-processing (adding the special tokens of the tokenizer, generating the attention mask and token type IDs)"),wt=c(),Ee=a("p"),Qs=n("As a reminder, here\u2019s another look at the overall process:"),$t=c(),I=a("div"),le=a("img"),en=c(),zt=a("img"),ra=c(),De=a("p"),up=n("The \u{1F917} Tokenizers library has been built to provide several options for each of those steps, which you can mix and match together. In this section we\u2019ll see how we can build a tokenizer from scratch, as opposed to training a new tokenizer from an old one as we did in "),tn=a("a"),mp=n("section 2"),kp=n(". You\u2019ll then be able to build any kind of tokenizer you can think of!"),aa=c(),f(bt.$$.fragment),ia=c(),Oe=a("p"),_p=n("More precisely, the library is built around a central "),Un=a("code"),wp=n("Tokenizer"),$p=n(" class with the building blocks regrouped in submodules:"),la=c(),T=a("ul"),pe=a("li"),Gn=a("code"),zp=n("normalizers"),bp=n(" contains all the possible types of "),Mn=a("code"),gp=n("Normalizer"),vp=n(" you can use (complete list "),gt=a("a"),Ep=n("here"),jp=n(")."),xp=c(),ce=a("li"),Xn=a("code"),yp=n("pre_tokenizers"),Tp=n(" contains all the possible types of "),Kn=a("code"),qp=n("PreTokenizer"),Pp=n(" you can use (complete list "),vt=a("a"),Cp=n("here"),Dp=n(")."),Op=c(),q=a("li"),Yn=a("code"),Lp=n("models"),Sp=n(" contains the various types of "),Hn=a("code"),Bp=n("Model"),Ap=n(" you can use, like "),Jn=a("code"),Np=n("BPE"),Wp=n(", "),Vn=a("code"),Fp=n("WordPiece"),Ip=n(", and "),Zn=a("code"),Rp=n("Unigram"),Up=n(" (complete list "),Et=a("a"),Gp=n("here"),Mp=n(")."),Xp=c(),he=a("li"),Qn=a("code"),Kp=n("trainers"),Yp=n(" contains all the different types of "),eo=a("code"),Hp=n("Trainer"),Jp=n(" you can use to train your model on a corpus (one per type of model; complete list "),jt=a("a"),Vp=n("here"),Zp=n(")."),Qp=c(),fe=a("li"),to=a("code"),ec=n("post_processors"),tc=n(" contains the various types of "),so=a("code"),sc=n("PostProcessor"),nc=n(" you can use (complete list "),xt=a("a"),oc=n("here"),rc=n(")."),ac=c(),de=a("li"),no=a("code"),ic=n("decoders"),lc=n(" contains the various types of "),oo=a("code"),pc=n("Decoder"),cc=n(" you can use to decode the outputs of tokenization (complete list "),yt=a("a"),hc=n("here"),fc=n(")."),pa=c(),Le=a("p"),dc=n("You can find the whole list of building blocks "),Tt=a("a"),uc=n("here"),mc=n("."),ca=c(),je=a("h2"),Se=a("a"),ro=a("span"),f(qt.$$.fragment),kc=c(),ao=a("span"),_c=n("Acquiring a corpus"),ha=c(),ue=a("p"),wc=n("To train our new tokenizer, we will use a small corpus of text (so the examples run fast). The steps for acquiring the corpus are similar to the ones we took at the "),sn=a("a"),$c=n("beginning of this chapter"),zc=n(", but this time we\u2019ll use the "),Pt=a("a"),bc=n("WikiText-2"),gc=n(" dataset:"),fa=c(),f(Ct.$$.fragment),da=c(),Be=a("p"),vc=n("The function "),io=a("code"),Ec=n("get_training_corpus()"),jc=n(" is a generator that will yield batches of 1,000 texts, which we will use to train the tokenizer."),ua=c(),nn=a("p"),xc=n("\u{1F917} Tokenizers can also be trained on text files directly. Here\u2019s how we can generate a text file containing all the texts/inputs from WikiText-2 that we can use locally:"),ma=c(),f(Dt.$$.fragment),ka=c(),on=a("p"),yc=n("Next we\u2019ll show you how to build your own BERT, GPT-2, and XLNet tokenizers, block by block. That will give us an example of each of the three main tokenization algorithms: WordPiece, BPE, and Unigram. Let\u2019s start with BERT!"),_a=c(),xe=a("h2"),Ae=a("a"),lo=a("span"),f(Ot.$$.fragment),Tc=c(),po=a("span"),qc=n("Building a WordPiece tokenizer from scratch"),wa=c(),E=a("p"),Pc=n("To build a tokenizer with the \u{1F917} Tokenizers library, we start by instantiating a "),co=a("code"),Cc=n("Tokenizer"),Dc=n(" object with a "),ho=a("code"),Oc=n("model"),Lc=n(", then set its "),fo=a("code"),Sc=n("normalizer"),Bc=n(", "),uo=a("code"),Ac=n("pre_tokenizer"),Nc=n(", "),mo=a("code"),Wc=n("post_processor"),Fc=n(", and "),ko=a("code"),Ic=n("decoder"),Rc=n(" attributes to the values we want."),$a=c(),Ne=a("p"),Uc=n("For this example, we\u2019ll create a "),_o=a("code"),Gc=n("Tokenizer"),Mc=n(" with a WordPiece model:"),za=c(),f(Lt.$$.fragment),ba=c(),R=a("p"),Xc=n("We have to specify the "),wo=a("code"),Kc=n("unk_token"),Yc=n(" so the model knows what to return when it encounters characters it hasn\u2019t seen before. Other arguments we can set here include the "),$o=a("code"),Hc=n("vocab"),Jc=n(" of our model (we\u2019re going to train the model, so we don\u2019t need to set this) and "),zo=a("code"),Vc=n("max_input_chars_per_word"),Zc=n(", which specifies a maximum length for each word (words longer than the value passed will be split)."),ga=c(),j=a("p"),Qc=n("The first step of tokenization is normalization, so let\u2019s begin with that. Since BERT is widely used, there is a "),bo=a("code"),eh=n("BertNormalizer"),th=n(" with the classic options we can set for BERT: "),go=a("code"),sh=n("lowercase"),nh=n(" and "),vo=a("code"),oh=n("strip_accents"),rh=n(", which are self-explanatory; "),Eo=a("code"),ah=n("clean_text"),ih=n(" to remove all control characters and replace repeating spaces with a single one; and "),jo=a("code"),lh=n("handle_chinese_chars"),ph=n(", which places spaces around Chinese characters. To replicate the "),xo=a("code"),ch=n("bert-base-uncased"),hh=n(" tokenizer, we can just set this normalizer:"),va=c(),f(St.$$.fragment),Ea=c(),U=a("p"),fh=n("Generally speaking, however, when building a new tokenizer you won\u2019t have access to such a handy normalizer already implemented in the \u{1F917} Tokenizers library \u2014 so let\u2019s see how to create the BERT normalizer by hand. The library provides a "),yo=a("code"),dh=n("Lowercase"),uh=n(" normalizer and a "),To=a("code"),mh=n("StripAccents"),kh=n(" normalizer, and you can compose several normalizers using a "),qo=a("code"),_h=n("Sequence"),wh=n(":"),ja=c(),f(Bt.$$.fragment),xa=c(),me=a("p"),$h=n("We\u2019re also using an "),Po=a("code"),zh=n("NFD"),bh=n(" Unicode normalizer, as otherwise the "),Co=a("code"),gh=n("StripAccents"),vh=n(" normalizer won\u2019t properly recognize the accented characters and thus won\u2019t strip them out."),ya=c(),ke=a("p"),Eh=n("As we\u2019ve seen before, we can use the "),Do=a("code"),jh=n("normalize_str()"),xh=n(" method of the "),Oo=a("code"),yh=n("normalizer"),Th=n(" to check out the effects it has on a given text:"),Ta=c(),f(At.$$.fragment),qa=c(),f(Nt.$$.fragment),Pa=c(),f(We.$$.fragment),Ca=c(),Fe=a("p"),qh=n("Next is the pre-tokenization step. Again, there is a prebuilt "),Lo=a("code"),Ph=n("BertPreTokenizer"),Ch=n(" that we can use:"),Da=c(),f(Wt.$$.fragment),Oa=c(),rn=a("p"),Dh=n("Or we can build it from scratch:"),La=c(),f(Ft.$$.fragment),Sa=c(),Ie=a("p"),Oh=n("Note that the "),So=a("code"),Lh=n("Whitespace"),Sh=n(" pre-tokenizer splits on whitespace and all characters that are not letters, digits, or the underscore character, so it technically splits on whitespace and punctuation:"),Ba=c(),f(It.$$.fragment),Aa=c(),f(Rt.$$.fragment),Na=c(),Re=a("p"),Bh=n("If you only want to split on whitespace, you should use the "),Bo=a("code"),Ah=n("WhitespaceSplit"),Nh=n(" pre-tokenizer instead:"),Wa=c(),f(Ut.$$.fragment),Fa=c(),f(Gt.$$.fragment),Ia=c(),Ue=a("p"),Wh=n("Like with normalizers, you can use a "),Ao=a("code"),Fh=n("Sequence"),Ih=n(" to compose several pre-tokenizers:"),Ra=c(),f(Mt.$$.fragment),Ua=c(),f(Xt.$$.fragment),Ga=c(),Ge=a("p"),Rh=n("The next step in the tokenization pipeline is running the inputs through the model. We already specified our model in the initialization, but we still need to train it, which will require a "),No=a("code"),Uh=n("WordPieceTrainer"),Gh=n(". The main thing to remember when instantiating a trainer in \u{1F917} Tokenizers is that you need to pass it all the special tokens you intend to use \u2014 otherwise it won\u2019t add them to the vocabulary, since they are not in the training corpus:"),Ma=c(),f(Kt.$$.fragment),Xa=c(),P=a("p"),Mh=n("As well as specifying the "),Wo=a("code"),Xh=n("vocab_size"),Kh=n(" and "),Fo=a("code"),Yh=n("special_tokens"),Hh=n(", we can set the "),Io=a("code"),Jh=n("min_frequency"),Vh=n(" (the number of times a token must appear to be included in the vocabulary) or change the "),Ro=a("code"),Zh=n("continuing_subword_prefix"),Qh=n(" (if we want to use something different from "),Uo=a("code"),ef=n("##"),tf=n(")."),Ka=c(),an=a("p"),sf=n("To train our model using the iterator we defined earlier, we just have to execute this command:"),Ya=c(),f(Yt.$$.fragment),Ha=c(),Me=a("p"),nf=n("We can also use text files to train our tokenizer, which would look like this (we reinitialize the model with an empty "),Go=a("code"),of=n("WordPiece"),rf=n(" beforehand):"),Ja=c(),f(Ht.$$.fragment),Va=c(),Xe=a("p"),af=n("In both cases, we can then test the tokenizer on a text by calling the "),Mo=a("code"),lf=n("encode()"),pf=n(" method:"),Za=c(),f(Jt.$$.fragment),Qa=c(),f(Vt.$$.fragment),ei=c(),b=a("p"),cf=n("The "),Xo=a("code"),hf=n("encoding"),ff=n(" obtained is an "),Ko=a("code"),df=n("Encoding"),uf=n(", which contains all the necessary outputs of the tokenizer in its various attributes: "),Yo=a("code"),mf=n("ids"),kf=n(", "),Ho=a("code"),_f=n("type_ids"),wf=n(", "),Jo=a("code"),$f=n("tokens"),zf=n(", "),Vo=a("code"),bf=n("offsets"),gf=n(", "),Zo=a("code"),vf=n("attention_mask"),Ef=n(", "),Qo=a("code"),jf=n("special_tokens_mask"),xf=n(", and "),er=a("code"),yf=n("overflowing"),Tf=n("."),ti=c(),C=a("p"),qf=n("The last step in the tokenization pipeline is post-processing. We need to add the "),tr=a("code"),Pf=n("[CLS]"),Cf=n(" token at the beginning and the "),sr=a("code"),Df=n("[SEP]"),Of=n(" token at the end (or after each sentence, if we have a pair of sentences). We will use a "),nr=a("code"),Lf=n("TemplateProcessor"),Sf=n(" for this, but first we need to know the IDs of the "),or=a("code"),Bf=n("[CLS]"),Af=n(" and "),rr=a("code"),Nf=n("[SEP]"),Wf=n(" tokens in the vocabulary:"),si=c(),f(Zt.$$.fragment),ni=c(),f(Qt.$$.fragment),oi=c(),G=a("p"),Ff=n("To write the template for the "),ar=a("code"),If=n("TemplateProcessor"),Rf=n(", we have to specify how to treat a single sentence and a pair of sentences. For both, we write the special tokens we want to use; the first (or single) sentence is represented by "),ir=a("code"),Uf=n("$A"),Gf=n(", while the second sentence (if encoding a pair) is represented by "),lr=a("code"),Mf=n("$B"),Xf=n(". For each of these (special tokens and sentences), we also specify the corresponding token type ID after a colon."),ri=c(),ln=a("p"),Kf=n("The classic BERT template is thus defined as follows:"),ai=c(),f(es.$$.fragment),ii=c(),pn=a("p"),Yf=n("Note that we need to pass along the IDs of the special tokens, so the tokenizer can properly convert them to their IDs."),li=c(),cn=a("p"),Hf=n("Once this is added, going back to our previous example will give:"),pi=c(),f(ts.$$.fragment),ci=c(),f(ss.$$.fragment),hi=c(),hn=a("p"),Jf=n("And on a pair of sentences, we get the proper result:"),fi=c(),f(ns.$$.fragment),di=c(),f(os.$$.fragment),ui=c(),fn=a("p"),Vf=n("We\u2019ve almost finished building this tokenizer from scratch \u2014 the last step is to include a decoder:"),mi=c(),f(rs.$$.fragment),ki=c(),Ke=a("p"),Zf=n("Let\u2019s test it on our previous "),pr=a("code"),Qf=n("encoding"),ed=n(":"),_i=c(),f(as.$$.fragment),wi=c(),f(is.$$.fragment),$i=c(),dn=a("p"),td=n("Great! We can save our tokenizer in a single JSON file like this:"),zi=c(),f(ls.$$.fragment),bi=c(),_e=a("p"),sd=n("We can then reload that file in a "),cr=a("code"),nd=n("Tokenizer"),od=n(" object with the "),hr=a("code"),rd=n("from_file()"),ad=n(" method:"),gi=c(),f(ps.$$.fragment),vi=c(),we=a("p"),id=n("To use this tokenizer in \u{1F917} Transformers, we have to wrap it in a "),fr=a("code"),ld=n("PreTrainedTokenizerFast"),pd=n(". We can either use the generic class or, if our tokenizer corresponds to an existing model, use that class (here, "),dr=a("code"),cd=n("BertTokenizerFast"),hd=n("). If you apply this lesson to build a brand new tokenizer, you will have to use the first option."),Ei=c(),D=a("p"),fd=n("To wrap the tokenizer in a "),ur=a("code"),dd=n("PreTrainedTokenizerFast"),ud=n(", we can either pass the tokenizer we built as a "),mr=a("code"),md=n("tokenizer_object"),kd=n(" or pass the tokenizer file we saved as "),kr=a("code"),_d=n("tokenizer_file"),wd=n(". The key thing to remember is that we have to manually set all the special tokens, since that class can\u2019t infer from the "),_r=a("code"),$d=n("tokenizer"),zd=n(" object which token is the mask token, the "),wr=a("code"),bd=n("[CLS]"),gd=n(" token, etc.:"),ji=c(),f(cs.$$.fragment),xi=c(),Ye=a("p"),vd=n("If you are using a specific tokenizer class (like "),$r=a("code"),Ed=n("BertTokenizerFast"),jd=n("), you will only need to specify the special tokens that are different from the default ones (here, none):"),yi=c(),f(hs.$$.fragment),Ti=c(),$e=a("p"),xd=n("You can then use this tokenizer like any other \u{1F917} Transformers tokenizer. You can save it with the "),zr=a("code"),yd=n("save_pretrained()"),Td=n(" method, or upload it to the Hub with the "),br=a("code"),qd=n("push_to_hub()"),Pd=n(" method."),qi=c(),un=a("p"),Cd=n("Now that we\u2019ve seen how to build a WordPiece tokenizer, let\u2019s do the same for a BPE tokenizer. We\u2019ll go a bit faster since you know all the steps, and only highlight the differences."),Pi=c(),ye=a("h2"),He=a("a"),gr=a("span"),f(fs.$$.fragment),Dd=c(),vr=a("span"),Od=n("Building a BPE tokenizer from scratch"),Ci=c(),Je=a("p"),Ld=n("Let\u2019s now build a GPT-2 tokenizer. Like for the BERT tokenizer, we start by initializing a "),Er=a("code"),Sd=n("Tokenizer"),Bd=n(" with a BPE model:"),Di=c(),f(ds.$$.fragment),Oi=c(),M=a("p"),Ad=n("Also like for BERT, we could initialize this model with a vocabulary if we had one (we would need to pass the "),jr=a("code"),Nd=n("vocab"),Wd=n(" and "),xr=a("code"),Fd=n("merges"),Id=n(" in this case), but since we will train from scratch, we don\u2019t need to do that. We also don\u2019t need to specify an "),yr=a("code"),Rd=n("unk_token"),Ud=n(" because GPT-2 uses byte-level BPE, which doesn\u2019t require it."),Li=c(),mn=a("p"),Gd=n("GPT-2 does not use a normalizer, so we skip that step and go directly to the pre-tokenization:"),Si=c(),f(us.$$.fragment),Bi=c(),Ve=a("p"),Md=n("The option we added to "),Tr=a("code"),Xd=n("ByteLevel"),Kd=n(" here is to not add a space at the beginning of a sentence (which is the default otherwise). We can have a look at the pre-tokenization of an example text like before:"),Ai=c(),f(ms.$$.fragment),Ni=c(),f(ks.$$.fragment),Wi=c(),kn=a("p"),Yd=n("Next is the model, which needs training. For GPT-2, the only special token is the end-of-text token:"),Fi=c(),f(_s.$$.fragment),Ii=c(),x=a("p"),Hd=n("Like with the "),qr=a("code"),Jd=n("WordPieceTrainer"),Vd=n(", as well as the "),Pr=a("code"),Zd=n("vocab_size"),Qd=n(" and "),Cr=a("code"),eu=n("special_tokens"),tu=n(", we can specify the "),Dr=a("code"),su=n("min_frequency"),nu=n(" if we want to, or if we have an end-of-word suffix (like "),Or=a("code"),ou=n("</w>"),ru=n("), we can set it with "),Lr=a("code"),au=n("end_of_word_suffix"),iu=n("."),Ri=c(),_n=a("p"),lu=n("This tokenizer can also be trained on text files:"),Ui=c(),f(ws.$$.fragment),Gi=c(),wn=a("p"),pu=n("Let\u2019s have a look at the tokenization of a sample text:"),Mi=c(),f($s.$$.fragment),Xi=c(),f(zs.$$.fragment),Ki=c(),$n=a("p"),cu=n("We apply the byte-level post-processing for the GPT-2 tokenizer as follows:"),Yi=c(),f(bs.$$.fragment),Hi=c(),ze=a("p"),hu=n("The "),Sr=a("code"),fu=n("trim_offsets = False"),du=n(" option indicates to the post-processor that we should leave the offsets of tokens that begin with \u2018\u0120\u2019 as they are: this way the start of the offsets will point to the space before the word, not the first character of the word (since the space is technically part of the token). Let\u2019s have a look at the result with the text we just encoded, where "),Br=a("code"),uu=n("'\u0120test'"),mu=n(" is the token at index 4:"),Ji=c(),f(gs.$$.fragment),Vi=c(),f(vs.$$.fragment),Zi=c(),zn=a("p"),ku=n("Finally, we add a byte-level decoder:"),Qi=c(),f(Es.$$.fragment),el=c(),bn=a("p"),_u=n("and we can double-check it works properly:"),tl=c(),f(js.$$.fragment),sl=c(),f(xs.$$.fragment),nl=c(),be=a("p"),wu=n("Great! Now that we\u2019re done, we can save the tokenizer like before, and wrap it in a "),Ar=a("code"),$u=n("PreTrainedTokenizerFast"),zu=n(" or "),Nr=a("code"),bu=n("GPT2TokenizerFast"),gu=n(" if we want to use it in \u{1F917} Transformers:"),ol=c(),f(ys.$$.fragment),rl=c(),gn=a("p"),vu=n("or:"),al=c(),f(Ts.$$.fragment),il=c(),vn=a("p"),Eu=n("As the last example, we\u2019ll show you how to build a Unigram tokenizer from scratch."),ll=c(),Te=a("h2"),Ze=a("a"),Wr=a("span"),f(qs.$$.fragment),ju=c(),Fr=a("span"),xu=n("Building a Unigram tokenizer from scratch"),pl=c(),Qe=a("p"),yu=n("Let\u2019s now build an XLNet tokenizer. Like for the previous tokenizers, we start by initializing a "),Ir=a("code"),Tu=n("Tokenizer"),qu=n(" with a Unigram model:"),cl=c(),f(Ps.$$.fragment),hl=c(),En=a("p"),Pu=n("Again, we could initialize this model with a vocabulary if we had one."),fl=c(),jn=a("p"),Cu=n("For the normalization, XLNet uses a few replacements (which come from SentencePiece):"),dl=c(),f(Cs.$$.fragment),ul=c(),X=a("p"),Du=n("This replaces "),Rr=a("code"),Ou=n("\u201C"),Lu=n(" and "),Ur=a("code"),Su=n("\u201D"),Bu=n(" with "),Gr=a("code"),Au=n("\u201D"),Nu=n(" and any sequence of two or more spaces with a single space, as well as removing the accents in the texts to tokenize."),ml=c(),et=a("p"),Wu=n("The pre-tokenizer to use for any SentencePiece tokenizer is "),Mr=a("code"),Fu=n("Metaspace"),Iu=n(":"),kl=c(),f(Ds.$$.fragment),_l=c(),xn=a("p"),Ru=n("We can have a look at the pre-tokenization of an example text like before:"),wl=c(),f(Os.$$.fragment),$l=c(),f(Ls.$$.fragment),zl=c(),yn=a("p"),Uu=n("Next is the model, which needs training. XLNet has quite a few special tokens:"),bl=c(),f(Ss.$$.fragment),gl=c(),O=a("p"),Gu=n("A very important argument not to forget for the "),Xr=a("code"),Mu=n("UnigramTrainer"),Xu=n(" is the "),Kr=a("code"),Ku=n("unk_token"),Yu=n(". We can also pass along other arguments specific to the Unigram algorithm, such as the "),Yr=a("code"),Hu=n("shrinking_factor"),Ju=n(" for each step where we remove tokens (defaults to 0.75) or the "),Hr=a("code"),Vu=n("max_piece_length"),Zu=n(" to specify the maximum length of a given token (defaults to 16)."),vl=c(),Tn=a("p"),Qu=n("This tokenizer can also be trained on text files:"),El=c(),f(Bs.$$.fragment),jl=c(),qn=a("p"),em=n("Let\u2019s have a look at the tokenization of a sample text:"),xl=c(),f(As.$$.fragment),yl=c(),f(Ns.$$.fragment),Tl=c(),K=a("p"),tm=n("A peculiarity of XLNet is that it puts the "),Jr=a("code"),sm=n("<cls>"),nm=n(" token at the end of the sentence, with a type ID of 2 (to distinguish it from the other tokens). It\u2019s padding on the left, as a result. We can deal with all the special tokens and token type IDs with a template, like for BERT, but first we have to get the IDs of the "),Vr=a("code"),om=n("<cls>"),rm=n(" and "),Zr=a("code"),am=n("<sep>"),im=n(" tokens:"),ql=c(),f(Ws.$$.fragment),Pl=c(),f(Fs.$$.fragment),Cl=c(),Pn=a("p"),lm=n("The template looks like this:"),Dl=c(),f(Is.$$.fragment),Ol=c(),Cn=a("p"),pm=n("And we can test it works by encoding a pair of sentences:"),Ll=c(),f(Rs.$$.fragment),Sl=c(),f(Us.$$.fragment),Bl=c(),tt=a("p"),cm=n("Finally, we add a "),Qr=a("code"),hm=n("Metaspace"),fm=n(" decoder:"),Al=c(),f(Gs.$$.fragment),Nl=c(),Y=a("p"),dm=n("and we\u2019re done with this tokenizer! We can save the tokenizer like before, and wrap it in a "),ea=a("code"),um=n("PreTrainedTokenizerFast"),mm=n(" or "),ta=a("code"),km=n("XLNetTokenizerFast"),_m=n(" if we want to use it in \u{1F917} Transformers. One thing to note when using "),sa=a("code"),wm=n("PreTrainedTokenizerFast"),$m=n(" is that on top of the special tokens, we need to tell the \u{1F917} Transformers library to pad on the left:"),Wl=c(),f(Ms.$$.fragment),Fl=c(),Dn=a("p"),zm=n("Or alternatively:"),Il=c(),f(Xs.$$.fragment),Rl=c(),On=a("p"),bm=n("Now that you have seen how the various building blocks are used to build existing tokenizers, you should be able to write any tokenizer you want with the \u{1F917} Tokenizers library and be able to use it in \u{1F917} Transformers."),this.h()},l(e){const r=W_('[data-svelte="svelte-1phssyn"]',document.head);z=i(r,"META",{name:!0,content:!0}),r.forEach(s),ae=h(e),A=i(e,"H1",{class:!0});var Ks=l(A);F=i(Ks,"A",{id:!0,class:!0,href:!0});var vm=l(F);Q=i(vm,"SPAN",{});var Em=l(Q);d(ee.$$.fragment,Em),Em.forEach(s),vm.forEach(s),ct=h(Ks),te=i(Ks,"SPAN",{});var jm=l(te);ht=o(jm,"Building a tokenizer, block by block"),jm.forEach(s),Ks.forEach(s),qe=h(e),d(N.$$.fragment,e),Pe=h(e),ie=i(e,"P",{});var xm=l(ie);ve=o(xm,"As we\u2019ve seen in the previous sections, tokenization comprises several steps:"),xm.forEach(s),Ce=h(e),y=i(e,"UL",{});var st=l(y);se=i(st,"LI",{});var ym=l(se);ft=o(ym,"Normalization (any cleanup of the text that is deemed necessary, such as removing spaces or accents, Unicode normalization, etc.)"),ym.forEach(s),dt=h(st),ne=i(st,"LI",{});var Tm=l(ne);ut=o(Tm,"Pre-tokenization (splitting the input into words)"),Tm.forEach(s),mt=h(st),oe=i(st,"LI",{});var qm=l(oe);kt=o(qm,"Running the input through the model (using the pre-tokenized words to produce a sequence of tokens)"),qm.forEach(s),_t=h(st),re=i(st,"LI",{});var Pm=l(re);g=o(Pm,"Post-processing (adding the special tokens of the tokenizer, generating the attention mask and token type IDs)"),Pm.forEach(s),st.forEach(s),wt=h(e),Ee=i(e,"P",{});var Cm=l(Ee);Qs=o(Cm,"As a reminder, here\u2019s another look at the overall process:"),Cm.forEach(s),$t=h(e),I=i(e,"DIV",{class:!0});var Gl=l(I);le=i(Gl,"IMG",{class:!0,src:!0,alt:!0}),en=h(Gl),zt=i(Gl,"IMG",{class:!0,src:!0,alt:!0}),Gl.forEach(s),ra=h(e),De=i(e,"P",{});var Ml=l(De);up=o(Ml,"The \u{1F917} Tokenizers library has been built to provide several options for each of those steps, which you can mix and match together. In this section we\u2019ll see how we can build a tokenizer from scratch, as opposed to training a new tokenizer from an old one as we did in "),tn=i(Ml,"A",{href:!0});var Dm=l(tn);mp=o(Dm,"section 2"),Dm.forEach(s),kp=o(Ml,". You\u2019ll then be able to build any kind of tokenizer you can think of!"),Ml.forEach(s),aa=h(e),d(bt.$$.fragment,e),ia=h(e),Oe=i(e,"P",{});var Xl=l(Oe);_p=o(Xl,"More precisely, the library is built around a central "),Un=i(Xl,"CODE",{});var Om=l(Un);wp=o(Om,"Tokenizer"),Om.forEach(s),$p=o(Xl," class with the building blocks regrouped in submodules:"),Xl.forEach(s),la=h(e),T=i(e,"UL",{});var H=l(T);pe=i(H,"LI",{});var Ys=l(pe);Gn=i(Ys,"CODE",{});var Lm=l(Gn);zp=o(Lm,"normalizers"),Lm.forEach(s),bp=o(Ys," contains all the possible types of "),Mn=i(Ys,"CODE",{});var Sm=l(Mn);gp=o(Sm,"Normalizer"),Sm.forEach(s),vp=o(Ys," you can use (complete list "),gt=i(Ys,"A",{href:!0,rel:!0});var Bm=l(gt);Ep=o(Bm,"here"),Bm.forEach(s),jp=o(Ys,")."),Ys.forEach(s),xp=h(H),ce=i(H,"LI",{});var Hs=l(ce);Xn=i(Hs,"CODE",{});var Am=l(Xn);yp=o(Am,"pre_tokenizers"),Am.forEach(s),Tp=o(Hs," contains all the possible types of "),Kn=i(Hs,"CODE",{});var Nm=l(Kn);qp=o(Nm,"PreTokenizer"),Nm.forEach(s),Pp=o(Hs," you can use (complete list "),vt=i(Hs,"A",{href:!0,rel:!0});var Wm=l(vt);Cp=o(Wm,"here"),Wm.forEach(s),Dp=o(Hs,")."),Hs.forEach(s),Op=h(H),q=i(H,"LI",{});var W=l(q);Yn=i(W,"CODE",{});var Fm=l(Yn);Lp=o(Fm,"models"),Fm.forEach(s),Sp=o(W," contains the various types of "),Hn=i(W,"CODE",{});var Im=l(Hn);Bp=o(Im,"Model"),Im.forEach(s),Ap=o(W," you can use, like "),Jn=i(W,"CODE",{});var Rm=l(Jn);Np=o(Rm,"BPE"),Rm.forEach(s),Wp=o(W,", "),Vn=i(W,"CODE",{});var Um=l(Vn);Fp=o(Um,"WordPiece"),Um.forEach(s),Ip=o(W,", and "),Zn=i(W,"CODE",{});var Gm=l(Zn);Rp=o(Gm,"Unigram"),Gm.forEach(s),Up=o(W," (complete list "),Et=i(W,"A",{href:!0,rel:!0});var Mm=l(Et);Gp=o(Mm,"here"),Mm.forEach(s),Mp=o(W,")."),W.forEach(s),Xp=h(H),he=i(H,"LI",{});var Js=l(he);Qn=i(Js,"CODE",{});var Xm=l(Qn);Kp=o(Xm,"trainers"),Xm.forEach(s),Yp=o(Js," contains all the different types of "),eo=i(Js,"CODE",{});var Km=l(eo);Hp=o(Km,"Trainer"),Km.forEach(s),Jp=o(Js," you can use to train your model on a corpus (one per type of model; complete list "),jt=i(Js,"A",{href:!0,rel:!0});var Ym=l(jt);Vp=o(Ym,"here"),Ym.forEach(s),Zp=o(Js,")."),Js.forEach(s),Qp=h(H),fe=i(H,"LI",{});var Vs=l(fe);to=i(Vs,"CODE",{});var Hm=l(to);ec=o(Hm,"post_processors"),Hm.forEach(s),tc=o(Vs," contains the various types of "),so=i(Vs,"CODE",{});var Jm=l(so);sc=o(Jm,"PostProcessor"),Jm.forEach(s),nc=o(Vs," you can use (complete list "),xt=i(Vs,"A",{href:!0,rel:!0});var Vm=l(xt);oc=o(Vm,"here"),Vm.forEach(s),rc=o(Vs,")."),Vs.forEach(s),ac=h(H),de=i(H,"LI",{});var Zs=l(de);no=i(Zs,"CODE",{});var Zm=l(no);ic=o(Zm,"decoders"),Zm.forEach(s),lc=o(Zs," contains the various types of "),oo=i(Zs,"CODE",{});var Qm=l(oo);pc=o(Qm,"Decoder"),Qm.forEach(s),cc=o(Zs," you can use to decode the outputs of tokenization (complete list "),yt=i(Zs,"A",{href:!0,rel:!0});var ek=l(yt);hc=o(ek,"here"),ek.forEach(s),fc=o(Zs,")."),Zs.forEach(s),H.forEach(s),pa=h(e),Le=i(e,"P",{});var Kl=l(Le);dc=o(Kl,"You can find the whole list of building blocks "),Tt=i(Kl,"A",{href:!0,rel:!0});var tk=l(Tt);uc=o(tk,"here"),tk.forEach(s),mc=o(Kl,"."),Kl.forEach(s),ca=h(e),je=i(e,"H2",{class:!0});var Yl=l(je);Se=i(Yl,"A",{id:!0,class:!0,href:!0});var sk=l(Se);ro=i(sk,"SPAN",{});var nk=l(ro);d(qt.$$.fragment,nk),nk.forEach(s),sk.forEach(s),kc=h(Yl),ao=i(Yl,"SPAN",{});var ok=l(ao);_c=o(ok,"Acquiring a corpus"),ok.forEach(s),Yl.forEach(s),ha=h(e),ue=i(e,"P",{});var Ln=l(ue);wc=o(Ln,"To train our new tokenizer, we will use a small corpus of text (so the examples run fast). The steps for acquiring the corpus are similar to the ones we took at the "),sn=i(Ln,"A",{href:!0});var rk=l(sn);$c=o(rk,"beginning of this chapter"),rk.forEach(s),zc=o(Ln,", but this time we\u2019ll use the "),Pt=i(Ln,"A",{href:!0,rel:!0});var ak=l(Pt);bc=o(ak,"WikiText-2"),ak.forEach(s),gc=o(Ln," dataset:"),Ln.forEach(s),fa=h(e),d(Ct.$$.fragment,e),da=h(e),Be=i(e,"P",{});var Hl=l(Be);vc=o(Hl,"The function "),io=i(Hl,"CODE",{});var ik=l(io);Ec=o(ik,"get_training_corpus()"),ik.forEach(s),jc=o(Hl," is a generator that will yield batches of 1,000 texts, which we will use to train the tokenizer."),Hl.forEach(s),ua=h(e),nn=i(e,"P",{});var lk=l(nn);xc=o(lk,"\u{1F917} Tokenizers can also be trained on text files directly. Here\u2019s how we can generate a text file containing all the texts/inputs from WikiText-2 that we can use locally:"),lk.forEach(s),ma=h(e),d(Dt.$$.fragment,e),ka=h(e),on=i(e,"P",{});var pk=l(on);yc=o(pk,"Next we\u2019ll show you how to build your own BERT, GPT-2, and XLNet tokenizers, block by block. That will give us an example of each of the three main tokenization algorithms: WordPiece, BPE, and Unigram. Let\u2019s start with BERT!"),pk.forEach(s),_a=h(e),xe=i(e,"H2",{class:!0});var Jl=l(xe);Ae=i(Jl,"A",{id:!0,class:!0,href:!0});var ck=l(Ae);lo=i(ck,"SPAN",{});var hk=l(lo);d(Ot.$$.fragment,hk),hk.forEach(s),ck.forEach(s),Tc=h(Jl),po=i(Jl,"SPAN",{});var fk=l(po);qc=o(fk,"Building a WordPiece tokenizer from scratch"),fk.forEach(s),Jl.forEach(s),wa=h(e),E=i(e,"P",{});var L=l(E);Pc=o(L,"To build a tokenizer with the \u{1F917} Tokenizers library, we start by instantiating a "),co=i(L,"CODE",{});var dk=l(co);Cc=o(dk,"Tokenizer"),dk.forEach(s),Dc=o(L," object with a "),ho=i(L,"CODE",{});var uk=l(ho);Oc=o(uk,"model"),uk.forEach(s),Lc=o(L,", then set its "),fo=i(L,"CODE",{});var mk=l(fo);Sc=o(mk,"normalizer"),mk.forEach(s),Bc=o(L,", "),uo=i(L,"CODE",{});var kk=l(uo);Ac=o(kk,"pre_tokenizer"),kk.forEach(s),Nc=o(L,", "),mo=i(L,"CODE",{});var _k=l(mo);Wc=o(_k,"post_processor"),_k.forEach(s),Fc=o(L,", and "),ko=i(L,"CODE",{});var wk=l(ko);Ic=o(wk,"decoder"),wk.forEach(s),Rc=o(L," attributes to the values we want."),L.forEach(s),$a=h(e),Ne=i(e,"P",{});var Vl=l(Ne);Uc=o(Vl,"For this example, we\u2019ll create a "),_o=i(Vl,"CODE",{});var $k=l(_o);Gc=o($k,"Tokenizer"),$k.forEach(s),Mc=o(Vl," with a WordPiece model:"),Vl.forEach(s),za=h(e),d(Lt.$$.fragment,e),ba=h(e),R=i(e,"P",{});var nt=l(R);Xc=o(nt,"We have to specify the "),wo=i(nt,"CODE",{});var zk=l(wo);Kc=o(zk,"unk_token"),zk.forEach(s),Yc=o(nt," so the model knows what to return when it encounters characters it hasn\u2019t seen before. Other arguments we can set here include the "),$o=i(nt,"CODE",{});var bk=l($o);Hc=o(bk,"vocab"),bk.forEach(s),Jc=o(nt," of our model (we\u2019re going to train the model, so we don\u2019t need to set this) and "),zo=i(nt,"CODE",{});var gk=l(zo);Vc=o(gk,"max_input_chars_per_word"),gk.forEach(s),Zc=o(nt,", which specifies a maximum length for each word (words longer than the value passed will be split)."),nt.forEach(s),ga=h(e),j=i(e,"P",{});var S=l(j);Qc=o(S,"The first step of tokenization is normalization, so let\u2019s begin with that. Since BERT is widely used, there is a "),bo=i(S,"CODE",{});var vk=l(bo);eh=o(vk,"BertNormalizer"),vk.forEach(s),th=o(S," with the classic options we can set for BERT: "),go=i(S,"CODE",{});var Ek=l(go);sh=o(Ek,"lowercase"),Ek.forEach(s),nh=o(S," and "),vo=i(S,"CODE",{});var jk=l(vo);oh=o(jk,"strip_accents"),jk.forEach(s),rh=o(S,", which are self-explanatory; "),Eo=i(S,"CODE",{});var xk=l(Eo);ah=o(xk,"clean_text"),xk.forEach(s),ih=o(S," to remove all control characters and replace repeating spaces with a single one; and "),jo=i(S,"CODE",{});var yk=l(jo);lh=o(yk,"handle_chinese_chars"),yk.forEach(s),ph=o(S,", which places spaces around Chinese characters. To replicate the "),xo=i(S,"CODE",{});var Tk=l(xo);ch=o(Tk,"bert-base-uncased"),Tk.forEach(s),hh=o(S," tokenizer, we can just set this normalizer:"),S.forEach(s),va=h(e),d(St.$$.fragment,e),Ea=h(e),U=i(e,"P",{});var ot=l(U);fh=o(ot,"Generally speaking, however, when building a new tokenizer you won\u2019t have access to such a handy normalizer already implemented in the \u{1F917} Tokenizers library \u2014 so let\u2019s see how to create the BERT normalizer by hand. The library provides a "),yo=i(ot,"CODE",{});var qk=l(yo);dh=o(qk,"Lowercase"),qk.forEach(s),uh=o(ot," normalizer and a "),To=i(ot,"CODE",{});var Pk=l(To);mh=o(Pk,"StripAccents"),Pk.forEach(s),kh=o(ot," normalizer, and you can compose several normalizers using a "),qo=i(ot,"CODE",{});var Ck=l(qo);_h=o(Ck,"Sequence"),Ck.forEach(s),wh=o(ot,":"),ot.forEach(s),ja=h(e),d(Bt.$$.fragment,e),xa=h(e),me=i(e,"P",{});var Sn=l(me);$h=o(Sn,"We\u2019re also using an "),Po=i(Sn,"CODE",{});var Dk=l(Po);zh=o(Dk,"NFD"),Dk.forEach(s),bh=o(Sn," Unicode normalizer, as otherwise the "),Co=i(Sn,"CODE",{});var Ok=l(Co);gh=o(Ok,"StripAccents"),Ok.forEach(s),vh=o(Sn," normalizer won\u2019t properly recognize the accented characters and thus won\u2019t strip them out."),Sn.forEach(s),ya=h(e),ke=i(e,"P",{});var Bn=l(ke);Eh=o(Bn,"As we\u2019ve seen before, we can use the "),Do=i(Bn,"CODE",{});var Lk=l(Do);jh=o(Lk,"normalize_str()"),Lk.forEach(s),xh=o(Bn," method of the "),Oo=i(Bn,"CODE",{});var Sk=l(Oo);yh=o(Sk,"normalizer"),Sk.forEach(s),Th=o(Bn," to check out the effects it has on a given text:"),Bn.forEach(s),Ta=h(e),d(At.$$.fragment,e),qa=h(e),d(Nt.$$.fragment,e),Pa=h(e),d(We.$$.fragment,e),Ca=h(e),Fe=i(e,"P",{});var Zl=l(Fe);qh=o(Zl,"Next is the pre-tokenization step. Again, there is a prebuilt "),Lo=i(Zl,"CODE",{});var Bk=l(Lo);Ph=o(Bk,"BertPreTokenizer"),Bk.forEach(s),Ch=o(Zl," that we can use:"),Zl.forEach(s),Da=h(e),d(Wt.$$.fragment,e),Oa=h(e),rn=i(e,"P",{});var Ak=l(rn);Dh=o(Ak,"Or we can build it from scratch:"),Ak.forEach(s),La=h(e),d(Ft.$$.fragment,e),Sa=h(e),Ie=i(e,"P",{});var Ql=l(Ie);Oh=o(Ql,"Note that the "),So=i(Ql,"CODE",{});var Nk=l(So);Lh=o(Nk,"Whitespace"),Nk.forEach(s),Sh=o(Ql," pre-tokenizer splits on whitespace and all characters that are not letters, digits, or the underscore character, so it technically splits on whitespace and punctuation:"),Ql.forEach(s),Ba=h(e),d(It.$$.fragment,e),Aa=h(e),d(Rt.$$.fragment,e),Na=h(e),Re=i(e,"P",{});var ep=l(Re);Bh=o(ep,"If you only want to split on whitespace, you should use the "),Bo=i(ep,"CODE",{});var Wk=l(Bo);Ah=o(Wk,"WhitespaceSplit"),Wk.forEach(s),Nh=o(ep," pre-tokenizer instead:"),ep.forEach(s),Wa=h(e),d(Ut.$$.fragment,e),Fa=h(e),d(Gt.$$.fragment,e),Ia=h(e),Ue=i(e,"P",{});var tp=l(Ue);Wh=o(tp,"Like with normalizers, you can use a "),Ao=i(tp,"CODE",{});var Fk=l(Ao);Fh=o(Fk,"Sequence"),Fk.forEach(s),Ih=o(tp," to compose several pre-tokenizers:"),tp.forEach(s),Ra=h(e),d(Mt.$$.fragment,e),Ua=h(e),d(Xt.$$.fragment,e),Ga=h(e),Ge=i(e,"P",{});var sp=l(Ge);Rh=o(sp,"The next step in the tokenization pipeline is running the inputs through the model. We already specified our model in the initialization, but we still need to train it, which will require a "),No=i(sp,"CODE",{});var Ik=l(No);Uh=o(Ik,"WordPieceTrainer"),Ik.forEach(s),Gh=o(sp,". The main thing to remember when instantiating a trainer in \u{1F917} Tokenizers is that you need to pass it all the special tokens you intend to use \u2014 otherwise it won\u2019t add them to the vocabulary, since they are not in the training corpus:"),sp.forEach(s),Ma=h(e),d(Kt.$$.fragment,e),Xa=h(e),P=i(e,"P",{});var J=l(P);Mh=o(J,"As well as specifying the "),Wo=i(J,"CODE",{});var Rk=l(Wo);Xh=o(Rk,"vocab_size"),Rk.forEach(s),Kh=o(J," and "),Fo=i(J,"CODE",{});var Uk=l(Fo);Yh=o(Uk,"special_tokens"),Uk.forEach(s),Hh=o(J,", we can set the "),Io=i(J,"CODE",{});var Gk=l(Io);Jh=o(Gk,"min_frequency"),Gk.forEach(s),Vh=o(J," (the number of times a token must appear to be included in the vocabulary) or change the "),Ro=i(J,"CODE",{});var Mk=l(Ro);Zh=o(Mk,"continuing_subword_prefix"),Mk.forEach(s),Qh=o(J," (if we want to use something different from "),Uo=i(J,"CODE",{});var Xk=l(Uo);ef=o(Xk,"##"),Xk.forEach(s),tf=o(J,")."),J.forEach(s),Ka=h(e),an=i(e,"P",{});var Kk=l(an);sf=o(Kk,"To train our model using the iterator we defined earlier, we just have to execute this command:"),Kk.forEach(s),Ya=h(e),d(Yt.$$.fragment,e),Ha=h(e),Me=i(e,"P",{});var np=l(Me);nf=o(np,"We can also use text files to train our tokenizer, which would look like this (we reinitialize the model with an empty "),Go=i(np,"CODE",{});var Yk=l(Go);of=o(Yk,"WordPiece"),Yk.forEach(s),rf=o(np," beforehand):"),np.forEach(s),Ja=h(e),d(Ht.$$.fragment,e),Va=h(e),Xe=i(e,"P",{});var op=l(Xe);af=o(op,"In both cases, we can then test the tokenizer on a text by calling the "),Mo=i(op,"CODE",{});var Hk=l(Mo);lf=o(Hk,"encode()"),Hk.forEach(s),pf=o(op," method:"),op.forEach(s),Za=h(e),d(Jt.$$.fragment,e),Qa=h(e),d(Vt.$$.fragment,e),ei=h(e),b=i(e,"P",{});var v=l(b);cf=o(v,"The "),Xo=i(v,"CODE",{});var Jk=l(Xo);hf=o(Jk,"encoding"),Jk.forEach(s),ff=o(v," obtained is an "),Ko=i(v,"CODE",{});var Vk=l(Ko);df=o(Vk,"Encoding"),Vk.forEach(s),uf=o(v,", which contains all the necessary outputs of the tokenizer in its various attributes: "),Yo=i(v,"CODE",{});var Zk=l(Yo);mf=o(Zk,"ids"),Zk.forEach(s),kf=o(v,", "),Ho=i(v,"CODE",{});var Qk=l(Ho);_f=o(Qk,"type_ids"),Qk.forEach(s),wf=o(v,", "),Jo=i(v,"CODE",{});var e2=l(Jo);$f=o(e2,"tokens"),e2.forEach(s),zf=o(v,", "),Vo=i(v,"CODE",{});var t2=l(Vo);bf=o(t2,"offsets"),t2.forEach(s),gf=o(v,", "),Zo=i(v,"CODE",{});var s2=l(Zo);vf=o(s2,"attention_mask"),s2.forEach(s),Ef=o(v,", "),Qo=i(v,"CODE",{});var n2=l(Qo);jf=o(n2,"special_tokens_mask"),n2.forEach(s),xf=o(v,", and "),er=i(v,"CODE",{});var o2=l(er);yf=o(o2,"overflowing"),o2.forEach(s),Tf=o(v,"."),v.forEach(s),ti=h(e),C=i(e,"P",{});var V=l(C);qf=o(V,"The last step in the tokenization pipeline is post-processing. We need to add the "),tr=i(V,"CODE",{});var r2=l(tr);Pf=o(r2,"[CLS]"),r2.forEach(s),Cf=o(V," token at the beginning and the "),sr=i(V,"CODE",{});var a2=l(sr);Df=o(a2,"[SEP]"),a2.forEach(s),Of=o(V," token at the end (or after each sentence, if we have a pair of sentences). We will use a "),nr=i(V,"CODE",{});var i2=l(nr);Lf=o(i2,"TemplateProcessor"),i2.forEach(s),Sf=o(V," for this, but first we need to know the IDs of the "),or=i(V,"CODE",{});var l2=l(or);Bf=o(l2,"[CLS]"),l2.forEach(s),Af=o(V," and "),rr=i(V,"CODE",{});var p2=l(rr);Nf=o(p2,"[SEP]"),p2.forEach(s),Wf=o(V," tokens in the vocabulary:"),V.forEach(s),si=h(e),d(Zt.$$.fragment,e),ni=h(e),d(Qt.$$.fragment,e),oi=h(e),G=i(e,"P",{});var rt=l(G);Ff=o(rt,"To write the template for the "),ar=i(rt,"CODE",{});var c2=l(ar);If=o(c2,"TemplateProcessor"),c2.forEach(s),Rf=o(rt,", we have to specify how to treat a single sentence and a pair of sentences. For both, we write the special tokens we want to use; the first (or single) sentence is represented by "),ir=i(rt,"CODE",{});var h2=l(ir);Uf=o(h2,"$A"),h2.forEach(s),Gf=o(rt,", while the second sentence (if encoding a pair) is represented by "),lr=i(rt,"CODE",{});var f2=l(lr);Mf=o(f2,"$B"),f2.forEach(s),Xf=o(rt,". For each of these (special tokens and sentences), we also specify the corresponding token type ID after a colon."),rt.forEach(s),ri=h(e),ln=i(e,"P",{});var d2=l(ln);Kf=o(d2,"The classic BERT template is thus defined as follows:"),d2.forEach(s),ai=h(e),d(es.$$.fragment,e),ii=h(e),pn=i(e,"P",{});var u2=l(pn);Yf=o(u2,"Note that we need to pass along the IDs of the special tokens, so the tokenizer can properly convert them to their IDs."),u2.forEach(s),li=h(e),cn=i(e,"P",{});var m2=l(cn);Hf=o(m2,"Once this is added, going back to our previous example will give:"),m2.forEach(s),pi=h(e),d(ts.$$.fragment,e),ci=h(e),d(ss.$$.fragment,e),hi=h(e),hn=i(e,"P",{});var k2=l(hn);Jf=o(k2,"And on a pair of sentences, we get the proper result:"),k2.forEach(s),fi=h(e),d(ns.$$.fragment,e),di=h(e),d(os.$$.fragment,e),ui=h(e),fn=i(e,"P",{});var _2=l(fn);Vf=o(_2,"We\u2019ve almost finished building this tokenizer from scratch \u2014 the last step is to include a decoder:"),_2.forEach(s),mi=h(e),d(rs.$$.fragment,e),ki=h(e),Ke=i(e,"P",{});var rp=l(Ke);Zf=o(rp,"Let\u2019s test it on our previous "),pr=i(rp,"CODE",{});var w2=l(pr);Qf=o(w2,"encoding"),w2.forEach(s),ed=o(rp,":"),rp.forEach(s),_i=h(e),d(as.$$.fragment,e),wi=h(e),d(is.$$.fragment,e),$i=h(e),dn=i(e,"P",{});var $2=l(dn);td=o($2,"Great! We can save our tokenizer in a single JSON file like this:"),$2.forEach(s),zi=h(e),d(ls.$$.fragment,e),bi=h(e),_e=i(e,"P",{});var An=l(_e);sd=o(An,"We can then reload that file in a "),cr=i(An,"CODE",{});var z2=l(cr);nd=o(z2,"Tokenizer"),z2.forEach(s),od=o(An," object with the "),hr=i(An,"CODE",{});var b2=l(hr);rd=o(b2,"from_file()"),b2.forEach(s),ad=o(An," method:"),An.forEach(s),gi=h(e),d(ps.$$.fragment,e),vi=h(e),we=i(e,"P",{});var Nn=l(we);id=o(Nn,"To use this tokenizer in \u{1F917} Transformers, we have to wrap it in a "),fr=i(Nn,"CODE",{});var g2=l(fr);ld=o(g2,"PreTrainedTokenizerFast"),g2.forEach(s),pd=o(Nn,". We can either use the generic class or, if our tokenizer corresponds to an existing model, use that class (here, "),dr=i(Nn,"CODE",{});var v2=l(dr);cd=o(v2,"BertTokenizerFast"),v2.forEach(s),hd=o(Nn,"). If you apply this lesson to build a brand new tokenizer, you will have to use the first option."),Nn.forEach(s),Ei=h(e),D=i(e,"P",{});var Z=l(D);fd=o(Z,"To wrap the tokenizer in a "),ur=i(Z,"CODE",{});var E2=l(ur);dd=o(E2,"PreTrainedTokenizerFast"),E2.forEach(s),ud=o(Z,", we can either pass the tokenizer we built as a "),mr=i(Z,"CODE",{});var j2=l(mr);md=o(j2,"tokenizer_object"),j2.forEach(s),kd=o(Z," or pass the tokenizer file we saved as "),kr=i(Z,"CODE",{});var x2=l(kr);_d=o(x2,"tokenizer_file"),x2.forEach(s),wd=o(Z,". The key thing to remember is that we have to manually set all the special tokens, since that class can\u2019t infer from the "),_r=i(Z,"CODE",{});var y2=l(_r);$d=o(y2,"tokenizer"),y2.forEach(s),zd=o(Z," object which token is the mask token, the "),wr=i(Z,"CODE",{});var T2=l(wr);bd=o(T2,"[CLS]"),T2.forEach(s),gd=o(Z," token, etc.:"),Z.forEach(s),ji=h(e),d(cs.$$.fragment,e),xi=h(e),Ye=i(e,"P",{});var ap=l(Ye);vd=o(ap,"If you are using a specific tokenizer class (like "),$r=i(ap,"CODE",{});var q2=l($r);Ed=o(q2,"BertTokenizerFast"),q2.forEach(s),jd=o(ap,"), you will only need to specify the special tokens that are different from the default ones (here, none):"),ap.forEach(s),yi=h(e),d(hs.$$.fragment,e),Ti=h(e),$e=i(e,"P",{});var Wn=l($e);xd=o(Wn,"You can then use this tokenizer like any other \u{1F917} Transformers tokenizer. You can save it with the "),zr=i(Wn,"CODE",{});var P2=l(zr);yd=o(P2,"save_pretrained()"),P2.forEach(s),Td=o(Wn," method, or upload it to the Hub with the "),br=i(Wn,"CODE",{});var C2=l(br);qd=o(C2,"push_to_hub()"),C2.forEach(s),Pd=o(Wn," method."),Wn.forEach(s),qi=h(e),un=i(e,"P",{});var D2=l(un);Cd=o(D2,"Now that we\u2019ve seen how to build a WordPiece tokenizer, let\u2019s do the same for a BPE tokenizer. We\u2019ll go a bit faster since you know all the steps, and only highlight the differences."),D2.forEach(s),Pi=h(e),ye=i(e,"H2",{class:!0});var ip=l(ye);He=i(ip,"A",{id:!0,class:!0,href:!0});var O2=l(He);gr=i(O2,"SPAN",{});var L2=l(gr);d(fs.$$.fragment,L2),L2.forEach(s),O2.forEach(s),Dd=h(ip),vr=i(ip,"SPAN",{});var S2=l(vr);Od=o(S2,"Building a BPE tokenizer from scratch"),S2.forEach(s),ip.forEach(s),Ci=h(e),Je=i(e,"P",{});var lp=l(Je);Ld=o(lp,"Let\u2019s now build a GPT-2 tokenizer. Like for the BERT tokenizer, we start by initializing a "),Er=i(lp,"CODE",{});var B2=l(Er);Sd=o(B2,"Tokenizer"),B2.forEach(s),Bd=o(lp," with a BPE model:"),lp.forEach(s),Di=h(e),d(ds.$$.fragment,e),Oi=h(e),M=i(e,"P",{});var at=l(M);Ad=o(at,"Also like for BERT, we could initialize this model with a vocabulary if we had one (we would need to pass the "),jr=i(at,"CODE",{});var A2=l(jr);Nd=o(A2,"vocab"),A2.forEach(s),Wd=o(at," and "),xr=i(at,"CODE",{});var N2=l(xr);Fd=o(N2,"merges"),N2.forEach(s),Id=o(at," in this case), but since we will train from scratch, we don\u2019t need to do that. We also don\u2019t need to specify an "),yr=i(at,"CODE",{});var W2=l(yr);Rd=o(W2,"unk_token"),W2.forEach(s),Ud=o(at," because GPT-2 uses byte-level BPE, which doesn\u2019t require it."),at.forEach(s),Li=h(e),mn=i(e,"P",{});var F2=l(mn);Gd=o(F2,"GPT-2 does not use a normalizer, so we skip that step and go directly to the pre-tokenization:"),F2.forEach(s),Si=h(e),d(us.$$.fragment,e),Bi=h(e),Ve=i(e,"P",{});var pp=l(Ve);Md=o(pp,"The option we added to "),Tr=i(pp,"CODE",{});var I2=l(Tr);Xd=o(I2,"ByteLevel"),I2.forEach(s),Kd=o(pp," here is to not add a space at the beginning of a sentence (which is the default otherwise). We can have a look at the pre-tokenization of an example text like before:"),pp.forEach(s),Ai=h(e),d(ms.$$.fragment,e),Ni=h(e),d(ks.$$.fragment,e),Wi=h(e),kn=i(e,"P",{});var R2=l(kn);Yd=o(R2,"Next is the model, which needs training. For GPT-2, the only special token is the end-of-text token:"),R2.forEach(s),Fi=h(e),d(_s.$$.fragment,e),Ii=h(e),x=i(e,"P",{});var B=l(x);Hd=o(B,"Like with the "),qr=i(B,"CODE",{});var U2=l(qr);Jd=o(U2,"WordPieceTrainer"),U2.forEach(s),Vd=o(B,", as well as the "),Pr=i(B,"CODE",{});var G2=l(Pr);Zd=o(G2,"vocab_size"),G2.forEach(s),Qd=o(B," and "),Cr=i(B,"CODE",{});var M2=l(Cr);eu=o(M2,"special_tokens"),M2.forEach(s),tu=o(B,", we can specify the "),Dr=i(B,"CODE",{});var X2=l(Dr);su=o(X2,"min_frequency"),X2.forEach(s),nu=o(B," if we want to, or if we have an end-of-word suffix (like "),Or=i(B,"CODE",{});var K2=l(Or);ou=o(K2,"</w>"),K2.forEach(s),ru=o(B,"), we can set it with "),Lr=i(B,"CODE",{});var Y2=l(Lr);au=o(Y2,"end_of_word_suffix"),Y2.forEach(s),iu=o(B,"."),B.forEach(s),Ri=h(e),_n=i(e,"P",{});var H2=l(_n);lu=o(H2,"This tokenizer can also be trained on text files:"),H2.forEach(s),Ui=h(e),d(ws.$$.fragment,e),Gi=h(e),wn=i(e,"P",{});var J2=l(wn);pu=o(J2,"Let\u2019s have a look at the tokenization of a sample text:"),J2.forEach(s),Mi=h(e),d($s.$$.fragment,e),Xi=h(e),d(zs.$$.fragment,e),Ki=h(e),$n=i(e,"P",{});var V2=l($n);cu=o(V2,"We apply the byte-level post-processing for the GPT-2 tokenizer as follows:"),V2.forEach(s),Yi=h(e),d(bs.$$.fragment,e),Hi=h(e),ze=i(e,"P",{});var Fn=l(ze);hu=o(Fn,"The "),Sr=i(Fn,"CODE",{});var Z2=l(Sr);fu=o(Z2,"trim_offsets = False"),Z2.forEach(s),du=o(Fn," option indicates to the post-processor that we should leave the offsets of tokens that begin with \u2018\u0120\u2019 as they are: this way the start of the offsets will point to the space before the word, not the first character of the word (since the space is technically part of the token). Let\u2019s have a look at the result with the text we just encoded, where "),Br=i(Fn,"CODE",{});var Q2=l(Br);uu=o(Q2,"'\u0120test'"),Q2.forEach(s),mu=o(Fn," is the token at index 4:"),Fn.forEach(s),Ji=h(e),d(gs.$$.fragment,e),Vi=h(e),d(vs.$$.fragment,e),Zi=h(e),zn=i(e,"P",{});var e_=l(zn);ku=o(e_,"Finally, we add a byte-level decoder:"),e_.forEach(s),Qi=h(e),d(Es.$$.fragment,e),el=h(e),bn=i(e,"P",{});var t_=l(bn);_u=o(t_,"and we can double-check it works properly:"),t_.forEach(s),tl=h(e),d(js.$$.fragment,e),sl=h(e),d(xs.$$.fragment,e),nl=h(e),be=i(e,"P",{});var In=l(be);wu=o(In,"Great! Now that we\u2019re done, we can save the tokenizer like before, and wrap it in a "),Ar=i(In,"CODE",{});var s_=l(Ar);$u=o(s_,"PreTrainedTokenizerFast"),s_.forEach(s),zu=o(In," or "),Nr=i(In,"CODE",{});var n_=l(Nr);bu=o(n_,"GPT2TokenizerFast"),n_.forEach(s),gu=o(In," if we want to use it in \u{1F917} Transformers:"),In.forEach(s),ol=h(e),d(ys.$$.fragment,e),rl=h(e),gn=i(e,"P",{});var o_=l(gn);vu=o(o_,"or:"),o_.forEach(s),al=h(e),d(Ts.$$.fragment,e),il=h(e),vn=i(e,"P",{});var r_=l(vn);Eu=o(r_,"As the last example, we\u2019ll show you how to build a Unigram tokenizer from scratch."),r_.forEach(s),ll=h(e),Te=i(e,"H2",{class:!0});var cp=l(Te);Ze=i(cp,"A",{id:!0,class:!0,href:!0});var a_=l(Ze);Wr=i(a_,"SPAN",{});var i_=l(Wr);d(qs.$$.fragment,i_),i_.forEach(s),a_.forEach(s),ju=h(cp),Fr=i(cp,"SPAN",{});var l_=l(Fr);xu=o(l_,"Building a Unigram tokenizer from scratch"),l_.forEach(s),cp.forEach(s),pl=h(e),Qe=i(e,"P",{});var hp=l(Qe);yu=o(hp,"Let\u2019s now build an XLNet tokenizer. Like for the previous tokenizers, we start by initializing a "),Ir=i(hp,"CODE",{});var p_=l(Ir);Tu=o(p_,"Tokenizer"),p_.forEach(s),qu=o(hp," with a Unigram model:"),hp.forEach(s),cl=h(e),d(Ps.$$.fragment,e),hl=h(e),En=i(e,"P",{});var c_=l(En);Pu=o(c_,"Again, we could initialize this model with a vocabulary if we had one."),c_.forEach(s),fl=h(e),jn=i(e,"P",{});var h_=l(jn);Cu=o(h_,"For the normalization, XLNet uses a few replacements (which come from SentencePiece):"),h_.forEach(s),dl=h(e),d(Cs.$$.fragment,e),ul=h(e),X=i(e,"P",{});var it=l(X);Du=o(it,"This replaces "),Rr=i(it,"CODE",{});var f_=l(Rr);Ou=o(f_,"\u201C"),f_.forEach(s),Lu=o(it," and "),Ur=i(it,"CODE",{});var d_=l(Ur);Su=o(d_,"\u201D"),d_.forEach(s),Bu=o(it," with "),Gr=i(it,"CODE",{});var u_=l(Gr);Au=o(u_,"\u201D"),u_.forEach(s),Nu=o(it," and any sequence of two or more spaces with a single space, as well as removing the accents in the texts to tokenize."),it.forEach(s),ml=h(e),et=i(e,"P",{});var fp=l(et);Wu=o(fp,"The pre-tokenizer to use for any SentencePiece tokenizer is "),Mr=i(fp,"CODE",{});var m_=l(Mr);Fu=o(m_,"Metaspace"),m_.forEach(s),Iu=o(fp,":"),fp.forEach(s),kl=h(e),d(Ds.$$.fragment,e),_l=h(e),xn=i(e,"P",{});var k_=l(xn);Ru=o(k_,"We can have a look at the pre-tokenization of an example text like before:"),k_.forEach(s),wl=h(e),d(Os.$$.fragment,e),$l=h(e),d(Ls.$$.fragment,e),zl=h(e),yn=i(e,"P",{});var __=l(yn);Uu=o(__,"Next is the model, which needs training. XLNet has quite a few special tokens:"),__.forEach(s),bl=h(e),d(Ss.$$.fragment,e),gl=h(e),O=i(e,"P",{});var ge=l(O);Gu=o(ge,"A very important argument not to forget for the "),Xr=i(ge,"CODE",{});var w_=l(Xr);Mu=o(w_,"UnigramTrainer"),w_.forEach(s),Xu=o(ge," is the "),Kr=i(ge,"CODE",{});var $_=l(Kr);Ku=o($_,"unk_token"),$_.forEach(s),Yu=o(ge,". We can also pass along other arguments specific to the Unigram algorithm, such as the "),Yr=i(ge,"CODE",{});var z_=l(Yr);Hu=o(z_,"shrinking_factor"),z_.forEach(s),Ju=o(ge," for each step where we remove tokens (defaults to 0.75) or the "),Hr=i(ge,"CODE",{});var b_=l(Hr);Vu=o(b_,"max_piece_length"),b_.forEach(s),Zu=o(ge," to specify the maximum length of a given token (defaults to 16)."),ge.forEach(s),vl=h(e),Tn=i(e,"P",{});var g_=l(Tn);Qu=o(g_,"This tokenizer can also be trained on text files:"),g_.forEach(s),El=h(e),d(Bs.$$.fragment,e),jl=h(e),qn=i(e,"P",{});var v_=l(qn);em=o(v_,"Let\u2019s have a look at the tokenization of a sample text:"),v_.forEach(s),xl=h(e),d(As.$$.fragment,e),yl=h(e),d(Ns.$$.fragment,e),Tl=h(e),K=i(e,"P",{});var lt=l(K);tm=o(lt,"A peculiarity of XLNet is that it puts the "),Jr=i(lt,"CODE",{});var E_=l(Jr);sm=o(E_,"<cls>"),E_.forEach(s),nm=o(lt," token at the end of the sentence, with a type ID of 2 (to distinguish it from the other tokens). It\u2019s padding on the left, as a result. We can deal with all the special tokens and token type IDs with a template, like for BERT, but first we have to get the IDs of the "),Vr=i(lt,"CODE",{});var j_=l(Vr);om=o(j_,"<cls>"),j_.forEach(s),rm=o(lt," and "),Zr=i(lt,"CODE",{});var x_=l(Zr);am=o(x_,"<sep>"),x_.forEach(s),im=o(lt," tokens:"),lt.forEach(s),ql=h(e),d(Ws.$$.fragment,e),Pl=h(e),d(Fs.$$.fragment,e),Cl=h(e),Pn=i(e,"P",{});var y_=l(Pn);lm=o(y_,"The template looks like this:"),y_.forEach(s),Dl=h(e),d(Is.$$.fragment,e),Ol=h(e),Cn=i(e,"P",{});var T_=l(Cn);pm=o(T_,"And we can test it works by encoding a pair of sentences:"),T_.forEach(s),Ll=h(e),d(Rs.$$.fragment,e),Sl=h(e),d(Us.$$.fragment,e),Bl=h(e),tt=i(e,"P",{});var dp=l(tt);cm=o(dp,"Finally, we add a "),Qr=i(dp,"CODE",{});var q_=l(Qr);hm=o(q_,"Metaspace"),q_.forEach(s),fm=o(dp," decoder:"),dp.forEach(s),Al=h(e),d(Gs.$$.fragment,e),Nl=h(e),Y=i(e,"P",{});var pt=l(Y);dm=o(pt,"and we\u2019re done with this tokenizer! We can save the tokenizer like before, and wrap it in a "),ea=i(pt,"CODE",{});var P_=l(ea);um=o(P_,"PreTrainedTokenizerFast"),P_.forEach(s),mm=o(pt," or "),ta=i(pt,"CODE",{});var C_=l(ta);km=o(C_,"XLNetTokenizerFast"),C_.forEach(s),_m=o(pt," if we want to use it in \u{1F917} Transformers. One thing to note when using "),sa=i(pt,"CODE",{});var D_=l(sa);wm=o(D_,"PreTrainedTokenizerFast"),D_.forEach(s),$m=o(pt," is that on top of the special tokens, we need to tell the \u{1F917} Transformers library to pad on the left:"),pt.forEach(s),Wl=h(e),d(Ms.$$.fragment,e),Fl=h(e),Dn=i(e,"P",{});var O_=l(Dn);zm=o(O_,"Or alternatively:"),O_.forEach(s),Il=h(e),d(Xs.$$.fragment,e),Rl=h(e),On=i(e,"P",{});var L_=l(On);bm=o(L_,"Now that you have seen how the various building blocks are used to build existing tokenizers, you should be able to write any tokenizer you want with the \u{1F917} Tokenizers library and be able to use it in \u{1F917} Transformers."),L_.forEach(s),this.h()},h(){$(z,"name","hf:doc:metadata"),$(z,"content",JSON.stringify(X_)),$(F,"id","building-a-tokenizer-block-by-block"),$(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(F,"href","#building-a-tokenizer-block-by-block"),$(A,"class","relative group"),$(le,"class","block dark:hidden"),S_(le.src,Rn="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline.svg")||$(le,"src",Rn),$(le,"alt","The tokenization pipeline."),$(zt,"class","hidden dark:block"),S_(zt.src,gm="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline-dark.svg")||$(zt,"src",gm),$(zt,"alt","The tokenization pipeline."),$(I,"class","flex justify-center"),$(tn,"href","/course/chapter6/2"),$(gt,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.normalizers"),$(gt,"rel","nofollow"),$(vt,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.pre_tokenizers"),$(vt,"rel","nofollow"),$(Et,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.models"),$(Et,"rel","nofollow"),$(jt,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.trainers"),$(jt,"rel","nofollow"),$(xt,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.processors"),$(xt,"rel","nofollow"),$(yt,"href","https://huggingface.co/docs/tokenizers/python/latest/components.html#decoders"),$(yt,"rel","nofollow"),$(Tt,"href","https://huggingface.co/docs/tokenizers/python/latest/components.html"),$(Tt,"rel","nofollow"),$(Se,"id","acquiring-a-corpus"),$(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(Se,"href","#acquiring-a-corpus"),$(je,"class","relative group"),$(sn,"href","/course/chapter6/2"),$(Pt,"href","https://huggingface.co/datasets/wikitext"),$(Pt,"rel","nofollow"),$(Ae,"id","building-a-wordpiece-tokenizer-from-scratch"),$(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(Ae,"href","#building-a-wordpiece-tokenizer-from-scratch"),$(xe,"class","relative group"),$(He,"id","building-a-bpe-tokenizer-from-scratch"),$(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(He,"href","#building-a-bpe-tokenizer-from-scratch"),$(ye,"class","relative group"),$(Ze,"id","building-a-unigram-tokenizer-from-scratch"),$(Ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(Ze,"href","#building-a-unigram-tokenizer-from-scratch"),$(Te,"class","relative group")},m(e,r){t(document.head,z),p(e,ae,r),p(e,A,r),t(A,F),t(F,Q),u(ee,Q,null),t(A,ct),t(A,te),t(te,ht),p(e,qe,r),u(N,e,r),p(e,Pe,r),p(e,ie,r),t(ie,ve),p(e,Ce,r),p(e,y,r),t(y,se),t(se,ft),t(y,dt),t(y,ne),t(ne,ut),t(y,mt),t(y,oe),t(oe,kt),t(y,_t),t(y,re),t(re,g),p(e,wt,r),p(e,Ee,r),t(Ee,Qs),p(e,$t,r),p(e,I,r),t(I,le),t(I,en),t(I,zt),p(e,ra,r),p(e,De,r),t(De,up),t(De,tn),t(tn,mp),t(De,kp),p(e,aa,r),u(bt,e,r),p(e,ia,r),p(e,Oe,r),t(Oe,_p),t(Oe,Un),t(Un,wp),t(Oe,$p),p(e,la,r),p(e,T,r),t(T,pe),t(pe,Gn),t(Gn,zp),t(pe,bp),t(pe,Mn),t(Mn,gp),t(pe,vp),t(pe,gt),t(gt,Ep),t(pe,jp),t(T,xp),t(T,ce),t(ce,Xn),t(Xn,yp),t(ce,Tp),t(ce,Kn),t(Kn,qp),t(ce,Pp),t(ce,vt),t(vt,Cp),t(ce,Dp),t(T,Op),t(T,q),t(q,Yn),t(Yn,Lp),t(q,Sp),t(q,Hn),t(Hn,Bp),t(q,Ap),t(q,Jn),t(Jn,Np),t(q,Wp),t(q,Vn),t(Vn,Fp),t(q,Ip),t(q,Zn),t(Zn,Rp),t(q,Up),t(q,Et),t(Et,Gp),t(q,Mp),t(T,Xp),t(T,he),t(he,Qn),t(Qn,Kp),t(he,Yp),t(he,eo),t(eo,Hp),t(he,Jp),t(he,jt),t(jt,Vp),t(he,Zp),t(T,Qp),t(T,fe),t(fe,to),t(to,ec),t(fe,tc),t(fe,so),t(so,sc),t(fe,nc),t(fe,xt),t(xt,oc),t(fe,rc),t(T,ac),t(T,de),t(de,no),t(no,ic),t(de,lc),t(de,oo),t(oo,pc),t(de,cc),t(de,yt),t(yt,hc),t(de,fc),p(e,pa,r),p(e,Le,r),t(Le,dc),t(Le,Tt),t(Tt,uc),t(Le,mc),p(e,ca,r),p(e,je,r),t(je,Se),t(Se,ro),u(qt,ro,null),t(je,kc),t(je,ao),t(ao,_c),p(e,ha,r),p(e,ue,r),t(ue,wc),t(ue,sn),t(sn,$c),t(ue,zc),t(ue,Pt),t(Pt,bc),t(ue,gc),p(e,fa,r),u(Ct,e,r),p(e,da,r),p(e,Be,r),t(Be,vc),t(Be,io),t(io,Ec),t(Be,jc),p(e,ua,r),p(e,nn,r),t(nn,xc),p(e,ma,r),u(Dt,e,r),p(e,ka,r),p(e,on,r),t(on,yc),p(e,_a,r),p(e,xe,r),t(xe,Ae),t(Ae,lo),u(Ot,lo,null),t(xe,Tc),t(xe,po),t(po,qc),p(e,wa,r),p(e,E,r),t(E,Pc),t(E,co),t(co,Cc),t(E,Dc),t(E,ho),t(ho,Oc),t(E,Lc),t(E,fo),t(fo,Sc),t(E,Bc),t(E,uo),t(uo,Ac),t(E,Nc),t(E,mo),t(mo,Wc),t(E,Fc),t(E,ko),t(ko,Ic),t(E,Rc),p(e,$a,r),p(e,Ne,r),t(Ne,Uc),t(Ne,_o),t(_o,Gc),t(Ne,Mc),p(e,za,r),u(Lt,e,r),p(e,ba,r),p(e,R,r),t(R,Xc),t(R,wo),t(wo,Kc),t(R,Yc),t(R,$o),t($o,Hc),t(R,Jc),t(R,zo),t(zo,Vc),t(R,Zc),p(e,ga,r),p(e,j,r),t(j,Qc),t(j,bo),t(bo,eh),t(j,th),t(j,go),t(go,sh),t(j,nh),t(j,vo),t(vo,oh),t(j,rh),t(j,Eo),t(Eo,ah),t(j,ih),t(j,jo),t(jo,lh),t(j,ph),t(j,xo),t(xo,ch),t(j,hh),p(e,va,r),u(St,e,r),p(e,Ea,r),p(e,U,r),t(U,fh),t(U,yo),t(yo,dh),t(U,uh),t(U,To),t(To,mh),t(U,kh),t(U,qo),t(qo,_h),t(U,wh),p(e,ja,r),u(Bt,e,r),p(e,xa,r),p(e,me,r),t(me,$h),t(me,Po),t(Po,zh),t(me,bh),t(me,Co),t(Co,gh),t(me,vh),p(e,ya,r),p(e,ke,r),t(ke,Eh),t(ke,Do),t(Do,jh),t(ke,xh),t(ke,Oo),t(Oo,yh),t(ke,Th),p(e,Ta,r),u(At,e,r),p(e,qa,r),u(Nt,e,r),p(e,Pa,r),u(We,e,r),p(e,Ca,r),p(e,Fe,r),t(Fe,qh),t(Fe,Lo),t(Lo,Ph),t(Fe,Ch),p(e,Da,r),u(Wt,e,r),p(e,Oa,r),p(e,rn,r),t(rn,Dh),p(e,La,r),u(Ft,e,r),p(e,Sa,r),p(e,Ie,r),t(Ie,Oh),t(Ie,So),t(So,Lh),t(Ie,Sh),p(e,Ba,r),u(It,e,r),p(e,Aa,r),u(Rt,e,r),p(e,Na,r),p(e,Re,r),t(Re,Bh),t(Re,Bo),t(Bo,Ah),t(Re,Nh),p(e,Wa,r),u(Ut,e,r),p(e,Fa,r),u(Gt,e,r),p(e,Ia,r),p(e,Ue,r),t(Ue,Wh),t(Ue,Ao),t(Ao,Fh),t(Ue,Ih),p(e,Ra,r),u(Mt,e,r),p(e,Ua,r),u(Xt,e,r),p(e,Ga,r),p(e,Ge,r),t(Ge,Rh),t(Ge,No),t(No,Uh),t(Ge,Gh),p(e,Ma,r),u(Kt,e,r),p(e,Xa,r),p(e,P,r),t(P,Mh),t(P,Wo),t(Wo,Xh),t(P,Kh),t(P,Fo),t(Fo,Yh),t(P,Hh),t(P,Io),t(Io,Jh),t(P,Vh),t(P,Ro),t(Ro,Zh),t(P,Qh),t(P,Uo),t(Uo,ef),t(P,tf),p(e,Ka,r),p(e,an,r),t(an,sf),p(e,Ya,r),u(Yt,e,r),p(e,Ha,r),p(e,Me,r),t(Me,nf),t(Me,Go),t(Go,of),t(Me,rf),p(e,Ja,r),u(Ht,e,r),p(e,Va,r),p(e,Xe,r),t(Xe,af),t(Xe,Mo),t(Mo,lf),t(Xe,pf),p(e,Za,r),u(Jt,e,r),p(e,Qa,r),u(Vt,e,r),p(e,ei,r),p(e,b,r),t(b,cf),t(b,Xo),t(Xo,hf),t(b,ff),t(b,Ko),t(Ko,df),t(b,uf),t(b,Yo),t(Yo,mf),t(b,kf),t(b,Ho),t(Ho,_f),t(b,wf),t(b,Jo),t(Jo,$f),t(b,zf),t(b,Vo),t(Vo,bf),t(b,gf),t(b,Zo),t(Zo,vf),t(b,Ef),t(b,Qo),t(Qo,jf),t(b,xf),t(b,er),t(er,yf),t(b,Tf),p(e,ti,r),p(e,C,r),t(C,qf),t(C,tr),t(tr,Pf),t(C,Cf),t(C,sr),t(sr,Df),t(C,Of),t(C,nr),t(nr,Lf),t(C,Sf),t(C,or),t(or,Bf),t(C,Af),t(C,rr),t(rr,Nf),t(C,Wf),p(e,si,r),u(Zt,e,r),p(e,ni,r),u(Qt,e,r),p(e,oi,r),p(e,G,r),t(G,Ff),t(G,ar),t(ar,If),t(G,Rf),t(G,ir),t(ir,Uf),t(G,Gf),t(G,lr),t(lr,Mf),t(G,Xf),p(e,ri,r),p(e,ln,r),t(ln,Kf),p(e,ai,r),u(es,e,r),p(e,ii,r),p(e,pn,r),t(pn,Yf),p(e,li,r),p(e,cn,r),t(cn,Hf),p(e,pi,r),u(ts,e,r),p(e,ci,r),u(ss,e,r),p(e,hi,r),p(e,hn,r),t(hn,Jf),p(e,fi,r),u(ns,e,r),p(e,di,r),u(os,e,r),p(e,ui,r),p(e,fn,r),t(fn,Vf),p(e,mi,r),u(rs,e,r),p(e,ki,r),p(e,Ke,r),t(Ke,Zf),t(Ke,pr),t(pr,Qf),t(Ke,ed),p(e,_i,r),u(as,e,r),p(e,wi,r),u(is,e,r),p(e,$i,r),p(e,dn,r),t(dn,td),p(e,zi,r),u(ls,e,r),p(e,bi,r),p(e,_e,r),t(_e,sd),t(_e,cr),t(cr,nd),t(_e,od),t(_e,hr),t(hr,rd),t(_e,ad),p(e,gi,r),u(ps,e,r),p(e,vi,r),p(e,we,r),t(we,id),t(we,fr),t(fr,ld),t(we,pd),t(we,dr),t(dr,cd),t(we,hd),p(e,Ei,r),p(e,D,r),t(D,fd),t(D,ur),t(ur,dd),t(D,ud),t(D,mr),t(mr,md),t(D,kd),t(D,kr),t(kr,_d),t(D,wd),t(D,_r),t(_r,$d),t(D,zd),t(D,wr),t(wr,bd),t(D,gd),p(e,ji,r),u(cs,e,r),p(e,xi,r),p(e,Ye,r),t(Ye,vd),t(Ye,$r),t($r,Ed),t(Ye,jd),p(e,yi,r),u(hs,e,r),p(e,Ti,r),p(e,$e,r),t($e,xd),t($e,zr),t(zr,yd),t($e,Td),t($e,br),t(br,qd),t($e,Pd),p(e,qi,r),p(e,un,r),t(un,Cd),p(e,Pi,r),p(e,ye,r),t(ye,He),t(He,gr),u(fs,gr,null),t(ye,Dd),t(ye,vr),t(vr,Od),p(e,Ci,r),p(e,Je,r),t(Je,Ld),t(Je,Er),t(Er,Sd),t(Je,Bd),p(e,Di,r),u(ds,e,r),p(e,Oi,r),p(e,M,r),t(M,Ad),t(M,jr),t(jr,Nd),t(M,Wd),t(M,xr),t(xr,Fd),t(M,Id),t(M,yr),t(yr,Rd),t(M,Ud),p(e,Li,r),p(e,mn,r),t(mn,Gd),p(e,Si,r),u(us,e,r),p(e,Bi,r),p(e,Ve,r),t(Ve,Md),t(Ve,Tr),t(Tr,Xd),t(Ve,Kd),p(e,Ai,r),u(ms,e,r),p(e,Ni,r),u(ks,e,r),p(e,Wi,r),p(e,kn,r),t(kn,Yd),p(e,Fi,r),u(_s,e,r),p(e,Ii,r),p(e,x,r),t(x,Hd),t(x,qr),t(qr,Jd),t(x,Vd),t(x,Pr),t(Pr,Zd),t(x,Qd),t(x,Cr),t(Cr,eu),t(x,tu),t(x,Dr),t(Dr,su),t(x,nu),t(x,Or),t(Or,ou),t(x,ru),t(x,Lr),t(Lr,au),t(x,iu),p(e,Ri,r),p(e,_n,r),t(_n,lu),p(e,Ui,r),u(ws,e,r),p(e,Gi,r),p(e,wn,r),t(wn,pu),p(e,Mi,r),u($s,e,r),p(e,Xi,r),u(zs,e,r),p(e,Ki,r),p(e,$n,r),t($n,cu),p(e,Yi,r),u(bs,e,r),p(e,Hi,r),p(e,ze,r),t(ze,hu),t(ze,Sr),t(Sr,fu),t(ze,du),t(ze,Br),t(Br,uu),t(ze,mu),p(e,Ji,r),u(gs,e,r),p(e,Vi,r),u(vs,e,r),p(e,Zi,r),p(e,zn,r),t(zn,ku),p(e,Qi,r),u(Es,e,r),p(e,el,r),p(e,bn,r),t(bn,_u),p(e,tl,r),u(js,e,r),p(e,sl,r),u(xs,e,r),p(e,nl,r),p(e,be,r),t(be,wu),t(be,Ar),t(Ar,$u),t(be,zu),t(be,Nr),t(Nr,bu),t(be,gu),p(e,ol,r),u(ys,e,r),p(e,rl,r),p(e,gn,r),t(gn,vu),p(e,al,r),u(Ts,e,r),p(e,il,r),p(e,vn,r),t(vn,Eu),p(e,ll,r),p(e,Te,r),t(Te,Ze),t(Ze,Wr),u(qs,Wr,null),t(Te,ju),t(Te,Fr),t(Fr,xu),p(e,pl,r),p(e,Qe,r),t(Qe,yu),t(Qe,Ir),t(Ir,Tu),t(Qe,qu),p(e,cl,r),u(Ps,e,r),p(e,hl,r),p(e,En,r),t(En,Pu),p(e,fl,r),p(e,jn,r),t(jn,Cu),p(e,dl,r),u(Cs,e,r),p(e,ul,r),p(e,X,r),t(X,Du),t(X,Rr),t(Rr,Ou),t(X,Lu),t(X,Ur),t(Ur,Su),t(X,Bu),t(X,Gr),t(Gr,Au),t(X,Nu),p(e,ml,r),p(e,et,r),t(et,Wu),t(et,Mr),t(Mr,Fu),t(et,Iu),p(e,kl,r),u(Ds,e,r),p(e,_l,r),p(e,xn,r),t(xn,Ru),p(e,wl,r),u(Os,e,r),p(e,$l,r),u(Ls,e,r),p(e,zl,r),p(e,yn,r),t(yn,Uu),p(e,bl,r),u(Ss,e,r),p(e,gl,r),p(e,O,r),t(O,Gu),t(O,Xr),t(Xr,Mu),t(O,Xu),t(O,Kr),t(Kr,Ku),t(O,Yu),t(O,Yr),t(Yr,Hu),t(O,Ju),t(O,Hr),t(Hr,Vu),t(O,Zu),p(e,vl,r),p(e,Tn,r),t(Tn,Qu),p(e,El,r),u(Bs,e,r),p(e,jl,r),p(e,qn,r),t(qn,em),p(e,xl,r),u(As,e,r),p(e,yl,r),u(Ns,e,r),p(e,Tl,r),p(e,K,r),t(K,tm),t(K,Jr),t(Jr,sm),t(K,nm),t(K,Vr),t(Vr,om),t(K,rm),t(K,Zr),t(Zr,am),t(K,im),p(e,ql,r),u(Ws,e,r),p(e,Pl,r),u(Fs,e,r),p(e,Cl,r),p(e,Pn,r),t(Pn,lm),p(e,Dl,r),u(Is,e,r),p(e,Ol,r),p(e,Cn,r),t(Cn,pm),p(e,Ll,r),u(Rs,e,r),p(e,Sl,r),u(Us,e,r),p(e,Bl,r),p(e,tt,r),t(tt,cm),t(tt,Qr),t(Qr,hm),t(tt,fm),p(e,Al,r),u(Gs,e,r),p(e,Nl,r),p(e,Y,r),t(Y,dm),t(Y,ea),t(ea,um),t(Y,mm),t(Y,ta),t(ta,km),t(Y,_m),t(Y,sa),t(sa,wm),t(Y,$m),p(e,Wl,r),u(Ms,e,r),p(e,Fl,r),p(e,Dn,r),t(Dn,zm),p(e,Il,r),u(Xs,e,r),p(e,Rl,r),p(e,On,r),t(On,bm),Ul=!0},p(e,[r]){const Ks={};r&2&&(Ks.$$scope={dirty:r,ctx:e}),We.$set(Ks)},i(e){Ul||(m(ee.$$.fragment,e),m(N.$$.fragment,e),m(bt.$$.fragment,e),m(qt.$$.fragment,e),m(Ct.$$.fragment,e),m(Dt.$$.fragment,e),m(Ot.$$.fragment,e),m(Lt.$$.fragment,e),m(St.$$.fragment,e),m(Bt.$$.fragment,e),m(At.$$.fragment,e),m(Nt.$$.fragment,e),m(We.$$.fragment,e),m(Wt.$$.fragment,e),m(Ft.$$.fragment,e),m(It.$$.fragment,e),m(Rt.$$.fragment,e),m(Ut.$$.fragment,e),m(Gt.$$.fragment,e),m(Mt.$$.fragment,e),m(Xt.$$.fragment,e),m(Kt.$$.fragment,e),m(Yt.$$.fragment,e),m(Ht.$$.fragment,e),m(Jt.$$.fragment,e),m(Vt.$$.fragment,e),m(Zt.$$.fragment,e),m(Qt.$$.fragment,e),m(es.$$.fragment,e),m(ts.$$.fragment,e),m(ss.$$.fragment,e),m(ns.$$.fragment,e),m(os.$$.fragment,e),m(rs.$$.fragment,e),m(as.$$.fragment,e),m(is.$$.fragment,e),m(ls.$$.fragment,e),m(ps.$$.fragment,e),m(cs.$$.fragment,e),m(hs.$$.fragment,e),m(fs.$$.fragment,e),m(ds.$$.fragment,e),m(us.$$.fragment,e),m(ms.$$.fragment,e),m(ks.$$.fragment,e),m(_s.$$.fragment,e),m(ws.$$.fragment,e),m($s.$$.fragment,e),m(zs.$$.fragment,e),m(bs.$$.fragment,e),m(gs.$$.fragment,e),m(vs.$$.fragment,e),m(Es.$$.fragment,e),m(js.$$.fragment,e),m(xs.$$.fragment,e),m(ys.$$.fragment,e),m(Ts.$$.fragment,e),m(qs.$$.fragment,e),m(Ps.$$.fragment,e),m(Cs.$$.fragment,e),m(Ds.$$.fragment,e),m(Os.$$.fragment,e),m(Ls.$$.fragment,e),m(Ss.$$.fragment,e),m(Bs.$$.fragment,e),m(As.$$.fragment,e),m(Ns.$$.fragment,e),m(Ws.$$.fragment,e),m(Fs.$$.fragment,e),m(Is.$$.fragment,e),m(Rs.$$.fragment,e),m(Us.$$.fragment,e),m(Gs.$$.fragment,e),m(Ms.$$.fragment,e),m(Xs.$$.fragment,e),Ul=!0)},o(e){k(ee.$$.fragment,e),k(N.$$.fragment,e),k(bt.$$.fragment,e),k(qt.$$.fragment,e),k(Ct.$$.fragment,e),k(Dt.$$.fragment,e),k(Ot.$$.fragment,e),k(Lt.$$.fragment,e),k(St.$$.fragment,e),k(Bt.$$.fragment,e),k(At.$$.fragment,e),k(Nt.$$.fragment,e),k(We.$$.fragment,e),k(Wt.$$.fragment,e),k(Ft.$$.fragment,e),k(It.$$.fragment,e),k(Rt.$$.fragment,e),k(Ut.$$.fragment,e),k(Gt.$$.fragment,e),k(Mt.$$.fragment,e),k(Xt.$$.fragment,e),k(Kt.$$.fragment,e),k(Yt.$$.fragment,e),k(Ht.$$.fragment,e),k(Jt.$$.fragment,e),k(Vt.$$.fragment,e),k(Zt.$$.fragment,e),k(Qt.$$.fragment,e),k(es.$$.fragment,e),k(ts.$$.fragment,e),k(ss.$$.fragment,e),k(ns.$$.fragment,e),k(os.$$.fragment,e),k(rs.$$.fragment,e),k(as.$$.fragment,e),k(is.$$.fragment,e),k(ls.$$.fragment,e),k(ps.$$.fragment,e),k(cs.$$.fragment,e),k(hs.$$.fragment,e),k(fs.$$.fragment,e),k(ds.$$.fragment,e),k(us.$$.fragment,e),k(ms.$$.fragment,e),k(ks.$$.fragment,e),k(_s.$$.fragment,e),k(ws.$$.fragment,e),k($s.$$.fragment,e),k(zs.$$.fragment,e),k(bs.$$.fragment,e),k(gs.$$.fragment,e),k(vs.$$.fragment,e),k(Es.$$.fragment,e),k(js.$$.fragment,e),k(xs.$$.fragment,e),k(ys.$$.fragment,e),k(Ts.$$.fragment,e),k(qs.$$.fragment,e),k(Ps.$$.fragment,e),k(Cs.$$.fragment,e),k(Ds.$$.fragment,e),k(Os.$$.fragment,e),k(Ls.$$.fragment,e),k(Ss.$$.fragment,e),k(Bs.$$.fragment,e),k(As.$$.fragment,e),k(Ns.$$.fragment,e),k(Ws.$$.fragment,e),k(Fs.$$.fragment,e),k(Is.$$.fragment,e),k(Rs.$$.fragment,e),k(Us.$$.fragment,e),k(Gs.$$.fragment,e),k(Ms.$$.fragment,e),k(Xs.$$.fragment,e),Ul=!1},d(e){s(z),e&&s(ae),e&&s(A),_(ee),e&&s(qe),_(N,e),e&&s(Pe),e&&s(ie),e&&s(Ce),e&&s(y),e&&s(wt),e&&s(Ee),e&&s($t),e&&s(I),e&&s(ra),e&&s(De),e&&s(aa),_(bt,e),e&&s(ia),e&&s(Oe),e&&s(la),e&&s(T),e&&s(pa),e&&s(Le),e&&s(ca),e&&s(je),_(qt),e&&s(ha),e&&s(ue),e&&s(fa),_(Ct,e),e&&s(da),e&&s(Be),e&&s(ua),e&&s(nn),e&&s(ma),_(Dt,e),e&&s(ka),e&&s(on),e&&s(_a),e&&s(xe),_(Ot),e&&s(wa),e&&s(E),e&&s($a),e&&s(Ne),e&&s(za),_(Lt,e),e&&s(ba),e&&s(R),e&&s(ga),e&&s(j),e&&s(va),_(St,e),e&&s(Ea),e&&s(U),e&&s(ja),_(Bt,e),e&&s(xa),e&&s(me),e&&s(ya),e&&s(ke),e&&s(Ta),_(At,e),e&&s(qa),_(Nt,e),e&&s(Pa),_(We,e),e&&s(Ca),e&&s(Fe),e&&s(Da),_(Wt,e),e&&s(Oa),e&&s(rn),e&&s(La),_(Ft,e),e&&s(Sa),e&&s(Ie),e&&s(Ba),_(It,e),e&&s(Aa),_(Rt,e),e&&s(Na),e&&s(Re),e&&s(Wa),_(Ut,e),e&&s(Fa),_(Gt,e),e&&s(Ia),e&&s(Ue),e&&s(Ra),_(Mt,e),e&&s(Ua),_(Xt,e),e&&s(Ga),e&&s(Ge),e&&s(Ma),_(Kt,e),e&&s(Xa),e&&s(P),e&&s(Ka),e&&s(an),e&&s(Ya),_(Yt,e),e&&s(Ha),e&&s(Me),e&&s(Ja),_(Ht,e),e&&s(Va),e&&s(Xe),e&&s(Za),_(Jt,e),e&&s(Qa),_(Vt,e),e&&s(ei),e&&s(b),e&&s(ti),e&&s(C),e&&s(si),_(Zt,e),e&&s(ni),_(Qt,e),e&&s(oi),e&&s(G),e&&s(ri),e&&s(ln),e&&s(ai),_(es,e),e&&s(ii),e&&s(pn),e&&s(li),e&&s(cn),e&&s(pi),_(ts,e),e&&s(ci),_(ss,e),e&&s(hi),e&&s(hn),e&&s(fi),_(ns,e),e&&s(di),_(os,e),e&&s(ui),e&&s(fn),e&&s(mi),_(rs,e),e&&s(ki),e&&s(Ke),e&&s(_i),_(as,e),e&&s(wi),_(is,e),e&&s($i),e&&s(dn),e&&s(zi),_(ls,e),e&&s(bi),e&&s(_e),e&&s(gi),_(ps,e),e&&s(vi),e&&s(we),e&&s(Ei),e&&s(D),e&&s(ji),_(cs,e),e&&s(xi),e&&s(Ye),e&&s(yi),_(hs,e),e&&s(Ti),e&&s($e),e&&s(qi),e&&s(un),e&&s(Pi),e&&s(ye),_(fs),e&&s(Ci),e&&s(Je),e&&s(Di),_(ds,e),e&&s(Oi),e&&s(M),e&&s(Li),e&&s(mn),e&&s(Si),_(us,e),e&&s(Bi),e&&s(Ve),e&&s(Ai),_(ms,e),e&&s(Ni),_(ks,e),e&&s(Wi),e&&s(kn),e&&s(Fi),_(_s,e),e&&s(Ii),e&&s(x),e&&s(Ri),e&&s(_n),e&&s(Ui),_(ws,e),e&&s(Gi),e&&s(wn),e&&s(Mi),_($s,e),e&&s(Xi),_(zs,e),e&&s(Ki),e&&s($n),e&&s(Yi),_(bs,e),e&&s(Hi),e&&s(ze),e&&s(Ji),_(gs,e),e&&s(Vi),_(vs,e),e&&s(Zi),e&&s(zn),e&&s(Qi),_(Es,e),e&&s(el),e&&s(bn),e&&s(tl),_(js,e),e&&s(sl),_(xs,e),e&&s(nl),e&&s(be),e&&s(ol),_(ys,e),e&&s(rl),e&&s(gn),e&&s(al),_(Ts,e),e&&s(il),e&&s(vn),e&&s(ll),e&&s(Te),_(qs),e&&s(pl),e&&s(Qe),e&&s(cl),_(Ps,e),e&&s(hl),e&&s(En),e&&s(fl),e&&s(jn),e&&s(dl),_(Cs,e),e&&s(ul),e&&s(X),e&&s(ml),e&&s(et),e&&s(kl),_(Ds,e),e&&s(_l),e&&s(xn),e&&s(wl),_(Os,e),e&&s($l),_(Ls,e),e&&s(zl),e&&s(yn),e&&s(bl),_(Ss,e),e&&s(gl),e&&s(O),e&&s(vl),e&&s(Tn),e&&s(El),_(Bs,e),e&&s(jl),e&&s(qn),e&&s(xl),_(As,e),e&&s(yl),_(Ns,e),e&&s(Tl),e&&s(K),e&&s(ql),_(Ws,e),e&&s(Pl),_(Fs,e),e&&s(Cl),e&&s(Pn),e&&s(Dl),_(Is,e),e&&s(Ol),e&&s(Cn),e&&s(Ll),_(Rs,e),e&&s(Sl),_(Us,e),e&&s(Bl),e&&s(tt),e&&s(Al),_(Gs,e),e&&s(Nl),e&&s(Y),e&&s(Wl),_(Ms,e),e&&s(Fl),e&&s(Dn),e&&s(Il),_(Xs,e),e&&s(Rl),e&&s(On)}}}const X_={local:"building-a-tokenizer-block-by-block",sections:[{local:"acquiring-a-corpus",title:"Acquiring a corpus"},{local:"building-a-wordpiece-tokenizer-from-scratch",title:"Building a WordPiece tokenizer from scratch"},{local:"building-a-bpe-tokenizer-from-scratch",title:"Building a BPE tokenizer from scratch"},{local:"building-a-unigram-tokenizer-from-scratch",title:"Building a Unigram tokenizer from scratch"}],title:"Building a tokenizer, block by block"};function K_(oa){return F_(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ew extends B_{constructor(z){super();A_(this,z,K_,M_,N_,{})}}export{ew as default,X_ as metadata};
