import{S as ne,i as re,s as se,e as n,k as f,w as ie,t as u,M as le,c as r,d as t,m as d,a as s,x as he,h as m,b as w,F as a,g as k,y as ce,L as fe,q as de,o as pe,B as ue,v as me}from"../../chunks/vendor-1e8b365d.js";import{I as ke}from"../../chunks/IconCopyLink-483c28ba.js";function _e(R){let h,L,c,p,y,_,U,z,q,T,v,M,I,b,N,g,o,E,j,C,x,G,K,P,W,F,$,H,J,B,O,A;return _=new ke({}),{c(){h=n("meta"),L=f(),c=n("h1"),p=n("a"),y=n("span"),ie(_.$$.fragment),U=f(),z=n("span"),q=u("Tokenizers, check!"),T=f(),v=n("p"),M=u("Great job finishing this chapter!"),I=f(),b=n("p"),N=u("After this deep dive into tokenizers, you should:"),g=f(),o=n("ul"),E=n("li"),j=u("Be able to train a new tokenizer using an old one as a template"),C=f(),x=n("li"),G=u("Understand how to use offsets to map tokens\u2019 positions to their original span of text"),K=f(),P=n("li"),W=u("Know the differences between BPE, WordPiece, and Unigram"),F=f(),$=n("li"),H=u("Be able to mix and match the blocks provided by the \u{1F917} Tokenizers library to build your own tokenizer"),J=f(),B=n("li"),O=u("Be able to use that tokenizer inside the \u{1F917} Transformers library"),this.h()},l(e){const i=le('[data-svelte="svelte-1phssyn"]',document.head);h=r(i,"META",{name:!0,content:!0}),i.forEach(t),L=d(e),c=r(e,"H1",{class:!0});var S=s(c);p=r(S,"A",{id:!0,class:!0,href:!0});var D=s(p);y=r(D,"SPAN",{});var Q=s(y);he(_.$$.fragment,Q),Q.forEach(t),D.forEach(t),U=d(S),z=r(S,"SPAN",{});var V=s(z);q=m(V,"Tokenizers, check!"),V.forEach(t),S.forEach(t),T=d(e),v=r(e,"P",{});var X=s(v);M=m(X,"Great job finishing this chapter!"),X.forEach(t),I=d(e),b=r(e,"P",{});var Y=s(b);N=m(Y,"After this deep dive into tokenizers, you should:"),Y.forEach(t),g=d(e),o=r(e,"UL",{});var l=s(o);E=r(l,"LI",{});var Z=s(E);j=m(Z,"Be able to train a new tokenizer using an old one as a template"),Z.forEach(t),C=d(l),x=r(l,"LI",{});var ee=s(x);G=m(ee,"Understand how to use offsets to map tokens\u2019 positions to their original span of text"),ee.forEach(t),K=d(l),P=r(l,"LI",{});var te=s(P);W=m(te,"Know the differences between BPE, WordPiece, and Unigram"),te.forEach(t),F=d(l),$=r(l,"LI",{});var ae=s($);H=m(ae,"Be able to mix and match the blocks provided by the \u{1F917} Tokenizers library to build your own tokenizer"),ae.forEach(t),J=d(l),B=r(l,"LI",{});var oe=s(B);O=m(oe,"Be able to use that tokenizer inside the \u{1F917} Transformers library"),oe.forEach(t),l.forEach(t),this.h()},h(){w(h,"name","hf:doc:metadata"),w(h,"content",JSON.stringify(ve)),w(p,"id","tokenizers-check"),w(p,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(p,"href","#tokenizers-check"),w(c,"class","relative group")},m(e,i){a(document.head,h),k(e,L,i),k(e,c,i),a(c,p),a(p,y),ce(_,y,null),a(c,U),a(c,z),a(z,q),k(e,T,i),k(e,v,i),a(v,M),k(e,I,i),k(e,b,i),a(b,N),k(e,g,i),k(e,o,i),a(o,E),a(E,j),a(o,C),a(o,x),a(x,G),a(o,K),a(o,P),a(P,W),a(o,F),a(o,$),a($,H),a(o,J),a(o,B),a(B,O),A=!0},p:fe,i(e){A||(de(_.$$.fragment,e),A=!0)},o(e){pe(_.$$.fragment,e),A=!1},d(e){t(h),e&&t(L),e&&t(c),ue(_),e&&t(T),e&&t(v),e&&t(I),e&&t(b),e&&t(g),e&&t(o)}}}const ve={local:"tokenizers-check",title:"Tokenizers, check!"};function be(R){return me(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ze extends ne{constructor(h){super();re(this,h,be,_e,se,{})}}export{ze as default,ve as metadata};
