import{S as pe,i as me,s as ue,e as n,k as h,w as se,t as d,M as de,c as s,d as t,m as c,a as i,x as ie,h as k,b as y,G as a,g as m,y as le,L as ke,q as he,o as ce,B as fe,v as _e}from"../../chunks/vendor-hf-doc-builder.js";import{I as ve}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as be}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function we(Q){let f,L,p,u,$,_,N,z,q,T,v,I,b,G,A,w,M,S,o,E,j,K,x,W,F,P,H,J,B,O,R,g,D,U;return _=new ve({}),v=new be({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),{c(){f=n("meta"),L=h(),p=n("h1"),u=n("a"),$=n("span"),se(_.$$.fragment),N=h(),z=n("span"),q=d("Tokenizers, check!"),T=h(),se(v.$$.fragment),I=h(),b=n("p"),G=d("Great job finishing this chapter!"),A=h(),w=n("p"),M=d("After this deep dive into tokenizers, you should:"),S=h(),o=n("ul"),E=n("li"),j=d("Be able to train a new tokenizer using an old one as a template"),K=h(),x=n("li"),W=d("Understand how to use offsets to map tokens\u2019 positions to their original span of text"),F=h(),P=n("li"),H=d("Know the differences between BPE, WordPiece, and Unigram"),J=h(),B=n("li"),O=d("Be able to mix and match the blocks provided by the \u{1F917} Tokenizers library to build your own tokenizer"),R=h(),g=n("li"),D=d("Be able to use that tokenizer inside the \u{1F917} Transformers library"),this.h()},l(e){const r=de('[data-svelte="svelte-1phssyn"]',document.head);f=s(r,"META",{name:!0,content:!0}),r.forEach(t),L=c(e),p=s(e,"H1",{class:!0});var C=i(p);u=s(C,"A",{id:!0,class:!0,href:!0});var V=i(u);$=s(V,"SPAN",{});var X=i($);ie(_.$$.fragment,X),X.forEach(t),V.forEach(t),N=c(C),z=s(C,"SPAN",{});var Y=i(z);q=k(Y,"Tokenizers, check!"),Y.forEach(t),C.forEach(t),T=c(e),ie(v.$$.fragment,e),I=c(e),b=s(e,"P",{});var Z=i(b);G=k(Z,"Great job finishing this chapter!"),Z.forEach(t),A=c(e),w=s(e,"P",{});var ee=i(w);M=k(ee,"After this deep dive into tokenizers, you should:"),ee.forEach(t),S=c(e),o=s(e,"UL",{});var l=i(o);E=s(l,"LI",{});var te=i(E);j=k(te,"Be able to train a new tokenizer using an old one as a template"),te.forEach(t),K=c(l),x=s(l,"LI",{});var ae=i(x);W=k(ae,"Understand how to use offsets to map tokens\u2019 positions to their original span of text"),ae.forEach(t),F=c(l),P=s(l,"LI",{});var oe=i(P);H=k(oe,"Know the differences between BPE, WordPiece, and Unigram"),oe.forEach(t),J=c(l),B=s(l,"LI",{});var re=i(B);O=k(re,"Be able to mix and match the blocks provided by the \u{1F917} Tokenizers library to build your own tokenizer"),re.forEach(t),R=c(l),g=s(l,"LI",{});var ne=i(g);D=k(ne,"Be able to use that tokenizer inside the \u{1F917} Transformers library"),ne.forEach(t),l.forEach(t),this.h()},h(){y(f,"name","hf:doc:metadata"),y(f,"content",JSON.stringify(ye)),y(u,"id","tokenizers-check"),y(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(u,"href","#tokenizers-check"),y(p,"class","relative group")},m(e,r){a(document.head,f),m(e,L,r),m(e,p,r),a(p,u),a(u,$),le(_,$,null),a(p,N),a(p,z),a(z,q),m(e,T,r),le(v,e,r),m(e,I,r),m(e,b,r),a(b,G),m(e,A,r),m(e,w,r),a(w,M),m(e,S,r),m(e,o,r),a(o,E),a(E,j),a(o,K),a(o,x),a(x,W),a(o,F),a(o,P),a(P,H),a(o,J),a(o,B),a(B,O),a(o,R),a(o,g),a(g,D),U=!0},p:ke,i(e){U||(he(_.$$.fragment,e),he(v.$$.fragment,e),U=!0)},o(e){ce(_.$$.fragment,e),ce(v.$$.fragment,e),U=!1},d(e){t(f),e&&t(L),e&&t(p),fe(_),e&&t(T),fe(v,e),e&&t(I),e&&t(b),e&&t(A),e&&t(w),e&&t(S),e&&t(o)}}}const ye={local:"tokenizers-check",title:"Tokenizers, check!"};function $e(Q){return _e(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pe extends pe{constructor(f){super();me(this,f,$e,we,ue,{})}}export{Pe as default,ye as metadata};
