import{S as Ae,i as Se,s as Be,e as n,k as v,w as ge,t as o,M as Ue,c as l,d as r,m as _,a as i,x as Ne,h as a,b as $,F as e,g as b,y as Ce,L as We,q as Fe,o as He,B as Je,v as Oe}from"../../chunks/vendor-1e8b365d.js";import{I as Re}from"../../chunks/IconCopyLink-483c28ba.js";function je(Ee){let f,C,u,E,x,z,j,L,D,W,w,G,F,k,K,T,Q,V,H,s,q,X,Y,I,Z,ee,p,te,A,re,oe,S,ae,ne,le,d,ie,B,se,ce,g,me,fe,ue,h,pe,U,de,he,N,ve,_e,J;return z=new Re({}),{c(){f=n("meta"),C=v(),u=n("h1"),E=n("a"),x=n("span"),ge(z.$$.fragment),j=v(),L=n("span"),D=o("*Tokenizer*, v\xE9rifi\xE9 !"),W=v(),w=n("p"),G=o("Bon travail pour finir ce chapitre !"),F=v(),k=n("p"),K=o("Apr\xE8s cette plong\xE9e en profondeur dans les "),T=n("em"),Q=o("tokenizers"),V=o(", vous devriez :"),H=v(),s=n("ul"),q=n("li"),X=o("\xEAtre capable d\u2019entra\xEEner un nouveau tokenizer en utilisant un ancien tokenizer comme mod\xE8le,"),Y=v(),I=n("li"),Z=o("comprendre comment utiliser les offsets pour faire correspondre la position des tokens \xE0 l\u2019\xE9tendue du texte d\u2019origine,"),ee=v(),p=n("li"),te=o("conna\xEEtre les diff\xE9rences entre BPE, "),A=n("em"),re=o("WordPiece"),oe=o(" et "),S=n("em"),ae=o("Unigram"),ne=o(","),le=v(),d=n("li"),ie=o("\xEAtre capable de combiner les blocs fournis par la biblioth\xE8que \u{1F917} "),B=n("em"),se=o("Tokenizers"),ce=o(" pour construire votre propre "),g=n("em"),me=o("tokenizer"),fe=o(","),ue=v(),h=n("li"),pe=o("\xEAtre capable d\u2019utiliser ce "),U=n("em"),de=o("tokenizer"),he=o(" dans la biblioth\xE8que \u{1F917} "),N=n("em"),ve=o("Transformers"),_e=o("."),this.h()},l(t){const c=Ue('[data-svelte="svelte-1phssyn"]',document.head);f=l(c,"META",{name:!0,content:!0}),c.forEach(r),C=_(t),u=l(t,"H1",{class:!0});var O=i(u);E=l(O,"A",{id:!0,class:!0,href:!0});var ke=i(E);x=l(ke,"SPAN",{});var be=i(x);Ne(z.$$.fragment,be),be.forEach(r),ke.forEach(r),j=_(O),L=l(O,"SPAN",{});var ze=i(L);D=a(ze,"*Tokenizer*, v\xE9rifi\xE9 !"),ze.forEach(r),O.forEach(r),W=_(t),w=l(t,"P",{});var we=i(w);G=a(we,"Bon travail pour finir ce chapitre !"),we.forEach(r),F=_(t),k=l(t,"P",{});var R=i(k);K=a(R,"Apr\xE8s cette plong\xE9e en profondeur dans les "),T=l(R,"EM",{});var ye=i(T);Q=a(ye,"tokenizers"),ye.forEach(r),V=a(R,", vous devriez :"),R.forEach(r),H=_(t),s=l(t,"UL",{});var m=i(s);q=l(m,"LI",{});var Pe=i(q);X=a(Pe,"\xEAtre capable d\u2019entra\xEEner un nouveau tokenizer en utilisant un ancien tokenizer comme mod\xE8le,"),Pe.forEach(r),Y=_(m),I=l(m,"LI",{});var Me=i(I);Z=a(Me,"comprendre comment utiliser les offsets pour faire correspondre la position des tokens \xE0 l\u2019\xE9tendue du texte d\u2019origine,"),Me.forEach(r),ee=_(m),p=l(m,"LI",{});var y=i(p);te=a(y,"conna\xEEtre les diff\xE9rences entre BPE, "),A=l(y,"EM",{});var $e=i(A);re=a($e,"WordPiece"),$e.forEach(r),oe=a(y," et "),S=l(y,"EM",{});var xe=i(S);ae=a(xe,"Unigram"),xe.forEach(r),ne=a(y,","),y.forEach(r),le=_(m),d=l(m,"LI",{});var P=i(d);ie=a(P,"\xEAtre capable de combiner les blocs fournis par la biblioth\xE8que \u{1F917} "),B=l(P,"EM",{});var Le=i(B);se=a(Le,"Tokenizers"),Le.forEach(r),ce=a(P," pour construire votre propre "),g=l(P,"EM",{});var Te=i(g);me=a(Te,"tokenizer"),Te.forEach(r),fe=a(P,","),P.forEach(r),ue=_(m),h=l(m,"LI",{});var M=i(h);pe=a(M,"\xEAtre capable d\u2019utiliser ce "),U=l(M,"EM",{});var qe=i(U);de=a(qe,"tokenizer"),qe.forEach(r),he=a(M," dans la biblioth\xE8que \u{1F917} "),N=l(M,"EM",{});var Ie=i(N);ve=a(Ie,"Transformers"),Ie.forEach(r),_e=a(M,"."),M.forEach(r),m.forEach(r),this.h()},h(){$(f,"name","hf:doc:metadata"),$(f,"content",JSON.stringify(De)),$(E,"id","tokenizer-vrifi"),$(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(E,"href","#tokenizer-vrifi"),$(u,"class","relative group")},m(t,c){e(document.head,f),b(t,C,c),b(t,u,c),e(u,E),e(E,x),Ce(z,x,null),e(u,j),e(u,L),e(L,D),b(t,W,c),b(t,w,c),e(w,G),b(t,F,c),b(t,k,c),e(k,K),e(k,T),e(T,Q),e(k,V),b(t,H,c),b(t,s,c),e(s,q),e(q,X),e(s,Y),e(s,I),e(I,Z),e(s,ee),e(s,p),e(p,te),e(p,A),e(A,re),e(p,oe),e(p,S),e(S,ae),e(p,ne),e(s,le),e(s,d),e(d,ie),e(d,B),e(B,se),e(d,ce),e(d,g),e(g,me),e(d,fe),e(s,ue),e(s,h),e(h,pe),e(h,U),e(U,de),e(h,he),e(h,N),e(N,ve),e(h,_e),J=!0},p:We,i(t){J||(Fe(z.$$.fragment,t),J=!0)},o(t){He(z.$$.fragment,t),J=!1},d(t){r(f),t&&r(C),t&&r(u),Je(z),t&&r(W),t&&r(w),t&&r(F),t&&r(k),t&&r(H),t&&r(s)}}}const De={local:"tokenizer-vrifi",title:"*Tokenizer*, v\xE9rifi\xE9 !"};function Ge(Ee){return Oe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ve extends Ae{constructor(f){super();Se(this,f,Ge,je,Be,{})}}export{Ve as default,De as metadata};
