import{S as Oe,i as Re,s as je,e as n,k as _,w as De,t as o,M as Fe,c as l,d as t,m as E,a as i,x as Ke,h as a,b as I,G as e,g as z,y as Qe,L as Ve,q as Xe,o as Ye,B as Ze,v as et}from"../../chunks/vendor-hf-doc-builder.js";import{I as tt}from"../../chunks/IconCopyLink-hf-doc-builder.js";function rt(Le){let f,J,p,k,T,M,Q,w,q,V,X,O,y,Y,R,b,Z,A,ee,te,j,s,S,re,oe,u,ae,B,ne,le,g,ie,se,ce,d,me,U,fe,pe,N,ue,de,he,h,ve,C,_e,Ee,W,ke,be,ze,v,Me,G,we,ye,H,Pe,$e,D;return M=new tt({}),{c(){f=n("meta"),J=_(),p=n("h1"),k=n("a"),T=n("span"),De(M.$$.fragment),Q=_(),w=n("span"),q=n("i"),V=o("Tokenizer"),X=o(", coch\xE9 !"),O=_(),y=n("p"),Y=o("Bon travail pour finir ce chapitre !"),R=_(),b=n("p"),Z=o("Apr\xE8s cette plong\xE9e en profondeur dans les "),A=n("em"),ee=o("tokenizers"),te=o(", vous devriez :"),j=_(),s=n("ul"),S=n("li"),re=o("\xEAtre capable d\u2019entra\xEEner un nouveau tokenizer en utilisant un ancien tokenizer comme mod\xE8le,"),oe=_(),u=n("li"),ae=o("comprendre comment utiliser les "),B=n("em"),ne=o("offsets"),le=o(" pour faire correspondre la position des "),g=n("em"),ie=o("tokens"),se=o(" \xE0 l\u2019\xE9tendue de texte d\u2019origine,"),ce=_(),d=n("li"),me=o("conna\xEEtre les diff\xE9rences entre BPE, "),U=n("em"),fe=o("WordPiece"),pe=o(" et "),N=n("em"),ue=o("Unigram"),de=o(","),he=_(),h=n("li"),ve=o("\xEAtre capable de combiner les blocs fournis par la biblioth\xE8que \u{1F917} "),C=n("em"),_e=o("Tokenizers"),Ee=o(" pour construire votre propre "),W=n("em"),ke=o("tokenizer"),be=o(","),ze=_(),v=n("li"),Me=o("\xEAtre capable d\u2019utiliser ce "),G=n("em"),we=o("tokenizer"),ye=o(" dans la biblioth\xE8que \u{1F917} "),H=n("em"),Pe=o("Transformers"),$e=o("."),this.h()},l(r){const c=Fe('[data-svelte="svelte-1phssyn"]',document.head);f=l(c,"META",{name:!0,content:!0}),c.forEach(t),J=E(r),p=l(r,"H1",{class:!0});var F=i(p);k=l(F,"A",{id:!0,class:!0,href:!0});var Ie=i(k);T=l(Ie,"SPAN",{});var Te=i(T);Ke(M.$$.fragment,Te),Te.forEach(t),Ie.forEach(t),Q=E(F),w=l(F,"SPAN",{});var xe=i(w);q=l(xe,"I",{});var qe=i(q);V=a(qe,"Tokenizer"),qe.forEach(t),X=a(xe,", coch\xE9 !"),xe.forEach(t),F.forEach(t),O=E(r),y=l(r,"P",{});var Ae=i(y);Y=a(Ae,"Bon travail pour finir ce chapitre !"),Ae.forEach(t),R=E(r),b=l(r,"P",{});var K=i(b);Z=a(K,"Apr\xE8s cette plong\xE9e en profondeur dans les "),A=l(K,"EM",{});var Se=i(A);ee=a(Se,"tokenizers"),Se.forEach(t),te=a(K,", vous devriez :"),K.forEach(t),j=E(r),s=l(r,"UL",{});var m=i(s);S=l(m,"LI",{});var Be=i(S);re=a(Be,"\xEAtre capable d\u2019entra\xEEner un nouveau tokenizer en utilisant un ancien tokenizer comme mod\xE8le,"),Be.forEach(t),oe=E(m),u=l(m,"LI",{});var P=i(u);ae=a(P,"comprendre comment utiliser les "),B=l(P,"EM",{});var ge=i(B);ne=a(ge,"offsets"),ge.forEach(t),le=a(P," pour faire correspondre la position des "),g=l(P,"EM",{});var Ue=i(g);ie=a(Ue,"tokens"),Ue.forEach(t),se=a(P," \xE0 l\u2019\xE9tendue de texte d\u2019origine,"),P.forEach(t),ce=E(m),d=l(m,"LI",{});var $=i(d);me=a($,"conna\xEEtre les diff\xE9rences entre BPE, "),U=l($,"EM",{});var Ne=i(U);fe=a(Ne,"WordPiece"),Ne.forEach(t),pe=a($," et "),N=l($,"EM",{});var Ce=i(N);ue=a(Ce,"Unigram"),Ce.forEach(t),de=a($,","),$.forEach(t),he=E(m),h=l(m,"LI",{});var x=i(h);ve=a(x,"\xEAtre capable de combiner les blocs fournis par la biblioth\xE8que \u{1F917} "),C=l(x,"EM",{});var We=i(C);_e=a(We,"Tokenizers"),We.forEach(t),Ee=a(x," pour construire votre propre "),W=l(x,"EM",{});var Ge=i(W);ke=a(Ge,"tokenizer"),Ge.forEach(t),be=a(x,","),x.forEach(t),ze=E(m),v=l(m,"LI",{});var L=i(v);Me=a(L,"\xEAtre capable d\u2019utiliser ce "),G=l(L,"EM",{});var He=i(G);we=a(He,"tokenizer"),He.forEach(t),ye=a(L," dans la biblioth\xE8que \u{1F917} "),H=l(L,"EM",{});var Je=i(H);Pe=a(Je,"Transformers"),Je.forEach(t),$e=a(L,"."),L.forEach(t),m.forEach(t),this.h()},h(){I(f,"name","hf:doc:metadata"),I(f,"content",JSON.stringify(ot)),I(k,"id","itokenizeri-coch"),I(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),I(k,"href","#itokenizeri-coch"),I(p,"class","relative group")},m(r,c){e(document.head,f),z(r,J,c),z(r,p,c),e(p,k),e(k,T),Qe(M,T,null),e(p,Q),e(p,w),e(w,q),e(q,V),e(w,X),z(r,O,c),z(r,y,c),e(y,Y),z(r,R,c),z(r,b,c),e(b,Z),e(b,A),e(A,ee),e(b,te),z(r,j,c),z(r,s,c),e(s,S),e(S,re),e(s,oe),e(s,u),e(u,ae),e(u,B),e(B,ne),e(u,le),e(u,g),e(g,ie),e(u,se),e(s,ce),e(s,d),e(d,me),e(d,U),e(U,fe),e(d,pe),e(d,N),e(N,ue),e(d,de),e(s,he),e(s,h),e(h,ve),e(h,C),e(C,_e),e(h,Ee),e(h,W),e(W,ke),e(h,be),e(s,ze),e(s,v),e(v,Me),e(v,G),e(G,we),e(v,ye),e(v,H),e(H,Pe),e(v,$e),D=!0},p:Ve,i(r){D||(Xe(M.$$.fragment,r),D=!0)},o(r){Ye(M.$$.fragment,r),D=!1},d(r){t(f),r&&t(J),r&&t(p),Ze(M),r&&t(O),r&&t(y),r&&t(R),r&&t(b),r&&t(j),r&&t(s)}}}const ot={local:"itokenizeri-coch",title:"<i>Tokenizer</i>, coch\xE9 !"};function at(Le){return et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class it extends Oe{constructor(f){super();Re(this,f,at,rt,je,{})}}export{it as default,ot as metadata};
