import{S as Xe,i as Ye,s as Ze,e as n,k as f,w as Re,t as o,M as et,c as l,d as r,m as p,a as i,x as je,h as a,b as T,G as e,g as k,y as De,L as tt,q as Ke,o as Qe,B as Ve,v as rt}from"../../chunks/vendor-hf-doc-builder.js";import{I as ot}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as at}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function nt(Te){let u,J,d,b,g,$,X,M,q,Y,Z,O,w,R,y,ee,j,z,te,A,re,oe,D,s,B,ae,ne,h,le,S,ie,se,C,ce,me,fe,v,pe,N,ue,de,U,he,ve,_e,_,Ee,W,ke,be,F,ze,$e,we,E,Me,G,ye,Pe,H,xe,Le,K;return $=new ot({}),w=new at({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),{c(){u=n("meta"),J=f(),d=n("h1"),b=n("a"),g=n("span"),Re($.$$.fragment),X=f(),M=n("span"),q=n("i"),Y=o("Tokenizer"),Z=o(", coch\xE9 !"),O=f(),Re(w.$$.fragment),R=f(),y=n("p"),ee=o("Bon travail pour finir ce chapitre !"),j=f(),z=n("p"),te=o("Apr\xE8s cette plong\xE9e en profondeur dans les "),A=n("em"),re=o("tokenizers"),oe=o(", vous devriez :"),D=f(),s=n("ul"),B=n("li"),ae=o("\xEAtre capable d\u2019entra\xEEner un nouveau tokenizer en utilisant un ancien tokenizer comme mod\xE8le,"),ne=f(),h=n("li"),le=o("comprendre comment utiliser les "),S=n("em"),ie=o("offsets"),se=o(" pour faire correspondre la position des "),C=n("em"),ce=o("tokens"),me=o(" \xE0 l\u2019\xE9tendue de texte d\u2019origine,"),fe=f(),v=n("li"),pe=o("conna\xEEtre les diff\xE9rences entre BPE, "),N=n("em"),ue=o("WordPiece"),de=o(" et "),U=n("em"),he=o("Unigram"),ve=o(","),_e=f(),_=n("li"),Ee=o("\xEAtre capable de combiner les blocs fournis par la biblioth\xE8que \u{1F917} "),W=n("em"),ke=o("Tokenizers"),be=o(" pour construire votre propre "),F=n("em"),ze=o("tokenizer"),$e=o(","),we=f(),E=n("li"),Me=o("\xEAtre capable d\u2019utiliser ce "),G=n("em"),ye=o("tokenizer"),Pe=o(" dans la biblioth\xE8que \u{1F917} "),H=n("em"),xe=o("Transformers"),Le=o("."),this.h()},l(t){const c=et('[data-svelte="svelte-1phssyn"]',document.head);u=l(c,"META",{name:!0,content:!0}),c.forEach(r),J=p(t),d=l(t,"H1",{class:!0});var Q=i(d);b=l(Q,"A",{id:!0,class:!0,href:!0});var ge=i(b);g=l(ge,"SPAN",{});var qe=i(g);je($.$$.fragment,qe),qe.forEach(r),ge.forEach(r),X=p(Q),M=l(Q,"SPAN",{});var Ie=i(M);q=l(Ie,"I",{});var Ae=i(q);Y=a(Ae,"Tokenizer"),Ae.forEach(r),Z=a(Ie,", coch\xE9 !"),Ie.forEach(r),Q.forEach(r),O=p(t),je(w.$$.fragment,t),R=p(t),y=l(t,"P",{});var Be=i(y);ee=a(Be,"Bon travail pour finir ce chapitre !"),Be.forEach(r),j=p(t),z=l(t,"P",{});var V=i(z);te=a(V,"Apr\xE8s cette plong\xE9e en profondeur dans les "),A=l(V,"EM",{});var Se=i(A);re=a(Se,"tokenizers"),Se.forEach(r),oe=a(V,", vous devriez :"),V.forEach(r),D=p(t),s=l(t,"UL",{});var m=i(s);B=l(m,"LI",{});var Ce=i(B);ae=a(Ce,"\xEAtre capable d\u2019entra\xEEner un nouveau tokenizer en utilisant un ancien tokenizer comme mod\xE8le,"),Ce.forEach(r),ne=p(m),h=l(m,"LI",{});var P=i(h);le=a(P,"comprendre comment utiliser les "),S=l(P,"EM",{});var Ne=i(S);ie=a(Ne,"offsets"),Ne.forEach(r),se=a(P," pour faire correspondre la position des "),C=l(P,"EM",{});var Ue=i(C);ce=a(Ue,"tokens"),Ue.forEach(r),me=a(P," \xE0 l\u2019\xE9tendue de texte d\u2019origine,"),P.forEach(r),fe=p(m),v=l(m,"LI",{});var x=i(v);pe=a(x,"conna\xEEtre les diff\xE9rences entre BPE, "),N=l(x,"EM",{});var We=i(N);ue=a(We,"WordPiece"),We.forEach(r),de=a(x," et "),U=l(x,"EM",{});var Fe=i(U);he=a(Fe,"Unigram"),Fe.forEach(r),ve=a(x,","),x.forEach(r),_e=p(m),_=l(m,"LI",{});var L=i(_);Ee=a(L,"\xEAtre capable de combiner les blocs fournis par la biblioth\xE8que \u{1F917} "),W=l(L,"EM",{});var Ge=i(W);ke=a(Ge,"Tokenizers"),Ge.forEach(r),be=a(L," pour construire votre propre "),F=l(L,"EM",{});var He=i(F);ze=a(He,"tokenizer"),He.forEach(r),$e=a(L,","),L.forEach(r),we=p(m),E=l(m,"LI",{});var I=i(E);Me=a(I,"\xEAtre capable d\u2019utiliser ce "),G=l(I,"EM",{});var Je=i(G);ye=a(Je,"tokenizer"),Je.forEach(r),Pe=a(I," dans la biblioth\xE8que \u{1F917} "),H=l(I,"EM",{});var Oe=i(H);xe=a(Oe,"Transformers"),Oe.forEach(r),Le=a(I,"."),I.forEach(r),m.forEach(r),this.h()},h(){T(u,"name","hf:doc:metadata"),T(u,"content",JSON.stringify(lt)),T(b,"id","itokenizeri-coch"),T(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(b,"href","#itokenizeri-coch"),T(d,"class","relative group")},m(t,c){e(document.head,u),k(t,J,c),k(t,d,c),e(d,b),e(b,g),De($,g,null),e(d,X),e(d,M),e(M,q),e(q,Y),e(M,Z),k(t,O,c),De(w,t,c),k(t,R,c),k(t,y,c),e(y,ee),k(t,j,c),k(t,z,c),e(z,te),e(z,A),e(A,re),e(z,oe),k(t,D,c),k(t,s,c),e(s,B),e(B,ae),e(s,ne),e(s,h),e(h,le),e(h,S),e(S,ie),e(h,se),e(h,C),e(C,ce),e(h,me),e(s,fe),e(s,v),e(v,pe),e(v,N),e(N,ue),e(v,de),e(v,U),e(U,he),e(v,ve),e(s,_e),e(s,_),e(_,Ee),e(_,W),e(W,ke),e(_,be),e(_,F),e(F,ze),e(_,$e),e(s,we),e(s,E),e(E,Me),e(E,G),e(G,ye),e(E,Pe),e(E,H),e(H,xe),e(E,Le),K=!0},p:tt,i(t){K||(Ke($.$$.fragment,t),Ke(w.$$.fragment,t),K=!0)},o(t){Qe($.$$.fragment,t),Qe(w.$$.fragment,t),K=!1},d(t){r(u),t&&r(J),t&&r(d),Ve($),t&&r(O),Ve(w,t),t&&r(R),t&&r(y),t&&r(j),t&&r(z),t&&r(D),t&&r(s)}}}const lt={local:"itokenizeri-coch",title:"<i>Tokenizer</i>, coch\xE9 !"};function it(Te){return rt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ft extends Xe{constructor(u){super();Ye(this,u,it,nt,Ze,{})}}export{ft as default,lt as metadata};
