import{S as _n,i as $n,s as kn,e as r,k as m,w as v,t as o,l as gn,M as jn,c as u,d as t,m as f,x as b,a as i,h as a,b as E,G as s,g as p,y as _,o as h,p as qn,q as g,B as $,v as wn,n as vn}from"../../chunks/vendor-hf-doc-builder.js";import{I as St}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as y}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as bn}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as En}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function xn(z){let c,d;return c=new bn({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter2/section6_tf.ipynb"}]}}),{c(){v(c.$$.fragment)},l(l){b(c.$$.fragment,l)},m(l,k){_(c,l,k),d=!0},i(l){d||(g(c.$$.fragment,l),d=!0)},o(l){h(c.$$.fragment,l),d=!1},d(l){$(c,l)}}}function yn(z){let c,d;return c=new bn({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter2/section6_pt.ipynb"}]}}),{c(){v(c.$$.fragment)},l(l){b(c.$$.fragment,l)},m(l,k){_(c,l,k),d=!0},i(l){d||(g(c.$$.fragment,l),d=!0)},o(l){h(c.$$.fragment,l),d=!1},d(l){$(c,l)}}}function zn(z){let c,d;return c=new y({props:{code:`
`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;So have I!&quot;</span>,
]  <span class="hljs-comment"># \xAB J&#x27;ai attendu un cours de HuggingFace toute ma vie. \xBB, \xAB Moi aussi ! \xBB</span>

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`}}),{c(){v(c.$$.fragment)},l(l){b(c.$$.fragment,l)},m(l,k){_(c,l,k),d=!0},i(l){d||(g(c.$$.fragment,l),d=!0)},o(l){h(c.$$.fragment,l),d=!1},d(l){$(c,l)}}}function Pn(z){let c,d;return c=new y({props:{code:`

`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;So have I!&quot;</span>,
]  <span class="hljs-comment"># \xAB J&#x27;ai attendu un cours de HuggingFace toute ma vie. \xBB, \xAB Moi aussi ! \xBB</span>


tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`}}),{c(){v(c.$$.fragment)},l(l){b(c.$$.fragment,l)},m(l,k){_(c,l,k),d=!0},i(l){d||(g(c.$$.fragment,l),d=!0)},o(l){h(c.$$.fragment,l),d=!1},d(l){$(c,l)}}}function Tn(z){let c,d,l,k,M,L,_e,U,ws,$e,Es,Ke,P,T,ie,A,xs,ke,ys,zs,je,Ps,Ts,Qe,I,Cs,we,Ss,As,Ee,Is,Fs,Ve,K,We,F,Ds,xe,Ms,Hs,ye,Ns,Ls,Xe,pe,Rs,Ye,Q,Ze,ce,Js,es,V,ss,R,Os,ze,Bs,Gs,ts,W,ns,me,Us,os,X,as,q,Ks,Pe,Qs,Vs,Te,Ws,Xs,Ce,Ys,Zs,Se,et,st,Ae,tt,nt,Ie,ot,at,ls,Y,rs,H,J,Fe,Z,lt,De,rt,us,O,ut,Me,it,pt,is,ee,ps,se,cs,fe,ct,ms,te,fs,ne,ds,j,mt,He,ft,dt,Ne,ht,gt,Le,qt,vt,Re,bt,_t,hs,N,B,Je,oe,$t,ae,kt,Oe,jt,wt,gs,x,Et,Be,xt,yt,Ge,zt,Pt,Ue,Tt,Ct,qs,C,S,de,vs;l=new En({props:{fw:z[0]}}),U=new St({});const At=[yn,xn],le=[];function It(e,n){return e[0]==="pt"?0:1}P=It(z),T=le[P]=At[P](z),K=new y({props:{code:`

`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>
<span class="hljs-comment"># J&#x27;ai attendu un cours d\u2019HuggingFace toute ma vie.</span>

model_inputs = tokenizer(sequence)`}}),Q=new y({props:{code:`
`,highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>
<span class="hljs-comment"># J&#x27;ai attendu un cours d\u2019HuggingFace toute ma vie.</span>


model_inputs = tokenizer(sequence)`}}),V=new y({props:{code:"",highlighted:`sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;So have I!&quot;</span>,
]  <span class="hljs-comment"># \xAB J&#x27;ai attendu un cours de HuggingFace toute ma vie. \xBB, \xAB Moi aussi ! \xBB</span>

model_inputs = tokenizer(sequences)`}}),W=new y({props:{code:`
`,highlighted:`<span class="hljs-comment"># Remplit les s\xE9quences jusqu&#x27;\xE0 la longueur maximale de la s\xE9quence</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Remplit les s\xE9quences jusqu&#x27;\xE0 la longueur maximale du mod\xE8le (512 pour BERT ou DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Remplit les s\xE9quences jusqu&#x27;\xE0 la longueur maximale sp\xE9cifi\xE9e</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`}}),X=new y({props:{code:`
`,highlighted:`sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;So have I!&quot;</span>,
]  <span class="hljs-comment"># \xAB J&#x27;ai attendu un cours de HuggingFace toute ma vie. \xBB, \xAB Moi aussi ! \xBB</span>

<span class="hljs-comment"># Tronque les s\xE9quences qui sont plus longues que la longueur maximale du mod\xE8le</span>
<span class="hljs-comment"># (512 pour BERT ou DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Tronque les s\xE9quences qui sont plus longues que la longueur maximale sp\xE9cifi\xE9e</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`}}),Y=new y({props:{code:`

`,highlighted:`sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;So have I!&quot;</span>,
]  <span class="hljs-comment"># \xAB J&#x27;ai attendu un cours de HuggingFace toute ma vie. \xBB, \xAB Moi aussi ! \xBB</span>

<span class="hljs-comment"># Retourne des tenseurs PyTorch</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Retourne des tenseurs TensorFlow</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Retourne des tableaux NumPy</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`}}),Z=new St({}),ee=new y({props:{code:`
`,highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>
<span class="hljs-comment"># \xAB J&#x27;ai attendu un cours de HuggingFace toute ma vie. \xBB</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`}}),se=new y({props:{code:`[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102]
[1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012]`,highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`}}),te=new y({props:{code:`print(tokenizer.decode(model_inputs["input_ids"]))
print(tokenizer.decode(ids))`,highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`}}),ne=new y({props:{code:`"[CLS] i've been waiting for a huggingface course my whole life. [SEP]"
"i've been waiting for a huggingface course my whole life."`,highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`}}),oe=new St({});const Ft=[Pn,zn],re=[];function Dt(e,n){return e[0]==="pt"?0:1}return C=Dt(z),S=re[C]=Ft[C](z),{c(){c=r("meta"),d=m(),v(l.$$.fragment),k=m(),M=r("h1"),L=r("a"),_e=r("span"),v(U.$$.fragment),ws=m(),$e=r("span"),Es=o("Tout assembler"),Ke=m(),T.c(),ie=m(),A=r("p"),xs=o("Dans les derni\xE8res sections, nous avons fait de notre mieux pour effectuer la plupart du travail manuellement. Nous avons explor\xE9 le fonctionnement des "),ke=r("em"),ys=o("tokenizers"),zs=o(" et examin\xE9 la tokenisation, la conversion en identifiants d\u2019entr\xE9e, le "),je=r("em"),Ps=o("padding"),Ts=o(", la troncature et les masques d\u2019attention."),Qe=m(),I=r("p"),Cs=o("Cependant, comme nous l\u2019avons vu dans la section 2, l\u2019API \u{1F917} "),we=r("em"),Ss=o("Transformers"),As=o(" peut g\xE9rer tout cela pour nous via une fonction dans laquelle nous allons nous plonger ici. Lorsque vous appelez votre "),Ee=r("code"),Is=o("tokenizer"),Fs=o(" directement sur la phrase, vous r\xE9cup\xE9rez des entr\xE9es qui sont pr\xEAtes \xE0 \xEAtre pass\xE9es dans votre mod\xE8le :"),Ve=m(),v(K.$$.fragment),We=m(),F=r("p"),Ds=o("Ici, la variable "),xe=r("code"),Ms=o("model_inputs"),Hs=o(" contient tout ce qui est n\xE9cessaire au bon fonctionnement d\u2019un mod\xE8le. Pour DistilBERT, cela inclut les identifiants d\u2019entr\xE9e ainsi que le masque d\u2019attention. D\u2019autres mod\xE8les qui acceptent des entr\xE9es suppl\xE9mentaires sont \xE9galement fournis par l\u2019objet "),ye=r("code"),Ns=o("tokenizer"),Ls=o("."),Xe=m(),pe=r("p"),Rs=o("Comme nous allons le voir dans les quelques exemples ci-dessous, cette m\xE9thode est tr\xE8s puissante. Premi\xE8rement, elle peut tokeniser une seule s\xE9quence :"),Ye=m(),v(Q.$$.fragment),Ze=m(),ce=r("p"),Js=o("Elle g\xE8re \xE9galement plusieurs s\xE9quences \xE0 la fois, sans modification de l\u2019API :"),es=m(),v(V.$$.fragment),ss=m(),R=r("p"),Os=o("Il est possible de faire du "),ze=r("em"),Bs=o("padding"),Gs=o(" selon plusieurs objectifs :"),ts=m(),v(W.$$.fragment),ns=m(),me=r("p"),Us=o("La fonction peut \xE9galement tronquer les s\xE9quences :"),os=m(),v(X.$$.fragment),as=m(),q=r("p"),Ks=o("L\u2019objet "),Pe=r("code"),Qs=o("tokenizer"),Vs=o(" peut g\xE9rer la conversion en des tenseurs de "),Te=r("em"),Ws=o("frameworks"),Xs=o(" sp\xE9cifiques. Ils peuvent ensuite \xEAtre directement envoy\xE9s au mod\xE8le. Par exemple, dans le code suivant, nous demandons au "),Ce=r("em"),Ys=o("tokenizer"),Zs=o(" de retourner des tenseurs PyTorch lorsque l\u2019on sp\xE9cifie "),Se=r("code"),et=o('"pt"'),st=o(", de retourner des tenseurs TensorFlow lorsque l\u2019on sp\xE9cifie "),Ae=r("code"),tt=o('"tf"'),nt=o(" et des tableaux NumPy lorsque l\u2019on indique "),Ie=r("code"),ot=o('"np"'),at=o(" :"),ls=m(),v(Y.$$.fragment),rs=m(),H=r("h2"),J=r("a"),Fe=r("span"),v(Z.$$.fragment),lt=m(),De=r("span"),rt=o("Jetons sp\xE9ciaux"),us=m(),O=r("p"),ut=o("Si nous jetons un coup d\u2019\u0153il aux identifiants d\u2019entr\xE9e renvoy\xE9s par le "),Me=r("em"),it=o("tokenizer"),pt=o(", nous verrons qu\u2019ils sont un peu diff\xE9rents de ceux que nous avions pr\xE9c\xE9demment :"),is=m(),v(ee.$$.fragment),ps=m(),v(se.$$.fragment),cs=m(),fe=r("p"),ct=o("Un identifiant symbolique a \xE9t\xE9 ajout\xE9 au d\xE9but ainsi qu\u2019un autre \xE0 la fin. D\xE9codons les deux s\xE9quences d\u2019identifiants ci-dessus pour voir de quoi il s\u2019agit :"),ms=m(),v(te.$$.fragment),fs=m(),v(ne.$$.fragment),ds=m(),j=r("p"),mt=o("Le "),He=r("em"),ft=o("tokenizer"),dt=o(" a ajout\xE9 le mot sp\xE9cial "),Ne=r("code"),ht=o("[CLS]"),gt=o(" au d\xE9but et le mot sp\xE9cial "),Le=r("code"),qt=o("[SEP]"),vt=o(" \xE0 la fin. C\u2019est parce que le mod\xE8le a \xE9t\xE9 pr\xE9-entra\xEEn\xE9 avec ces mots, donc pour obtenir les m\xEAmes r\xE9sultats pour l\u2019inf\xE9rence, nous devons \xE9galement les ajouter. Notez que certains mod\xE8les n\u2019ajoutent pas de mots sp\xE9ciaux, ou en ajoutent des diff\xE9rents. Les mod\xE8les peuvent aussi ajouter ces mots sp\xE9ciaux seulement au d\xE9but, ou seulement \xE0 la fin. Dans tous les cas, le "),Re=r("em"),bt=o("tokenizer"),_t=o(" sait lesquels sont attendus et s\u2019en occupe pour vous."),hs=m(),N=r("h2"),B=r("a"),Je=r("span"),v(oe.$$.fragment),$t=m(),ae=r("span"),kt=o("Conclusion : du "),Oe=r("i"),jt=o("tokenizer"),wt=o(" au mod\xE8le"),gs=m(),x=r("p"),Et=o("Maintenant que nous avons vu toutes les \xE9tapes individuelles que l\u2019objet "),Be=r("code"),xt=o("tokenizer"),yt=o(" utilise lorsqu\u2019il est appliqu\xE9 sur des textes, voyons une derni\xE8re fois comment il peut g\xE9rer plusieurs s\xE9quences ("),Ge=r("em"),zt=o("padding"),Pt=o("), de tr\xE8s longues s\xE9quences ("),Ue=r("em"),Tt=o("troncation"),Ct=o(") et plusieurs types de tenseurs avec son API principale :"),qs=m(),S.c(),de=gn(),this.h()},l(e){const n=jn('[data-svelte="svelte-1phssyn"]',document.head);c=u(n,"META",{name:!0,content:!0}),n.forEach(t),d=f(e),b(l.$$.fragment,e),k=f(e),M=u(e,"H1",{class:!0});var ue=i(M);L=u(ue,"A",{id:!0,class:!0,href:!0});var he=i(L);_e=u(he,"SPAN",{});var ge=i(_e);b(U.$$.fragment,ge),ge.forEach(t),he.forEach(t),ws=f(ue),$e=u(ue,"SPAN",{});var Mt=i($e);Es=a(Mt,"Tout assembler"),Mt.forEach(t),ue.forEach(t),Ke=f(e),T.l(e),ie=f(e),A=u(e,"P",{});var qe=i(A);xs=a(qe,"Dans les derni\xE8res sections, nous avons fait de notre mieux pour effectuer la plupart du travail manuellement. Nous avons explor\xE9 le fonctionnement des "),ke=u(qe,"EM",{});var Ht=i(ke);ys=a(Ht,"tokenizers"),Ht.forEach(t),zs=a(qe," et examin\xE9 la tokenisation, la conversion en identifiants d\u2019entr\xE9e, le "),je=u(qe,"EM",{});var Nt=i(je);Ps=a(Nt,"padding"),Nt.forEach(t),Ts=a(qe,", la troncature et les masques d\u2019attention."),qe.forEach(t),Qe=f(e),I=u(e,"P",{});var ve=i(I);Cs=a(ve,"Cependant, comme nous l\u2019avons vu dans la section 2, l\u2019API \u{1F917} "),we=u(ve,"EM",{});var Lt=i(we);Ss=a(Lt,"Transformers"),Lt.forEach(t),As=a(ve," peut g\xE9rer tout cela pour nous via une fonction dans laquelle nous allons nous plonger ici. Lorsque vous appelez votre "),Ee=u(ve,"CODE",{});var Rt=i(Ee);Is=a(Rt,"tokenizer"),Rt.forEach(t),Fs=a(ve," directement sur la phrase, vous r\xE9cup\xE9rez des entr\xE9es qui sont pr\xEAtes \xE0 \xEAtre pass\xE9es dans votre mod\xE8le :"),ve.forEach(t),Ve=f(e),b(K.$$.fragment,e),We=f(e),F=u(e,"P",{});var be=i(F);Ds=a(be,"Ici, la variable "),xe=u(be,"CODE",{});var Jt=i(xe);Ms=a(Jt,"model_inputs"),Jt.forEach(t),Hs=a(be," contient tout ce qui est n\xE9cessaire au bon fonctionnement d\u2019un mod\xE8le. Pour DistilBERT, cela inclut les identifiants d\u2019entr\xE9e ainsi que le masque d\u2019attention. D\u2019autres mod\xE8les qui acceptent des entr\xE9es suppl\xE9mentaires sont \xE9galement fournis par l\u2019objet "),ye=u(be,"CODE",{});var Ot=i(ye);Ns=a(Ot,"tokenizer"),Ot.forEach(t),Ls=a(be,"."),be.forEach(t),Xe=f(e),pe=u(e,"P",{});var Bt=i(pe);Rs=a(Bt,"Comme nous allons le voir dans les quelques exemples ci-dessous, cette m\xE9thode est tr\xE8s puissante. Premi\xE8rement, elle peut tokeniser une seule s\xE9quence :"),Bt.forEach(t),Ye=f(e),b(Q.$$.fragment,e),Ze=f(e),ce=u(e,"P",{});var Gt=i(ce);Js=a(Gt,"Elle g\xE8re \xE9galement plusieurs s\xE9quences \xE0 la fois, sans modification de l\u2019API :"),Gt.forEach(t),es=f(e),b(V.$$.fragment,e),ss=f(e),R=u(e,"P",{});var bs=i(R);Os=a(bs,"Il est possible de faire du "),ze=u(bs,"EM",{});var Ut=i(ze);Bs=a(Ut,"padding"),Ut.forEach(t),Gs=a(bs," selon plusieurs objectifs :"),bs.forEach(t),ts=f(e),b(W.$$.fragment,e),ns=f(e),me=u(e,"P",{});var Kt=i(me);Us=a(Kt,"La fonction peut \xE9galement tronquer les s\xE9quences :"),Kt.forEach(t),os=f(e),b(X.$$.fragment,e),as=f(e),q=u(e,"P",{});var w=i(q);Ks=a(w,"L\u2019objet "),Pe=u(w,"CODE",{});var Qt=i(Pe);Qs=a(Qt,"tokenizer"),Qt.forEach(t),Vs=a(w," peut g\xE9rer la conversion en des tenseurs de "),Te=u(w,"EM",{});var Vt=i(Te);Ws=a(Vt,"frameworks"),Vt.forEach(t),Xs=a(w," sp\xE9cifiques. Ils peuvent ensuite \xEAtre directement envoy\xE9s au mod\xE8le. Par exemple, dans le code suivant, nous demandons au "),Ce=u(w,"EM",{});var Wt=i(Ce);Ys=a(Wt,"tokenizer"),Wt.forEach(t),Zs=a(w," de retourner des tenseurs PyTorch lorsque l\u2019on sp\xE9cifie "),Se=u(w,"CODE",{});var Xt=i(Se);et=a(Xt,'"pt"'),Xt.forEach(t),st=a(w,", de retourner des tenseurs TensorFlow lorsque l\u2019on sp\xE9cifie "),Ae=u(w,"CODE",{});var Yt=i(Ae);tt=a(Yt,'"tf"'),Yt.forEach(t),nt=a(w," et des tableaux NumPy lorsque l\u2019on indique "),Ie=u(w,"CODE",{});var Zt=i(Ie);ot=a(Zt,'"np"'),Zt.forEach(t),at=a(w," :"),w.forEach(t),ls=f(e),b(Y.$$.fragment,e),rs=f(e),H=u(e,"H2",{class:!0});var _s=i(H);J=u(_s,"A",{id:!0,class:!0,href:!0});var en=i(J);Fe=u(en,"SPAN",{});var sn=i(Fe);b(Z.$$.fragment,sn),sn.forEach(t),en.forEach(t),lt=f(_s),De=u(_s,"SPAN",{});var tn=i(De);rt=a(tn,"Jetons sp\xE9ciaux"),tn.forEach(t),_s.forEach(t),us=f(e),O=u(e,"P",{});var $s=i(O);ut=a($s,"Si nous jetons un coup d\u2019\u0153il aux identifiants d\u2019entr\xE9e renvoy\xE9s par le "),Me=u($s,"EM",{});var nn=i(Me);it=a(nn,"tokenizer"),nn.forEach(t),pt=a($s,", nous verrons qu\u2019ils sont un peu diff\xE9rents de ceux que nous avions pr\xE9c\xE9demment :"),$s.forEach(t),is=f(e),b(ee.$$.fragment,e),ps=f(e),b(se.$$.fragment,e),cs=f(e),fe=u(e,"P",{});var on=i(fe);ct=a(on,"Un identifiant symbolique a \xE9t\xE9 ajout\xE9 au d\xE9but ainsi qu\u2019un autre \xE0 la fin. D\xE9codons les deux s\xE9quences d\u2019identifiants ci-dessus pour voir de quoi il s\u2019agit :"),on.forEach(t),ms=f(e),b(te.$$.fragment,e),fs=f(e),b(ne.$$.fragment,e),ds=f(e),j=u(e,"P",{});var D=i(j);mt=a(D,"Le "),He=u(D,"EM",{});var an=i(He);ft=a(an,"tokenizer"),an.forEach(t),dt=a(D," a ajout\xE9 le mot sp\xE9cial "),Ne=u(D,"CODE",{});var ln=i(Ne);ht=a(ln,"[CLS]"),ln.forEach(t),gt=a(D," au d\xE9but et le mot sp\xE9cial "),Le=u(D,"CODE",{});var rn=i(Le);qt=a(rn,"[SEP]"),rn.forEach(t),vt=a(D," \xE0 la fin. C\u2019est parce que le mod\xE8le a \xE9t\xE9 pr\xE9-entra\xEEn\xE9 avec ces mots, donc pour obtenir les m\xEAmes r\xE9sultats pour l\u2019inf\xE9rence, nous devons \xE9galement les ajouter. Notez que certains mod\xE8les n\u2019ajoutent pas de mots sp\xE9ciaux, ou en ajoutent des diff\xE9rents. Les mod\xE8les peuvent aussi ajouter ces mots sp\xE9ciaux seulement au d\xE9but, ou seulement \xE0 la fin. Dans tous les cas, le "),Re=u(D,"EM",{});var un=i(Re);bt=a(un,"tokenizer"),un.forEach(t),_t=a(D," sait lesquels sont attendus et s\u2019en occupe pour vous."),D.forEach(t),hs=f(e),N=u(e,"H2",{class:!0});var ks=i(N);B=u(ks,"A",{id:!0,class:!0,href:!0});var pn=i(B);Je=u(pn,"SPAN",{});var cn=i(Je);b(oe.$$.fragment,cn),cn.forEach(t),pn.forEach(t),$t=f(ks),ae=u(ks,"SPAN",{});var js=i(ae);kt=a(js,"Conclusion : du "),Oe=u(js,"I",{});var mn=i(Oe);jt=a(mn,"tokenizer"),mn.forEach(t),wt=a(js," au mod\xE8le"),js.forEach(t),ks.forEach(t),gs=f(e),x=u(e,"P",{});var G=i(x);Et=a(G,"Maintenant que nous avons vu toutes les \xE9tapes individuelles que l\u2019objet "),Be=u(G,"CODE",{});var fn=i(Be);xt=a(fn,"tokenizer"),fn.forEach(t),yt=a(G," utilise lorsqu\u2019il est appliqu\xE9 sur des textes, voyons une derni\xE8re fois comment il peut g\xE9rer plusieurs s\xE9quences ("),Ge=u(G,"EM",{});var dn=i(Ge);zt=a(dn,"padding"),dn.forEach(t),Pt=a(G,"), de tr\xE8s longues s\xE9quences ("),Ue=u(G,"EM",{});var hn=i(Ue);Tt=a(hn,"troncation"),hn.forEach(t),Ct=a(G,") et plusieurs types de tenseurs avec son API principale :"),G.forEach(t),qs=f(e),S.l(e),de=gn(),this.h()},h(){E(c,"name","hf:doc:metadata"),E(c,"content",JSON.stringify(Cn)),E(L,"id","tout-assembler"),E(L,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(L,"href","#tout-assembler"),E(M,"class","relative group"),E(J,"id","jetons-spciaux"),E(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(J,"href","#jetons-spciaux"),E(H,"class","relative group"),E(B,"id","conclusion-du-itokenizeri-au-modle"),E(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(B,"href","#conclusion-du-itokenizeri-au-modle"),E(N,"class","relative group")},m(e,n){s(document.head,c),p(e,d,n),_(l,e,n),p(e,k,n),p(e,M,n),s(M,L),s(L,_e),_(U,_e,null),s(M,ws),s(M,$e),s($e,Es),p(e,Ke,n),le[P].m(e,n),p(e,ie,n),p(e,A,n),s(A,xs),s(A,ke),s(ke,ys),s(A,zs),s(A,je),s(je,Ps),s(A,Ts),p(e,Qe,n),p(e,I,n),s(I,Cs),s(I,we),s(we,Ss),s(I,As),s(I,Ee),s(Ee,Is),s(I,Fs),p(e,Ve,n),_(K,e,n),p(e,We,n),p(e,F,n),s(F,Ds),s(F,xe),s(xe,Ms),s(F,Hs),s(F,ye),s(ye,Ns),s(F,Ls),p(e,Xe,n),p(e,pe,n),s(pe,Rs),p(e,Ye,n),_(Q,e,n),p(e,Ze,n),p(e,ce,n),s(ce,Js),p(e,es,n),_(V,e,n),p(e,ss,n),p(e,R,n),s(R,Os),s(R,ze),s(ze,Bs),s(R,Gs),p(e,ts,n),_(W,e,n),p(e,ns,n),p(e,me,n),s(me,Us),p(e,os,n),_(X,e,n),p(e,as,n),p(e,q,n),s(q,Ks),s(q,Pe),s(Pe,Qs),s(q,Vs),s(q,Te),s(Te,Ws),s(q,Xs),s(q,Ce),s(Ce,Ys),s(q,Zs),s(q,Se),s(Se,et),s(q,st),s(q,Ae),s(Ae,tt),s(q,nt),s(q,Ie),s(Ie,ot),s(q,at),p(e,ls,n),_(Y,e,n),p(e,rs,n),p(e,H,n),s(H,J),s(J,Fe),_(Z,Fe,null),s(H,lt),s(H,De),s(De,rt),p(e,us,n),p(e,O,n),s(O,ut),s(O,Me),s(Me,it),s(O,pt),p(e,is,n),_(ee,e,n),p(e,ps,n),_(se,e,n),p(e,cs,n),p(e,fe,n),s(fe,ct),p(e,ms,n),_(te,e,n),p(e,fs,n),_(ne,e,n),p(e,ds,n),p(e,j,n),s(j,mt),s(j,He),s(He,ft),s(j,dt),s(j,Ne),s(Ne,ht),s(j,gt),s(j,Le),s(Le,qt),s(j,vt),s(j,Re),s(Re,bt),s(j,_t),p(e,hs,n),p(e,N,n),s(N,B),s(B,Je),_(oe,Je,null),s(N,$t),s(N,ae),s(ae,kt),s(ae,Oe),s(Oe,jt),s(ae,wt),p(e,gs,n),p(e,x,n),s(x,Et),s(x,Be),s(Be,xt),s(x,yt),s(x,Ge),s(Ge,zt),s(x,Pt),s(x,Ue),s(Ue,Tt),s(x,Ct),p(e,qs,n),re[C].m(e,n),p(e,de,n),vs=!0},p(e,[n]){const ue={};n&1&&(ue.fw=e[0]),l.$set(ue);let he=P;P=It(e),P!==he&&(vn(),h(le[he],1,1,()=>{le[he]=null}),qn(),T=le[P],T||(T=le[P]=At[P](e),T.c()),g(T,1),T.m(ie.parentNode,ie));let ge=C;C=Dt(e),C!==ge&&(vn(),h(re[ge],1,1,()=>{re[ge]=null}),qn(),S=re[C],S||(S=re[C]=Ft[C](e),S.c()),g(S,1),S.m(de.parentNode,de))},i(e){vs||(g(l.$$.fragment,e),g(U.$$.fragment,e),g(T),g(K.$$.fragment,e),g(Q.$$.fragment,e),g(V.$$.fragment,e),g(W.$$.fragment,e),g(X.$$.fragment,e),g(Y.$$.fragment,e),g(Z.$$.fragment,e),g(ee.$$.fragment,e),g(se.$$.fragment,e),g(te.$$.fragment,e),g(ne.$$.fragment,e),g(oe.$$.fragment,e),g(S),vs=!0)},o(e){h(l.$$.fragment,e),h(U.$$.fragment,e),h(T),h(K.$$.fragment,e),h(Q.$$.fragment,e),h(V.$$.fragment,e),h(W.$$.fragment,e),h(X.$$.fragment,e),h(Y.$$.fragment,e),h(Z.$$.fragment,e),h(ee.$$.fragment,e),h(se.$$.fragment,e),h(te.$$.fragment,e),h(ne.$$.fragment,e),h(oe.$$.fragment,e),h(S),vs=!1},d(e){t(c),e&&t(d),$(l,e),e&&t(k),e&&t(M),$(U),e&&t(Ke),le[P].d(e),e&&t(ie),e&&t(A),e&&t(Qe),e&&t(I),e&&t(Ve),$(K,e),e&&t(We),e&&t(F),e&&t(Xe),e&&t(pe),e&&t(Ye),$(Q,e),e&&t(Ze),e&&t(ce),e&&t(es),$(V,e),e&&t(ss),e&&t(R),e&&t(ts),$(W,e),e&&t(ns),e&&t(me),e&&t(os),$(X,e),e&&t(as),e&&t(q),e&&t(ls),$(Y,e),e&&t(rs),e&&t(H),$(Z),e&&t(us),e&&t(O),e&&t(is),$(ee,e),e&&t(ps),$(se,e),e&&t(cs),e&&t(fe),e&&t(ms),$(te,e),e&&t(fs),$(ne,e),e&&t(ds),e&&t(j),e&&t(hs),e&&t(N),$(oe),e&&t(gs),e&&t(x),e&&t(qs),re[C].d(e),e&&t(de)}}}const Cn={local:"tout-assembler",sections:[{local:"jetons-spciaux",title:"Jetons sp\xE9ciaux"},{local:"conclusion-du-itokenizeri-au-modle",title:"Conclusion : du <i>tokenizer</i> au mod\xE8le"}],title:"Tout assembler"};function Sn(z,c,d){let l="pt";return wn(()=>{const k=new URLSearchParams(window.location.search);d(0,l=k.get("fw")||"pt")}),[l]}class Hn extends _n{constructor(c){super();$n(this,c,Sn,Tn,kn,{})}}export{Hn as default,Cn as metadata};
