import{S as Sm,i as Bm,s as Rm,e as l,k as r,w as b,t as e,M as Fm,c,d as t,m as p,x as k,a as h,h as a,b as _,N as qm,G as s,g as o,y as v,o as d,p as jc,q as f,B as j,v as Am,n as _c}from"../../chunks/vendor-hf-doc-builder.js";import{T as ep}from"../../chunks/Tip-hf-doc-builder.js";import{Y as $c}from"../../chunks/Youtube-hf-doc-builder.js";import{I as St}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as y}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Pm}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as Nm}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function Mm(D){let g,x;return g=new Pm({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section3_tf.ipynb"}]}}),{c(){b(g.$$.fragment)},l(m){k(g.$$.fragment,m)},m(m,$){v(g,m,$),x=!0},i(m){x||(f(g.$$.fragment,m),x=!0)},o(m){d(g.$$.fragment,m),x=!1},d(m){j(g,m)}}}function Xm(D){let g,x;return g=new Pm({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section3_pt.ipynb"}]}}),{c(){b(g.$$.fragment)},l(m){k(g.$$.fragment,m)},m(m,$){v(g,m,$),x=!0},i(m){x||(f(g.$$.fragment,m),x=!0)},o(m){d(g.$$.fragment,m),x=!1},d(m){j(g,m)}}}function Hm(D){let g,x;return{c(){g=l("p"),x=e("\u26A0\uFE0F Khi tokenize m\u1ED9t c\xE2u, b\u1EA1n kh\xF4ng ph\u1EA3i l\xFAc n\xE0o c\u0169ng th\u1EA5y s\u1EF1 kh\xE1c bi\u1EC7t v\u1EC1 t\u1ED1c \u0111\u1ED9 gi\u1EEFa c\xE1c phi\xEAn b\u1EA3n ch\u1EADm v\xE0 nhanh c\u1EE7a c\xF9ng m\u1ED9t tr\xECnh tokenize. Tr\xEAn th\u1EF1c t\u1EBF, phi\xEAn b\u1EA3n nhanh c\xF3 th\u1EC3 ch\u1EADm h\u01A1n! Ch\u1EC9 khi tokenize nhi\u1EC1u v\u0103n b\u1EA3n song song c\xF9ng m\u1ED9t l\xFAc, b\u1EA1n m\u1EDBi c\xF3 th\u1EC3 th\u1EA5y r\xF5 s\u1EF1 kh\xE1c bi\u1EC7t.")},l(m){g=c(m,"P",{});var $=h(g);x=a($,"\u26A0\uFE0F Khi tokenize m\u1ED9t c\xE2u, b\u1EA1n kh\xF4ng ph\u1EA3i l\xFAc n\xE0o c\u0169ng th\u1EA5y s\u1EF1 kh\xE1c bi\u1EC7t v\u1EC1 t\u1ED1c \u0111\u1ED9 gi\u1EEFa c\xE1c phi\xEAn b\u1EA3n ch\u1EADm v\xE0 nhanh c\u1EE7a c\xF9ng m\u1ED9t tr\xECnh tokenize. Tr\xEAn th\u1EF1c t\u1EBF, phi\xEAn b\u1EA3n nhanh c\xF3 th\u1EC3 ch\u1EADm h\u01A1n! Ch\u1EC9 khi tokenize nhi\u1EC1u v\u0103n b\u1EA3n song song c\xF9ng m\u1ED9t l\xFAc, b\u1EA1n m\u1EDBi c\xF3 th\u1EC3 th\u1EA5y r\xF5 s\u1EF1 kh\xE1c bi\u1EC7t."),$.forEach(t)},m(m,$){o(m,g,$),s(g,x)},d(m){m&&t(g)}}}function Gm(D){let g,x,m,$,q,w,S,A,H,P,K,O,F,I;return{c(){g=l("p"),x=e("Kh\xE1i ni\u1EC7m v\u1EC1 m\u1ED9t t\u1EEB r\u1EA5t l\xE0 ph\u1EE9c t\u1EA1p. V\xED d\u1EE5: \u201CI\u2019ll\u201D (t\u1EEB r\xFAt g\u1ECDn c\u1EE7a \u201CI will\u201D) c\xF3 \u0111\u01B0\u1EE3c t\xEDnh l\xE0 m\u1ED9t hay hai t\u1EEB? N\xF3 th\u1EF1c s\u1EF1 ph\u1EE5 thu\u1ED9c v\xE0o tr\xECnh tokenize v\xE0 ho\u1EA1t \u0111\u1ED9ng ti\u1EC1n tokenize m\xE0 n\xF3 \xE1p d\u1EE5ng. M\u1ED9t s\u1ED1 tokenizer ch\u1EC9 t\xE1ch ra tr\xEAn kho\u1EA3ng tr\u1EAFng, v\xEC v\u1EADy h\u1ECD s\u1EBD coi \u0111\xE2y l\xE0 m\u1ED9t t\u1EEB. Nh\u1EEFng ng\u01B0\u1EDDi kh\xE1c s\u1EED d\u1EE5ng d\u1EA5u c\xE2u \u1EDF \u0111\u1EA7u kho\u1EA3ng tr\u1EAFng, v\xEC v\u1EADy s\u1EBD coi n\xF3 l\xE0 hai t\u1EEB."),m=r(),$=l("p"),q=e("\u270F\uFE0F "),w=l("strong"),S=e("Th\u1EED nghi\u1EC7m th\xF4i!"),A=e(" T\u1EA1o tokenizer t\u1EEB c\xE1c checkpoints "),H=l("code"),P=e("bert-base-cased"),K=e(" v\xE0"),O=l("code"),F=e("roberta-base"),I=e(" v\xE0 tokenize \u201D81s\u201D v\u1EDBi ch\xFAng. B\u1EA1n quan s\xE1t th\u1EA5y g\xEC? ID t\u1EEB l\xE0 g\xEC?")},l(T){g=c(T,"P",{});var C=h(g);x=a(C,"Kh\xE1i ni\u1EC7m v\u1EC1 m\u1ED9t t\u1EEB r\u1EA5t l\xE0 ph\u1EE9c t\u1EA1p. V\xED d\u1EE5: \u201CI\u2019ll\u201D (t\u1EEB r\xFAt g\u1ECDn c\u1EE7a \u201CI will\u201D) c\xF3 \u0111\u01B0\u1EE3c t\xEDnh l\xE0 m\u1ED9t hay hai t\u1EEB? N\xF3 th\u1EF1c s\u1EF1 ph\u1EE5 thu\u1ED9c v\xE0o tr\xECnh tokenize v\xE0 ho\u1EA1t \u0111\u1ED9ng ti\u1EC1n tokenize m\xE0 n\xF3 \xE1p d\u1EE5ng. M\u1ED9t s\u1ED1 tokenizer ch\u1EC9 t\xE1ch ra tr\xEAn kho\u1EA3ng tr\u1EAFng, v\xEC v\u1EADy h\u1ECD s\u1EBD coi \u0111\xE2y l\xE0 m\u1ED9t t\u1EEB. Nh\u1EEFng ng\u01B0\u1EDDi kh\xE1c s\u1EED d\u1EE5ng d\u1EA5u c\xE2u \u1EDF \u0111\u1EA7u kho\u1EA3ng tr\u1EAFng, v\xEC v\u1EADy s\u1EBD coi n\xF3 l\xE0 hai t\u1EEB."),C.forEach(t),m=p(T),$=c(T,"P",{});var N=h($);q=a(N,"\u270F\uFE0F "),w=c(N,"STRONG",{});var B=h(w);S=a(B,"Th\u1EED nghi\u1EC7m th\xF4i!"),B.forEach(t),A=a(N," T\u1EA1o tokenizer t\u1EEB c\xE1c checkpoints "),H=c(N,"CODE",{});var Q=h(H);P=a(Q,"bert-base-cased"),Q.forEach(t),K=a(N," v\xE0"),O=c(N,"CODE",{});var M=h(O);F=a(M,"roberta-base"),M.forEach(t),I=a(N," v\xE0 tokenize \u201D81s\u201D v\u1EDBi ch\xFAng. B\u1EA1n quan s\xE1t th\u1EA5y g\xEC? ID t\u1EEB l\xE0 g\xEC?"),N.forEach(t)},m(T,C){o(T,g,C),s(g,x),o(T,m,C),o(T,$,C),s($,q),s($,w),s(w,S),s($,A),s($,H),s(H,P),s($,K),s($,O),s(O,F),s($,I)},d(T){T&&t(g),T&&t(m),T&&t($)}}}function Lm(D){let g,x,m,$,q;return{c(){g=l("p"),x=e("\u270F\uFE0F "),m=l("strong"),$=e("Th\u1EED nghi\u1EC7m th\xF4i"),q=e(" T\u1EA1o v\u0103n b\u1EA3n m\u1EABu c\u1EE7a ri\xEAng b\u1EA1n v\xE0 xem li\u1EC7u b\u1EA1n c\xF3 th\u1EC3 hi\u1EC3u nh\u1EEFng token n\xE0o \u0111\u01B0\u1EE3c li\xEAn k\u1EBFt v\u1EDBi ID t\u1EEB, c\u0169ng nh\u01B0 c\xE1ch tr\xEDch xu\u1EA5t k\xFD t\u1EF1 k\xE9o d\xE0i cho m\u1ED9t t\u1EEB. \u0110\u1EC3 c\xF3 \u0111i\u1EC3m th\u01B0\u1EDFng, h\xE3y th\u1EED s\u1EED d\u1EE5ng hai c\xE2u l\xE0m \u0111\u1EA7u v\xE0o v\xE0 xem li\u1EC7u ID c\xE2u c\xF3 ph\xF9 h\u1EE3p v\u1EDBi b\u1EA1n kh\xF4ng.")},l(w){g=c(w,"P",{});var S=h(g);x=a(S,"\u270F\uFE0F "),m=c(S,"STRONG",{});var A=h(m);$=a(A,"Th\u1EED nghi\u1EC7m th\xF4i"),A.forEach(t),q=a(S," T\u1EA1o v\u0103n b\u1EA3n m\u1EABu c\u1EE7a ri\xEAng b\u1EA1n v\xE0 xem li\u1EC7u b\u1EA1n c\xF3 th\u1EC3 hi\u1EC3u nh\u1EEFng token n\xE0o \u0111\u01B0\u1EE3c li\xEAn k\u1EBFt v\u1EDBi ID t\u1EEB, c\u0169ng nh\u01B0 c\xE1ch tr\xEDch xu\u1EA5t k\xFD t\u1EF1 k\xE9o d\xE0i cho m\u1ED9t t\u1EEB. \u0110\u1EC3 c\xF3 \u0111i\u1EC3m th\u01B0\u1EDFng, h\xE3y th\u1EED s\u1EED d\u1EE5ng hai c\xE2u l\xE0m \u0111\u1EA7u v\xE0o v\xE0 xem li\u1EC7u ID c\xE2u c\xF3 ph\xF9 h\u1EE3p v\u1EDBi b\u1EA1n kh\xF4ng."),S.forEach(t)},m(w,S){o(w,g,S),s(g,x),s(g,m),s(m,$),s(g,q)},d(w){w&&t(g)}}}function Vm(D){let g,x;return g=new $c({props:{id:"PrX4CjrVnNc"}}),{c(){b(g.$$.fragment)},l(m){k(g.$$.fragment,m)},m(m,$){v(g,m,$),x=!0},i(m){x||(f(g.$$.fragment,m),x=!0)},o(m){d(g.$$.fragment,m),x=!1},d(m){j(g,m)}}}function Km(D){let g,x;return g=new $c({props:{id:"0E7ltQB7fM8"}}),{c(){b(g.$$.fragment)},l(m){k(g.$$.fragment,m)},m(m,$){v(g,m,$),x=!0},i(m){x||(f(g.$$.fragment,m),x=!0)},o(m){d(g.$$.fragment,m),x=!1},d(m){j(g,m)}}}function Ym(D){let g,x,m,$,q,w,S,A,H,P,K,O,F,I,T,C,N,B,Q,M,nn;return P=new y({props:{code:`from transformers import AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="tf")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
outputs = model(**inputs)`}}),B=new y({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),M=new y({props:{code:`(1, 19)
(1, 19, 9)`,highlighted:`(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>)
(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>)`}}),{c(){g=l("p"),x=e("\u0110\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n tokenize \u0111\u1EA7u v\xE0o c\u1EE7a ch\xFAng ta v\xE0 truy\u1EC1n ch\xFAng v\xE0o m\xF4 h\xECnh. \u0110\xE2y ch\xEDnh x\xE1c l\xE0 nh\u1EEFng g\xEC ta \u0111\xE3 l\xE0m \u1EDF "),m=l("a"),$=e("Ch\u01B0\u01A1ng 2"),q=e("; ta kh\u1EDFi t\u1EA1o tokenizer v\xE0 m\xF4 h\xECnh s\u1EED d\u1EE5ng l\u1EDBp "),w=l("code"),S=e("TFAutoXxx"),A=e(" v\xE0 sau \u0111\xF3 d\xF9ng ch\xFAng v\xE0o c\xE1c m\u1EABu c\u1EE7a m\xECnh:"),H=r(),b(P.$$.fragment),K=r(),O=l("p"),F=e("V\xEC ta d\xF9ng "),I=l("code"),T=e("TFAutoModelForTokenClassification"),C=e(" \u1EDF \u0111\xE2y, ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c t\u1EADp h\u1EE3p c\xE1c logits cho t\u1EEBng token c\u1EE7a chu\u1ED7i \u0111\u1EA7u v\xE0o:"),N=r(),b(B.$$.fragment),Q=r(),b(M.$$.fragment),this.h()},l(u){g=c(u,"P",{});var E=h(g);x=a(E,"\u0110\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n tokenize \u0111\u1EA7u v\xE0o c\u1EE7a ch\xFAng ta v\xE0 truy\u1EC1n ch\xFAng v\xE0o m\xF4 h\xECnh. \u0110\xE2y ch\xEDnh x\xE1c l\xE0 nh\u1EEFng g\xEC ta \u0111\xE3 l\xE0m \u1EDF "),m=c(E,"A",{href:!0});var fn=h(m);$=a(fn,"Ch\u01B0\u01A1ng 2"),fn.forEach(t),q=a(E,"; ta kh\u1EDFi t\u1EA1o tokenizer v\xE0 m\xF4 h\xECnh s\u1EED d\u1EE5ng l\u1EDBp "),w=c(E,"CODE",{});var zn=h(w);S=a(zn,"TFAutoXxx"),zn.forEach(t),A=a(E," v\xE0 sau \u0111\xF3 d\xF9ng ch\xFAng v\xE0o c\xE1c m\u1EABu c\u1EE7a m\xECnh:"),E.forEach(t),H=p(u),k(P.$$.fragment,u),K=p(u),O=c(u,"P",{});var ln=h(O);F=a(ln,"V\xEC ta d\xF9ng "),I=c(ln,"CODE",{});var on=h(I);T=a(on,"TFAutoModelForTokenClassification"),on.forEach(t),C=a(ln," \u1EDF \u0111\xE2y, ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c t\u1EADp h\u1EE3p c\xE1c logits cho t\u1EEBng token c\u1EE7a chu\u1ED7i \u0111\u1EA7u v\xE0o:"),ln.forEach(t),N=p(u),k(B.$$.fragment,u),Q=p(u),k(M.$$.fragment,u),this.h()},h(){_(m,"href","/course/chapter2")},m(u,E){o(u,g,E),s(g,x),s(g,m),s(m,$),s(g,q),s(g,w),s(w,S),s(g,A),o(u,H,E),v(P,u,E),o(u,K,E),o(u,O,E),s(O,F),s(O,I),s(I,T),s(O,C),o(u,N,E),v(B,u,E),o(u,Q,E),v(M,u,E),nn=!0},i(u){nn||(f(P.$$.fragment,u),f(B.$$.fragment,u),f(M.$$.fragment,u),nn=!0)},o(u){d(P.$$.fragment,u),d(B.$$.fragment,u),d(M.$$.fragment,u),nn=!1},d(u){u&&t(g),u&&t(H),j(P,u),u&&t(K),u&&t(O),u&&t(N),j(B,u),u&&t(Q),j(M,u)}}}function Qm(D){let g,x,m,$,q,w,S,A,H,P,K,O,F,I,T,C,N,B,Q,M,nn;return P=new y({props:{code:`from transformers import AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="pt")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
outputs = model(**inputs)`}}),B=new y({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),M=new y({props:{code:`torch.Size([1, 19])
torch.Size([1, 19, 9])`,highlighted:`torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>])
torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>])`}}),{c(){g=l("p"),x=e("\u0110\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n tokenize \u0111\u1EA7u v\xE0o c\u1EE7a ch\xFAng ta v\xE0 truy\u1EC1n ch\xFAng v\xE0o m\xF4 h\xECnh. \u0110\xE2y ch\xEDnh x\xE1c l\xE0 nh\u1EEFng g\xEC ta \u0111\xE3 l\xE0m \u1EDF "),m=l("a"),$=e("Ch\u01B0\u01A1ng 2"),q=e("; ta kh\u1EDFi t\u1EA1o tokenizer v\xE0 m\xF4 h\xECnh s\u1EED d\u1EE5ng l\u1EDBp "),w=l("code"),S=e("AutoXxx"),A=e(" v\xE0 sau \u0111\xF3 d\xF9ng ch\xFAng v\xE0o c\xE1c m\u1EABu c\u1EE7a m\xECnh:"),H=r(),b(P.$$.fragment),K=r(),O=l("p"),F=e("V\xEC ch\xFAng ta s\u1EED d\u1EE5ng "),I=l("code"),T=e("AutoModelForTokenClassification"),C=e(" \u1EDF \u0111\xE2y,ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c t\u1EADp h\u1EE3p c\xE1c logits cho t\u1EEBng token c\u1EE7a chu\u1ED7i \u0111\u1EA7u v\xE0o:"),N=r(),b(B.$$.fragment),Q=r(),b(M.$$.fragment),this.h()},l(u){g=c(u,"P",{});var E=h(g);x=a(E,"\u0110\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n tokenize \u0111\u1EA7u v\xE0o c\u1EE7a ch\xFAng ta v\xE0 truy\u1EC1n ch\xFAng v\xE0o m\xF4 h\xECnh. \u0110\xE2y ch\xEDnh x\xE1c l\xE0 nh\u1EEFng g\xEC ta \u0111\xE3 l\xE0m \u1EDF "),m=c(E,"A",{href:!0});var fn=h(m);$=a(fn,"Ch\u01B0\u01A1ng 2"),fn.forEach(t),q=a(E,"; ta kh\u1EDFi t\u1EA1o tokenizer v\xE0 m\xF4 h\xECnh s\u1EED d\u1EE5ng l\u1EDBp "),w=c(E,"CODE",{});var zn=h(w);S=a(zn,"AutoXxx"),zn.forEach(t),A=a(E," v\xE0 sau \u0111\xF3 d\xF9ng ch\xFAng v\xE0o c\xE1c m\u1EABu c\u1EE7a m\xECnh:"),E.forEach(t),H=p(u),k(P.$$.fragment,u),K=p(u),O=c(u,"P",{});var ln=h(O);F=a(ln,"V\xEC ch\xFAng ta s\u1EED d\u1EE5ng "),I=c(ln,"CODE",{});var on=h(I);T=a(on,"AutoModelForTokenClassification"),on.forEach(t),C=a(ln," \u1EDF \u0111\xE2y,ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c t\u1EADp h\u1EE3p c\xE1c logits cho t\u1EEBng token c\u1EE7a chu\u1ED7i \u0111\u1EA7u v\xE0o:"),ln.forEach(t),N=p(u),k(B.$$.fragment,u),Q=p(u),k(M.$$.fragment,u),this.h()},h(){_(m,"href","/course/chapter2")},m(u,E){o(u,g,E),s(g,x),s(g,m),s(m,$),s(g,q),s(g,w),s(w,S),s(g,A),o(u,H,E),v(P,u,E),o(u,K,E),o(u,O,E),s(O,F),s(O,I),s(I,T),s(O,C),o(u,N,E),v(B,u,E),o(u,Q,E),v(M,u,E),nn=!0},i(u){nn||(f(P.$$.fragment,u),f(B.$$.fragment,u),f(M.$$.fragment,u),nn=!0)},o(u){d(P.$$.fragment,u),d(B.$$.fragment,u),d(M.$$.fragment,u),nn=!1},d(u){u&&t(g),u&&t(H),j(P,u),u&&t(K),u&&t(O),u&&t(N),j(B,u),u&&t(Q),j(M,u)}}}function Um(D){let g,x;return g=new y({props:{code:`import tensorflow as tf

probabilities = tf.math.softmax(outputs.logits, axis=-1)[0]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-1)[0]
predictions = predictions.numpy().tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

probabilities = tf.math.softmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
predictions = predictions.numpy().tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){b(g.$$.fragment)},l(m){k(g.$$.fragment,m)},m(m,$){v(g,m,$),x=!0},i(m){x||(f(g.$$.fragment,m),x=!0)},o(m){d(g.$$.fragment,m),x=!1},d(m){j(g,m)}}}function Jm(D){let g,x;return g=new y({props:{code:`import torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)[0].tolist()
predictions = outputs.logits.argmax(dim=-1)[0].tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
predictions = outputs.logits.argmax(dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){b(g.$$.fragment)},l(m){k(g.$$.fragment,m)},m(m,$){v(g,m,$),x=!0},i(m){x||(f(g.$$.fragment,m),x=!0)},o(m){d(g.$$.fragment,m),x=!1},d(m){j(g,m)}}}function Wm(D){let g,x,m,$,q,w,S,A,H,P,K,O,F,I,T,C,N,B,Q,M,nn,u,E,fn,zn,ln,on,yc,Ec,Pa,hs,Sa,An,wc,nt,Cc,Oc,Ba,Nn,Bt,In,st,zc,Ic,tt,Tc,Dc,Rt,qc,is,Tn,et,Ft,Pc,Sc,at,Bc,Rc,lt,Fc,Ac,Dn,ct,At,Nc,Mc,ht,Xc,Hc,it,Gc,Ra,Mn,Fa,qn,Xn,Nt,os,Lc,Mt,Vc,Aa,rs,Na,Hn,Kc,Xt,Yc,Qc,Ma,$n,Uc,Ht,Jc,Wc,Gt,Zc,nh,Xa,ot,sh,Ha,ps,Ga,Gn,th,Lt,eh,ah,La,gs,Va,sn,lh,Vt,ch,hh,Kt,ih,oh,Yt,rh,ph,Qt,gh,mh,Ka,ms,Ya,us,Qa,Ln,uh,Ut,dh,fh,Ua,ds,Ja,fs,Wa,rt,xh,Za,xs,nl,bs,sl,yn,bh,Jt,kh,vh,Wt,jh,_h,tl,ks,el,vs,al,U,$h,Zt,yh,Eh,ne,wh,Ch,se,Oh,zh,te,Ih,Th,ee,Dh,qh,ll,Vn,cl,En,Ph,ae,Sh,Bh,le,Rh,Fh,hl,G,Ah,ce,Nh,Mh,he,Xh,Hh,ie,Gh,Lh,oe,Vh,Kh,re,Yh,Qh,pe,Uh,Jh,il,js,ol,_s,rl,rn,Wh,ge,Zh,ni,me,si,ti,ue,ei,ai,pl,Kn,gl,Pn,Yn,de,$s,li,pt,ci,fe,hi,ml,tn,ii,gt,oi,ri,xe,pi,gi,mt,mi,ui,be,di,fi,ul,xn,bn,ut,Sn,Qn,ke,ys,xi,ve,bi,dl,Un,ki,Es,je,vi,ji,fl,ws,xl,Cs,bl,dt,_i,kl,Os,vl,zs,jl,Y,_e,$i,yi,$e,Ei,wi,ye,Ci,Oi,Ee,zi,Ii,we,Ti,Di,Ce,qi,Pi,_l,wn,Jn,Oe,Si,Bi,ze,Ri,Fi,Ai,ft,Ie,Ni,Mi,Xi,Wn,Te,Hi,Gi,De,Li,Vi,$l,Zn,Ki,qe,Yi,Qi,yl,Bn,ns,Pe,Is,Ui,Se,Ji,El,kn,vn,xt,bt,Wi,wl,jn,_n,kt,Ts,Cl,ss,Zi,Be,no,so,Ol,Ds,zl,qs,Il,z,to,Re,eo,ao,Fe,lo,co,Ae,ho,io,Ne,oo,ro,Me,po,go,Xe,mo,uo,He,fo,xo,Ge,bo,ko,Le,vo,jo,Ve,_o,$o,Ke,yo,Eo,Tl,X,wo,Ye,Co,Oo,Qe,zo,Io,Ue,To,Do,Je,qo,Po,We,So,Bo,Ze,Ro,Fo,na,Ao,No,sa,Mo,Xo,Dl,Rn,Ps,ap,Ho,Ss,lp,ql,ts,Go,ta,Lo,Vo,Pl,Bs,Sl,Rs,Bl,J,Ko,ea,Yo,Qo,aa,Uo,Jo,la,Wo,Zo,ca,nr,sr,ha,tr,er,Rl,Fs,Fl,As,Al,pn,ar,ia,lr,cr,oa,hr,ir,ra,or,rr,Nl,Ns,Ml,es,pr,pa,gr,mr,Xl,Ms,Hl,vt,ur,Gl,Xs,Ll,Hs,Vl,jt,dr,Kl,Fn,as,ga,Gs,fr,ma,xr,Yl,L,br,ua,kr,vr,da,jr,_r,fa,$r,yr,xa,Er,wr,ba,Cr,Or,ka,zr,Ir,Ql,W,Tr,va,Dr,qr,ja,Pr,Sr,_a,Br,Rr,$a,Fr,Ar,ya,Nr,Mr,Ul,Ls,Jl,Vs,Wl,Z,Xr,Ea,Hr,Gr,wa,Lr,Vr,Ca,Kr,Yr,Oa,Qr,Ur,za,Jr,Wr,Zl,Ks,nc,_t,Zr,sc,Ys,tc,$t,np,ec;m=new Nm({props:{fw:D[0]}}),A=new St({});const cp=[Xm,Mm],Qs=[];function hp(n,i){return n[0]==="pt"?0:1}F=hp(D),I=Qs[F]=cp[F](D),hs=new $c({props:{id:"g8quOxoqhHQ"}}),Mn=new ep({props:{warning:!0,$$slots:{default:[Hm]},$$scope:{ctx:D}}}),os=new St({}),rs=new $c({props:{id:"3umI3tm27Vw"}}),ps=new y({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
encoding = tokenizer(example)
print(type(encoding))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
encoding = tokenizer(example)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(encoding))`}}),gs=new y({props:{code:"<class 'transformers.tokenization_utils_base.BatchEncoding'>",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;transformers.tokenization_utils_base.BatchEncoding&#x27;</span>&gt;'}}),ms=new y({props:{code:"tokenizer.is_fast",highlighted:"tokenizer.is_fast"}}),us=new y({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),ds=new y({props:{code:"encoding.is_fast",highlighted:"encoding.is_fast"}}),fs=new y({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),xs=new y({props:{code:"encoding.tokens()",highlighted:"encoding.tokens()"}}),bs=new y({props:{code:`['[CLS]', 'My', 'name', 'is', 'S', '##yl', '##va', '##in', 'and', 'I', 'work', 'at', 'Hu', '##gging', 'Face', 'in',
 'Brooklyn', '.', '[SEP]']`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;My&#x27;</span>, <span class="hljs-string">&#x27;name&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&#x27;work&#x27;</span>, <span class="hljs-string">&#x27;at&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>,
 <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),ks=new y({props:{code:"encoding.word_ids()",highlighted:"encoding.word_ids()"}}),vs=new y({props:{code:"[None, 0, 1, 2, 3, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, None]",highlighted:'[<span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>, <span class="hljs-number">10</span>, <span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-literal">None</span>]'}}),Vn=new ep({props:{$$slots:{default:[Gm]},$$scope:{ctx:D}}}),js=new y({props:{code:`start, end = encoding.word_to_chars(3)
example[start:end]`,highlighted:`start, end = encoding.word_to_chars(<span class="hljs-number">3</span>)
example[start:end]`}}),_s=new y({props:{code:"Sylvain",highlighted:"Sylvain"}}),Kn=new ep({props:{$$slots:{default:[Lm]},$$scope:{ctx:D}}}),$s=new St({});const ip=[Km,Vm],Us=[];function op(n,i){return n[0]==="pt"?0:1}xn=op(D),bn=Us[xn]=ip[xn](D),ys=new St({}),ws=new y({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),Cs=new y({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Os=new y({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification", aggregation_strategy="simple")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>, aggregation_strategy=<span class="hljs-string">&quot;simple&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),zs=new y({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Is=new St({});const rp=[Qm,Ym],Js=[];function pp(n,i){return n[0]==="pt"?0:1}kn=pp(D),vn=Js[kn]=rp[kn](D);const gp=[Jm,Um],Ws=[];function mp(n,i){return n[0]==="pt"?0:1}return jn=mp(D),_n=Ws[jn]=gp[jn](D),Ts=new y({props:{code:"[0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 6, 0, 8, 0, 0]",highlighted:'[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">0</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]'}}),Ds=new y({props:{code:"model.config.id2label",highlighted:"model.config.id2label"}}),qs=new y({props:{code:`{0: 'O',
 1: 'B-MISC',
 2: 'I-MISC',
 3: 'B-PER',
 4: 'I-PER',
 5: 'B-ORG',
 6: 'I-ORG',
 7: 'B-LOC',
 8: 'I-LOC'}`,highlighted:`{<span class="hljs-number">0</span>: <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-number">1</span>: <span class="hljs-string">&#x27;B-MISC&#x27;</span>,
 <span class="hljs-number">2</span>: <span class="hljs-string">&#x27;I-MISC&#x27;</span>,
 <span class="hljs-number">3</span>: <span class="hljs-string">&#x27;B-PER&#x27;</span>,
 <span class="hljs-number">4</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>,
 <span class="hljs-number">5</span>: <span class="hljs-string">&#x27;B-ORG&#x27;</span>,
 <span class="hljs-number">6</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>,
 <span class="hljs-number">7</span>: <span class="hljs-string">&#x27;B-LOC&#x27;</span>,
 <span class="hljs-number">8</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>}`}}),Bs=new y({props:{code:`results = []
tokens = inputs.tokens()

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        results.append(
            {"entity": label, "score": probabilities[idx][pred], "word": tokens[idx]}
        )

print(results)`,highlighted:`results = []
tokens = inputs.tokens()

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        results.append(
            {<span class="hljs-string">&quot;entity&quot;</span>: label, <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred], <span class="hljs-string">&quot;word&quot;</span>: tokens[idx]}
        )

<span class="hljs-built_in">print</span>(results)`}}),Rs=new y({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S'},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl'},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va'},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in'},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu'},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging'},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face'},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn'}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>}]`}}),Fs=new y({props:{code:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
inputs_with_offsets["offset_mapping"]`,highlighted:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]`}}),As=new y({props:{code:`[(0, 0), (0, 2), (3, 7), (8, 10), (11, 12), (12, 14), (14, 16), (16, 18), (19, 22), (23, 24), (25, 29), (30, 32),
 (33, 35), (35, 40), (41, 45), (46, 48), (49, 57), (57, 58), (0, 0)]`,highlighted:`[(<span class="hljs-number">0</span>, <span class="hljs-number">0</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">2</span>), (<span class="hljs-number">3</span>, <span class="hljs-number">7</span>), (<span class="hljs-number">8</span>, <span class="hljs-number">10</span>), (<span class="hljs-number">11</span>, <span class="hljs-number">12</span>), (<span class="hljs-number">12</span>, <span class="hljs-number">14</span>), (<span class="hljs-number">14</span>, <span class="hljs-number">16</span>), (<span class="hljs-number">16</span>, <span class="hljs-number">18</span>), (<span class="hljs-number">19</span>, <span class="hljs-number">22</span>), (<span class="hljs-number">23</span>, <span class="hljs-number">24</span>), (<span class="hljs-number">25</span>, <span class="hljs-number">29</span>), (<span class="hljs-number">30</span>, <span class="hljs-number">32</span>),
 (<span class="hljs-number">33</span>, <span class="hljs-number">35</span>), (<span class="hljs-number">35</span>, <span class="hljs-number">40</span>), (<span class="hljs-number">41</span>, <span class="hljs-number">45</span>), (<span class="hljs-number">46</span>, <span class="hljs-number">48</span>), (<span class="hljs-number">49</span>, <span class="hljs-number">57</span>), (<span class="hljs-number">57</span>, <span class="hljs-number">58</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">0</span>)]`}}),Ns=new y({props:{code:"example[12:14]",highlighted:'example[<span class="hljs-number">12</span>:<span class="hljs-number">14</span>]'}}),Ms=new y({props:{code:"yl",highlighted:"yl"}}),Xs=new y({props:{code:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        start, end = offsets[idx]
        results.append(
            {
                "entity": label,
                "score": probabilities[idx][pred],
                "word": tokens[idx],
                "start": start,
                "end": end,
            }
        )

print(results)`,highlighted:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        start, end = offsets[idx]
        results.append(
            {
                <span class="hljs-string">&quot;entity&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred],
                <span class="hljs-string">&quot;word&quot;</span>: tokens[idx],
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )

<span class="hljs-built_in">print</span>(results)`}}),Hs=new y({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Gs=new St({}),Ls=new y({props:{code:"example[33:45]",highlighted:'example[<span class="hljs-number">33</span>:<span class="hljs-number">45</span>]'}}),Vs=new y({props:{code:"Hugging Face",highlighted:"Hugging Face"}}),Ks=new y({props:{code:`import numpy as np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

idx = 0
while idx < len(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    if label != "O":
        # Xo\xE1 B- ho\u1EB7c I-
        label = label[2:]
        start, _ = offsets[idx]

        # L\u1EA5y t\u1EA5t c\u1EA3 c\xE1c tokens c\xF3 nh\xE3n I-
        all_scores = []
        while (
            idx < len(predictions)
            and model.config.id2label[predictions[idx]] == f"I-{label}"
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += 1

        # \u0110i\u1EC3m l\xE0 gi\xE1 tr\u1ECB trung b\xECnh c\u1EE7a t\u1EA5t c\u1EA3 \u0111i\u1EC3m c\u1EE7a c\xE1c token trong th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c nh\xF3m \u0111\xF3
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                "entity_group": label,
                "score": score,
                "word": word,
                "start": start,
                "end": end,
            }
        )
    idx += 1

print(results)`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

idx = <span class="hljs-number">0</span>
<span class="hljs-keyword">while</span> idx &lt; <span class="hljs-built_in">len</span>(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        <span class="hljs-comment"># Xo\xE1 B- ho\u1EB7c I-</span>
        label = label[<span class="hljs-number">2</span>:]
        start, _ = offsets[idx]

        <span class="hljs-comment"># L\u1EA5y t\u1EA5t c\u1EA3 c\xE1c tokens c\xF3 nh\xE3n I-</span>
        all_scores = []
        <span class="hljs-keyword">while</span> (
            idx &lt; <span class="hljs-built_in">len</span>(predictions)
            <span class="hljs-keyword">and</span> model.config.id2label[predictions[idx]] == <span class="hljs-string">f&quot;I-<span class="hljs-subst">{label}</span>&quot;</span>
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += <span class="hljs-number">1</span>

        <span class="hljs-comment"># \u0110i\u1EC3m l\xE0 gi\xE1 tr\u1ECB trung b\xECnh c\u1EE7a t\u1EA5t c\u1EA3 \u0111i\u1EC3m c\u1EE7a c\xE1c token trong th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c nh\xF3m \u0111\xF3</span>
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                <span class="hljs-string">&quot;entity_group&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: score,
                <span class="hljs-string">&quot;word&quot;</span>: word,
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )
    idx += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(results)`}}),Ys=new y({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),{c(){g=l("meta"),x=r(),b(m.$$.fragment),$=r(),q=l("h1"),w=l("a"),S=l("span"),b(A.$$.fragment),H=r(),P=l("span"),K=e("S\u1EE9c m\u1EA1nh \u0111\u1EB7c bi\u1EC7t c\u1EE7a tokenizer nhanh"),O=r(),I.c(),T=r(),C=l("p"),N=e("Trong ph\u1EA7n n\xE0y, ch\xFAng ta s\u1EBD xem x\xE9t k\u1EF9 h\u01A1n c\xE1c kh\u1EA3 n\u0103ng c\u1EE7a tokenizer trong \u{1F917} Transformers. Cho \u0111\u1EBFn nay, ch\xFAng ta ch\u1EC9 s\u1EED d\u1EE5ng ch\xFAng \u0111\u1EC3 tokenize \u0111\u1EA7u v\xE0o ho\u1EB7c gi\u1EA3i m\xE3 ID tr\u1EDF l\u1EA1i th\xE0nh v\u0103n b\u1EA3n, nh\u01B0ng c\xE1c tr\xECnh tokenize - \u0111\u1EB7c bi\u1EC7t l\xE0 nh\u1EEFng tr\xECnh tokenize \u0111\u01B0\u1EE3c h\u1ED7 tr\u1EE3 b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers - c\xF3 th\u1EC3 l\xE0m \u0111\u01B0\u1EE3c nhi\u1EC1u h\u01A1n th\u1EBF. \u0110\u1EC3 minh h\u1ECDa c\xE1c t\xEDnh n\u0103ng b\u1ED5 sung n\xE0y, ch\xFAng ta s\u1EBD kh\xE1m ph\xE1 c\xE1ch t\xE1i t\u1EA1o k\u1EBFt qu\u1EA3 c\u1EE7a "),B=l("code"),Q=e("token-classification"),M=e(" (m\xE0 ch\xFAng ta g\u1ECDi l\xE0 "),nn=l("code"),u=e("ner"),E=e(") v\xE0 "),fn=l("code"),zn=e("question-answering"),ln=e(" ch\xFAng ta g\u1EB7p ph\u1EA3i l\u1EA7n \u0111\u1EA7u ti\xEAn trong "),on=l("a"),yc=e("Ch\u01B0\u01A1ng 1"),Ec=e("."),Pa=r(),b(hs.$$.fragment),Sa=r(),An=l("p"),wc=e("Trong ph\u1EA7n th\u1EA3o lu\u1EADn k\u1EBF ti\u1EBFp, ch\xFAng ta s\u1EBD ph\xE2n bi\u1EC7t gi\u1EEFa c\xE1c lo\u1EA1i tokenizer \u201Cch\u1EADm\u201D v\xE0 \u201Cnhanh\u201D. Phi\xEAn b\u1EA3n ch\u1EADm l\xE0 nh\u1EEFng phi\xEAn b\u1EA3n \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng Python b\xEAn trong th\u01B0 vi\u1EC7n \u{1F917} Transformers, trong khi phi\xEAn b\u1EA3n nhanh l\xE0 nh\u1EEFng phi\xEAn b\u1EA3n \u0111\u01B0\u1EE3c cung c\u1EA5p b\u1EDFi \u{1F917} Tokenizers, \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng Rust. N\u1EBFu b\u1EA1n nh\u1EDB b\u1EA3ng t\u1EEB "),nt=l("a"),Cc=e("Ch\u01B0\u01A1ng 5"),Oc=e(" b\xE1o c\xE1o kho\u1EA3ng th\u1EDDi gian tokenize nhanh v\xE0 ch\u1EADm c\u1EA7n \u0111\u1EC3 tokenize Drug Review Dataset, b\u1EA1n n\xEAn bi\u1EBFt l\xFD do t\u1EA1i sao ch\xFAng t\xF4i g\u1ECDi ch\xFAng l\xE0 nhanh v\xE0 ch\u1EADm :"),Ba=r(),Nn=l("table"),Bt=l("thead"),In=l("tr"),st=l("th"),zc=e("Tokenizer nhanh"),Ic=r(),tt=l("th"),Tc=e("Tokenizer ch\u1EADm"),Dc=r(),Rt=l("th"),qc=r(),is=l("tbody"),Tn=l("tr"),et=l("td"),Ft=l("code"),Pc=e("batched=True"),Sc=r(),at=l("td"),Bc=e("10.8s"),Rc=r(),lt=l("td"),Fc=e("4min41s"),Ac=r(),Dn=l("tr"),ct=l("td"),At=l("code"),Nc=e("batched=False"),Mc=r(),ht=l("td"),Xc=e("59.2s"),Hc=r(),it=l("td"),Gc=e("5min3s"),Ra=r(),b(Mn.$$.fragment),Fa=r(),qn=l("h2"),Xn=l("a"),Nt=l("span"),b(os.$$.fragment),Lc=r(),Mt=l("span"),Vc=e("M\xE3 ho\xE1 theo l\xF4"),Aa=r(),b(rs.$$.fragment),Na=r(),Hn=l("p"),Kc=e("\u0110\u1EA7u ra c\u1EE7a tokenizer kh\xF4ng ph\u1EA3i l\xE0 m\u1ED9t t\u1EEB \u0111i\u1EC3n Python \u0111\u01A1n gi\u1EA3n; nh\u1EEFng g\xEC ch\xFAng ta nh\u1EADn \u0111\u01B0\u1EE3c th\u1EF1c s\u1EF1 l\xE0 m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng "),Xt=l("code"),Yc=e("BatchEncoding"),Qc=e(" \u0111\u1EB7c bi\u1EC7t. \u0110\xF3 l\xE0 m\u1ED9t l\u1EDBp con c\u1EE7a t\u1EEB \u0111i\u1EC3n (\u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao tr\u01B0\u1EDBc \u0111\xE2y ch\xFAng ta c\xF3 th\u1EC3 l\u1EADp ch\u1EC9 m\u1EE5c v\xE0o k\u1EBFt qu\u1EA3 \u0111\xF3 m\xE0 kh\xF4ng g\u1EB7p b\u1EA5t k\u1EF3 v\u1EA5n \u0111\u1EC1 g\xEC), nh\u01B0ng v\u1EDBi c\xE1c ph\u01B0\u01A1ng th\u1EE9c b\u1ED5 sung h\u1EA7u h\u1EBFt \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi c\xE1c tr\xECnh tokenize nhanh."),Ma=r(),$n=l("p"),Uc=e("B\xEAn c\u1EA1nh kh\u1EA3 n\u0103ng song song h\xF3a c\u1EE7a ch\xFAng, ch\u1EE9c n\u0103ng ch\xEDnh c\u1EE7a c\xE1c tr\xECnh tokenize nhanh l\xE0 ch\xFAng lu\xF4n theo d\xF5i kho\u1EA3ng v\u0103n b\u1EA3n ban \u0111\u1EA7u m\xE0 ta tokenize - m\u1ED9t t\xEDnh n\u0103ng \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),Ht=l("em"),Jc=e("offset mapping"),Wc=e(" hay "),Gt=l("em"),Zc=e("\xE1nh x\u1EA1 b\xF9 tr\u1EEB"),nh=e(". \u0110i\u1EC1u n\xE0y l\u1EA7n l\u01B0\u1EE3t m\u1EDF kh\xF3a c\xE1c t\xEDnh n\u0103ng nh\u01B0 \xE1nh x\u1EA1 t\u1EEBng t\u1EEB v\u1EDBi token m\xE0 n\xF3 t\u1EA1o ra ho\u1EB7c \xE1nh x\u1EA1 t\u1EEBng k\xFD t\u1EF1 c\u1EE7a v\u0103n b\u1EA3n g\u1ED1c v\u1EDBi token b\xEAn trong v\xE0 ng\u01B0\u1EE3c l\u1EA1i."),Xa=r(),ot=l("p"),sh=e("C\xF9ng xem m\u1ED9t m\u1EABu:"),Ha=r(),b(ps.$$.fragment),Ga=r(),Gn=l("p"),th=e("Nh\u01B0 \u0111\xE3 \u0111\u1EC1 c\u1EADp tr\u01B0\u1EDBc \u0111\xE2y, ch\xFAng ta nh\u1EADn \u0111\u01B0\u1EE3c m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng "),Lt=l("code"),eh=e("BatchEncoding"),ah=e(" trong \u0111\u1EA7u ra c\u1EE7a tr\xECnh tokenize:"),La=r(),b(gs.$$.fragment),Va=r(),sn=l("p"),lh=e("V\xEC l\u1EDBp "),Vt=l("code"),ch=e("AutoTokenizer"),hh=e(" ch\u1ECDn m\u1ED9t tr\xECnh tokenizer nhanh theo m\u1EB7c \u0111\u1ECBnh, ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng c\xE1c ph\u01B0\u01A1ng th\u1EE9c b\u1ED5 sung m\xE0 \u0111\u1ED1i t\u01B0\u1EE3ng "),Kt=l("code"),ih=e("BatchEncoding"),oh=e(" n\xE0y cung c\u1EA5p. Ch\xFAng ta c\xF3 hai c\xE1ch \u0111\u1EC3 ki\u1EC3m tra xem tr\xECnh tokenize l\xE0 nhanh hay ch\u1EADm. Ch\xFAng ta c\xF3 th\u1EC3 ki\u1EC3m tra b\u1EB1ng thu\u1ED9c t\xEDnh "),Yt=l("code"),rh=e("is_fast"),ph=e(" c\u1EE7a "),Qt=l("code"),gh=e("tokenizer"),mh=e(":"),Ka=r(),b(ms.$$.fragment),Ya=r(),b(us.$$.fragment),Qa=r(),Ln=l("p"),uh=e("ho\u1EB7c ki\u1EC3m tra c\xF9ng thu\u1ED9c t\xEDnh \u0111\xF3 c\u1EE7a "),Ut=l("code"),dh=e("encoding"),fh=e(":"),Ua=r(),b(ds.$$.fragment),Ja=r(),b(fs.$$.fragment),Wa=r(),rt=l("p"),xh=e("H\xE3y xem nh\u1EEFng g\xEC m\u1ED9t tokenizer nhanh cho ph\xE9p ch\xFAng ta l\xE0m. \u0110\u1EA7u ti\xEAn, ch\xFAng t\xF4i c\xF3 th\u1EC3 truy c\u1EADp token m\xE0 kh\xF4ng c\u1EA7n ph\u1EA3i chuy\u1EC3n \u0111\u1ED5i ID tr\u1EDF l\u1EA1i token:"),Za=r(),b(xs.$$.fragment),nl=r(),b(bs.$$.fragment),sl=r(),yn=l("p"),bh=e("Trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, token \u1EDF ch\u1EC9 m\u1EE5c 5 l\xE0 "),Jt=l("code"),kh=e("##yl"),vh=e(", l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a t\u1EEB \u201CSylvain\u201D trong c\xE2u g\u1ED1c. Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Wt=l("code"),jh=e("word_ids()"),_h=e(" \u0111\u1EC3 l\u1EA5y ch\u1EC9 m\u1EE5c c\u1EE7a t\u1EEB m\xE0 m\u1ED7i token \u0111\u1EBFn t\u1EEB:"),tl=r(),b(ks.$$.fragment),el=r(),b(vs.$$.fragment),al=r(),U=l("p"),$h=e("Ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y r\u1EB1ng c\xE1c token \u0111\u1EB7c bi\u1EC7t c\u1EE7a tr\xECnh tokenize nh\u01B0 "),Zt=l("code"),yh=e("[CLS]"),Eh=e(" v\xE0 "),ne=l("code"),wh=e("[SEP]"),Ch=e(" \u0111\u01B0\u1EE3c \xE1nh x\u1EA1 th\xE0nh "),se=l("code"),Oh=e("None"),zh=e(", v\xE0 sau \u0111\xF3 m\u1ED7i token \u0111\u01B0\u1EE3c \xE1nh x\u1EA1 t\u1EDBi t\u1EEB m\xE0 n\xF3 b\u1EAFt ngu\u1ED3n. \u0110i\u1EC1u n\xE0y \u0111\u1EB7c bi\u1EC7t h\u1EEFu \xEDch \u0111\u1EC3 x\xE1c \u0111\u1ECBnh xem m\u1ED9t token n\u1EB1m \u1EDF \u0111\u1EA7u m\u1ED9t t\u1EEB hay n\u1EBFu hai token c\xF3 trong c\xF9ng thu\u1ED9c m\u1ED9t t\u1EEB. Ch\xFAng ta c\xF3 th\u1EC3 d\u1EF1a v\xE0o ti\u1EC1n t\u1ED1 "),te=l("code"),Ih=e("##"),Th=e(" cho \u0111i\u1EC1u \u0111\xF3, nh\u01B0ng n\xF3 ch\u1EC9 ho\u1EA1t \u0111\u1ED9ng \u0111\u1ED1i v\u1EDBi c\xE1c tokenize ki\u1EC3u BERT; ph\u01B0\u01A1ng ph\xE1p n\xE0y ho\u1EA1t \u0111\u1ED9ng v\u1EDBi b\u1EA5t k\u1EF3 lo\u1EA1i tokenizer n\xE0o mi\u1EC5n n\xF3 l\xE0 phi\xEAn b\u1EA3n nhanh. Trong ch\u01B0\u01A1ng ti\u1EBFp theo, ch\xFAng ta s\u1EBD xem c\xE1ch ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng kh\u1EA3 n\u0103ng n\xE0y \u0111\u1EC3 \xE1p d\u1EE5ng nh\xE3n ch\xFAng ta c\xF3 cho m\u1ED7i t\u1EEB \u0111\xFAng c\xE1ch v\u1EDBi c\xE1c token trong c\xE1c t\xE1c v\u1EE5 nh\u01B0 nh\u1EADn d\u1EA1ng th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c \u0111\u1EB7t t\xEAn (NER) v\xE0  g\xE1n nh\xE3n t\u1EEB lo\u1EA1i (POS). Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng n\xF3 \u0111\u1EC3 che gi\u1EA5u t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EBFn t\u1EEB c\xF9ng m\u1ED9t t\u1EEB trong m\xF4 h\xECnh ng\xF4n ng\u1EEF \u0111\u01B0\u1EE3c che (m\u1ED9t k\u1EF9 thu\u1EADt \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),ee=l("em"),Dh=e("whole word masking"),qh=e(")."),ll=r(),b(Vn.$$.fragment),cl=r(),En=l("p"),Ph=e("T\u01B0\u01A1ng t\u1EF1, c\xF3 m\u1ED9t ph\u01B0\u01A1ng th\u1EE9c "),ae=l("code"),Sh=e("question_ids()"),Bh=e(" m\xE0 ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 \xE1nh x\u1EA1 token \u0111\u1EBFn c\xE2u m\xE0 n\xF3 b\u1EAFt ngu\u1ED3n (m\u1EB7c d\xF9 trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, "),le=l("code"),Rh=e("token_type_ids"),Fh=e(" \u0111\u01B0\u1EE3c tr\u1EA3 v\u1EC1 b\u1EDFi tokenizer c\xF3 th\u1EC3 cung c\u1EA5p cho ch\xFAng ta c\xF9ng m\u1ED9t th\xF4ng tin)."),hl=r(),G=l("p"),Ah=e("Cu\u1ED1i c\xF9ng, ch\xFAng ta c\xF3 th\u1EC3 \xE1nh x\u1EA1 b\u1EA5t k\u1EF3 t\u1EEB ho\u1EB7c token n\xE0o v\u1EDBi c\xE1c k\xFD t\u1EF1 trong v\u0103n b\u1EA3n g\u1ED1c v\xE0 ng\u01B0\u1EE3c l\u1EA1i, th\xF4ng qua c\xE1c ph\u01B0\u01A1ng th\u1EE9c "),ce=l("code"),Nh=e("word_to_chars()"),Mh=e(" ho\u1EB7c "),he=l("code"),Xh=e("token_to_chars()"),Hh=e(" v\xE0 "),ie=l("code"),Gh=e("char_to_word()"),Lh=e(" ho\u1EB7c "),oe=l("code"),Vh=e("char_to_token()"),Kh=e(". V\xED d\u1EE5: ph\u01B0\u01A1ng th\u1EE9c "),re=l("code"),Yh=e("word_ids()"),Qh=e(" cho ch\xFAng ta bi\u1EBFt r\u1EB1ng "),pe=l("code"),Uh=e("##yl"),Jh=e(" l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a t\u1EEB \u1EDF ch\u1EC9 m\u1EE5c 3, nh\u01B0ng t\u1EEB \u0111\xF3 n\u1EB1m trong c\xE2u n\xE0o? Ch\xFAng ta c\xF3 th\u1EC3 t\xECm hi\u1EC3u nh\u01B0 th\u1EBF n\xE0y:"),il=r(),b(js.$$.fragment),ol=r(),b(_s.$$.fragment),rl=r(),rn=l("p"),Wh=e("Nh\u01B0 \u0111\xE3 \u0111\u1EC1 c\u1EADp tr\u01B0\u1EDBc \u0111\xF3, t\u1EA5t c\u1EA3 \u0111i\u1EC1u n\xE0y th\u1EF1c t\u1EBF \u0111\u01B0\u1EE3c h\u1ED7 tr\u1EE3 b\u1EDFi l\xE0 tr\xECnh tokenizer nhanh k\u1EBFt h\u1EE3p kho\u1EA3ng v\u0103n b\u1EA3n m\xE0 m\u1ED7i token \u0111\u1EBFn t\u1EEB danh s\xE1ch "),ge=l("em"),Zh=e("offset"),ni=e(" hay "),me=l("em"),si=e("offset"),ti=e(". \u0110\u1EC3 minh h\u1ECDa vi\u1EC7c s\u1EED d\u1EE5ng ch\xFAng, ti\u1EBFp theo, ch\xFAng t\xF4i s\u1EBD h\u01B0\u1EDBng d\u1EABn b\u1EA1n c\xE1ch sao ch\xE9p c\xE1c k\u1EBFt qu\u1EA3 c\u1EE7a "),ue=l("code"),ei=e("token-classification"),ai=e(" theo c\xE1ch th\u1EE7 c\xF4ng."),pl=r(),b(Kn.$$.fragment),gl=r(),Pn=l("h2"),Yn=l("a"),de=l("span"),b($s.$$.fragment),li=r(),pt=l("span"),ci=e("B\xEAn trong pipeline "),fe=l("code"),hi=e("token-classification"),ml=r(),tn=l("p"),ii=e("Trong "),gt=l("a"),oi=e("Ch\u01B0\u01A1ng 1"),ri=e(", ch\xFAng ta l\u1EA7n \u0111\u1EA7u \u0111\u01B0\u1EE3c th\u1EED \xE1p d\u1EE5ng NER - t\xE1c v\u1EE5 x\xE1c \u0111\u1ECBnh nh\u1EEFng ph\u1EA7n n\xE0o c\u1EE7a v\u0103n b\u1EA3n t\u01B0\u01A1ng \u1EE9ng v\u1EDBi c\xE1c th\u1EF1c th\u1EC3 nh\u01B0 ng\u01B0\u1EDDi, \u0111\u1ECBa \u0111i\u1EC3m ho\u1EB7c t\u1ED5 ch\u1EE9c - v\u1EDBi "),xe=l("code"),pi=e("pipeline()"),gi=e(" c\u1EE7a \u{1F917} Transformers. Sau \u0111\xF3, trong "),mt=l("a"),mi=e("Ch\u01B0\u01A1ng 2"),ui=e(", ch\xFAng ta \u0111\xE3 th\u1EA5y c\xE1ch m\u1ED9t pipeline nh\xF3m ba giai \u0111o\u1EA1n c\u1EA7n thi\u1EBFt l\u1EA1i v\u1EDBi nhau \u0111\u1EC3 nh\u1EADn c\xE1c d\u1EF1 \u0111o\xE1n t\u1EEB m\u1ED9t v\u0103n b\u1EA3n th\xF4: tokenize, chuy\u1EC3n c\xE1c \u0111\u1EA7u v\xE0o qua m\xF4 h\xECnh v\xE0 h\u1EADu x\u1EED l\xFD. Hai b\u01B0\u1EDBc \u0111\u1EA7u ti\xEAn trong quy tr\xECnh "),be=l("code"),di=e("token-classification"),fi=e(" c\u0169ng gi\u1ED1ng nh\u01B0 trong b\u1EA5t k\u1EF3 quy tr\xECnh n\xE0o kh\xE1c, nh\u01B0ng qu\xE1 tr\xECnh h\u1EADu x\u1EED l\xFD ph\u1EE9c t\u1EA1p h\u01A1n m\u1ED9t ch\xFAt - h\xE3y c\xF9ng xem c\xE1ch th\u1EF1c hi\u1EC7n!"),ul=r(),bn.c(),ut=r(),Sn=l("h3"),Qn=l("a"),ke=l("span"),b(ys.$$.fragment),xi=r(),ve=l("span"),bi=e("Nh\u1EADn k\u1EBFt qu\u1EA3 c\u01A1 s\u1EDF v\u1EDBi baseline"),dl=r(),Un=l("p"),ki=e("Tr\u01B0\u1EDBc ti\xEAn, h\xE3y l\u1EA5y m\u1ED9t pipeline ph\xE2n lo\u1EA1i token ch\xFAng ta c\xF3 th\u1EC3 nh\u1EADn \u0111\u01B0\u1EE3c m\u1ED9t s\u1ED1 k\u1EBFt qu\u1EA3 \u0111\u1EC3 so s\xE1nh theo c\xE1ch th\u1EE7 c\xF4ng. M\xF4 h\xECnh \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng theo m\u1EB7c \u0111\u1ECBnh l\xE0 "),Es=l("a"),je=l("code"),vi=e("dbmdz/bert-large-cased-finetuned-conll03-english"),ji=e("; n\xF3 th\u1EF1c hi\u1EC7n NER tr\xEAn c\xE1c c\xE2u:"),fl=r(),b(ws.$$.fragment),xl=r(),b(Cs.$$.fragment),bl=r(),dt=l("p"),_i=e("M\xF4 h\xECnh \u0111\xE3 x\xE1c \u0111\u1ECBnh \u0111\xFAng m\u1ED7i token do \u201CSylvain\u201D t\u1EA1o ra l\xE0 m\u1ED9t ng\u01B0\u1EDDi, m\u1ED7i token \u0111\u01B0\u1EE3c t\u1EA1o b\u1EDFi \u201CHugging Face\u201D l\xE0 m\u1ED9t t\u1ED5 ch\u1EE9c v\xE0 token \u201CBrooklyn\u201D l\xE0 m\u1ED9t \u0111\u1ECBa \u0111i\u1EC3m. Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 y\xEAu c\u1EA7u pipeline nh\xF3m c\xE1c token t\u01B0\u01A1ng \u1EE9ng v\u1EDBi c\xF9ng m\u1ED9t th\u1EF1c th\u1EC3 l\u1EA1i v\u1EDBi nhau:"),kl=r(),b(Os.$$.fragment),vl=r(),b(zs.$$.fragment),jl=r(),Y=l("p"),_e=l("code"),$i=e("aggregation_strategy"),yi=e(" \u0111\u01B0\u1EE3c ch\u1ECDn s\u1EBD thay \u0111\u1ED5i \u0111i\u1EC3m \u0111\u01B0\u1EE3c t\xEDnh cho m\u1ED7i th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c nh\xF3m l\u1EA1i, V\u1EDBi "),$e=l("code"),Ei=e('"simple"'),wi=e(", \u0111i\u1EC3m ch\u1EC9 l\xE0 gi\xE1 tr\u1ECB trung b\xECnh c\u1EE7a \u0111i\u1EC3m c\u1EE7a m\u1ED7i token trong th\u1EF1c th\u1EC3 \u0111\xE3 cho: v\xED d\u1EE5: \u0111i\u1EC3m c\u1EE7a \u201CSylvain\u201D l\xE0 trung b\xECnh \u0111i\u1EC3m m\xE0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong v\xED d\u1EE5 tr\u01B0\u1EDBc cho c\xE1c token "),ye=l("code"),Ci=e("S"),Oi=e(" , "),Ee=l("code"),zi=e("##yl"),Ii=e(","),we=l("code"),Ti=e("## va"),Di=e(" v\xE0 "),Ce=l("code"),qi=e("##in"),Pi=e(". C\xE1c chi\u1EBFn l\u01B0\u1EE3c c\xF3 s\u1EB5n kh\xE1c l\xE0:"),_l=r(),wn=l("ul"),Jn=l("li"),Oe=l("code"),Si=e('"first"'),Bi=e(", trong \u0111\xF3 \u0111i\u1EC3m c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 \u0111i\u1EC3m c\u1EE7a token \u0111\u1EA7u ti\xEAn c\u1EE7a th\u1EF1c th\u1EC3 \u0111\xF3 (v\xEC v\u1EADy \u0111\u1ED1i v\u1EDBi \u201CSylvain\u201D, n\xF3 s\u1EBD l\xE0 0.993828, \u0111i\u1EC3m c\u1EE7a token "),ze=l("code"),Ri=e("S"),Fi=e(")"),Ai=r(),ft=l("li"),Ie=l("code"),Ni=e('"max"'),Mi=e(", trong \u0111\xF3 \u0111i\u1EC3m c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 \u0111i\u1EC3m t\u1ED1i \u0111a c\u1EE7a c\xE1c token trong th\u1EF1c th\u1EBF \u0111\xF3 (v\xEC v\u1EADy v\u1EDBi \u201CHugging Face\u201D s\u1EBD l\xE0 0.98879766, \u0111i\u1EC3m c\u1EE7a \u201CFace\u201D)"),Xi=r(),Wn=l("li"),Te=l("code"),Hi=e('"average"'),Gi=e(", trong \u0111\xF3 \u0111i\u1EC3m c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 \u0111i\u1EC3m trung b\xECnh c\u1EE7a c\xE1c t\u1EEB t\u1EA1o n\xEAn th\u1EF1c th\u1EC3 \u0111\xF3 (v\xED d\u1EE5 v\u1EDBi \u201CSylvain\u201D th\xEC ph\u01B0\u01A1ng ph\xE1p n\xE0y s\u1EBD kh\xF4ng c\xF3 s\u1EF1 kh\xE1c bi\u1EC7t so v\u1EDBi ph\u01B0\u01A1ng ph\xE1p "),De=l("code"),Li=e('"simple"'),Vi=e(", nh\u01B0ng v\u1EDBi \u201CHugging Face\u201D, \u0111i\u1EC3m tr\u1EA3 v\u1EC1 s\u1EBD l\xE0 0.9819, \u0111i\u1EC3m trung b\xECnh c\u1EE7a \u201CHugging\u201D, 0.975, v\xE0 \u201CFace\u201D, 0.98879)"),$l=r(),Zn=l("p"),Ki=e("Gi\u1EDD ch\xFAng ta h\xE3y xem l\xE0m th\u1EBF n\xE0o \u0111\u1EC3 c\xF3 \u0111\u01B0\u1EE3c nh\u1EEFng k\u1EBFt qu\u1EA3 n\xE0y m\xE0 kh\xF4ng c\u1EA7n s\u1EED d\u1EE5ng h\xE0m"),qe=l("code"),Yi=e("pipeline()"),Qi=e("!"),yl=r(),Bn=l("h3"),ns=l("a"),Pe=l("span"),b(Is.$$.fragment),Ui=r(),Se=l("span"),Ji=e("T\u1EEB \u0111\u1EA7u v\xE0o t\u1EDBi d\u1EF1 \u0111o\xE1n"),El=r(),vn.c(),xt=r(),bt=l("p"),Wi=e("Ch\xFAng ta c\xF3 m\u1ED9t l\xF4 v\u1EDBi 1 chu\u1ED7i g\u1ED3m 19 token v\xE0 m\xF4 h\xECnh c\xF3 9 nh\xE3n kh\xE1c nhau, v\xEC v\u1EADy \u0111\u1EA7u ra c\u1EE7a m\xF4 h\xECnh c\xF3 h\xECnh d\u1EA1ng 1 x 19 x 9. Gi\u1ED1ng nh\u01B0 \u0111\u1ED1i v\u1EDBi pipeline ph\xE2n lo\u1EA1i v\u0103n b\u1EA3n, ch\xFAng ta s\u1EED d\u1EE5ng h\xE0m softmax \u0111\u1EC3 chuy\u1EC3n \u0111\u1ED5i c\xE1c logits \u0111\xF3 theo x\xE1c su\u1EA5t, v\xE0 ch\xFAng ta l\u1EA5y argmax \u0111\u1EC3 nh\u1EADn d\u1EF1 \u0111o\xE1n (l\u01B0u \xFD r\u1EB1ng ta c\xF3 th\u1EC3 l\u1EA5y argmax tr\xEAn logits v\xEC softmax kh\xF4ng thay \u0111\u1ED5i th\u1EE9 t\u1EF1):"),wl=r(),_n.c(),kt=r(),b(Ts.$$.fragment),Cl=r(),ss=l("p"),Zi=e("Thu\u1ED9c t\xEDnh "),Be=l("code"),no=e("model.config.id2label"),so=e(" ch\u1EE9a \xE1nh x\u1EA1 c\xE1c ch\u1EC9 m\u1EE5c t\u1EDBi c\xE1c nh\xE3n m\xE0 ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 hi\u1EC3u c\xE1c d\u1EF1 \u0111o\xE1n:"),Ol=r(),b(Ds.$$.fragment),zl=r(),b(qs.$$.fragment),Il=r(),z=l("p"),to=e("Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, c\xF3 9 nh\xE3n: "),Re=l("code"),eo=e("O"),ao=e(" l\xE0 nh\xE3n cho c\xE1c token kh\xF4ng n\u1EB1m trong b\u1EA5t k\u1EF3 th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c \u0111\u1EB7t t\xEAn n\xE0o (n\xF3 l\xE0 vi\u1EBFt t\u1EAFt c\u1EE7a \u201Coutside\u201D hay \u201Cb\xEAn ngo\xE0i\u201D) v\xE0 sau \u0111\xF3 ch\xFAng ta c\xF3 hai nh\xE3n cho m\u1ED7i lo\u1EA1i th\u1EF1c th\u1EC3 (linh tinh, ng\u01B0\u1EDDi , t\u1ED5 ch\u1EE9c v\xE0 v\u1ECB tr\xED). Nh\xE3n "),Fe=l("code"),lo=e("B-XXX"),co=e(" cho bi\u1EBFt token n\u1EB1m \u1EDF \u0111\u1EA7u th\u1EF1c th\u1EC3 "),Ae=l("code"),ho=e("XXX"),io=e(" v\xE0 nh\xE3n "),Ne=l("code"),oo=e("I-XXX"),ro=e(" cho bi\u1EBFt token n\u1EB1m b\xEAn trong th\u1EF1c th\u1EC3 "),Me=l("code"),po=e("XXX"),go=e(". V\xED d\u1EE5: trong m\u1EABu hi\u1EC7n t\u1EA1i, ch\xFAng ta k\xEC v\u1ECDng m\xF4 h\xECnh ph\xE2n lo\u1EA1i token "),Xe=l("code"),mo=e("S"),uo=e(" l\xE0 "),He=l("code"),fo=e("B-PER"),xo=e(" (b\u1EAFt \u0111\u1EA7u c\u1EE7a m\u1ED9t th\u1EF1c th\u1EC3 ng\u01B0\u1EDDi) v\xE0 c\xE1c token "),Ge=l("code"),bo=e("##yl"),ko=e(","),Le=l("code"),vo=e("##va"),jo=e(" v\xE0 "),Ve=l("code"),_o=e("##in"),$o=e(" l\xE0 "),Ke=l("code"),yo=e("I-PER"),Eo=e(" (b\xEAn trong m\u1ED9t th\u1EF1c th\u1EC3 ng\u01B0\u1EDDi)."),Tl=r(),X=l("p"),wo=e("B\u1EA1n c\xF3 th\u1EC3 ngh\u0129 r\u1EB1ng m\xF4 h\xECnh \u0111\xE3 sai trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y v\xEC n\xF3 \u0111\xE3 g\u1EAFn nh\xE3n "),Ye=l("code"),Co=e("I-PER"),Oo=e(" cho c\u1EA3 b\u1ED1n token n\xE0y, nh\u01B0ng \u0111i\u1EC1u \u0111\xF3 kh\xF4ng ho\xE0n to\xE0n \u0111\xFAng. Th\u1EF1c t\u1EBF c\xF3 hai \u0111\u1ECBnh d\u1EA1ng cho c\xE1c nh\xE3n "),Qe=l("code"),zo=e("B-"),Io=e(" v\xE0 "),Ue=l("code"),To=e("I-"),Do=e(" \u0111\xF3 l\xE0: "),Je=l("em"),qo=e("IOB1"),Po=e(" v\xE0 "),We=l("em"),So=e("IOB2"),Bo=e(". \u0110\u1ECBnh d\u1EA1ng IOB2 (m\xE0u h\u1ED3ng b\xEAn d\u01B0\u1EDBi), l\xE0 \u0111\u1ECBnh d\u1EA1ng ch\xFAng ta \u0111\xE3 gi\u1EDBi thi\u1EC7u trong khi \u1EDF \u0111\u1ECBnh d\u1EA1ng IOB1 (m\xE0u xanh lam), c\xE1c nh\xE3n b\u1EAFt \u0111\u1EA7u b\u1EB1ng "),Ze=l("code"),Ro=e("B-"),Fo=e(" ch\u1EC9 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 ph\xE2n t\xE1ch hai th\u1EF1c th\u1EC3 li\u1EC1n k\u1EC1 c\xF9ng lo\u1EA1i. M\xF4 h\xECnh ch\xFAng t\xF4i \u0111ang s\u1EED d\u1EE5ng \u0111\xE3 \u0111\u01B0\u1EE3c tinh ch\u1EC9nh tr\xEAn t\u1EADp d\u1EEF li\u1EC7u b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng \u0111\u1ECBnh d\u1EA1ng \u0111\xF3, \u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao n\xF3 g\xE1n nh\xE3n "),na=l("code"),Ao=e("I-PER"),No=e(" cho m\xE3 th\xF4ng b\xE1o "),sa=l("code"),Mo=e("S"),Xo=e("."),Dl=r(),Rn=l("div"),Ps=l("img"),Ho=r(),Ss=l("img"),ql=r(),ts=l("p"),Go=e("V\u1EDBi ph\xE9p \xE1nh x\u1EA1 n\xE0y, ch\xFAng ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC1 t\xE1i t\u1EA1o l\u1EA1i (g\u1EA7n nh\u01B0 ho\xE0n to\xE0n) k\u1EBFt qu\u1EA3 c\u1EE7a pipeline \u0111\u1EA7u \u2014 ta ch\u1EC9 c\u1EA7n l\u1EA5y \u0111i\u1EC3m v\xE0 nh\xE3n c\u1EE7a m\u1ED7i token m\xE0 kh\xF4ng \u0111\u01B0\u1EE3c ph\xE2n v\xE0o "),ta=l("code"),Lo=e("O"),Vo=e(":"),Pl=r(),b(Bs.$$.fragment),Sl=r(),b(Rs.$$.fragment),Bl=r(),J=l("p"),Ko=e("\u0110i\u1EC1u n\xE0y r\u1EA5t gi\u1ED1ng v\u1EDBi nh\u1EEFng g\xEC ta \u0111\xE3 c\xF3 tr\u01B0\u1EDBc \u0111\xE2y, ngo\u1EA1i tr\u1EEB m\u1ED9t ngo\u1EA1i l\u1EC7: pipeline c\u0169ng cung c\u1EA5p th\xF4ng tin v\u1EC1 \u0111i\u1EC3m "),ea=l("code"),Yo=e("start"),Qo=e(" hay "),aa=l("code"),Uo=e("b\u1EAFt \u0111\u1EA7u"),Jo=e(" v\xE0 "),la=l("code"),Wo=e("end"),Zo=e("  hay "),ca=l("code"),nr=e("k\u1EBFt th\xFAc"),sr=e(" c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 trong c\xE2u g\u1ED1c. \u0110\xE2y l\xE0 l\xFAc \xE1nh x\u1EA1 b\xF9 tr\u1EEB c\u1EE7a ch\xFAng ta s\u1EBD ph\xE1t huy t\xE1c d\u1EE5ng. \u0110\u1EC3 c\xF3 \u0111\u01B0\u1EE3c offset, ch\xFAng ta ch\u1EC9 c\u1EA7n \u0111\u1EB7t "),ha=l("code"),tr=e("return_offsets_mapping=True"),er=e(" khi ch\xFAng ta \xE1p d\u1EE5ng tokenizer cho c\xE1c \u0111\u1EA7u v\xE0o c\u1EE7a m\xECnh:"),Rl=r(),b(Fs.$$.fragment),Fl=r(),b(As.$$.fragment),Al=r(),pn=l("p"),ar=e("M\u1ED7i tuple l\xE0 kho\u1EA3ng v\u0103n b\u1EA3n t\u01B0\u01A1ng \u1EE9ng v\u1EDBi m\u1ED7i token, trong \u0111\xF3 "),ia=l("code"),lr=e("(0, 0)"),cr=e(" \u0111\u01B0\u1EE3c d\xE0nh ri\xEAng cho c\xE1c token \u0111\u1EB7c bi\u1EC7t. Tr\u01B0\u1EDBc \u0111\xE2y, ch\xFAng ta \u0111\xE3 th\u1EA5y r\u1EB1ng token \u1EDF ch\u1EC9 m\u1EE5c 5 l\xE0 "),oa=l("code"),hr=e("##yl"),ir=e(", c\xF3 "),ra=l("code"),or=e("(12, 14)"),rr=e(" l\xE0 c\xE1c ph\u1EA7n b\xF9 \u1EDF \u0111\xE2y. N\u1EBFu ch\xFAng ta l\u1EA5y ph\u1EA7n t\u01B0\u01A1ng \u1EE9ng trong m\u1EABu c\u1EE7a m\xECnh:"),Nl=r(),b(Ns.$$.fragment),Ml=r(),es=l("p"),pr=e("ta nh\u1EADn \u0111\u01B0\u1EE3c kho\u1EA3ng v\u0103n b\u1EA3n th\xEDch h\u1EE3p m\xE0 kh\xF4ng c\xF3 "),pa=l("code"),gr=e("##"),mr=e(":"),Xl=r(),b(Ms.$$.fragment),Hl=r(),vt=l("p"),ur=e("S\u1EED d\u1EE5ng \u0111i\u1EC1u n\xE0y, b\xE2y gi\u1EDD ch\xFAng ta c\xF3 th\u1EC3 ho\xE0n th\xE0nh c\xE1c k\u1EBFt qu\u1EA3 tr\u01B0\u1EDBc \u0111\xF3:"),Gl=r(),b(Xs.$$.fragment),Ll=r(),b(Hs.$$.fragment),Vl=r(),jt=l("p"),dr=e("\u0110\xE2y gi\u1ED1ng nh\u01B0 nh\u1EEFng g\xEC ch\xFAng ta c\xF3 \u0111\u01B0\u1EE3c t\u1EEB pipeline \u0111\u1EA7u ti\xEAn!"),Kl=r(),Fn=l("h3"),as=l("a"),ga=l("span"),b(Gs.$$.fragment),fr=r(),ma=l("span"),xr=e("Nh\xF3m c\xE1c th\u1EF1c th\u1EC3"),Yl=r(),L=l("p"),br=e("S\u1EED d\u1EE5ng c\xE1c offset \u0111\u1EC3 x\xE1c \u0111\u1ECBnh \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u v\xE0 k\u1EBFt th\xFAc cho m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 r\u1EA5t ti\u1EC7n d\u1EE5ng, nh\u01B0ng th\xF4ng tin \u0111\xF3 kh\xF4ng ho\xE0n to\xE0n c\u1EA7n thi\u1EBFt. Tuy nhi\xEAn, khi ch\xFAng ta mu\u1ED1n nh\xF3m c\xE1c th\u1EF1c th\u1EC3 l\u1EA1i v\u1EDBi nhau, vi\u1EC7c offset s\u1EBD gi\xFAp ch\xFAng ta ti\u1EBFt ki\u1EC7m r\u1EA5t nhi\u1EC1u \u0111o\u1EA1n m\xE3 l\u1ED9n x\u1ED9n. V\xED d\u1EE5: n\u1EBFu ch\xFAng ta mu\u1ED1n nh\xF3m c\xE1c token "),ua=l("code"),kr=e("Hu"),vr=e(", "),da=l("code"),jr=e("##gging"),_r=e(" v\xE0 "),fa=l("code"),$r=e("Face"),yr=e(" l\u1EA1i v\u1EDBi nhau, ch\xFAng ta c\xF3 th\u1EC3 \u0111\u01B0a ra c\xE1c quy t\u1EAFc \u0111\u1EB7c bi\u1EC7t n\xF3i r\u1EB1ng hai token \u0111\u1EA7u ti\xEAn ph\u1EA3i \u0111\u01B0\u1EE3c \u0111\xEDnh k\xE8m trong khi x\xF3a d\u1EA5u "),xa=l("code"),Er=e("##"),wr=e(" v\xE0 "),ba=l("code"),Cr=e("Face"),Or=e(" n\xEAn \u0111\u01B0\u1EE3c th\xEAm m\u1ED9t kho\u1EA3ng tr\u1EAFng v\xEC n\xF3 kh\xF4ng b\u1EAFt \u0111\u1EA7u b\u1EB1ng "),ka=l("code"),zr=e("##"),Ir=e(" - nh\u01B0ng \u0111i\u1EC1u \u0111\xF3 s\u1EBD ch\u1EC9 ho\u1EA1t \u0111\u1ED9ng \u0111\u1ED1i v\u1EDBi lo\u1EA1i tokenizer c\u1EE5 th\u1EC3 n\xE0y. Ch\xFAng ta s\u1EBD ph\u1EA3i vi\u1EBFt m\u1ED9t b\u1ED9 quy t\u1EAFc kh\xE1c cho SentencePiece ho\u1EB7c Byte-Pair-Encoding (s\u1EBD \u0111\u01B0\u1EE3c th\u1EA3o lu\u1EADn \u1EDF ph\u1EA7n sau c\u1EE7a ch\u01B0\u01A1ng n\xE0y)."),Ql=r(),W=l("p"),Tr=e("V\u1EDBi offset, t\u1EA5t c\u1EA3 m\xE3 t\xF9y ch\u1EC9nh \u0111\xF3 s\u1EBD bi\u1EBFn m\u1EA5t: ch\xFAng ta ch\u1EC9 c\xF3 th\u1EC3 l\u1EA5y kho\u1EA3ng trong v\u0103n b\u1EA3n g\u1ED1c b\u1EAFt \u0111\u1EA7u b\u1EB1ng token \u0111\u1EA7u ti\xEAn v\xE0 k\u1EBFt th\xFAc b\u1EB1ng token cu\u1ED1i c\xF9ng. V\xEC v\u1EADy, trong tr\u01B0\u1EDDng h\u1EE3p c\xE1c m\xE3 th\xF4ng b\xE1o "),va=l("code"),Dr=e("Hu"),qr=e(", "),ja=l("code"),Pr=e("##gging"),Sr=e(" v\xE0 "),_a=l("code"),Br=e("Face"),Rr=e(", ch\xFAng ta n\xEAn b\u1EAFt \u0111\u1EA7u \u1EDF k\xFD t\u1EF1 33 (\u0111\u1EA7u c\u1EE7a "),$a=l("code"),Fr=e("Hu"),Ar=e(") v\xE0 k\u1EBFt th\xFAc tr\u01B0\u1EDBc k\xFD t\u1EF1 45 (cu\u1ED1i c\u1EE7a "),ya=l("code"),Nr=e("Face"),Mr=e(") :"),Ul=r(),b(Ls.$$.fragment),Jl=r(),b(Vs.$$.fragment),Wl=r(),Z=l("p"),Xr=e("\u0110\u1EC3 vi\u1EBFt \u0111o\u1EA1n m\xE3 h\u1EADu x\u1EED l\xFD c\xE1c d\u1EF1 \u0111o\xE1n trong khi nh\xF3m c\xE1c th\u1EF1c th\u1EC3, ta s\u1EBD nh\xF3m c\xE1c th\u1EF1c th\u1EC3 li\xEAn ti\u1EBFp v\xE0 c\xF3 nh\xE3n "),Ea=l("code"),Hr=e("I-XXX"),Gr=e(" v\u1EDBi nhau tr\u1EEB khi n\xF3 l\xE0 t\u1EEB \u0111\u1EA7u ti\xEAn, \u0111\u01B0\u1EE3c g\xE1n nh\xE3n "),wa=l("code"),Lr=e("B-XXX"),Vr=e(" ho\u1EB7c "),Ca=l("code"),Kr=e("I-XXX"),Yr=e(" (\u0111\u1EC3 ta c\xF3 th\u1EC3 d\u1EEBng nh\xF3m m\u1ED9t th\u1EF1c th\u1EC3 khi nh\u1EADn \u0111\u01B0\u1EE3c "),Oa=l("code"),Qr=e("O"),Ur=e(", m\u1ED9t ki\u1EC3u th\u1EF1c th\u1EC3 m\u1EDBi, ho\u1EB7c m\u1ED9t "),za=l("code"),Jr=e("B-XXX"),Wr=e(" cho ta bi\u1EBFt th\u1EF1c th\u1EC3 c\xF3 ki\u1EC3u gi\u1ED1ng v\u1EDBi \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u):"),Zl=r(),b(Ks.$$.fragment),nc=r(),_t=l("p"),Zr=e("V\xE0 ch\xFAng ta nh\u1EADn \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 t\u01B0\u01A1ng t\u1EF1 nh\u01B0 v\u1EDBi pipeline th\u1EE9 hai c\u1EE7a m\xECnh!"),sc=r(),b(Ys.$$.fragment),tc=r(),$t=l("p"),np=e("M\u1ED9t v\xED d\u1EE5 kh\xE1c v\u1EC1 t\xE1c v\u1EE5 m\xE0 nh\u1EEFng offset n\xE0y c\u1EF1c k\u1EF3 h\u1EEFu \xEDch l\xE0 h\u1ECFi \u0111\xE1p. \u0110\xE0o s\xE2u v\xE0o pipeline n\xE0y, ch\xFAng ta s\u1EBD th\u1EF1c hi\u1EC7n trong ph\u1EA7n ti\u1EBFp theo, c\u0169ng s\u1EBD cho ph\xE9p ch\xFAng ta xem x\xE9t m\u1ED9t t\xEDnh n\u0103ng cu\u1ED1i c\xF9ng c\u1EE7a c\xE1c tokenizers trong th\u01B0 vi\u1EC7n \u{1F917} Transformers: x\u1EED l\xFD c\xE1c token tr\xE0n khi ch\xFAng ta c\u1EAFt b\u1EDBt m\u1ED9t \u0111\u1EA7u v\xE0o \u0111\u1EBFn m\u1ED9t \u0111\u1ED9 d\xE0i nh\u1EA5t \u0111\u1ECBnh."),this.h()},l(n){const i=Fm('[data-svelte="svelte-1phssyn"]',document.head);g=c(i,"META",{name:!0,content:!0}),i.forEach(t),x=p(n),k(m.$$.fragment,n),$=p(n),q=c(n,"H1",{class:!0});var Zs=h(q);w=c(Zs,"A",{id:!0,class:!0,href:!0});var yt=h(w);S=c(yt,"SPAN",{});var Ia=h(S);k(A.$$.fragment,Ia),Ia.forEach(t),yt.forEach(t),H=p(Zs),P=c(Zs,"SPAN",{});var Ta=h(P);K=a(Ta,"S\u1EE9c m\u1EA1nh \u0111\u1EB7c bi\u1EC7t c\u1EE7a tokenizer nhanh"),Ta.forEach(t),Zs.forEach(t),O=p(n),I.l(n),T=p(n),C=c(n,"P",{});var cn=h(C);N=a(cn,"Trong ph\u1EA7n n\xE0y, ch\xFAng ta s\u1EBD xem x\xE9t k\u1EF9 h\u01A1n c\xE1c kh\u1EA3 n\u0103ng c\u1EE7a tokenizer trong \u{1F917} Transformers. Cho \u0111\u1EBFn nay, ch\xFAng ta ch\u1EC9 s\u1EED d\u1EE5ng ch\xFAng \u0111\u1EC3 tokenize \u0111\u1EA7u v\xE0o ho\u1EB7c gi\u1EA3i m\xE3 ID tr\u1EDF l\u1EA1i th\xE0nh v\u0103n b\u1EA3n, nh\u01B0ng c\xE1c tr\xECnh tokenize - \u0111\u1EB7c bi\u1EC7t l\xE0 nh\u1EEFng tr\xECnh tokenize \u0111\u01B0\u1EE3c h\u1ED7 tr\u1EE3 b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers - c\xF3 th\u1EC3 l\xE0m \u0111\u01B0\u1EE3c nhi\u1EC1u h\u01A1n th\u1EBF. \u0110\u1EC3 minh h\u1ECDa c\xE1c t\xEDnh n\u0103ng b\u1ED5 sung n\xE0y, ch\xFAng ta s\u1EBD kh\xE1m ph\xE1 c\xE1ch t\xE1i t\u1EA1o k\u1EBFt qu\u1EA3 c\u1EE7a "),B=c(cn,"CODE",{});var Et=h(B);Q=a(Et,"token-classification"),Et.forEach(t),M=a(cn," (m\xE0 ch\xFAng ta g\u1ECDi l\xE0 "),nn=c(cn,"CODE",{});var wt=h(nn);u=a(wt,"ner"),wt.forEach(t),E=a(cn,") v\xE0 "),fn=c(cn,"CODE",{});var Ct=h(fn);zn=a(Ct,"question-answering"),Ct.forEach(t),ln=a(cn," ch\xFAng ta g\u1EB7p ph\u1EA3i l\u1EA7n \u0111\u1EA7u ti\xEAn trong "),on=c(cn,"A",{href:!0});var up=h(on);yc=a(up,"Ch\u01B0\u01A1ng 1"),up.forEach(t),Ec=a(cn,"."),cn.forEach(t),Pa=p(n),k(hs.$$.fragment,n),Sa=p(n),An=c(n,"P",{});var ac=h(An);wc=a(ac,"Trong ph\u1EA7n th\u1EA3o lu\u1EADn k\u1EBF ti\u1EBFp, ch\xFAng ta s\u1EBD ph\xE2n bi\u1EC7t gi\u1EEFa c\xE1c lo\u1EA1i tokenizer \u201Cch\u1EADm\u201D v\xE0 \u201Cnhanh\u201D. Phi\xEAn b\u1EA3n ch\u1EADm l\xE0 nh\u1EEFng phi\xEAn b\u1EA3n \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng Python b\xEAn trong th\u01B0 vi\u1EC7n \u{1F917} Transformers, trong khi phi\xEAn b\u1EA3n nhanh l\xE0 nh\u1EEFng phi\xEAn b\u1EA3n \u0111\u01B0\u1EE3c cung c\u1EA5p b\u1EDFi \u{1F917} Tokenizers, \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng Rust. N\u1EBFu b\u1EA1n nh\u1EDB b\u1EA3ng t\u1EEB "),nt=c(ac,"A",{href:!0});var dp=h(nt);Cc=a(dp,"Ch\u01B0\u01A1ng 5"),dp.forEach(t),Oc=a(ac," b\xE1o c\xE1o kho\u1EA3ng th\u1EDDi gian tokenize nhanh v\xE0 ch\u1EADm c\u1EA7n \u0111\u1EC3 tokenize Drug Review Dataset, b\u1EA1n n\xEAn bi\u1EBFt l\xFD do t\u1EA1i sao ch\xFAng t\xF4i g\u1ECDi ch\xFAng l\xE0 nhanh v\xE0 ch\u1EADm :"),ac.forEach(t),Ba=p(n),Nn=c(n,"TABLE",{});var lc=h(Nn);Bt=c(lc,"THEAD",{});var fp=h(Bt);In=c(fp,"TR",{});var Ot=h(In);st=c(Ot,"TH",{align:!0});var xp=h(st);zc=a(xp,"Tokenizer nhanh"),xp.forEach(t),Ic=p(Ot),tt=c(Ot,"TH",{align:!0});var bp=h(tt);Tc=a(bp,"Tokenizer ch\u1EADm"),bp.forEach(t),Dc=p(Ot),Rt=c(Ot,"TH",{align:!0}),h(Rt).forEach(t),Ot.forEach(t),fp.forEach(t),qc=p(lc),is=c(lc,"TBODY",{});var cc=h(is);Tn=c(cc,"TR",{});var zt=h(Tn);et=c(zt,"TD",{align:!0});var kp=h(et);Ft=c(kp,"CODE",{});var vp=h(Ft);Pc=a(vp,"batched=True"),vp.forEach(t),kp.forEach(t),Sc=p(zt),at=c(zt,"TD",{align:!0});var jp=h(at);Bc=a(jp,"10.8s"),jp.forEach(t),Rc=p(zt),lt=c(zt,"TD",{align:!0});var _p=h(lt);Fc=a(_p,"4min41s"),_p.forEach(t),zt.forEach(t),Ac=p(cc),Dn=c(cc,"TR",{});var It=h(Dn);ct=c(It,"TD",{align:!0});var $p=h(ct);At=c($p,"CODE",{});var yp=h(At);Nc=a(yp,"batched=False"),yp.forEach(t),$p.forEach(t),Mc=p(It),ht=c(It,"TD",{align:!0});var Ep=h(ht);Xc=a(Ep,"59.2s"),Ep.forEach(t),Hc=p(It),it=c(It,"TD",{align:!0});var wp=h(it);Gc=a(wp,"5min3s"),wp.forEach(t),It.forEach(t),cc.forEach(t),lc.forEach(t),Ra=p(n),k(Mn.$$.fragment,n),Fa=p(n),qn=c(n,"H2",{class:!0});var hc=h(qn);Xn=c(hc,"A",{id:!0,class:!0,href:!0});var Cp=h(Xn);Nt=c(Cp,"SPAN",{});var Op=h(Nt);k(os.$$.fragment,Op),Op.forEach(t),Cp.forEach(t),Lc=p(hc),Mt=c(hc,"SPAN",{});var zp=h(Mt);Vc=a(zp,"M\xE3 ho\xE1 theo l\xF4"),zp.forEach(t),hc.forEach(t),Aa=p(n),k(rs.$$.fragment,n),Na=p(n),Hn=c(n,"P",{});var ic=h(Hn);Kc=a(ic,"\u0110\u1EA7u ra c\u1EE7a tokenizer kh\xF4ng ph\u1EA3i l\xE0 m\u1ED9t t\u1EEB \u0111i\u1EC3n Python \u0111\u01A1n gi\u1EA3n; nh\u1EEFng g\xEC ch\xFAng ta nh\u1EADn \u0111\u01B0\u1EE3c th\u1EF1c s\u1EF1 l\xE0 m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng "),Xt=c(ic,"CODE",{});var Ip=h(Xt);Yc=a(Ip,"BatchEncoding"),Ip.forEach(t),Qc=a(ic," \u0111\u1EB7c bi\u1EC7t. \u0110\xF3 l\xE0 m\u1ED9t l\u1EDBp con c\u1EE7a t\u1EEB \u0111i\u1EC3n (\u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao tr\u01B0\u1EDBc \u0111\xE2y ch\xFAng ta c\xF3 th\u1EC3 l\u1EADp ch\u1EC9 m\u1EE5c v\xE0o k\u1EBFt qu\u1EA3 \u0111\xF3 m\xE0 kh\xF4ng g\u1EB7p b\u1EA5t k\u1EF3 v\u1EA5n \u0111\u1EC1 g\xEC), nh\u01B0ng v\u1EDBi c\xE1c ph\u01B0\u01A1ng th\u1EE9c b\u1ED5 sung h\u1EA7u h\u1EBFt \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi c\xE1c tr\xECnh tokenize nhanh."),ic.forEach(t),Ma=p(n),$n=c(n,"P",{});var Tt=h($n);Uc=a(Tt,"B\xEAn c\u1EA1nh kh\u1EA3 n\u0103ng song song h\xF3a c\u1EE7a ch\xFAng, ch\u1EE9c n\u0103ng ch\xEDnh c\u1EE7a c\xE1c tr\xECnh tokenize nhanh l\xE0 ch\xFAng lu\xF4n theo d\xF5i kho\u1EA3ng v\u0103n b\u1EA3n ban \u0111\u1EA7u m\xE0 ta tokenize - m\u1ED9t t\xEDnh n\u0103ng \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),Ht=c(Tt,"EM",{});var Tp=h(Ht);Jc=a(Tp,"offset mapping"),Tp.forEach(t),Wc=a(Tt," hay "),Gt=c(Tt,"EM",{});var Dp=h(Gt);Zc=a(Dp,"\xE1nh x\u1EA1 b\xF9 tr\u1EEB"),Dp.forEach(t),nh=a(Tt,". \u0110i\u1EC1u n\xE0y l\u1EA7n l\u01B0\u1EE3t m\u1EDF kh\xF3a c\xE1c t\xEDnh n\u0103ng nh\u01B0 \xE1nh x\u1EA1 t\u1EEBng t\u1EEB v\u1EDBi token m\xE0 n\xF3 t\u1EA1o ra ho\u1EB7c \xE1nh x\u1EA1 t\u1EEBng k\xFD t\u1EF1 c\u1EE7a v\u0103n b\u1EA3n g\u1ED1c v\u1EDBi token b\xEAn trong v\xE0 ng\u01B0\u1EE3c l\u1EA1i."),Tt.forEach(t),Xa=p(n),ot=c(n,"P",{});var qp=h(ot);sh=a(qp,"C\xF9ng xem m\u1ED9t m\u1EABu:"),qp.forEach(t),Ha=p(n),k(ps.$$.fragment,n),Ga=p(n),Gn=c(n,"P",{});var oc=h(Gn);th=a(oc,"Nh\u01B0 \u0111\xE3 \u0111\u1EC1 c\u1EADp tr\u01B0\u1EDBc \u0111\xE2y, ch\xFAng ta nh\u1EADn \u0111\u01B0\u1EE3c m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng "),Lt=c(oc,"CODE",{});var Pp=h(Lt);eh=a(Pp,"BatchEncoding"),Pp.forEach(t),ah=a(oc," trong \u0111\u1EA7u ra c\u1EE7a tr\xECnh tokenize:"),oc.forEach(t),La=p(n),k(gs.$$.fragment,n),Va=p(n),sn=c(n,"P",{});var Cn=h(sn);lh=a(Cn,"V\xEC l\u1EDBp "),Vt=c(Cn,"CODE",{});var Sp=h(Vt);ch=a(Sp,"AutoTokenizer"),Sp.forEach(t),hh=a(Cn," ch\u1ECDn m\u1ED9t tr\xECnh tokenizer nhanh theo m\u1EB7c \u0111\u1ECBnh, ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng c\xE1c ph\u01B0\u01A1ng th\u1EE9c b\u1ED5 sung m\xE0 \u0111\u1ED1i t\u01B0\u1EE3ng "),Kt=c(Cn,"CODE",{});var Bp=h(Kt);ih=a(Bp,"BatchEncoding"),Bp.forEach(t),oh=a(Cn," n\xE0y cung c\u1EA5p. Ch\xFAng ta c\xF3 hai c\xE1ch \u0111\u1EC3 ki\u1EC3m tra xem tr\xECnh tokenize l\xE0 nhanh hay ch\u1EADm. Ch\xFAng ta c\xF3 th\u1EC3 ki\u1EC3m tra b\u1EB1ng thu\u1ED9c t\xEDnh "),Yt=c(Cn,"CODE",{});var Rp=h(Yt);rh=a(Rp,"is_fast"),Rp.forEach(t),ph=a(Cn," c\u1EE7a "),Qt=c(Cn,"CODE",{});var Fp=h(Qt);gh=a(Fp,"tokenizer"),Fp.forEach(t),mh=a(Cn,":"),Cn.forEach(t),Ka=p(n),k(ms.$$.fragment,n),Ya=p(n),k(us.$$.fragment,n),Qa=p(n),Ln=c(n,"P",{});var rc=h(Ln);uh=a(rc,"ho\u1EB7c ki\u1EC3m tra c\xF9ng thu\u1ED9c t\xEDnh \u0111\xF3 c\u1EE7a "),Ut=c(rc,"CODE",{});var Ap=h(Ut);dh=a(Ap,"encoding"),Ap.forEach(t),fh=a(rc,":"),rc.forEach(t),Ua=p(n),k(ds.$$.fragment,n),Ja=p(n),k(fs.$$.fragment,n),Wa=p(n),rt=c(n,"P",{});var Np=h(rt);xh=a(Np,"H\xE3y xem nh\u1EEFng g\xEC m\u1ED9t tokenizer nhanh cho ph\xE9p ch\xFAng ta l\xE0m. \u0110\u1EA7u ti\xEAn, ch\xFAng t\xF4i c\xF3 th\u1EC3 truy c\u1EADp token m\xE0 kh\xF4ng c\u1EA7n ph\u1EA3i chuy\u1EC3n \u0111\u1ED5i ID tr\u1EDF l\u1EA1i token:"),Np.forEach(t),Za=p(n),k(xs.$$.fragment,n),nl=p(n),k(bs.$$.fragment,n),sl=p(n),yn=c(n,"P",{});var Dt=h(yn);bh=a(Dt,"Trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, token \u1EDF ch\u1EC9 m\u1EE5c 5 l\xE0 "),Jt=c(Dt,"CODE",{});var Mp=h(Jt);kh=a(Mp,"##yl"),Mp.forEach(t),vh=a(Dt,", l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a t\u1EEB \u201CSylvain\u201D trong c\xE2u g\u1ED1c. Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Wt=c(Dt,"CODE",{});var Xp=h(Wt);jh=a(Xp,"word_ids()"),Xp.forEach(t),_h=a(Dt," \u0111\u1EC3 l\u1EA5y ch\u1EC9 m\u1EE5c c\u1EE7a t\u1EEB m\xE0 m\u1ED7i token \u0111\u1EBFn t\u1EEB:"),Dt.forEach(t),tl=p(n),k(ks.$$.fragment,n),el=p(n),k(vs.$$.fragment,n),al=p(n),U=c(n,"P",{});var gn=h(U);$h=a(gn,"Ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y r\u1EB1ng c\xE1c token \u0111\u1EB7c bi\u1EC7t c\u1EE7a tr\xECnh tokenize nh\u01B0 "),Zt=c(gn,"CODE",{});var Hp=h(Zt);yh=a(Hp,"[CLS]"),Hp.forEach(t),Eh=a(gn," v\xE0 "),ne=c(gn,"CODE",{});var Gp=h(ne);wh=a(Gp,"[SEP]"),Gp.forEach(t),Ch=a(gn," \u0111\u01B0\u1EE3c \xE1nh x\u1EA1 th\xE0nh "),se=c(gn,"CODE",{});var Lp=h(se);Oh=a(Lp,"None"),Lp.forEach(t),zh=a(gn,", v\xE0 sau \u0111\xF3 m\u1ED7i token \u0111\u01B0\u1EE3c \xE1nh x\u1EA1 t\u1EDBi t\u1EEB m\xE0 n\xF3 b\u1EAFt ngu\u1ED3n. \u0110i\u1EC1u n\xE0y \u0111\u1EB7c bi\u1EC7t h\u1EEFu \xEDch \u0111\u1EC3 x\xE1c \u0111\u1ECBnh xem m\u1ED9t token n\u1EB1m \u1EDF \u0111\u1EA7u m\u1ED9t t\u1EEB hay n\u1EBFu hai token c\xF3 trong c\xF9ng thu\u1ED9c m\u1ED9t t\u1EEB. Ch\xFAng ta c\xF3 th\u1EC3 d\u1EF1a v\xE0o ti\u1EC1n t\u1ED1 "),te=c(gn,"CODE",{});var Vp=h(te);Ih=a(Vp,"##"),Vp.forEach(t),Th=a(gn," cho \u0111i\u1EC1u \u0111\xF3, nh\u01B0ng n\xF3 ch\u1EC9 ho\u1EA1t \u0111\u1ED9ng \u0111\u1ED1i v\u1EDBi c\xE1c tokenize ki\u1EC3u BERT; ph\u01B0\u01A1ng ph\xE1p n\xE0y ho\u1EA1t \u0111\u1ED9ng v\u1EDBi b\u1EA5t k\u1EF3 lo\u1EA1i tokenizer n\xE0o mi\u1EC5n n\xF3 l\xE0 phi\xEAn b\u1EA3n nhanh. Trong ch\u01B0\u01A1ng ti\u1EBFp theo, ch\xFAng ta s\u1EBD xem c\xE1ch ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng kh\u1EA3 n\u0103ng n\xE0y \u0111\u1EC3 \xE1p d\u1EE5ng nh\xE3n ch\xFAng ta c\xF3 cho m\u1ED7i t\u1EEB \u0111\xFAng c\xE1ch v\u1EDBi c\xE1c token trong c\xE1c t\xE1c v\u1EE5 nh\u01B0 nh\u1EADn d\u1EA1ng th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c \u0111\u1EB7t t\xEAn (NER) v\xE0  g\xE1n nh\xE3n t\u1EEB lo\u1EA1i (POS). Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng n\xF3 \u0111\u1EC3 che gi\u1EA5u t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EBFn t\u1EEB c\xF9ng m\u1ED9t t\u1EEB trong m\xF4 h\xECnh ng\xF4n ng\u1EEF \u0111\u01B0\u1EE3c che (m\u1ED9t k\u1EF9 thu\u1EADt \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),ee=c(gn,"EM",{});var Kp=h(ee);Dh=a(Kp,"whole word masking"),Kp.forEach(t),qh=a(gn,")."),gn.forEach(t),ll=p(n),k(Vn.$$.fragment,n),cl=p(n),En=c(n,"P",{});var qt=h(En);Ph=a(qt,"T\u01B0\u01A1ng t\u1EF1, c\xF3 m\u1ED9t ph\u01B0\u01A1ng th\u1EE9c "),ae=c(qt,"CODE",{});var Yp=h(ae);Sh=a(Yp,"question_ids()"),Yp.forEach(t),Bh=a(qt," m\xE0 ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 \xE1nh x\u1EA1 token \u0111\u1EBFn c\xE2u m\xE0 n\xF3 b\u1EAFt ngu\u1ED3n (m\u1EB7c d\xF9 trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, "),le=c(qt,"CODE",{});var Qp=h(le);Rh=a(Qp,"token_type_ids"),Qp.forEach(t),Fh=a(qt," \u0111\u01B0\u1EE3c tr\u1EA3 v\u1EC1 b\u1EDFi tokenizer c\xF3 th\u1EC3 cung c\u1EA5p cho ch\xFAng ta c\xF9ng m\u1ED9t th\xF4ng tin)."),qt.forEach(t),hl=p(n),G=c(n,"P",{});var en=h(G);Ah=a(en,"Cu\u1ED1i c\xF9ng, ch\xFAng ta c\xF3 th\u1EC3 \xE1nh x\u1EA1 b\u1EA5t k\u1EF3 t\u1EEB ho\u1EB7c token n\xE0o v\u1EDBi c\xE1c k\xFD t\u1EF1 trong v\u0103n b\u1EA3n g\u1ED1c v\xE0 ng\u01B0\u1EE3c l\u1EA1i, th\xF4ng qua c\xE1c ph\u01B0\u01A1ng th\u1EE9c "),ce=c(en,"CODE",{});var Up=h(ce);Nh=a(Up,"word_to_chars()"),Up.forEach(t),Mh=a(en," ho\u1EB7c "),he=c(en,"CODE",{});var Jp=h(he);Xh=a(Jp,"token_to_chars()"),Jp.forEach(t),Hh=a(en," v\xE0 "),ie=c(en,"CODE",{});var Wp=h(ie);Gh=a(Wp,"char_to_word()"),Wp.forEach(t),Lh=a(en," ho\u1EB7c "),oe=c(en,"CODE",{});var Zp=h(oe);Vh=a(Zp,"char_to_token()"),Zp.forEach(t),Kh=a(en,". V\xED d\u1EE5: ph\u01B0\u01A1ng th\u1EE9c "),re=c(en,"CODE",{});var ng=h(re);Yh=a(ng,"word_ids()"),ng.forEach(t),Qh=a(en," cho ch\xFAng ta bi\u1EBFt r\u1EB1ng "),pe=c(en,"CODE",{});var sg=h(pe);Uh=a(sg,"##yl"),sg.forEach(t),Jh=a(en," l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a t\u1EEB \u1EDF ch\u1EC9 m\u1EE5c 3, nh\u01B0ng t\u1EEB \u0111\xF3 n\u1EB1m trong c\xE2u n\xE0o? Ch\xFAng ta c\xF3 th\u1EC3 t\xECm hi\u1EC3u nh\u01B0 th\u1EBF n\xE0y:"),en.forEach(t),il=p(n),k(js.$$.fragment,n),ol=p(n),k(_s.$$.fragment,n),rl=p(n),rn=c(n,"P",{});var ls=h(rn);Wh=a(ls,"Nh\u01B0 \u0111\xE3 \u0111\u1EC1 c\u1EADp tr\u01B0\u1EDBc \u0111\xF3, t\u1EA5t c\u1EA3 \u0111i\u1EC1u n\xE0y th\u1EF1c t\u1EBF \u0111\u01B0\u1EE3c h\u1ED7 tr\u1EE3 b\u1EDFi l\xE0 tr\xECnh tokenizer nhanh k\u1EBFt h\u1EE3p kho\u1EA3ng v\u0103n b\u1EA3n m\xE0 m\u1ED7i token \u0111\u1EBFn t\u1EEB danh s\xE1ch "),ge=c(ls,"EM",{});var tg=h(ge);Zh=a(tg,"offset"),tg.forEach(t),ni=a(ls," hay "),me=c(ls,"EM",{});var eg=h(me);si=a(eg,"offset"),eg.forEach(t),ti=a(ls,". \u0110\u1EC3 minh h\u1ECDa vi\u1EC7c s\u1EED d\u1EE5ng ch\xFAng, ti\u1EBFp theo, ch\xFAng t\xF4i s\u1EBD h\u01B0\u1EDBng d\u1EABn b\u1EA1n c\xE1ch sao ch\xE9p c\xE1c k\u1EBFt qu\u1EA3 c\u1EE7a "),ue=c(ls,"CODE",{});var ag=h(ue);ei=a(ag,"token-classification"),ag.forEach(t),ai=a(ls," theo c\xE1ch th\u1EE7 c\xF4ng."),ls.forEach(t),pl=p(n),k(Kn.$$.fragment,n),gl=p(n),Pn=c(n,"H2",{class:!0});var pc=h(Pn);Yn=c(pc,"A",{id:!0,class:!0,href:!0});var lg=h(Yn);de=c(lg,"SPAN",{});var cg=h(de);k($s.$$.fragment,cg),cg.forEach(t),lg.forEach(t),li=p(pc),pt=c(pc,"SPAN",{});var sp=h(pt);ci=a(sp,"B\xEAn trong pipeline "),fe=c(sp,"CODE",{});var hg=h(fe);hi=a(hg,"token-classification"),hg.forEach(t),sp.forEach(t),pc.forEach(t),ml=p(n),tn=c(n,"P",{});var On=h(tn);ii=a(On,"Trong "),gt=c(On,"A",{href:!0});var ig=h(gt);oi=a(ig,"Ch\u01B0\u01A1ng 1"),ig.forEach(t),ri=a(On,", ch\xFAng ta l\u1EA7n \u0111\u1EA7u \u0111\u01B0\u1EE3c th\u1EED \xE1p d\u1EE5ng NER - t\xE1c v\u1EE5 x\xE1c \u0111\u1ECBnh nh\u1EEFng ph\u1EA7n n\xE0o c\u1EE7a v\u0103n b\u1EA3n t\u01B0\u01A1ng \u1EE9ng v\u1EDBi c\xE1c th\u1EF1c th\u1EC3 nh\u01B0 ng\u01B0\u1EDDi, \u0111\u1ECBa \u0111i\u1EC3m ho\u1EB7c t\u1ED5 ch\u1EE9c - v\u1EDBi "),xe=c(On,"CODE",{});var og=h(xe);pi=a(og,"pipeline()"),og.forEach(t),gi=a(On," c\u1EE7a \u{1F917} Transformers. Sau \u0111\xF3, trong "),mt=c(On,"A",{href:!0});var rg=h(mt);mi=a(rg,"Ch\u01B0\u01A1ng 2"),rg.forEach(t),ui=a(On,", ch\xFAng ta \u0111\xE3 th\u1EA5y c\xE1ch m\u1ED9t pipeline nh\xF3m ba giai \u0111o\u1EA1n c\u1EA7n thi\u1EBFt l\u1EA1i v\u1EDBi nhau \u0111\u1EC3 nh\u1EADn c\xE1c d\u1EF1 \u0111o\xE1n t\u1EEB m\u1ED9t v\u0103n b\u1EA3n th\xF4: tokenize, chuy\u1EC3n c\xE1c \u0111\u1EA7u v\xE0o qua m\xF4 h\xECnh v\xE0 h\u1EADu x\u1EED l\xFD. Hai b\u01B0\u1EDBc \u0111\u1EA7u ti\xEAn trong quy tr\xECnh "),be=c(On,"CODE",{});var pg=h(be);di=a(pg,"token-classification"),pg.forEach(t),fi=a(On," c\u0169ng gi\u1ED1ng nh\u01B0 trong b\u1EA5t k\u1EF3 quy tr\xECnh n\xE0o kh\xE1c, nh\u01B0ng qu\xE1 tr\xECnh h\u1EADu x\u1EED l\xFD ph\u1EE9c t\u1EA1p h\u01A1n m\u1ED9t ch\xFAt - h\xE3y c\xF9ng xem c\xE1ch th\u1EF1c hi\u1EC7n!"),On.forEach(t),ul=p(n),bn.l(n),ut=p(n),Sn=c(n,"H3",{class:!0});var gc=h(Sn);Qn=c(gc,"A",{id:!0,class:!0,href:!0});var gg=h(Qn);ke=c(gg,"SPAN",{});var mg=h(ke);k(ys.$$.fragment,mg),mg.forEach(t),gg.forEach(t),xi=p(gc),ve=c(gc,"SPAN",{});var ug=h(ve);bi=a(ug,"Nh\u1EADn k\u1EBFt qu\u1EA3 c\u01A1 s\u1EDF v\u1EDBi baseline"),ug.forEach(t),gc.forEach(t),dl=p(n),Un=c(n,"P",{});var mc=h(Un);ki=a(mc,"Tr\u01B0\u1EDBc ti\xEAn, h\xE3y l\u1EA5y m\u1ED9t pipeline ph\xE2n lo\u1EA1i token ch\xFAng ta c\xF3 th\u1EC3 nh\u1EADn \u0111\u01B0\u1EE3c m\u1ED9t s\u1ED1 k\u1EBFt qu\u1EA3 \u0111\u1EC3 so s\xE1nh theo c\xE1ch th\u1EE7 c\xF4ng. M\xF4 h\xECnh \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng theo m\u1EB7c \u0111\u1ECBnh l\xE0 "),Es=c(mc,"A",{href:!0,rel:!0});var dg=h(Es);je=c(dg,"CODE",{});var fg=h(je);vi=a(fg,"dbmdz/bert-large-cased-finetuned-conll03-english"),fg.forEach(t),dg.forEach(t),ji=a(mc,"; n\xF3 th\u1EF1c hi\u1EC7n NER tr\xEAn c\xE1c c\xE2u:"),mc.forEach(t),fl=p(n),k(ws.$$.fragment,n),xl=p(n),k(Cs.$$.fragment,n),bl=p(n),dt=c(n,"P",{});var xg=h(dt);_i=a(xg,"M\xF4 h\xECnh \u0111\xE3 x\xE1c \u0111\u1ECBnh \u0111\xFAng m\u1ED7i token do \u201CSylvain\u201D t\u1EA1o ra l\xE0 m\u1ED9t ng\u01B0\u1EDDi, m\u1ED7i token \u0111\u01B0\u1EE3c t\u1EA1o b\u1EDFi \u201CHugging Face\u201D l\xE0 m\u1ED9t t\u1ED5 ch\u1EE9c v\xE0 token \u201CBrooklyn\u201D l\xE0 m\u1ED9t \u0111\u1ECBa \u0111i\u1EC3m. Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 y\xEAu c\u1EA7u pipeline nh\xF3m c\xE1c token t\u01B0\u01A1ng \u1EE9ng v\u1EDBi c\xF9ng m\u1ED9t th\u1EF1c th\u1EC3 l\u1EA1i v\u1EDBi nhau:"),xg.forEach(t),kl=p(n),k(Os.$$.fragment,n),vl=p(n),k(zs.$$.fragment,n),jl=p(n),Y=c(n,"P",{});var hn=h(Y);_e=c(hn,"CODE",{});var bg=h(_e);$i=a(bg,"aggregation_strategy"),bg.forEach(t),yi=a(hn," \u0111\u01B0\u1EE3c ch\u1ECDn s\u1EBD thay \u0111\u1ED5i \u0111i\u1EC3m \u0111\u01B0\u1EE3c t\xEDnh cho m\u1ED7i th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c nh\xF3m l\u1EA1i, V\u1EDBi "),$e=c(hn,"CODE",{});var kg=h($e);Ei=a(kg,'"simple"'),kg.forEach(t),wi=a(hn,", \u0111i\u1EC3m ch\u1EC9 l\xE0 gi\xE1 tr\u1ECB trung b\xECnh c\u1EE7a \u0111i\u1EC3m c\u1EE7a m\u1ED7i token trong th\u1EF1c th\u1EC3 \u0111\xE3 cho: v\xED d\u1EE5: \u0111i\u1EC3m c\u1EE7a \u201CSylvain\u201D l\xE0 trung b\xECnh \u0111i\u1EC3m m\xE0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong v\xED d\u1EE5 tr\u01B0\u1EDBc cho c\xE1c token "),ye=c(hn,"CODE",{});var vg=h(ye);Ci=a(vg,"S"),vg.forEach(t),Oi=a(hn," , "),Ee=c(hn,"CODE",{});var jg=h(Ee);zi=a(jg,"##yl"),jg.forEach(t),Ii=a(hn,","),we=c(hn,"CODE",{});var _g=h(we);Ti=a(_g,"## va"),_g.forEach(t),Di=a(hn," v\xE0 "),Ce=c(hn,"CODE",{});var $g=h(Ce);qi=a($g,"##in"),$g.forEach(t),Pi=a(hn,". C\xE1c chi\u1EBFn l\u01B0\u1EE3c c\xF3 s\u1EB5n kh\xE1c l\xE0:"),hn.forEach(t),_l=p(n),wn=c(n,"UL",{});var Pt=h(wn);Jn=c(Pt,"LI",{});var Da=h(Jn);Oe=c(Da,"CODE",{});var yg=h(Oe);Si=a(yg,'"first"'),yg.forEach(t),Bi=a(Da,", trong \u0111\xF3 \u0111i\u1EC3m c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 \u0111i\u1EC3m c\u1EE7a token \u0111\u1EA7u ti\xEAn c\u1EE7a th\u1EF1c th\u1EC3 \u0111\xF3 (v\xEC v\u1EADy \u0111\u1ED1i v\u1EDBi \u201CSylvain\u201D, n\xF3 s\u1EBD l\xE0 0.993828, \u0111i\u1EC3m c\u1EE7a token "),ze=c(Da,"CODE",{});var Eg=h(ze);Ri=a(Eg,"S"),Eg.forEach(t),Fi=a(Da,")"),Da.forEach(t),Ai=p(Pt),ft=c(Pt,"LI",{});var tp=h(ft);Ie=c(tp,"CODE",{});var wg=h(Ie);Ni=a(wg,'"max"'),wg.forEach(t),Mi=a(tp,", trong \u0111\xF3 \u0111i\u1EC3m c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 \u0111i\u1EC3m t\u1ED1i \u0111a c\u1EE7a c\xE1c token trong th\u1EF1c th\u1EBF \u0111\xF3 (v\xEC v\u1EADy v\u1EDBi \u201CHugging Face\u201D s\u1EBD l\xE0 0.98879766, \u0111i\u1EC3m c\u1EE7a \u201CFace\u201D)"),tp.forEach(t),Xi=p(Pt),Wn=c(Pt,"LI",{});var qa=h(Wn);Te=c(qa,"CODE",{});var Cg=h(Te);Hi=a(Cg,'"average"'),Cg.forEach(t),Gi=a(qa,", trong \u0111\xF3 \u0111i\u1EC3m c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 \u0111i\u1EC3m trung b\xECnh c\u1EE7a c\xE1c t\u1EEB t\u1EA1o n\xEAn th\u1EF1c th\u1EC3 \u0111\xF3 (v\xED d\u1EE5 v\u1EDBi \u201CSylvain\u201D th\xEC ph\u01B0\u01A1ng ph\xE1p n\xE0y s\u1EBD kh\xF4ng c\xF3 s\u1EF1 kh\xE1c bi\u1EC7t so v\u1EDBi ph\u01B0\u01A1ng ph\xE1p "),De=c(qa,"CODE",{});var Og=h(De);Li=a(Og,'"simple"'),Og.forEach(t),Vi=a(qa,", nh\u01B0ng v\u1EDBi \u201CHugging Face\u201D, \u0111i\u1EC3m tr\u1EA3 v\u1EC1 s\u1EBD l\xE0 0.9819, \u0111i\u1EC3m trung b\xECnh c\u1EE7a \u201CHugging\u201D, 0.975, v\xE0 \u201CFace\u201D, 0.98879)"),qa.forEach(t),Pt.forEach(t),$l=p(n),Zn=c(n,"P",{});var uc=h(Zn);Ki=a(uc,"Gi\u1EDD ch\xFAng ta h\xE3y xem l\xE0m th\u1EBF n\xE0o \u0111\u1EC3 c\xF3 \u0111\u01B0\u1EE3c nh\u1EEFng k\u1EBFt qu\u1EA3 n\xE0y m\xE0 kh\xF4ng c\u1EA7n s\u1EED d\u1EE5ng h\xE0m"),qe=c(uc,"CODE",{});var zg=h(qe);Yi=a(zg,"pipeline()"),zg.forEach(t),Qi=a(uc,"!"),uc.forEach(t),yl=p(n),Bn=c(n,"H3",{class:!0});var dc=h(Bn);ns=c(dc,"A",{id:!0,class:!0,href:!0});var Ig=h(ns);Pe=c(Ig,"SPAN",{});var Tg=h(Pe);k(Is.$$.fragment,Tg),Tg.forEach(t),Ig.forEach(t),Ui=p(dc),Se=c(dc,"SPAN",{});var Dg=h(Se);Ji=a(Dg,"T\u1EEB \u0111\u1EA7u v\xE0o t\u1EDBi d\u1EF1 \u0111o\xE1n"),Dg.forEach(t),dc.forEach(t),El=p(n),vn.l(n),xt=p(n),bt=c(n,"P",{});var qg=h(bt);Wi=a(qg,"Ch\xFAng ta c\xF3 m\u1ED9t l\xF4 v\u1EDBi 1 chu\u1ED7i g\u1ED3m 19 token v\xE0 m\xF4 h\xECnh c\xF3 9 nh\xE3n kh\xE1c nhau, v\xEC v\u1EADy \u0111\u1EA7u ra c\u1EE7a m\xF4 h\xECnh c\xF3 h\xECnh d\u1EA1ng 1 x 19 x 9. Gi\u1ED1ng nh\u01B0 \u0111\u1ED1i v\u1EDBi pipeline ph\xE2n lo\u1EA1i v\u0103n b\u1EA3n, ch\xFAng ta s\u1EED d\u1EE5ng h\xE0m softmax \u0111\u1EC3 chuy\u1EC3n \u0111\u1ED5i c\xE1c logits \u0111\xF3 theo x\xE1c su\u1EA5t, v\xE0 ch\xFAng ta l\u1EA5y argmax \u0111\u1EC3 nh\u1EADn d\u1EF1 \u0111o\xE1n (l\u01B0u \xFD r\u1EB1ng ta c\xF3 th\u1EC3 l\u1EA5y argmax tr\xEAn logits v\xEC softmax kh\xF4ng thay \u0111\u1ED5i th\u1EE9 t\u1EF1):"),qg.forEach(t),wl=p(n),_n.l(n),kt=p(n),k(Ts.$$.fragment,n),Cl=p(n),ss=c(n,"P",{});var fc=h(ss);Zi=a(fc,"Thu\u1ED9c t\xEDnh "),Be=c(fc,"CODE",{});var Pg=h(Be);no=a(Pg,"model.config.id2label"),Pg.forEach(t),so=a(fc," ch\u1EE9a \xE1nh x\u1EA1 c\xE1c ch\u1EC9 m\u1EE5c t\u1EDBi c\xE1c nh\xE3n m\xE0 ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 hi\u1EC3u c\xE1c d\u1EF1 \u0111o\xE1n:"),fc.forEach(t),Ol=p(n),k(Ds.$$.fragment,n),zl=p(n),k(qs.$$.fragment,n),Il=p(n),z=c(n,"P",{});var R=h(z);to=a(R,"Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, c\xF3 9 nh\xE3n: "),Re=c(R,"CODE",{});var Sg=h(Re);eo=a(Sg,"O"),Sg.forEach(t),ao=a(R," l\xE0 nh\xE3n cho c\xE1c token kh\xF4ng n\u1EB1m trong b\u1EA5t k\u1EF3 th\u1EF1c th\u1EC3 \u0111\u01B0\u1EE3c \u0111\u1EB7t t\xEAn n\xE0o (n\xF3 l\xE0 vi\u1EBFt t\u1EAFt c\u1EE7a \u201Coutside\u201D hay \u201Cb\xEAn ngo\xE0i\u201D) v\xE0 sau \u0111\xF3 ch\xFAng ta c\xF3 hai nh\xE3n cho m\u1ED7i lo\u1EA1i th\u1EF1c th\u1EC3 (linh tinh, ng\u01B0\u1EDDi , t\u1ED5 ch\u1EE9c v\xE0 v\u1ECB tr\xED). Nh\xE3n "),Fe=c(R,"CODE",{});var Bg=h(Fe);lo=a(Bg,"B-XXX"),Bg.forEach(t),co=a(R," cho bi\u1EBFt token n\u1EB1m \u1EDF \u0111\u1EA7u th\u1EF1c th\u1EC3 "),Ae=c(R,"CODE",{});var Rg=h(Ae);ho=a(Rg,"XXX"),Rg.forEach(t),io=a(R," v\xE0 nh\xE3n "),Ne=c(R,"CODE",{});var Fg=h(Ne);oo=a(Fg,"I-XXX"),Fg.forEach(t),ro=a(R," cho bi\u1EBFt token n\u1EB1m b\xEAn trong th\u1EF1c th\u1EC3 "),Me=c(R,"CODE",{});var Ag=h(Me);po=a(Ag,"XXX"),Ag.forEach(t),go=a(R,". V\xED d\u1EE5: trong m\u1EABu hi\u1EC7n t\u1EA1i, ch\xFAng ta k\xEC v\u1ECDng m\xF4 h\xECnh ph\xE2n lo\u1EA1i token "),Xe=c(R,"CODE",{});var Ng=h(Xe);mo=a(Ng,"S"),Ng.forEach(t),uo=a(R," l\xE0 "),He=c(R,"CODE",{});var Mg=h(He);fo=a(Mg,"B-PER"),Mg.forEach(t),xo=a(R," (b\u1EAFt \u0111\u1EA7u c\u1EE7a m\u1ED9t th\u1EF1c th\u1EC3 ng\u01B0\u1EDDi) v\xE0 c\xE1c token "),Ge=c(R,"CODE",{});var Xg=h(Ge);bo=a(Xg,"##yl"),Xg.forEach(t),ko=a(R,","),Le=c(R,"CODE",{});var Hg=h(Le);vo=a(Hg,"##va"),Hg.forEach(t),jo=a(R," v\xE0 "),Ve=c(R,"CODE",{});var Gg=h(Ve);_o=a(Gg,"##in"),Gg.forEach(t),$o=a(R," l\xE0 "),Ke=c(R,"CODE",{});var Lg=h(Ke);yo=a(Lg,"I-PER"),Lg.forEach(t),Eo=a(R," (b\xEAn trong m\u1ED9t th\u1EF1c th\u1EC3 ng\u01B0\u1EDDi)."),R.forEach(t),Tl=p(n),X=c(n,"P",{});var V=h(X);wo=a(V,"B\u1EA1n c\xF3 th\u1EC3 ngh\u0129 r\u1EB1ng m\xF4 h\xECnh \u0111\xE3 sai trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y v\xEC n\xF3 \u0111\xE3 g\u1EAFn nh\xE3n "),Ye=c(V,"CODE",{});var Vg=h(Ye);Co=a(Vg,"I-PER"),Vg.forEach(t),Oo=a(V," cho c\u1EA3 b\u1ED1n token n\xE0y, nh\u01B0ng \u0111i\u1EC1u \u0111\xF3 kh\xF4ng ho\xE0n to\xE0n \u0111\xFAng. Th\u1EF1c t\u1EBF c\xF3 hai \u0111\u1ECBnh d\u1EA1ng cho c\xE1c nh\xE3n "),Qe=c(V,"CODE",{});var Kg=h(Qe);zo=a(Kg,"B-"),Kg.forEach(t),Io=a(V," v\xE0 "),Ue=c(V,"CODE",{});var Yg=h(Ue);To=a(Yg,"I-"),Yg.forEach(t),Do=a(V," \u0111\xF3 l\xE0: "),Je=c(V,"EM",{});var Qg=h(Je);qo=a(Qg,"IOB1"),Qg.forEach(t),Po=a(V," v\xE0 "),We=c(V,"EM",{});var Ug=h(We);So=a(Ug,"IOB2"),Ug.forEach(t),Bo=a(V,". \u0110\u1ECBnh d\u1EA1ng IOB2 (m\xE0u h\u1ED3ng b\xEAn d\u01B0\u1EDBi), l\xE0 \u0111\u1ECBnh d\u1EA1ng ch\xFAng ta \u0111\xE3 gi\u1EDBi thi\u1EC7u trong khi \u1EDF \u0111\u1ECBnh d\u1EA1ng IOB1 (m\xE0u xanh lam), c\xE1c nh\xE3n b\u1EAFt \u0111\u1EA7u b\u1EB1ng "),Ze=c(V,"CODE",{});var Jg=h(Ze);Ro=a(Jg,"B-"),Jg.forEach(t),Fo=a(V," ch\u1EC9 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 ph\xE2n t\xE1ch hai th\u1EF1c th\u1EC3 li\u1EC1n k\u1EC1 c\xF9ng lo\u1EA1i. M\xF4 h\xECnh ch\xFAng t\xF4i \u0111ang s\u1EED d\u1EE5ng \u0111\xE3 \u0111\u01B0\u1EE3c tinh ch\u1EC9nh tr\xEAn t\u1EADp d\u1EEF li\u1EC7u b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng \u0111\u1ECBnh d\u1EA1ng \u0111\xF3, \u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao n\xF3 g\xE1n nh\xE3n "),na=c(V,"CODE",{});var Wg=h(na);Ao=a(Wg,"I-PER"),Wg.forEach(t),No=a(V," cho m\xE3 th\xF4ng b\xE1o "),sa=c(V,"CODE",{});var Zg=h(sa);Mo=a(Zg,"S"),Zg.forEach(t),Xo=a(V,"."),V.forEach(t),Dl=p(n),Rn=c(n,"DIV",{class:!0});var xc=h(Rn);Ps=c(xc,"IMG",{class:!0,src:!0,alt:!0}),Ho=p(xc),Ss=c(xc,"IMG",{class:!0,src:!0,alt:!0}),xc.forEach(t),ql=p(n),ts=c(n,"P",{});var bc=h(ts);Go=a(bc,"V\u1EDBi ph\xE9p \xE1nh x\u1EA1 n\xE0y, ch\xFAng ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC1 t\xE1i t\u1EA1o l\u1EA1i (g\u1EA7n nh\u01B0 ho\xE0n to\xE0n) k\u1EBFt qu\u1EA3 c\u1EE7a pipeline \u0111\u1EA7u \u2014 ta ch\u1EC9 c\u1EA7n l\u1EA5y \u0111i\u1EC3m v\xE0 nh\xE3n c\u1EE7a m\u1ED7i token m\xE0 kh\xF4ng \u0111\u01B0\u1EE3c ph\xE2n v\xE0o "),ta=c(bc,"CODE",{});var nm=h(ta);Lo=a(nm,"O"),nm.forEach(t),Vo=a(bc,":"),bc.forEach(t),Pl=p(n),k(Bs.$$.fragment,n),Sl=p(n),k(Rs.$$.fragment,n),Bl=p(n),J=c(n,"P",{});var mn=h(J);Ko=a(mn,"\u0110i\u1EC1u n\xE0y r\u1EA5t gi\u1ED1ng v\u1EDBi nh\u1EEFng g\xEC ta \u0111\xE3 c\xF3 tr\u01B0\u1EDBc \u0111\xE2y, ngo\u1EA1i tr\u1EEB m\u1ED9t ngo\u1EA1i l\u1EC7: pipeline c\u0169ng cung c\u1EA5p th\xF4ng tin v\u1EC1 \u0111i\u1EC3m "),ea=c(mn,"CODE",{});var sm=h(ea);Yo=a(sm,"start"),sm.forEach(t),Qo=a(mn," hay "),aa=c(mn,"CODE",{});var tm=h(aa);Uo=a(tm,"b\u1EAFt \u0111\u1EA7u"),tm.forEach(t),Jo=a(mn," v\xE0 "),la=c(mn,"CODE",{});var em=h(la);Wo=a(em,"end"),em.forEach(t),Zo=a(mn,"  hay "),ca=c(mn,"CODE",{});var am=h(ca);nr=a(am,"k\u1EBFt th\xFAc"),am.forEach(t),sr=a(mn," c\u1EE7a m\u1ED7i th\u1EF1c th\u1EC3 trong c\xE2u g\u1ED1c. \u0110\xE2y l\xE0 l\xFAc \xE1nh x\u1EA1 b\xF9 tr\u1EEB c\u1EE7a ch\xFAng ta s\u1EBD ph\xE1t huy t\xE1c d\u1EE5ng. \u0110\u1EC3 c\xF3 \u0111\u01B0\u1EE3c offset, ch\xFAng ta ch\u1EC9 c\u1EA7n \u0111\u1EB7t "),ha=c(mn,"CODE",{});var lm=h(ha);tr=a(lm,"return_offsets_mapping=True"),lm.forEach(t),er=a(mn," khi ch\xFAng ta \xE1p d\u1EE5ng tokenizer cho c\xE1c \u0111\u1EA7u v\xE0o c\u1EE7a m\xECnh:"),mn.forEach(t),Rl=p(n),k(Fs.$$.fragment,n),Fl=p(n),k(As.$$.fragment,n),Al=p(n),pn=c(n,"P",{});var cs=h(pn);ar=a(cs,"M\u1ED7i tuple l\xE0 kho\u1EA3ng v\u0103n b\u1EA3n t\u01B0\u01A1ng \u1EE9ng v\u1EDBi m\u1ED7i token, trong \u0111\xF3 "),ia=c(cs,"CODE",{});var cm=h(ia);lr=a(cm,"(0, 0)"),cm.forEach(t),cr=a(cs," \u0111\u01B0\u1EE3c d\xE0nh ri\xEAng cho c\xE1c token \u0111\u1EB7c bi\u1EC7t. Tr\u01B0\u1EDBc \u0111\xE2y, ch\xFAng ta \u0111\xE3 th\u1EA5y r\u1EB1ng token \u1EDF ch\u1EC9 m\u1EE5c 5 l\xE0 "),oa=c(cs,"CODE",{});var hm=h(oa);hr=a(hm,"##yl"),hm.forEach(t),ir=a(cs,", c\xF3 "),ra=c(cs,"CODE",{});var im=h(ra);or=a(im,"(12, 14)"),im.forEach(t),rr=a(cs," l\xE0 c\xE1c ph\u1EA7n b\xF9 \u1EDF \u0111\xE2y. N\u1EBFu ch\xFAng ta l\u1EA5y ph\u1EA7n t\u01B0\u01A1ng \u1EE9ng trong m\u1EABu c\u1EE7a m\xECnh:"),cs.forEach(t),Nl=p(n),k(Ns.$$.fragment,n),Ml=p(n),es=c(n,"P",{});var kc=h(es);pr=a(kc,"ta nh\u1EADn \u0111\u01B0\u1EE3c kho\u1EA3ng v\u0103n b\u1EA3n th\xEDch h\u1EE3p m\xE0 kh\xF4ng c\xF3 "),pa=c(kc,"CODE",{});var om=h(pa);gr=a(om,"##"),om.forEach(t),mr=a(kc,":"),kc.forEach(t),Xl=p(n),k(Ms.$$.fragment,n),Hl=p(n),vt=c(n,"P",{});var rm=h(vt);ur=a(rm,"S\u1EED d\u1EE5ng \u0111i\u1EC1u n\xE0y, b\xE2y gi\u1EDD ch\xFAng ta c\xF3 th\u1EC3 ho\xE0n th\xE0nh c\xE1c k\u1EBFt qu\u1EA3 tr\u01B0\u1EDBc \u0111\xF3:"),rm.forEach(t),Gl=p(n),k(Xs.$$.fragment,n),Ll=p(n),k(Hs.$$.fragment,n),Vl=p(n),jt=c(n,"P",{});var pm=h(jt);dr=a(pm,"\u0110\xE2y gi\u1ED1ng nh\u01B0 nh\u1EEFng g\xEC ch\xFAng ta c\xF3 \u0111\u01B0\u1EE3c t\u1EEB pipeline \u0111\u1EA7u ti\xEAn!"),pm.forEach(t),Kl=p(n),Fn=c(n,"H3",{class:!0});var vc=h(Fn);as=c(vc,"A",{id:!0,class:!0,href:!0});var gm=h(as);ga=c(gm,"SPAN",{});var mm=h(ga);k(Gs.$$.fragment,mm),mm.forEach(t),gm.forEach(t),fr=p(vc),ma=c(vc,"SPAN",{});var um=h(ma);xr=a(um,"Nh\xF3m c\xE1c th\u1EF1c th\u1EC3"),um.forEach(t),vc.forEach(t),Yl=p(n),L=c(n,"P",{});var an=h(L);br=a(an,"S\u1EED d\u1EE5ng c\xE1c offset \u0111\u1EC3 x\xE1c \u0111\u1ECBnh \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u v\xE0 k\u1EBFt th\xFAc cho m\u1ED7i th\u1EF1c th\u1EC3 l\xE0 r\u1EA5t ti\u1EC7n d\u1EE5ng, nh\u01B0ng th\xF4ng tin \u0111\xF3 kh\xF4ng ho\xE0n to\xE0n c\u1EA7n thi\u1EBFt. Tuy nhi\xEAn, khi ch\xFAng ta mu\u1ED1n nh\xF3m c\xE1c th\u1EF1c th\u1EC3 l\u1EA1i v\u1EDBi nhau, vi\u1EC7c offset s\u1EBD gi\xFAp ch\xFAng ta ti\u1EBFt ki\u1EC7m r\u1EA5t nhi\u1EC1u \u0111o\u1EA1n m\xE3 l\u1ED9n x\u1ED9n. V\xED d\u1EE5: n\u1EBFu ch\xFAng ta mu\u1ED1n nh\xF3m c\xE1c token "),ua=c(an,"CODE",{});var dm=h(ua);kr=a(dm,"Hu"),dm.forEach(t),vr=a(an,", "),da=c(an,"CODE",{});var fm=h(da);jr=a(fm,"##gging"),fm.forEach(t),_r=a(an," v\xE0 "),fa=c(an,"CODE",{});var xm=h(fa);$r=a(xm,"Face"),xm.forEach(t),yr=a(an," l\u1EA1i v\u1EDBi nhau, ch\xFAng ta c\xF3 th\u1EC3 \u0111\u01B0a ra c\xE1c quy t\u1EAFc \u0111\u1EB7c bi\u1EC7t n\xF3i r\u1EB1ng hai token \u0111\u1EA7u ti\xEAn ph\u1EA3i \u0111\u01B0\u1EE3c \u0111\xEDnh k\xE8m trong khi x\xF3a d\u1EA5u "),xa=c(an,"CODE",{});var bm=h(xa);Er=a(bm,"##"),bm.forEach(t),wr=a(an," v\xE0 "),ba=c(an,"CODE",{});var km=h(ba);Cr=a(km,"Face"),km.forEach(t),Or=a(an," n\xEAn \u0111\u01B0\u1EE3c th\xEAm m\u1ED9t kho\u1EA3ng tr\u1EAFng v\xEC n\xF3 kh\xF4ng b\u1EAFt \u0111\u1EA7u b\u1EB1ng "),ka=c(an,"CODE",{});var vm=h(ka);zr=a(vm,"##"),vm.forEach(t),Ir=a(an," - nh\u01B0ng \u0111i\u1EC1u \u0111\xF3 s\u1EBD ch\u1EC9 ho\u1EA1t \u0111\u1ED9ng \u0111\u1ED1i v\u1EDBi lo\u1EA1i tokenizer c\u1EE5 th\u1EC3 n\xE0y. Ch\xFAng ta s\u1EBD ph\u1EA3i vi\u1EBFt m\u1ED9t b\u1ED9 quy t\u1EAFc kh\xE1c cho SentencePiece ho\u1EB7c Byte-Pair-Encoding (s\u1EBD \u0111\u01B0\u1EE3c th\u1EA3o lu\u1EADn \u1EDF ph\u1EA7n sau c\u1EE7a ch\u01B0\u01A1ng n\xE0y)."),an.forEach(t),Ql=p(n),W=c(n,"P",{});var un=h(W);Tr=a(un,"V\u1EDBi offset, t\u1EA5t c\u1EA3 m\xE3 t\xF9y ch\u1EC9nh \u0111\xF3 s\u1EBD bi\u1EBFn m\u1EA5t: ch\xFAng ta ch\u1EC9 c\xF3 th\u1EC3 l\u1EA5y kho\u1EA3ng trong v\u0103n b\u1EA3n g\u1ED1c b\u1EAFt \u0111\u1EA7u b\u1EB1ng token \u0111\u1EA7u ti\xEAn v\xE0 k\u1EBFt th\xFAc b\u1EB1ng token cu\u1ED1i c\xF9ng. V\xEC v\u1EADy, trong tr\u01B0\u1EDDng h\u1EE3p c\xE1c m\xE3 th\xF4ng b\xE1o "),va=c(un,"CODE",{});var jm=h(va);Dr=a(jm,"Hu"),jm.forEach(t),qr=a(un,", "),ja=c(un,"CODE",{});var _m=h(ja);Pr=a(_m,"##gging"),_m.forEach(t),Sr=a(un," v\xE0 "),_a=c(un,"CODE",{});var $m=h(_a);Br=a($m,"Face"),$m.forEach(t),Rr=a(un,", ch\xFAng ta n\xEAn b\u1EAFt \u0111\u1EA7u \u1EDF k\xFD t\u1EF1 33 (\u0111\u1EA7u c\u1EE7a "),$a=c(un,"CODE",{});var ym=h($a);Fr=a(ym,"Hu"),ym.forEach(t),Ar=a(un,") v\xE0 k\u1EBFt th\xFAc tr\u01B0\u1EDBc k\xFD t\u1EF1 45 (cu\u1ED1i c\u1EE7a "),ya=c(un,"CODE",{});var Em=h(ya);Nr=a(Em,"Face"),Em.forEach(t),Mr=a(un,") :"),un.forEach(t),Ul=p(n),k(Ls.$$.fragment,n),Jl=p(n),k(Vs.$$.fragment,n),Wl=p(n),Z=c(n,"P",{});var dn=h(Z);Xr=a(dn,"\u0110\u1EC3 vi\u1EBFt \u0111o\u1EA1n m\xE3 h\u1EADu x\u1EED l\xFD c\xE1c d\u1EF1 \u0111o\xE1n trong khi nh\xF3m c\xE1c th\u1EF1c th\u1EC3, ta s\u1EBD nh\xF3m c\xE1c th\u1EF1c th\u1EC3 li\xEAn ti\u1EBFp v\xE0 c\xF3 nh\xE3n "),Ea=c(dn,"CODE",{});var wm=h(Ea);Hr=a(wm,"I-XXX"),wm.forEach(t),Gr=a(dn," v\u1EDBi nhau tr\u1EEB khi n\xF3 l\xE0 t\u1EEB \u0111\u1EA7u ti\xEAn, \u0111\u01B0\u1EE3c g\xE1n nh\xE3n "),wa=c(dn,"CODE",{});var Cm=h(wa);Lr=a(Cm,"B-XXX"),Cm.forEach(t),Vr=a(dn," ho\u1EB7c "),Ca=c(dn,"CODE",{});var Om=h(Ca);Kr=a(Om,"I-XXX"),Om.forEach(t),Yr=a(dn," (\u0111\u1EC3 ta c\xF3 th\u1EC3 d\u1EEBng nh\xF3m m\u1ED9t th\u1EF1c th\u1EC3 khi nh\u1EADn \u0111\u01B0\u1EE3c "),Oa=c(dn,"CODE",{});var zm=h(Oa);Qr=a(zm,"O"),zm.forEach(t),Ur=a(dn,", m\u1ED9t ki\u1EC3u th\u1EF1c th\u1EC3 m\u1EDBi, ho\u1EB7c m\u1ED9t "),za=c(dn,"CODE",{});var Im=h(za);Jr=a(Im,"B-XXX"),Im.forEach(t),Wr=a(dn," cho ta bi\u1EBFt th\u1EF1c th\u1EC3 c\xF3 ki\u1EC3u gi\u1ED1ng v\u1EDBi \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u):"),dn.forEach(t),Zl=p(n),k(Ks.$$.fragment,n),nc=p(n),_t=c(n,"P",{});var Tm=h(_t);Zr=a(Tm,"V\xE0 ch\xFAng ta nh\u1EADn \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 t\u01B0\u01A1ng t\u1EF1 nh\u01B0 v\u1EDBi pipeline th\u1EE9 hai c\u1EE7a m\xECnh!"),Tm.forEach(t),sc=p(n),k(Ys.$$.fragment,n),tc=p(n),$t=c(n,"P",{});var Dm=h($t);np=a(Dm,"M\u1ED9t v\xED d\u1EE5 kh\xE1c v\u1EC1 t\xE1c v\u1EE5 m\xE0 nh\u1EEFng offset n\xE0y c\u1EF1c k\u1EF3 h\u1EEFu \xEDch l\xE0 h\u1ECFi \u0111\xE1p. \u0110\xE0o s\xE2u v\xE0o pipeline n\xE0y, ch\xFAng ta s\u1EBD th\u1EF1c hi\u1EC7n trong ph\u1EA7n ti\u1EBFp theo, c\u0169ng s\u1EBD cho ph\xE9p ch\xFAng ta xem x\xE9t m\u1ED9t t\xEDnh n\u0103ng cu\u1ED1i c\xF9ng c\u1EE7a c\xE1c tokenizers trong th\u01B0 vi\u1EC7n \u{1F917} Transformers: x\u1EED l\xFD c\xE1c token tr\xE0n khi ch\xFAng ta c\u1EAFt b\u1EDBt m\u1ED9t \u0111\u1EA7u v\xE0o \u0111\u1EBFn m\u1ED9t \u0111\u1ED9 d\xE0i nh\u1EA5t \u0111\u1ECBnh."),Dm.forEach(t),this.h()},h(){_(g,"name","hf:doc:metadata"),_(g,"content",JSON.stringify(Zm)),_(w,"id","sc-mnh-c-bit-ca-tokenizer-nhanh"),_(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(w,"href","#sc-mnh-c-bit-ca-tokenizer-nhanh"),_(q,"class","relative group"),_(on,"href","/course/chapter1"),_(nt,"href","/course/chapter5/3"),_(st,"align","center"),_(tt,"align","center"),_(Rt,"align","center"),_(et,"align","center"),_(at,"align","center"),_(lt,"align","center"),_(ct,"align","center"),_(ht,"align","center"),_(it,"align","center"),_(Xn,"id","m-ho-theo-l"),_(Xn,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Xn,"href","#m-ho-theo-l"),_(qn,"class","relative group"),_(Yn,"id","bn-trong-pipeline-tokenclassification"),_(Yn,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Yn,"href","#bn-trong-pipeline-tokenclassification"),_(Pn,"class","relative group"),_(gt,"href","/course/chapter1"),_(mt,"href","/course/chapter2"),_(Qn,"id","nhn-kt-qu-c-s-vi-baseline"),_(Qn,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Qn,"href","#nhn-kt-qu-c-s-vi-baseline"),_(Sn,"class","relative group"),_(Es,"href","https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english"),_(Es,"rel","nofollow"),_(ns,"id","t-u-vo-ti-d-on"),_(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(ns,"href","#t-u-vo-ti-d-on"),_(Bn,"class","relative group"),_(Ps,"class","block dark:hidden"),qm(Ps.src,ap="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions.svg")||_(Ps,"src",ap),_(Ps,"alt","IOB1 vs IOB2 format"),_(Ss,"class","hidden dark:block"),qm(Ss.src,lp="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions-dark.svg")||_(Ss,"src",lp),_(Ss,"alt","IOB1 vs IOB2 format"),_(Rn,"class","flex justify-center"),_(as,"id","nhm-cc-thc-th"),_(as,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(as,"href","#nhm-cc-thc-th"),_(Fn,"class","relative group")},m(n,i){s(document.head,g),o(n,x,i),v(m,n,i),o(n,$,i),o(n,q,i),s(q,w),s(w,S),v(A,S,null),s(q,H),s(q,P),s(P,K),o(n,O,i),Qs[F].m(n,i),o(n,T,i),o(n,C,i),s(C,N),s(C,B),s(B,Q),s(C,M),s(C,nn),s(nn,u),s(C,E),s(C,fn),s(fn,zn),s(C,ln),s(C,on),s(on,yc),s(C,Ec),o(n,Pa,i),v(hs,n,i),o(n,Sa,i),o(n,An,i),s(An,wc),s(An,nt),s(nt,Cc),s(An,Oc),o(n,Ba,i),o(n,Nn,i),s(Nn,Bt),s(Bt,In),s(In,st),s(st,zc),s(In,Ic),s(In,tt),s(tt,Tc),s(In,Dc),s(In,Rt),s(Nn,qc),s(Nn,is),s(is,Tn),s(Tn,et),s(et,Ft),s(Ft,Pc),s(Tn,Sc),s(Tn,at),s(at,Bc),s(Tn,Rc),s(Tn,lt),s(lt,Fc),s(is,Ac),s(is,Dn),s(Dn,ct),s(ct,At),s(At,Nc),s(Dn,Mc),s(Dn,ht),s(ht,Xc),s(Dn,Hc),s(Dn,it),s(it,Gc),o(n,Ra,i),v(Mn,n,i),o(n,Fa,i),o(n,qn,i),s(qn,Xn),s(Xn,Nt),v(os,Nt,null),s(qn,Lc),s(qn,Mt),s(Mt,Vc),o(n,Aa,i),v(rs,n,i),o(n,Na,i),o(n,Hn,i),s(Hn,Kc),s(Hn,Xt),s(Xt,Yc),s(Hn,Qc),o(n,Ma,i),o(n,$n,i),s($n,Uc),s($n,Ht),s(Ht,Jc),s($n,Wc),s($n,Gt),s(Gt,Zc),s($n,nh),o(n,Xa,i),o(n,ot,i),s(ot,sh),o(n,Ha,i),v(ps,n,i),o(n,Ga,i),o(n,Gn,i),s(Gn,th),s(Gn,Lt),s(Lt,eh),s(Gn,ah),o(n,La,i),v(gs,n,i),o(n,Va,i),o(n,sn,i),s(sn,lh),s(sn,Vt),s(Vt,ch),s(sn,hh),s(sn,Kt),s(Kt,ih),s(sn,oh),s(sn,Yt),s(Yt,rh),s(sn,ph),s(sn,Qt),s(Qt,gh),s(sn,mh),o(n,Ka,i),v(ms,n,i),o(n,Ya,i),v(us,n,i),o(n,Qa,i),o(n,Ln,i),s(Ln,uh),s(Ln,Ut),s(Ut,dh),s(Ln,fh),o(n,Ua,i),v(ds,n,i),o(n,Ja,i),v(fs,n,i),o(n,Wa,i),o(n,rt,i),s(rt,xh),o(n,Za,i),v(xs,n,i),o(n,nl,i),v(bs,n,i),o(n,sl,i),o(n,yn,i),s(yn,bh),s(yn,Jt),s(Jt,kh),s(yn,vh),s(yn,Wt),s(Wt,jh),s(yn,_h),o(n,tl,i),v(ks,n,i),o(n,el,i),v(vs,n,i),o(n,al,i),o(n,U,i),s(U,$h),s(U,Zt),s(Zt,yh),s(U,Eh),s(U,ne),s(ne,wh),s(U,Ch),s(U,se),s(se,Oh),s(U,zh),s(U,te),s(te,Ih),s(U,Th),s(U,ee),s(ee,Dh),s(U,qh),o(n,ll,i),v(Vn,n,i),o(n,cl,i),o(n,En,i),s(En,Ph),s(En,ae),s(ae,Sh),s(En,Bh),s(En,le),s(le,Rh),s(En,Fh),o(n,hl,i),o(n,G,i),s(G,Ah),s(G,ce),s(ce,Nh),s(G,Mh),s(G,he),s(he,Xh),s(G,Hh),s(G,ie),s(ie,Gh),s(G,Lh),s(G,oe),s(oe,Vh),s(G,Kh),s(G,re),s(re,Yh),s(G,Qh),s(G,pe),s(pe,Uh),s(G,Jh),o(n,il,i),v(js,n,i),o(n,ol,i),v(_s,n,i),o(n,rl,i),o(n,rn,i),s(rn,Wh),s(rn,ge),s(ge,Zh),s(rn,ni),s(rn,me),s(me,si),s(rn,ti),s(rn,ue),s(ue,ei),s(rn,ai),o(n,pl,i),v(Kn,n,i),o(n,gl,i),o(n,Pn,i),s(Pn,Yn),s(Yn,de),v($s,de,null),s(Pn,li),s(Pn,pt),s(pt,ci),s(pt,fe),s(fe,hi),o(n,ml,i),o(n,tn,i),s(tn,ii),s(tn,gt),s(gt,oi),s(tn,ri),s(tn,xe),s(xe,pi),s(tn,gi),s(tn,mt),s(mt,mi),s(tn,ui),s(tn,be),s(be,di),s(tn,fi),o(n,ul,i),Us[xn].m(n,i),o(n,ut,i),o(n,Sn,i),s(Sn,Qn),s(Qn,ke),v(ys,ke,null),s(Sn,xi),s(Sn,ve),s(ve,bi),o(n,dl,i),o(n,Un,i),s(Un,ki),s(Un,Es),s(Es,je),s(je,vi),s(Un,ji),o(n,fl,i),v(ws,n,i),o(n,xl,i),v(Cs,n,i),o(n,bl,i),o(n,dt,i),s(dt,_i),o(n,kl,i),v(Os,n,i),o(n,vl,i),v(zs,n,i),o(n,jl,i),o(n,Y,i),s(Y,_e),s(_e,$i),s(Y,yi),s(Y,$e),s($e,Ei),s(Y,wi),s(Y,ye),s(ye,Ci),s(Y,Oi),s(Y,Ee),s(Ee,zi),s(Y,Ii),s(Y,we),s(we,Ti),s(Y,Di),s(Y,Ce),s(Ce,qi),s(Y,Pi),o(n,_l,i),o(n,wn,i),s(wn,Jn),s(Jn,Oe),s(Oe,Si),s(Jn,Bi),s(Jn,ze),s(ze,Ri),s(Jn,Fi),s(wn,Ai),s(wn,ft),s(ft,Ie),s(Ie,Ni),s(ft,Mi),s(wn,Xi),s(wn,Wn),s(Wn,Te),s(Te,Hi),s(Wn,Gi),s(Wn,De),s(De,Li),s(Wn,Vi),o(n,$l,i),o(n,Zn,i),s(Zn,Ki),s(Zn,qe),s(qe,Yi),s(Zn,Qi),o(n,yl,i),o(n,Bn,i),s(Bn,ns),s(ns,Pe),v(Is,Pe,null),s(Bn,Ui),s(Bn,Se),s(Se,Ji),o(n,El,i),Js[kn].m(n,i),o(n,xt,i),o(n,bt,i),s(bt,Wi),o(n,wl,i),Ws[jn].m(n,i),o(n,kt,i),v(Ts,n,i),o(n,Cl,i),o(n,ss,i),s(ss,Zi),s(ss,Be),s(Be,no),s(ss,so),o(n,Ol,i),v(Ds,n,i),o(n,zl,i),v(qs,n,i),o(n,Il,i),o(n,z,i),s(z,to),s(z,Re),s(Re,eo),s(z,ao),s(z,Fe),s(Fe,lo),s(z,co),s(z,Ae),s(Ae,ho),s(z,io),s(z,Ne),s(Ne,oo),s(z,ro),s(z,Me),s(Me,po),s(z,go),s(z,Xe),s(Xe,mo),s(z,uo),s(z,He),s(He,fo),s(z,xo),s(z,Ge),s(Ge,bo),s(z,ko),s(z,Le),s(Le,vo),s(z,jo),s(z,Ve),s(Ve,_o),s(z,$o),s(z,Ke),s(Ke,yo),s(z,Eo),o(n,Tl,i),o(n,X,i),s(X,wo),s(X,Ye),s(Ye,Co),s(X,Oo),s(X,Qe),s(Qe,zo),s(X,Io),s(X,Ue),s(Ue,To),s(X,Do),s(X,Je),s(Je,qo),s(X,Po),s(X,We),s(We,So),s(X,Bo),s(X,Ze),s(Ze,Ro),s(X,Fo),s(X,na),s(na,Ao),s(X,No),s(X,sa),s(sa,Mo),s(X,Xo),o(n,Dl,i),o(n,Rn,i),s(Rn,Ps),s(Rn,Ho),s(Rn,Ss),o(n,ql,i),o(n,ts,i),s(ts,Go),s(ts,ta),s(ta,Lo),s(ts,Vo),o(n,Pl,i),v(Bs,n,i),o(n,Sl,i),v(Rs,n,i),o(n,Bl,i),o(n,J,i),s(J,Ko),s(J,ea),s(ea,Yo),s(J,Qo),s(J,aa),s(aa,Uo),s(J,Jo),s(J,la),s(la,Wo),s(J,Zo),s(J,ca),s(ca,nr),s(J,sr),s(J,ha),s(ha,tr),s(J,er),o(n,Rl,i),v(Fs,n,i),o(n,Fl,i),v(As,n,i),o(n,Al,i),o(n,pn,i),s(pn,ar),s(pn,ia),s(ia,lr),s(pn,cr),s(pn,oa),s(oa,hr),s(pn,ir),s(pn,ra),s(ra,or),s(pn,rr),o(n,Nl,i),v(Ns,n,i),o(n,Ml,i),o(n,es,i),s(es,pr),s(es,pa),s(pa,gr),s(es,mr),o(n,Xl,i),v(Ms,n,i),o(n,Hl,i),o(n,vt,i),s(vt,ur),o(n,Gl,i),v(Xs,n,i),o(n,Ll,i),v(Hs,n,i),o(n,Vl,i),o(n,jt,i),s(jt,dr),o(n,Kl,i),o(n,Fn,i),s(Fn,as),s(as,ga),v(Gs,ga,null),s(Fn,fr),s(Fn,ma),s(ma,xr),o(n,Yl,i),o(n,L,i),s(L,br),s(L,ua),s(ua,kr),s(L,vr),s(L,da),s(da,jr),s(L,_r),s(L,fa),s(fa,$r),s(L,yr),s(L,xa),s(xa,Er),s(L,wr),s(L,ba),s(ba,Cr),s(L,Or),s(L,ka),s(ka,zr),s(L,Ir),o(n,Ql,i),o(n,W,i),s(W,Tr),s(W,va),s(va,Dr),s(W,qr),s(W,ja),s(ja,Pr),s(W,Sr),s(W,_a),s(_a,Br),s(W,Rr),s(W,$a),s($a,Fr),s(W,Ar),s(W,ya),s(ya,Nr),s(W,Mr),o(n,Ul,i),v(Ls,n,i),o(n,Jl,i),v(Vs,n,i),o(n,Wl,i),o(n,Z,i),s(Z,Xr),s(Z,Ea),s(Ea,Hr),s(Z,Gr),s(Z,wa),s(wa,Lr),s(Z,Vr),s(Z,Ca),s(Ca,Kr),s(Z,Yr),s(Z,Oa),s(Oa,Qr),s(Z,Ur),s(Z,za),s(za,Jr),s(Z,Wr),o(n,Zl,i),v(Ks,n,i),o(n,nc,i),o(n,_t,i),s(_t,Zr),o(n,sc,i),v(Ys,n,i),o(n,tc,i),o(n,$t,i),s($t,np),ec=!0},p(n,[i]){const Zs={};i&1&&(Zs.fw=n[0]),m.$set(Zs);let yt=F;F=hp(n),F!==yt&&(_c(),d(Qs[yt],1,1,()=>{Qs[yt]=null}),jc(),I=Qs[F],I||(I=Qs[F]=cp[F](n),I.c()),f(I,1),I.m(T.parentNode,T));const Ia={};i&2&&(Ia.$$scope={dirty:i,ctx:n}),Mn.$set(Ia);const Ta={};i&2&&(Ta.$$scope={dirty:i,ctx:n}),Vn.$set(Ta);const cn={};i&2&&(cn.$$scope={dirty:i,ctx:n}),Kn.$set(cn);let Et=xn;xn=op(n),xn!==Et&&(_c(),d(Us[Et],1,1,()=>{Us[Et]=null}),jc(),bn=Us[xn],bn||(bn=Us[xn]=ip[xn](n),bn.c()),f(bn,1),bn.m(ut.parentNode,ut));let wt=kn;kn=pp(n),kn!==wt&&(_c(),d(Js[wt],1,1,()=>{Js[wt]=null}),jc(),vn=Js[kn],vn||(vn=Js[kn]=rp[kn](n),vn.c()),f(vn,1),vn.m(xt.parentNode,xt));let Ct=jn;jn=mp(n),jn!==Ct&&(_c(),d(Ws[Ct],1,1,()=>{Ws[Ct]=null}),jc(),_n=Ws[jn],_n||(_n=Ws[jn]=gp[jn](n),_n.c()),f(_n,1),_n.m(kt.parentNode,kt))},i(n){ec||(f(m.$$.fragment,n),f(A.$$.fragment,n),f(I),f(hs.$$.fragment,n),f(Mn.$$.fragment,n),f(os.$$.fragment,n),f(rs.$$.fragment,n),f(ps.$$.fragment,n),f(gs.$$.fragment,n),f(ms.$$.fragment,n),f(us.$$.fragment,n),f(ds.$$.fragment,n),f(fs.$$.fragment,n),f(xs.$$.fragment,n),f(bs.$$.fragment,n),f(ks.$$.fragment,n),f(vs.$$.fragment,n),f(Vn.$$.fragment,n),f(js.$$.fragment,n),f(_s.$$.fragment,n),f(Kn.$$.fragment,n),f($s.$$.fragment,n),f(bn),f(ys.$$.fragment,n),f(ws.$$.fragment,n),f(Cs.$$.fragment,n),f(Os.$$.fragment,n),f(zs.$$.fragment,n),f(Is.$$.fragment,n),f(vn),f(_n),f(Ts.$$.fragment,n),f(Ds.$$.fragment,n),f(qs.$$.fragment,n),f(Bs.$$.fragment,n),f(Rs.$$.fragment,n),f(Fs.$$.fragment,n),f(As.$$.fragment,n),f(Ns.$$.fragment,n),f(Ms.$$.fragment,n),f(Xs.$$.fragment,n),f(Hs.$$.fragment,n),f(Gs.$$.fragment,n),f(Ls.$$.fragment,n),f(Vs.$$.fragment,n),f(Ks.$$.fragment,n),f(Ys.$$.fragment,n),ec=!0)},o(n){d(m.$$.fragment,n),d(A.$$.fragment,n),d(I),d(hs.$$.fragment,n),d(Mn.$$.fragment,n),d(os.$$.fragment,n),d(rs.$$.fragment,n),d(ps.$$.fragment,n),d(gs.$$.fragment,n),d(ms.$$.fragment,n),d(us.$$.fragment,n),d(ds.$$.fragment,n),d(fs.$$.fragment,n),d(xs.$$.fragment,n),d(bs.$$.fragment,n),d(ks.$$.fragment,n),d(vs.$$.fragment,n),d(Vn.$$.fragment,n),d(js.$$.fragment,n),d(_s.$$.fragment,n),d(Kn.$$.fragment,n),d($s.$$.fragment,n),d(bn),d(ys.$$.fragment,n),d(ws.$$.fragment,n),d(Cs.$$.fragment,n),d(Os.$$.fragment,n),d(zs.$$.fragment,n),d(Is.$$.fragment,n),d(vn),d(_n),d(Ts.$$.fragment,n),d(Ds.$$.fragment,n),d(qs.$$.fragment,n),d(Bs.$$.fragment,n),d(Rs.$$.fragment,n),d(Fs.$$.fragment,n),d(As.$$.fragment,n),d(Ns.$$.fragment,n),d(Ms.$$.fragment,n),d(Xs.$$.fragment,n),d(Hs.$$.fragment,n),d(Gs.$$.fragment,n),d(Ls.$$.fragment,n),d(Vs.$$.fragment,n),d(Ks.$$.fragment,n),d(Ys.$$.fragment,n),ec=!1},d(n){t(g),n&&t(x),j(m,n),n&&t($),n&&t(q),j(A),n&&t(O),Qs[F].d(n),n&&t(T),n&&t(C),n&&t(Pa),j(hs,n),n&&t(Sa),n&&t(An),n&&t(Ba),n&&t(Nn),n&&t(Ra),j(Mn,n),n&&t(Fa),n&&t(qn),j(os),n&&t(Aa),j(rs,n),n&&t(Na),n&&t(Hn),n&&t(Ma),n&&t($n),n&&t(Xa),n&&t(ot),n&&t(Ha),j(ps,n),n&&t(Ga),n&&t(Gn),n&&t(La),j(gs,n),n&&t(Va),n&&t(sn),n&&t(Ka),j(ms,n),n&&t(Ya),j(us,n),n&&t(Qa),n&&t(Ln),n&&t(Ua),j(ds,n),n&&t(Ja),j(fs,n),n&&t(Wa),n&&t(rt),n&&t(Za),j(xs,n),n&&t(nl),j(bs,n),n&&t(sl),n&&t(yn),n&&t(tl),j(ks,n),n&&t(el),j(vs,n),n&&t(al),n&&t(U),n&&t(ll),j(Vn,n),n&&t(cl),n&&t(En),n&&t(hl),n&&t(G),n&&t(il),j(js,n),n&&t(ol),j(_s,n),n&&t(rl),n&&t(rn),n&&t(pl),j(Kn,n),n&&t(gl),n&&t(Pn),j($s),n&&t(ml),n&&t(tn),n&&t(ul),Us[xn].d(n),n&&t(ut),n&&t(Sn),j(ys),n&&t(dl),n&&t(Un),n&&t(fl),j(ws,n),n&&t(xl),j(Cs,n),n&&t(bl),n&&t(dt),n&&t(kl),j(Os,n),n&&t(vl),j(zs,n),n&&t(jl),n&&t(Y),n&&t(_l),n&&t(wn),n&&t($l),n&&t(Zn),n&&t(yl),n&&t(Bn),j(Is),n&&t(El),Js[kn].d(n),n&&t(xt),n&&t(bt),n&&t(wl),Ws[jn].d(n),n&&t(kt),j(Ts,n),n&&t(Cl),n&&t(ss),n&&t(Ol),j(Ds,n),n&&t(zl),j(qs,n),n&&t(Il),n&&t(z),n&&t(Tl),n&&t(X),n&&t(Dl),n&&t(Rn),n&&t(ql),n&&t(ts),n&&t(Pl),j(Bs,n),n&&t(Sl),j(Rs,n),n&&t(Bl),n&&t(J),n&&t(Rl),j(Fs,n),n&&t(Fl),j(As,n),n&&t(Al),n&&t(pn),n&&t(Nl),j(Ns,n),n&&t(Ml),n&&t(es),n&&t(Xl),j(Ms,n),n&&t(Hl),n&&t(vt),n&&t(Gl),j(Xs,n),n&&t(Ll),j(Hs,n),n&&t(Vl),n&&t(jt),n&&t(Kl),n&&t(Fn),j(Gs),n&&t(Yl),n&&t(L),n&&t(Ql),n&&t(W),n&&t(Ul),j(Ls,n),n&&t(Jl),j(Vs,n),n&&t(Wl),n&&t(Z),n&&t(Zl),j(Ks,n),n&&t(nc),n&&t(_t),n&&t(sc),j(Ys,n),n&&t(tc),n&&t($t)}}}const Zm={local:"sc-mnh-c-bit-ca-tokenizer-nhanh",sections:[{local:"m-ho-theo-l",title:"M\xE3 ho\xE1 theo l\xF4"},{local:"bn-trong-pipeline-tokenclassification",sections:[{local:"nhn-kt-qu-c-s-vi-baseline",title:"Nh\u1EADn k\u1EBFt qu\u1EA3 c\u01A1 s\u1EDF v\u1EDBi baseline"},{local:"t-u-vo-ti-d-on",title:"T\u1EEB \u0111\u1EA7u v\xE0o t\u1EDBi d\u1EF1 \u0111o\xE1n"},{local:"nhm-cc-thc-th",title:"Nh\xF3m c\xE1c th\u1EF1c th\u1EC3"}],title:"B\xEAn trong pipeline `token-classification`"}],title:"S\u1EE9c m\u1EA1nh \u0111\u1EB7c bi\u1EC7t c\u1EE7a tokenizer nhanh"};function nu(D,g,x){let m="pt";return Am(()=>{const $=new URLSearchParams(window.location.search);x(0,m=$.get("fw")||"pt")}),[m]}class iu extends Sm{constructor(g){super();Bm(this,g,nu,Wm,Rm,{})}}export{iu as default,Zm as metadata};
