import{S as ut,i as ft,s as pt,e as i,k as c,w as ot,t as v,M as vt,c as o,d as e,m as h,a as s,x as st,h as k,b as $,G as n,g as f,y as lt,L as kt,q as ct,o as ht,B as mt,v as dt}from"../../chunks/vendor-hf-doc-builder.js";import{I as _t}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as gt}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function bt(Q){let m,L,u,p,y,d,U,z,q,S,_,T,g,H,I,b,M,B,a,E,W,F,w,G,J,C,O,R,x,j,D,P,K,A;return d=new _t({}),_=new gt({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),{c(){m=i("meta"),L=c(),u=i("h1"),p=i("a"),y=i("span"),ot(d.$$.fragment),U=c(),z=i("span"),q=v("Tokenizers, ki\u1EC3m tra n\xE0o!"),S=c(),ot(_.$$.fragment),T=c(),g=i("p"),H=v("Ch\xFAc m\u1EEBng b\u1EA1n \u0111\xE3 ho\xE0n th\xE0nh ch\u01B0\u01A1ng n\xE0y!"),I=c(),b=i("p"),M=v("Sau khi t\xECm hi\u1EC3u s\xE2u v\u1EC1 tokenizer, b\u1EA1n n\xEAn:"),B=c(),a=i("ul"),E=i("li"),W=v("C\xF3 th\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng m\u1ED9t c\xE1i c\u0169 l\xE0m m\u1EABu"),F=c(),w=i("li"),G=v("Hi\u1EC3u c\xE1ch s\u1EED d\u1EE5ng hi\u1EC7u s\u1ED1 \u0111\u1EC3 \xE1nh x\u1EA1 v\u1ECB tr\xED c\u1EE7a token v\u1EDBi kho\u1EA3ng v\u0103n b\u1EA3n ban \u0111\u1EA7u c\u1EE7a ch\xFAng"),J=c(),C=i("li"),O=v("Bi\u1EBFt s\u1EF1 kh\xE1c bi\u1EC7t gi\u1EEFa BPE, WordPiece v\xE0 Unigram"),R=c(),x=i("li"),j=v("C\xF3 th\u1EC3 tr\u1ED9n v\xE0 k\u1EBFt h\u1EE3p c\xE1c kh\u1ED1i \u0111\u01B0\u1EE3c cung c\u1EA5p b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers \u0111\u1EC3 x\xE2y d\u1EF1ng tokenizer c\u1EE7a ri\xEAng b\u1EA1n"),D=c(),P=i("li"),K=v("C\xF3 th\u1EC3 s\u1EED d\u1EE5ng tokenizer \u0111\xF3 trong th\u01B0 vi\u1EC7n \u{1F917} Transformers"),this.h()},l(t){const r=vt('[data-svelte="svelte-1phssyn"]',document.head);m=o(r,"META",{name:!0,content:!0}),r.forEach(e),L=h(t),u=o(t,"H1",{class:!0});var N=s(u);p=o(N,"A",{id:!0,class:!0,href:!0});var V=s(p);y=o(V,"SPAN",{});var X=s(y);st(d.$$.fragment,X),X.forEach(e),V.forEach(e),U=h(N),z=o(N,"SPAN",{});var Y=s(z);q=k(Y,"Tokenizers, ki\u1EC3m tra n\xE0o!"),Y.forEach(e),N.forEach(e),S=h(t),st(_.$$.fragment,t),T=h(t),g=o(t,"P",{});var Z=s(g);H=k(Z,"Ch\xFAc m\u1EEBng b\u1EA1n \u0111\xE3 ho\xE0n th\xE0nh ch\u01B0\u01A1ng n\xE0y!"),Z.forEach(e),I=h(t),b=o(t,"P",{});var tt=s(b);M=k(tt,"Sau khi t\xECm hi\u1EC3u s\xE2u v\u1EC1 tokenizer, b\u1EA1n n\xEAn:"),tt.forEach(e),B=h(t),a=o(t,"UL",{});var l=s(a);E=o(l,"LI",{});var et=s(E);W=k(et,"C\xF3 th\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng m\u1ED9t c\xE1i c\u0169 l\xE0m m\u1EABu"),et.forEach(e),F=h(l),w=o(l,"LI",{});var nt=s(w);G=k(nt,"Hi\u1EC3u c\xE1ch s\u1EED d\u1EE5ng hi\u1EC7u s\u1ED1 \u0111\u1EC3 \xE1nh x\u1EA1 v\u1ECB tr\xED c\u1EE7a token v\u1EDBi kho\u1EA3ng v\u0103n b\u1EA3n ban \u0111\u1EA7u c\u1EE7a ch\xFAng"),nt.forEach(e),J=h(l),C=o(l,"LI",{});var at=s(C);O=k(at,"Bi\u1EBFt s\u1EF1 kh\xE1c bi\u1EC7t gi\u1EEFa BPE, WordPiece v\xE0 Unigram"),at.forEach(e),R=h(l),x=o(l,"LI",{});var rt=s(x);j=k(rt,"C\xF3 th\u1EC3 tr\u1ED9n v\xE0 k\u1EBFt h\u1EE3p c\xE1c kh\u1ED1i \u0111\u01B0\u1EE3c cung c\u1EA5p b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers \u0111\u1EC3 x\xE2y d\u1EF1ng tokenizer c\u1EE7a ri\xEAng b\u1EA1n"),rt.forEach(e),D=h(l),P=o(l,"LI",{});var it=s(P);K=k(it,"C\xF3 th\u1EC3 s\u1EED d\u1EE5ng tokenizer \u0111\xF3 trong th\u01B0 vi\u1EC7n \u{1F917} Transformers"),it.forEach(e),l.forEach(e),this.h()},h(){$(m,"name","hf:doc:metadata"),$(m,"content",JSON.stringify($t)),$(p,"id","tokenizers-kim-tra-no"),$(p,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(p,"href","#tokenizers-kim-tra-no"),$(u,"class","relative group")},m(t,r){n(document.head,m),f(t,L,r),f(t,u,r),n(u,p),n(p,y),lt(d,y,null),n(u,U),n(u,z),n(z,q),f(t,S,r),lt(_,t,r),f(t,T,r),f(t,g,r),n(g,H),f(t,I,r),f(t,b,r),n(b,M),f(t,B,r),f(t,a,r),n(a,E),n(E,W),n(a,F),n(a,w),n(w,G),n(a,J),n(a,C),n(C,O),n(a,R),n(a,x),n(x,j),n(a,D),n(a,P),n(P,K),A=!0},p:kt,i(t){A||(ct(d.$$.fragment,t),ct(_.$$.fragment,t),A=!0)},o(t){ht(d.$$.fragment,t),ht(_.$$.fragment,t),A=!1},d(t){e(m),t&&e(L),t&&e(u),mt(d),t&&e(S),mt(_,t),t&&e(T),t&&e(g),t&&e(I),t&&e(b),t&&e(B),t&&e(a)}}}const $t={local:"tokenizers-kim-tra-no",title:"Tokenizers, ki\u1EC3m tra n\xE0o!"};function yt(Q){return dt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xt extends ut{constructor(m){super();ft(this,m,yt,bt,pt,{})}}export{xt as default,$t as metadata};
