import{S as Ee,i as xe,s as Te,e as o,k as p,w as Ie,t as n,M as Pe,c as r,d as a,m as d,a as i,x as Le,h as s,b as c,F as t,g as f,y as $e,L as Ae,q as Ce,o as Se,B as qe,v as He}from"../../chunks/vendor-1e8b365d.js";import{I as Ne}from"../../chunks/IconCopyLink-483c28ba.js";function We(fe){let w,q,m,k,P,_,R,L,j,H,v,D,E,G,K,N,u,Q,g,V,X,z,Y,Z,W,x,ee,F,h,$,te,ae,A,oe,re,C,ne,ie,S,se,J,y,le,T,he,ce,M;return _=new Ne({}),{c(){w=o("meta"),q=p(),m=o("h1"),k=o("a"),P=o("span"),Ie(_.$$.fragment),R=p(),L=o("span"),j=n("Introduction"),H=p(),v=o("p"),D=n("In "),E=o("a"),G=n("Chapter 3"),K=n(", we looked at how to fine-tune a model on a given task. When we do that, we use the same tokenizer that the model was pretrained with \u2014 but what do we do when we want to train a model from scratch? In these cases, using a tokenizer that was pretrained on a corpus from another domain or language is typically suboptimal. For example, a tokenizer that\u2019s trained on an English corpus will perform poorly on a corpus of Japanese texts because the use of spaces and punctuation is very different in the two languages."),N=p(),u=o("p"),Q=n("In this chapter, you will learn how to train a brand new tokenizer on a corpus of texts, so it can then be used to pretrain a language model. This will all be done with the help of the "),g=o("a"),V=n("\u{1F917} Tokenizers"),X=n(" library, which provides the \u201Cfast\u201D tokenizers in the "),z=o("a"),Y=n("\u{1F917} Transformers"),Z=n(" library. We\u2019ll take a close look at the features that this library provides, and explore how the fast tokenizers differ from the \u201Cslow\u201D versions."),W=p(),x=o("p"),ee=n("Topics we will cover include:"),F=p(),h=o("ul"),$=o("li"),te=n("How to train a new tokenizer similar to the one used by a given checkpoint on a new corpus of texts"),ae=p(),A=o("li"),oe=n("The special features of fast tokenizers"),re=p(),C=o("li"),ne=n("The differences between the three main subword tokenization algorithms used in NLP today"),ie=p(),S=o("li"),se=n("How to build a tokenizer from scratch with the \u{1F917} Tokenizers library and train it on some data"),J=p(),y=o("p"),le=n("The techniques introduced in this chapter will prepare you for the section in "),T=o("a"),he=n("Chapter 7"),ce=n(" where we look at creating a language model for Python source code. Let\u2019s start by looking at what it means to \u201Ctrain\u201D a tokenizer in the first place."),this.h()},l(e){const l=Pe('[data-svelte="svelte-1phssyn"]',document.head);w=r(l,"META",{name:!0,content:!0}),l.forEach(a),q=d(e),m=r(e,"H1",{class:!0});var U=i(m);k=r(U,"A",{id:!0,class:!0,href:!0});var ue=i(k);P=r(ue,"SPAN",{});var pe=i(P);Le(_.$$.fragment,pe),pe.forEach(a),ue.forEach(a),R=d(U),L=r(U,"SPAN",{});var de=i(L);j=s(de,"Introduction"),de.forEach(a),U.forEach(a),H=d(e),v=r(e,"P",{});var B=i(v);D=s(B,"In "),E=r(B,"A",{href:!0});var we=i(E);G=s(we,"Chapter 3"),we.forEach(a),K=s(B,", we looked at how to fine-tune a model on a given task. When we do that, we use the same tokenizer that the model was pretrained with \u2014 but what do we do when we want to train a model from scratch? In these cases, using a tokenizer that was pretrained on a corpus from another domain or language is typically suboptimal. For example, a tokenizer that\u2019s trained on an English corpus will perform poorly on a corpus of Japanese texts because the use of spaces and punctuation is very different in the two languages."),B.forEach(a),N=d(e),u=r(e,"P",{});var I=i(u);Q=s(I,"In this chapter, you will learn how to train a brand new tokenizer on a corpus of texts, so it can then be used to pretrain a language model. This will all be done with the help of the "),g=r(I,"A",{href:!0,rel:!0});var me=i(g);V=s(me,"\u{1F917} Tokenizers"),me.forEach(a),X=s(I," library, which provides the \u201Cfast\u201D tokenizers in the "),z=r(I,"A",{href:!0,rel:!0});var ke=i(z);Y=s(ke,"\u{1F917} Transformers"),ke.forEach(a),Z=s(I," library. We\u2019ll take a close look at the features that this library provides, and explore how the fast tokenizers differ from the \u201Cslow\u201D versions."),I.forEach(a),W=d(e),x=r(e,"P",{});var ve=i(x);ee=s(ve,"Topics we will cover include:"),ve.forEach(a),F=d(e),h=r(e,"UL",{});var b=i(h);$=r(b,"LI",{});var ye=i($);te=s(ye,"How to train a new tokenizer similar to the one used by a given checkpoint on a new corpus of texts"),ye.forEach(a),ae=d(b),A=r(b,"LI",{});var be=i(A);oe=s(be,"The special features of fast tokenizers"),be.forEach(a),re=d(b),C=r(b,"LI",{});var _e=i(C);ne=s(_e,"The differences between the three main subword tokenization algorithms used in NLP today"),_e.forEach(a),ie=d(b),S=r(b,"LI",{});var ge=i(S);se=s(ge,"How to build a tokenizer from scratch with the \u{1F917} Tokenizers library and train it on some data"),ge.forEach(a),b.forEach(a),J=d(e),y=r(e,"P",{});var O=i(y);le=s(O,"The techniques introduced in this chapter will prepare you for the section in "),T=r(O,"A",{href:!0});var ze=i(T);he=s(ze,"Chapter 7"),ze.forEach(a),ce=s(O," where we look at creating a language model for Python source code. Let\u2019s start by looking at what it means to \u201Ctrain\u201D a tokenizer in the first place."),O.forEach(a),this.h()},h(){c(w,"name","hf:doc:metadata"),c(w,"content",JSON.stringify(Fe)),c(k,"id","introduction"),c(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(k,"href","#introduction"),c(m,"class","relative group"),c(E,"href","/course/chapter3"),c(g,"href","https://github.com/huggingface/tokenizers"),c(g,"rel","nofollow"),c(z,"href","https://github.com/huggingface/transformers"),c(z,"rel","nofollow"),c(T,"href","/course/chapter7/6")},m(e,l){t(document.head,w),f(e,q,l),f(e,m,l),t(m,k),t(k,P),$e(_,P,null),t(m,R),t(m,L),t(L,j),f(e,H,l),f(e,v,l),t(v,D),t(v,E),t(E,G),t(v,K),f(e,N,l),f(e,u,l),t(u,Q),t(u,g),t(g,V),t(u,X),t(u,z),t(z,Y),t(u,Z),f(e,W,l),f(e,x,l),t(x,ee),f(e,F,l),f(e,h,l),t(h,$),t($,te),t(h,ae),t(h,A),t(A,oe),t(h,re),t(h,C),t(C,ne),t(h,ie),t(h,S),t(S,se),f(e,J,l),f(e,y,l),t(y,le),t(y,T),t(T,he),t(y,ce),M=!0},p:Ae,i(e){M||(Ce(_.$$.fragment,e),M=!0)},o(e){Se(_.$$.fragment,e),M=!1},d(e){a(w),e&&a(q),e&&a(m),qe(_),e&&a(H),e&&a(v),e&&a(N),e&&a(u),e&&a(W),e&&a(x),e&&a(F),e&&a(h),e&&a(J),e&&a(y)}}}const Fe={local:"introduction",title:"Introduction"};function Je(fe){return He(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Be extends Ee{constructor(w){super();xe(this,w,Je,We,Te,{})}}export{Be as default,Fe as metadata};
