import{S as ad,i as od,s as rd,e as o,k as i,w as h,t as l,M as nd,c as r,d as t,m as d,a as n,x as g,h as s,b as c,F as e,g as p,y as u,q as _,o as v,B as k}from"../../chunks/vendor-4833417e.js";import{T as td}from"../../chunks/Tip-fffd6df1.js";import{D as P}from"../../chunks/Docstring-4f315ed9.js";import{I as ye}from"../../chunks/IconCopyLink-4b81c553.js";function ld(We){let f,z,b,E,O,y,H,N,S,V,x,F,D,R;return{c(){f=o("p"),z=l(`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the `),b=o("code"),E=l('"special_tokens_mask"'),O=l(" key, as returned by a "),y=o("a"),H=l("PreTrainedTokenizer"),N=l(` or a
`),S=o("a"),V=l("PreTrainedTokenizerFast"),x=l(" with the argument "),F=o("code"),D=l("return_special_tokens_mask=True"),R=l("."),this.h()},l(j){f=r(j,"P",{});var $=n(f);z=s($,`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the `),b=r($,"CODE",{});var Ct=n(b);E=s(Ct,'"special_tokens_mask"'),Ct.forEach(t),O=s($," key, as returned by a "),y=r($,"A",{href:!0});var Tt=n(y);H=s(Tt,"PreTrainedTokenizer"),Tt.forEach(t),N=s($,` or a
`),S=r($,"A",{href:!0});var Ve=n(S);V=s(Ve,"PreTrainedTokenizerFast"),Ve.forEach(t),x=s($," with the argument "),F=r($,"CODE",{});var U=n(F);D=s(U,"return_special_tokens_mask=True"),U.forEach(t),R=s($,"."),$.forEach(t),this.h()},h(){c(y,"href","/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),c(S,"href","/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast")},m(j,$){p(j,f,$),e(f,z),e(f,b),e(b,E),e(f,O),e(f,y),e(y,H),e(f,N),e(f,S),e(S,V),e(f,x),e(f,F),e(F,D),e(f,R)},d(j){j&&t(f)}}}function sd(We){let f,z,b,E,O,y,H,N,S,V,x;return{c(){f=o("p"),z=l("This collator relies on details of the implementation of subword tokenization by "),b=o("a"),E=l("BertTokenizer"),O=l(`, specifically
that subword tokens are prefixed with `),y=o("em"),H=l("##"),N=l(`. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to `),S=o("code"),V=l(".DataCollatorForLanguageModeling"),x=l("."),this.h()},l(F){f=r(F,"P",{});var D=n(f);z=s(D,"This collator relies on details of the implementation of subword tokenization by "),b=r(D,"A",{href:!0});var R=n(b);E=s(R,"BertTokenizer"),R.forEach(t),O=s(D,`, specifically
that subword tokens are prefixed with `),y=r(D,"EM",{});var j=n(y);H=s(j,"##"),j.forEach(t),N=s(D,`. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to `),S=r(D,"CODE",{});var $=n(S);V=s($,".DataCollatorForLanguageModeling"),$.forEach(t),x=s(D,"."),D.forEach(t),this.h()},h(){c(b,"href","/docs/transformers/pr_15839/en/model_doc/bert#transformers.BertTokenizer")},m(F,D){p(F,f,D),e(f,z),e(f,b),e(b,E),e(f,O),e(f,y),e(y,H),e(f,N),e(f,S),e(S,V),e(f,x)},d(F){F&&t(f)}}}function id(We){let f,z,b,E,O,y,H,N,S,V,x,F,D,R,j,$,Ct,Tt,Ve,U,rr,Pt,nr,lr,po,X,sr,St,ir,dr,Ft,cr,mr,fo,ne,De,Rt,je,pr,Xt,fr,ho,W,Be,hr,Jt,gr,ur,Ke,Lt,Gt,_r,vr,kr,zt,Qt,br,$r,yr,Yt,Dr,go,le,Ee,Zt,He,Er,ea,wr,uo,L,Ue,xr,ta,Cr,Tr,Re,At,aa,Pr,Sr,Fr,qt,oa,Lr,zr,Ar,ra,qr,Ir,na,Mr,_o,se,we,la,Xe,Or,sa,Nr,vo,ie,Je,Wr,ia,Vr,ko,de,xe,da,Ge,jr,ca,Br,bo,ce,Qe,Kr,ma,Hr,$o,me,Ce,pa,Ye,Ur,fa,Rr,yo,pe,Ze,Xr,ha,Jr,Do,fe,Te,ga,et,Gr,ua,Qr,Eo,C,tt,Yr,_a,Zr,en,Pe,tn,Se,at,an,va,on,rn,Fe,ot,nn,ka,ln,sn,Le,rt,dn,ba,cn,wo,he,ze,$a,nt,mn,ya,pn,xo,w,lt,fn,Da,hn,gn,st,Ea,un,_n,wa,vn,kn,Ae,bn,qe,it,$n,xa,yn,Dn,Ie,dt,En,Ca,wn,xn,Me,ct,Cn,Ta,Tn,Co,ge,Oe,Pa,mt,Pn,Sa,Sn,To,T,pt,Fn,Fa,Ln,zn,ft,La,An,qn,za,In,Mn,J,ht,On,Aa,Nn,Wn,A,gt,Vn,qa,jn,Bn,Kn,ue,Hn,Ia,Un,Rn,Ma,Xn,Jn,Gn,ut,Qn,Oa,Yn,Zn,el,G,tl,Na,al,ol,Wa,rl,nl,Va,ll,sl,_e,il,ja,dl,cl,Ba,ml,pl,fl,Q,_t,hl,Ka,gl,ul,q,vt,_l,Ha,vl,kl,bl,ve,$l,Ua,yl,Dl,Ra,El,wl,xl,kt,Cl,Xa,Tl,Pl,Sl,Y,Fl,Ja,Ll,zl,Ga,Al,ql,Qa,Il,Ml,ke,Ol,Ya,Nl,Wl,Za,Vl,jl,Bl,Z,bt,Kl,eo,Hl,Ul,I,$t,Rl,to,Xl,Jl,Gl,be,Ql,ao,Yl,Zl,oo,es,ts,as,yt,os,ro,rs,ns,ls,ee,ss,no,is,ds,lo,cs,ms,so,ps,fs,$e,hs,io,gs,us,co,_s,vs,Po;return y=new ye({}),je=new ye({}),Be=new P({props:{name:"transformers.default_data_collator",anchor:"transformers.default_data_collator",parameters:[{name:"features",val:": typing.List[InputDataClass]"},{name:"return_tensors",val:" = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L48"}}),He=new ye({}),Ue=new P({props:{name:"class transformers.DefaultDataCollator",anchor:"transformers.DefaultDataCollator",parameters:[{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L74",parametersDescription:[{anchor:"transformers.DefaultDataCollator.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}]}}),Xe=new ye({}),Je=new P({props:{name:"class transformers.DataCollatorWithPadding",anchor:"transformers.DataCollatorWithPadding",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.file_utils.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L212",parametersDescription:[{anchor:"transformers.DataCollatorWithPadding.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorWithPadding.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_15839/en/internal/file_utils#transformers.file_utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code>: Pad to the longest sequence in the batch (or no padding if only a single sequence
if provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code> (default): No padding (i.e., can output a batch with sequences of different
lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorWithPadding.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorWithPadding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorWithPadding.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}]}}),Ge=new ye({}),Qe=new P({props:{name:"class transformers.DataCollatorForTokenClassification",anchor:"transformers.DataCollatorForTokenClassification",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.file_utils.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L264",parametersDescription:[{anchor:"transformers.DataCollatorForTokenClassification.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForTokenClassification.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_15839/en/internal/file_utils#transformers.file_utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code>: Pad to the longest sequence in the batch (or no padding if only a single sequence
if provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code> (default): No padding (i.e., can output a batch with sequences of different
lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForTokenClassification.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForTokenClassification.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForTokenClassification.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForTokenClassification.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}]}}),Ye=new ye({}),Ze=new P({props:{name:"class transformers.DataCollatorForSeq2Seq",anchor:"transformers.DataCollatorForSeq2Seq",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"model",val:": typing.Optional[typing.Any] = None"},{name:"padding",val:": typing.Union[bool, str, transformers.file_utils.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L514",parametersDescription:[{anchor:"transformers.DataCollatorForSeq2Seq.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForSeq2Seq.model",description:`<strong>model</strong> (<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>) &#x2014;
The model that is being trained. If set and has the <em>prepare_decoder_input_ids_from_labels</em>, use it to
prepare the <em>decoder_input_ids</em></p>
<p>This is useful when using <em>label_smoothing</em> to avoid calculating loss twice.`,name:"model"},{anchor:"transformers.DataCollatorForSeq2Seq.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_15839/en/internal/file_utils#transformers.file_utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code>: Pad to the longest sequence in the batch (or no padding if only a single sequence
is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code> (default): No padding (i.e., can output a batch with sequences of different
lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForSeq2Seq.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForSeq2Seq.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForSeq2Seq.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForSeq2Seq.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}]}}),et=new ye({}),tt=new P({props:{name:"class transformers.DataCollatorForLanguageModeling",anchor:"transformers.DataCollatorForLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L607",parametersDescription:[{anchor:"transformers.DataCollatorForLanguageModeling.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm",description:`<strong>mlm</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to use masked language modeling. If set to <code>False</code>, the labels are the same as the inputs
with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked
tokens and the value to predict for the masked token.`,name:"mlm"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm_probability",description:`<strong>mlm_probability</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) &#x2014;
The probability with which to (randomly) mask tokens in the input, when <code>mlm</code> is set to <code>True</code>.`,name:"mlm_probability"},{anchor:"transformers.DataCollatorForLanguageModeling.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForLanguageModeling.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}]}}),Pe=new td({props:{$$slots:{default:[ld]},$$scope:{ctx:We}}}),at=new P({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L805"}}),ot=new P({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"vocab_size",val:""},{name:"mask_token_id",val:""},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L659"}}),rt=new P({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L748"}}),nt=new ye({}),lt=new P({props:{name:"class transformers.DataCollatorForWholeWordMask",anchor:"transformers.DataCollatorForWholeWordMask",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L846"}}),Ae=new td({props:{$$slots:{default:[sd]},$$scope:{ctx:We}}}),it=new P({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L1072"}}),dt=new P({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L1031"}}),ct=new P({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L992"}}),mt=new ye({}),pt=new P({props:{name:"class transformers.DataCollatorForPermutationLanguageModeling",anchor:"transformers.DataCollatorForPermutationLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"plm_probability",val:": float = 0.16666666666666666"},{name:"max_span_length",val:": int = 5"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L1197"}}),ht=new P({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L1436"}}),_t=new P({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L1328"}}),bt=new P({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/data/data_collator.py#L1231"}}),{c(){f=o("meta"),z=i(),b=o("h1"),E=o("a"),O=o("span"),h(y.$$.fragment),H=i(),N=o("span"),S=l("Data Collator"),V=i(),x=o("p"),F=l(`Data collators are objects that will form a batch by using a list of dataset elements as input. These elements are of
the same type as the elements of `),D=o("code"),R=l("train_dataset"),j=l(" or "),$=o("code"),Ct=l("eval_dataset"),Tt=l("."),Ve=i(),U=o("p"),rr=l(`To be able to build batches, data collators may apply some processing (like padding). Some of them (like
`),Pt=o("a"),nr=l("DataCollatorForLanguageModeling"),lr=l(`) also apply some random data augmentation (like random masking)
on the formed batch.`),po=i(),X=o("p"),sr=l("Examples of use can be found in the "),St=o("a"),ir=l("example scripts"),dr=l(" or "),Ft=o("a"),cr=l("example notebooks"),mr=l("."),fo=i(),ne=o("h2"),De=o("a"),Rt=o("span"),h(je.$$.fragment),pr=i(),Xt=o("span"),fr=l("Default data collator"),ho=i(),W=o("div"),h(Be.$$.fragment),hr=i(),Jt=o("p"),gr=l(`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),ur=i(),Ke=o("ul"),Lt=o("li"),Gt=o("code"),_r=l("label"),vr=l(": handles a single value (int or float) per object"),kr=i(),zt=o("li"),Qt=o("code"),br=l("label_ids"),$r=l(": handles a list of values per object"),yr=i(),Yt=o("p"),Dr=l(`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),go=i(),le=o("h2"),Ee=o("a"),Zt=o("span"),h(He.$$.fragment),Er=i(),ea=o("span"),wr=l("DefaultDataCollator"),uo=i(),L=o("div"),h(Ue.$$.fragment),xr=i(),ta=o("p"),Cr=l(`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),Tr=i(),Re=o("ul"),At=o("li"),aa=o("code"),Pr=l("label"),Sr=l(": handles a single value (int or float) per object"),Fr=i(),qt=o("li"),oa=o("code"),Lr=l("label_ids"),zr=l(": handles a list of values per object"),Ar=i(),ra=o("p"),qr=l(`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),Ir=i(),na=o("p"),Mr=l(`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`),_o=i(),se=o("h2"),we=o("a"),la=o("span"),h(Xe.$$.fragment),Or=i(),sa=o("span"),Nr=l("DataCollatorWithPadding"),vo=i(),ie=o("div"),h(Je.$$.fragment),Wr=i(),ia=o("p"),Vr=l("Data collator that will dynamically pad the inputs received."),ko=i(),de=o("h2"),xe=o("a"),da=o("span"),h(Ge.$$.fragment),jr=i(),ca=o("span"),Br=l("DataCollatorForTokenClassification"),bo=i(),ce=o("div"),h(Qe.$$.fragment),Kr=i(),ma=o("p"),Hr=l("Data collator that will dynamically pad the inputs received, as well as the labels."),$o=i(),me=o("h2"),Ce=o("a"),pa=o("span"),h(Ye.$$.fragment),Ur=i(),fa=o("span"),Rr=l("DataCollatorForSeq2Seq"),yo=i(),pe=o("div"),h(Ze.$$.fragment),Xr=i(),ha=o("p"),Jr=l("Data collator that will dynamically pad the inputs received, as well as the labels."),Do=i(),fe=o("h2"),Te=o("a"),ga=o("span"),h(et.$$.fragment),Gr=i(),ua=o("span"),Qr=l("DataCollatorForLanguageModeling"),Eo=i(),C=o("div"),h(tt.$$.fragment),Yr=i(),_a=o("p"),Zr=l(`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`),en=i(),h(Pe.$$.fragment),tn=i(),Se=o("div"),h(at.$$.fragment),an=i(),va=o("p"),on=l("Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),rn=i(),Fe=o("div"),h(ot.$$.fragment),nn=i(),ka=o("p"),ln=l("Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),sn=i(),Le=o("div"),h(rt.$$.fragment),dn=i(),ba=o("p"),cn=l("Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),wo=i(),he=o("h2"),ze=o("a"),$a=o("span"),h(nt.$$.fragment),mn=i(),ya=o("span"),pn=l("DataCollatorForWholeWordMask"),xo=i(),w=o("div"),h(lt.$$.fragment),fn=i(),Da=o("p"),hn=l("Data collator used for language modeling that masks entire words."),gn=i(),st=o("ul"),Ea=o("li"),un=l("collates batches of tensors, honoring their tokenizer\u2019s pad_token"),_n=i(),wa=o("li"),vn=l("preprocesses batches for masked language modeling"),kn=i(),h(Ae.$$.fragment),bn=i(),qe=o("div"),h(it.$$.fragment),$n=i(),xa=o("p"),yn=l(`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),Dn=i(),Ie=o("div"),h(dt.$$.fragment),En=i(),Ca=o("p"),wn=l(`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),xn=i(),Me=o("div"),h(ct.$$.fragment),Cn=i(),Ta=o("p"),Tn=l(`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),Co=i(),ge=o("h2"),Oe=o("a"),Pa=o("span"),h(mt.$$.fragment),Pn=i(),Sa=o("span"),Sn=l("DataCollatorForPermutationLanguageModeling"),To=i(),T=o("div"),h(pt.$$.fragment),Fn=i(),Fa=o("p"),Ln=l("Data collator used for permutation language modeling."),zn=i(),ft=o("ul"),La=o("li"),An=l("collates batches of tensors, honoring their tokenizer\u2019s pad_token"),qn=i(),za=o("li"),In=l("preprocesses batches for permutation language modeling with procedures specific to XLNet"),Mn=i(),J=o("div"),h(ht.$$.fragment),On=i(),Aa=o("p"),Nn=l("The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),Wn=i(),A=o("ol"),gt=o("li"),Vn=l("Start from the beginning of the sequence by setting "),qa=o("code"),jn=l("cur_len = 0"),Bn=l(" (number of tokens processed so far)."),Kn=i(),ue=o("li"),Hn=l("Sample a "),Ia=o("code"),Un=l("span_length"),Rn=l(" from the interval "),Ma=o("code"),Xn=l("[1, max_span_length]"),Jn=l(" (length of span of tokens to be masked)"),Gn=i(),ut=o("li"),Qn=l("Reserve a context of length "),Oa=o("code"),Yn=l("context_length = span_length / plm_probability"),Zn=l(` to surround span to be
masked`),el=i(),G=o("li"),tl=l("Sample a starting point "),Na=o("code"),al=l("start_index"),ol=l(" from the interval "),Wa=o("code"),rl=l("[cur_len, cur_len + context_length - span_length]"),nl=l(" and mask tokens "),Va=o("code"),ll=l("start_index:start_index + span_length"),sl=i(),_e=o("li"),il=l("Set "),ja=o("code"),dl=l("cur_len = cur_len + context_length"),cl=l(". If "),Ba=o("code"),ml=l("cur_len < max_len"),pl=l(` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),fl=i(),Q=o("div"),h(_t.$$.fragment),hl=i(),Ka=o("p"),gl=l("The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),ul=i(),q=o("ol"),vt=o("li"),_l=l("Start from the beginning of the sequence by setting "),Ha=o("code"),vl=l("cur_len = 0"),kl=l(" (number of tokens processed so far)."),bl=i(),ve=o("li"),$l=l("Sample a "),Ua=o("code"),yl=l("span_length"),Dl=l(" from the interval "),Ra=o("code"),El=l("[1, max_span_length]"),wl=l(" (length of span of tokens to be masked)"),xl=i(),kt=o("li"),Cl=l("Reserve a context of length "),Xa=o("code"),Tl=l("context_length = span_length / plm_probability"),Pl=l(` to surround span to be
masked`),Sl=i(),Y=o("li"),Fl=l("Sample a starting point "),Ja=o("code"),Ll=l("start_index"),zl=l(" from the interval "),Ga=o("code"),Al=l("[cur_len, cur_len + context_length - span_length]"),ql=l(" and mask tokens "),Qa=o("code"),Il=l("start_index:start_index + span_length"),Ml=i(),ke=o("li"),Ol=l("Set "),Ya=o("code"),Nl=l("cur_len = cur_len + context_length"),Wl=l(". If "),Za=o("code"),Vl=l("cur_len < max_len"),jl=l(` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Bl=i(),Z=o("div"),h(bt.$$.fragment),Kl=i(),eo=o("p"),Hl=l("The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),Ul=i(),I=o("ol"),$t=o("li"),Rl=l("Start from the beginning of the sequence by setting "),to=o("code"),Xl=l("cur_len = 0"),Jl=l(" (number of tokens processed so far)."),Gl=i(),be=o("li"),Ql=l("Sample a "),ao=o("code"),Yl=l("span_length"),Zl=l(" from the interval "),oo=o("code"),es=l("[1, max_span_length]"),ts=l(" (length of span of tokens to be masked)"),as=i(),yt=o("li"),os=l("Reserve a context of length "),ro=o("code"),rs=l("context_length = span_length / plm_probability"),ns=l(` to surround span to be
masked`),ls=i(),ee=o("li"),ss=l("Sample a starting point "),no=o("code"),is=l("start_index"),ds=l(" from the interval "),lo=o("code"),cs=l("[cur_len, cur_len + context_length - span_length]"),ms=l(" and mask tokens "),so=o("code"),ps=l("start_index:start_index + span_length"),fs=i(),$e=o("li"),hs=l("Set "),io=o("code"),gs=l("cur_len = cur_len + context_length"),us=l(". If "),co=o("code"),_s=l("cur_len < max_len"),vs=l(` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),this.h()},l(a){const m=nd('[data-svelte="svelte-1phssyn"]',document.head);f=r(m,"META",{name:!0,content:!0}),m.forEach(t),z=d(a),b=r(a,"H1",{class:!0});var Dt=n(b);E=r(Dt,"A",{id:!0,class:!0,href:!0});var mo=n(E);O=r(mo,"SPAN",{});var Ds=n(O);g(y.$$.fragment,Ds),Ds.forEach(t),mo.forEach(t),H=d(Dt),N=r(Dt,"SPAN",{});var Es=n(N);S=s(Es,"Data Collator"),Es.forEach(t),Dt.forEach(t),V=d(a),x=r(a,"P",{});var It=n(x);F=s(It,`Data collators are objects that will form a batch by using a list of dataset elements as input. These elements are of
the same type as the elements of `),D=r(It,"CODE",{});var ws=n(D);R=s(ws,"train_dataset"),ws.forEach(t),j=s(It," or "),$=r(It,"CODE",{});var xs=n($);Ct=s(xs,"eval_dataset"),xs.forEach(t),Tt=s(It,"."),It.forEach(t),Ve=d(a),U=r(a,"P",{});var So=n(U);rr=s(So,`To be able to build batches, data collators may apply some processing (like padding). Some of them (like
`),Pt=r(So,"A",{href:!0});var Cs=n(Pt);nr=s(Cs,"DataCollatorForLanguageModeling"),Cs.forEach(t),lr=s(So,`) also apply some random data augmentation (like random masking)
on the formed batch.`),So.forEach(t),po=d(a),X=r(a,"P",{});var Mt=n(X);sr=s(Mt,"Examples of use can be found in the "),St=r(Mt,"A",{href:!0});var Ts=n(St);ir=s(Ts,"example scripts"),Ts.forEach(t),dr=s(Mt," or "),Ft=r(Mt,"A",{href:!0});var Ps=n(Ft);cr=s(Ps,"example notebooks"),Ps.forEach(t),mr=s(Mt,"."),Mt.forEach(t),fo=d(a),ne=r(a,"H2",{class:!0});var Fo=n(ne);De=r(Fo,"A",{id:!0,class:!0,href:!0});var Ss=n(De);Rt=r(Ss,"SPAN",{});var Fs=n(Rt);g(je.$$.fragment,Fs),Fs.forEach(t),Ss.forEach(t),pr=d(Fo),Xt=r(Fo,"SPAN",{});var Ls=n(Xt);fr=s(Ls,"Default data collator"),Ls.forEach(t),Fo.forEach(t),ho=d(a),W=r(a,"DIV",{class:!0});var Ne=n(W);g(Be.$$.fragment,Ne),hr=d(Ne),Jt=r(Ne,"P",{});var zs=n(Jt);gr=s(zs,`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),zs.forEach(t),ur=d(Ne),Ke=r(Ne,"UL",{});var Lo=n(Ke);Lt=r(Lo,"LI",{});var ks=n(Lt);Gt=r(ks,"CODE",{});var As=n(Gt);_r=s(As,"label"),As.forEach(t),vr=s(ks,": handles a single value (int or float) per object"),ks.forEach(t),kr=d(Lo),zt=r(Lo,"LI",{});var bs=n(zt);Qt=r(bs,"CODE",{});var qs=n(Qt);br=s(qs,"label_ids"),qs.forEach(t),$r=s(bs,": handles a list of values per object"),bs.forEach(t),Lo.forEach(t),yr=d(Ne),Yt=r(Ne,"P",{});var Is=n(Yt);Dr=s(Is,`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),Is.forEach(t),Ne.forEach(t),go=d(a),le=r(a,"H2",{class:!0});var zo=n(le);Ee=r(zo,"A",{id:!0,class:!0,href:!0});var Ms=n(Ee);Zt=r(Ms,"SPAN",{});var Os=n(Zt);g(He.$$.fragment,Os),Os.forEach(t),Ms.forEach(t),Er=d(zo),ea=r(zo,"SPAN",{});var Ns=n(ea);wr=s(Ns,"DefaultDataCollator"),Ns.forEach(t),zo.forEach(t),uo=d(a),L=r(a,"DIV",{class:!0});var te=n(L);g(Ue.$$.fragment,te),xr=d(te),ta=r(te,"P",{});var Ws=n(ta);Cr=s(Ws,`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),Ws.forEach(t),Tr=d(te),Re=r(te,"UL",{});var Ao=n(Re);At=r(Ao,"LI",{});var $s=n(At);aa=r($s,"CODE",{});var Vs=n(aa);Pr=s(Vs,"label"),Vs.forEach(t),Sr=s($s,": handles a single value (int or float) per object"),$s.forEach(t),Fr=d(Ao),qt=r(Ao,"LI",{});var ys=n(qt);oa=r(ys,"CODE",{});var js=n(oa);Lr=s(js,"label_ids"),js.forEach(t),zr=s(ys,": handles a list of values per object"),ys.forEach(t),Ao.forEach(t),Ar=d(te),ra=r(te,"P",{});var Bs=n(ra);qr=s(Bs,`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),Bs.forEach(t),Ir=d(te),na=r(te,"P",{});var Ks=n(na);Mr=s(Ks,`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`),Ks.forEach(t),te.forEach(t),_o=d(a),se=r(a,"H2",{class:!0});var qo=n(se);we=r(qo,"A",{id:!0,class:!0,href:!0});var Hs=n(we);la=r(Hs,"SPAN",{});var Us=n(la);g(Xe.$$.fragment,Us),Us.forEach(t),Hs.forEach(t),Or=d(qo),sa=r(qo,"SPAN",{});var Rs=n(sa);Nr=s(Rs,"DataCollatorWithPadding"),Rs.forEach(t),qo.forEach(t),vo=d(a),ie=r(a,"DIV",{class:!0});var Io=n(ie);g(Je.$$.fragment,Io),Wr=d(Io),ia=r(Io,"P",{});var Xs=n(ia);Vr=s(Xs,"Data collator that will dynamically pad the inputs received."),Xs.forEach(t),Io.forEach(t),ko=d(a),de=r(a,"H2",{class:!0});var Mo=n(de);xe=r(Mo,"A",{id:!0,class:!0,href:!0});var Js=n(xe);da=r(Js,"SPAN",{});var Gs=n(da);g(Ge.$$.fragment,Gs),Gs.forEach(t),Js.forEach(t),jr=d(Mo),ca=r(Mo,"SPAN",{});var Qs=n(ca);Br=s(Qs,"DataCollatorForTokenClassification"),Qs.forEach(t),Mo.forEach(t),bo=d(a),ce=r(a,"DIV",{class:!0});var Oo=n(ce);g(Qe.$$.fragment,Oo),Kr=d(Oo),ma=r(Oo,"P",{});var Ys=n(ma);Hr=s(Ys,"Data collator that will dynamically pad the inputs received, as well as the labels."),Ys.forEach(t),Oo.forEach(t),$o=d(a),me=r(a,"H2",{class:!0});var No=n(me);Ce=r(No,"A",{id:!0,class:!0,href:!0});var Zs=n(Ce);pa=r(Zs,"SPAN",{});var ei=n(pa);g(Ye.$$.fragment,ei),ei.forEach(t),Zs.forEach(t),Ur=d(No),fa=r(No,"SPAN",{});var ti=n(fa);Rr=s(ti,"DataCollatorForSeq2Seq"),ti.forEach(t),No.forEach(t),yo=d(a),pe=r(a,"DIV",{class:!0});var Wo=n(pe);g(Ze.$$.fragment,Wo),Xr=d(Wo),ha=r(Wo,"P",{});var ai=n(ha);Jr=s(ai,"Data collator that will dynamically pad the inputs received, as well as the labels."),ai.forEach(t),Wo.forEach(t),Do=d(a),fe=r(a,"H2",{class:!0});var Vo=n(fe);Te=r(Vo,"A",{id:!0,class:!0,href:!0});var oi=n(Te);ga=r(oi,"SPAN",{});var ri=n(ga);g(et.$$.fragment,ri),ri.forEach(t),oi.forEach(t),Gr=d(Vo),ua=r(Vo,"SPAN",{});var ni=n(ua);Qr=s(ni,"DataCollatorForLanguageModeling"),ni.forEach(t),Vo.forEach(t),Eo=d(a),C=r(a,"DIV",{class:!0});var B=n(C);g(tt.$$.fragment,B),Yr=d(B),_a=r(B,"P",{});var li=n(_a);Zr=s(li,`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`),li.forEach(t),en=d(B),g(Pe.$$.fragment,B),tn=d(B),Se=r(B,"DIV",{class:!0});var jo=n(Se);g(at.$$.fragment,jo),an=d(jo),va=r(jo,"P",{});var si=n(va);on=s(si,"Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),si.forEach(t),jo.forEach(t),rn=d(B),Fe=r(B,"DIV",{class:!0});var Bo=n(Fe);g(ot.$$.fragment,Bo),nn=d(Bo),ka=r(Bo,"P",{});var ii=n(ka);ln=s(ii,"Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),ii.forEach(t),Bo.forEach(t),sn=d(B),Le=r(B,"DIV",{class:!0});var Ko=n(Le);g(rt.$$.fragment,Ko),dn=d(Ko),ba=r(Ko,"P",{});var di=n(ba);cn=s(di,"Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),di.forEach(t),Ko.forEach(t),B.forEach(t),wo=d(a),he=r(a,"H2",{class:!0});var Ho=n(he);ze=r(Ho,"A",{id:!0,class:!0,href:!0});var ci=n(ze);$a=r(ci,"SPAN",{});var mi=n($a);g(nt.$$.fragment,mi),mi.forEach(t),ci.forEach(t),mn=d(Ho),ya=r(Ho,"SPAN",{});var pi=n(ya);pn=s(pi,"DataCollatorForWholeWordMask"),pi.forEach(t),Ho.forEach(t),xo=d(a),w=r(a,"DIV",{class:!0});var M=n(w);g(lt.$$.fragment,M),fn=d(M),Da=r(M,"P",{});var fi=n(Da);hn=s(fi,"Data collator used for language modeling that masks entire words."),fi.forEach(t),gn=d(M),st=r(M,"UL",{});var Uo=n(st);Ea=r(Uo,"LI",{});var hi=n(Ea);un=s(hi,"collates batches of tensors, honoring their tokenizer\u2019s pad_token"),hi.forEach(t),_n=d(Uo),wa=r(Uo,"LI",{});var gi=n(wa);vn=s(gi,"preprocesses batches for masked language modeling"),gi.forEach(t),Uo.forEach(t),kn=d(M),g(Ae.$$.fragment,M),bn=d(M),qe=r(M,"DIV",{class:!0});var Ro=n(qe);g(it.$$.fragment,Ro),$n=d(Ro),xa=r(Ro,"P",{});var ui=n(xa);yn=s(ui,`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),ui.forEach(t),Ro.forEach(t),Dn=d(M),Ie=r(M,"DIV",{class:!0});var Xo=n(Ie);g(dt.$$.fragment,Xo),En=d(Xo),Ca=r(Xo,"P",{});var _i=n(Ca);wn=s(_i,`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),_i.forEach(t),Xo.forEach(t),xn=d(M),Me=r(M,"DIV",{class:!0});var Jo=n(Me);g(ct.$$.fragment,Jo),Cn=d(Jo),Ta=r(Jo,"P",{});var vi=n(Ta);Tn=s(vi,`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),vi.forEach(t),Jo.forEach(t),M.forEach(t),Co=d(a),ge=r(a,"H2",{class:!0});var Go=n(ge);Oe=r(Go,"A",{id:!0,class:!0,href:!0});var ki=n(Oe);Pa=r(ki,"SPAN",{});var bi=n(Pa);g(mt.$$.fragment,bi),bi.forEach(t),ki.forEach(t),Pn=d(Go),Sa=r(Go,"SPAN",{});var $i=n(Sa);Sn=s($i,"DataCollatorForPermutationLanguageModeling"),$i.forEach(t),Go.forEach(t),To=d(a),T=r(a,"DIV",{class:!0});var K=n(T);g(pt.$$.fragment,K),Fn=d(K),Fa=r(K,"P",{});var yi=n(Fa);Ln=s(yi,"Data collator used for permutation language modeling."),yi.forEach(t),zn=d(K),ft=r(K,"UL",{});var Qo=n(ft);La=r(Qo,"LI",{});var Di=n(La);An=s(Di,"collates batches of tensors, honoring their tokenizer\u2019s pad_token"),Di.forEach(t),qn=d(Qo),za=r(Qo,"LI",{});var Ei=n(za);In=s(Ei,"preprocesses batches for permutation language modeling with procedures specific to XLNet"),Ei.forEach(t),Qo.forEach(t),Mn=d(K),J=r(K,"DIV",{class:!0});var Ot=n(J);g(ht.$$.fragment,Ot),On=d(Ot),Aa=r(Ot,"P",{});var wi=n(Aa);Nn=s(wi,"The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),wi.forEach(t),Wn=d(Ot),A=r(Ot,"OL",{start:!0});var ae=n(A);gt=r(ae,"LI",{});var Yo=n(gt);Vn=s(Yo,"Start from the beginning of the sequence by setting "),qa=r(Yo,"CODE",{});var xi=n(qa);jn=s(xi,"cur_len = 0"),xi.forEach(t),Bn=s(Yo," (number of tokens processed so far)."),Yo.forEach(t),Kn=d(ae),ue=r(ae,"LI",{});var Nt=n(ue);Hn=s(Nt,"Sample a "),Ia=r(Nt,"CODE",{});var Ci=n(Ia);Un=s(Ci,"span_length"),Ci.forEach(t),Rn=s(Nt," from the interval "),Ma=r(Nt,"CODE",{});var Ti=n(Ma);Xn=s(Ti,"[1, max_span_length]"),Ti.forEach(t),Jn=s(Nt," (length of span of tokens to be masked)"),Nt.forEach(t),Gn=d(ae),ut=r(ae,"LI",{});var Zo=n(ut);Qn=s(Zo,"Reserve a context of length "),Oa=r(Zo,"CODE",{});var Pi=n(Oa);Yn=s(Pi,"context_length = span_length / plm_probability"),Pi.forEach(t),Zn=s(Zo,` to surround span to be
masked`),Zo.forEach(t),el=d(ae),G=r(ae,"LI",{});var Et=n(G);tl=s(Et,"Sample a starting point "),Na=r(Et,"CODE",{});var Si=n(Na);al=s(Si,"start_index"),Si.forEach(t),ol=s(Et," from the interval "),Wa=r(Et,"CODE",{});var Fi=n(Wa);rl=s(Fi,"[cur_len, cur_len + context_length - span_length]"),Fi.forEach(t),nl=s(Et," and mask tokens "),Va=r(Et,"CODE",{});var Li=n(Va);ll=s(Li,"start_index:start_index + span_length"),Li.forEach(t),Et.forEach(t),sl=d(ae),_e=r(ae,"LI",{});var Wt=n(_e);il=s(Wt,"Set "),ja=r(Wt,"CODE",{});var zi=n(ja);dl=s(zi,"cur_len = cur_len + context_length"),zi.forEach(t),cl=s(Wt,". If "),Ba=r(Wt,"CODE",{});var Ai=n(Ba);ml=s(Ai,"cur_len < max_len"),Ai.forEach(t),pl=s(Wt,` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Wt.forEach(t),ae.forEach(t),Ot.forEach(t),fl=d(K),Q=r(K,"DIV",{class:!0});var Vt=n(Q);g(_t.$$.fragment,Vt),hl=d(Vt),Ka=r(Vt,"P",{});var qi=n(Ka);gl=s(qi,"The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),qi.forEach(t),ul=d(Vt),q=r(Vt,"OL",{start:!0});var oe=n(q);vt=r(oe,"LI",{});var er=n(vt);_l=s(er,"Start from the beginning of the sequence by setting "),Ha=r(er,"CODE",{});var Ii=n(Ha);vl=s(Ii,"cur_len = 0"),Ii.forEach(t),kl=s(er," (number of tokens processed so far)."),er.forEach(t),bl=d(oe),ve=r(oe,"LI",{});var jt=n(ve);$l=s(jt,"Sample a "),Ua=r(jt,"CODE",{});var Mi=n(Ua);yl=s(Mi,"span_length"),Mi.forEach(t),Dl=s(jt," from the interval "),Ra=r(jt,"CODE",{});var Oi=n(Ra);El=s(Oi,"[1, max_span_length]"),Oi.forEach(t),wl=s(jt," (length of span of tokens to be masked)"),jt.forEach(t),xl=d(oe),kt=r(oe,"LI",{});var tr=n(kt);Cl=s(tr,"Reserve a context of length "),Xa=r(tr,"CODE",{});var Ni=n(Xa);Tl=s(Ni,"context_length = span_length / plm_probability"),Ni.forEach(t),Pl=s(tr,` to surround span to be
masked`),tr.forEach(t),Sl=d(oe),Y=r(oe,"LI",{});var wt=n(Y);Fl=s(wt,"Sample a starting point "),Ja=r(wt,"CODE",{});var Wi=n(Ja);Ll=s(Wi,"start_index"),Wi.forEach(t),zl=s(wt," from the interval "),Ga=r(wt,"CODE",{});var Vi=n(Ga);Al=s(Vi,"[cur_len, cur_len + context_length - span_length]"),Vi.forEach(t),ql=s(wt," and mask tokens "),Qa=r(wt,"CODE",{});var ji=n(Qa);Il=s(ji,"start_index:start_index + span_length"),ji.forEach(t),wt.forEach(t),Ml=d(oe),ke=r(oe,"LI",{});var Bt=n(ke);Ol=s(Bt,"Set "),Ya=r(Bt,"CODE",{});var Bi=n(Ya);Nl=s(Bi,"cur_len = cur_len + context_length"),Bi.forEach(t),Wl=s(Bt,". If "),Za=r(Bt,"CODE",{});var Ki=n(Za);Vl=s(Ki,"cur_len < max_len"),Ki.forEach(t),jl=s(Bt,` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Bt.forEach(t),oe.forEach(t),Vt.forEach(t),Bl=d(K),Z=r(K,"DIV",{class:!0});var Kt=n(Z);g(bt.$$.fragment,Kt),Kl=d(Kt),eo=r(Kt,"P",{});var Hi=n(eo);Hl=s(Hi,"The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),Hi.forEach(t),Ul=d(Kt),I=r(Kt,"OL",{start:!0});var re=n(I);$t=r(re,"LI",{});var ar=n($t);Rl=s(ar,"Start from the beginning of the sequence by setting "),to=r(ar,"CODE",{});var Ui=n(to);Xl=s(Ui,"cur_len = 0"),Ui.forEach(t),Jl=s(ar," (number of tokens processed so far)."),ar.forEach(t),Gl=d(re),be=r(re,"LI",{});var Ht=n(be);Ql=s(Ht,"Sample a "),ao=r(Ht,"CODE",{});var Ri=n(ao);Yl=s(Ri,"span_length"),Ri.forEach(t),Zl=s(Ht," from the interval "),oo=r(Ht,"CODE",{});var Xi=n(oo);es=s(Xi,"[1, max_span_length]"),Xi.forEach(t),ts=s(Ht," (length of span of tokens to be masked)"),Ht.forEach(t),as=d(re),yt=r(re,"LI",{});var or=n(yt);os=s(or,"Reserve a context of length "),ro=r(or,"CODE",{});var Ji=n(ro);rs=s(Ji,"context_length = span_length / plm_probability"),Ji.forEach(t),ns=s(or,` to surround span to be
masked`),or.forEach(t),ls=d(re),ee=r(re,"LI",{});var xt=n(ee);ss=s(xt,"Sample a starting point "),no=r(xt,"CODE",{});var Gi=n(no);is=s(Gi,"start_index"),Gi.forEach(t),ds=s(xt," from the interval "),lo=r(xt,"CODE",{});var Qi=n(lo);cs=s(Qi,"[cur_len, cur_len + context_length - span_length]"),Qi.forEach(t),ms=s(xt," and mask tokens "),so=r(xt,"CODE",{});var Yi=n(so);ps=s(Yi,"start_index:start_index + span_length"),Yi.forEach(t),xt.forEach(t),fs=d(re),$e=r(re,"LI",{});var Ut=n($e);hs=s(Ut,"Set "),io=r(Ut,"CODE",{});var Zi=n(io);gs=s(Zi,"cur_len = cur_len + context_length"),Zi.forEach(t),us=s(Ut,". If "),co=r(Ut,"CODE",{});var ed=n(co);_s=s(ed,"cur_len < max_len"),ed.forEach(t),vs=s(Ut,` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Ut.forEach(t),re.forEach(t),Kt.forEach(t),K.forEach(t),this.h()},h(){c(f,"name","hf:doc:metadata"),c(f,"content",JSON.stringify(dd)),c(E,"id","data-collator"),c(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(E,"href","#data-collator"),c(b,"class","relative group"),c(Pt,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling"),c(St,"href","../examples"),c(Ft,"href","../notebooks"),c(De,"id","transformers.default_data_collator"),c(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(De,"href","#transformers.default_data_collator"),c(ne,"class","relative group"),c(W,"class","docstring"),c(Ee,"id","transformers.DefaultDataCollator"),c(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ee,"href","#transformers.DefaultDataCollator"),c(le,"class","relative group"),c(L,"class","docstring"),c(we,"id","transformers.DataCollatorWithPadding"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#transformers.DataCollatorWithPadding"),c(se,"class","relative group"),c(ie,"class","docstring"),c(xe,"id","transformers.DataCollatorForTokenClassification"),c(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(xe,"href","#transformers.DataCollatorForTokenClassification"),c(de,"class","relative group"),c(ce,"class","docstring"),c(Ce,"id","transformers.DataCollatorForSeq2Seq"),c(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ce,"href","#transformers.DataCollatorForSeq2Seq"),c(me,"class","relative group"),c(pe,"class","docstring"),c(Te,"id","transformers.DataCollatorForLanguageModeling"),c(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Te,"href","#transformers.DataCollatorForLanguageModeling"),c(fe,"class","relative group"),c(Se,"class","docstring"),c(Fe,"class","docstring"),c(Le,"class","docstring"),c(C,"class","docstring"),c(ze,"id","transformers.DataCollatorForWholeWordMask"),c(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ze,"href","#transformers.DataCollatorForWholeWordMask"),c(he,"class","relative group"),c(qe,"class","docstring"),c(Ie,"class","docstring"),c(Me,"class","docstring"),c(w,"class","docstring"),c(Oe,"id","transformers.DataCollatorForPermutationLanguageModeling"),c(Oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Oe,"href","#transformers.DataCollatorForPermutationLanguageModeling"),c(ge,"class","relative group"),c(A,"start","0"),c(J,"class","docstring"),c(q,"start","0"),c(Q,"class","docstring"),c(I,"start","0"),c(Z,"class","docstring"),c(T,"class","docstring")},m(a,m){e(document.head,f),p(a,z,m),p(a,b,m),e(b,E),e(E,O),u(y,O,null),e(b,H),e(b,N),e(N,S),p(a,V,m),p(a,x,m),e(x,F),e(x,D),e(D,R),e(x,j),e(x,$),e($,Ct),e(x,Tt),p(a,Ve,m),p(a,U,m),e(U,rr),e(U,Pt),e(Pt,nr),e(U,lr),p(a,po,m),p(a,X,m),e(X,sr),e(X,St),e(St,ir),e(X,dr),e(X,Ft),e(Ft,cr),e(X,mr),p(a,fo,m),p(a,ne,m),e(ne,De),e(De,Rt),u(je,Rt,null),e(ne,pr),e(ne,Xt),e(Xt,fr),p(a,ho,m),p(a,W,m),u(Be,W,null),e(W,hr),e(W,Jt),e(Jt,gr),e(W,ur),e(W,Ke),e(Ke,Lt),e(Lt,Gt),e(Gt,_r),e(Lt,vr),e(Ke,kr),e(Ke,zt),e(zt,Qt),e(Qt,br),e(zt,$r),e(W,yr),e(W,Yt),e(Yt,Dr),p(a,go,m),p(a,le,m),e(le,Ee),e(Ee,Zt),u(He,Zt,null),e(le,Er),e(le,ea),e(ea,wr),p(a,uo,m),p(a,L,m),u(Ue,L,null),e(L,xr),e(L,ta),e(ta,Cr),e(L,Tr),e(L,Re),e(Re,At),e(At,aa),e(aa,Pr),e(At,Sr),e(Re,Fr),e(Re,qt),e(qt,oa),e(oa,Lr),e(qt,zr),e(L,Ar),e(L,ra),e(ra,qr),e(L,Ir),e(L,na),e(na,Mr),p(a,_o,m),p(a,se,m),e(se,we),e(we,la),u(Xe,la,null),e(se,Or),e(se,sa),e(sa,Nr),p(a,vo,m),p(a,ie,m),u(Je,ie,null),e(ie,Wr),e(ie,ia),e(ia,Vr),p(a,ko,m),p(a,de,m),e(de,xe),e(xe,da),u(Ge,da,null),e(de,jr),e(de,ca),e(ca,Br),p(a,bo,m),p(a,ce,m),u(Qe,ce,null),e(ce,Kr),e(ce,ma),e(ma,Hr),p(a,$o,m),p(a,me,m),e(me,Ce),e(Ce,pa),u(Ye,pa,null),e(me,Ur),e(me,fa),e(fa,Rr),p(a,yo,m),p(a,pe,m),u(Ze,pe,null),e(pe,Xr),e(pe,ha),e(ha,Jr),p(a,Do,m),p(a,fe,m),e(fe,Te),e(Te,ga),u(et,ga,null),e(fe,Gr),e(fe,ua),e(ua,Qr),p(a,Eo,m),p(a,C,m),u(tt,C,null),e(C,Yr),e(C,_a),e(_a,Zr),e(C,en),u(Pe,C,null),e(C,tn),e(C,Se),u(at,Se,null),e(Se,an),e(Se,va),e(va,on),e(C,rn),e(C,Fe),u(ot,Fe,null),e(Fe,nn),e(Fe,ka),e(ka,ln),e(C,sn),e(C,Le),u(rt,Le,null),e(Le,dn),e(Le,ba),e(ba,cn),p(a,wo,m),p(a,he,m),e(he,ze),e(ze,$a),u(nt,$a,null),e(he,mn),e(he,ya),e(ya,pn),p(a,xo,m),p(a,w,m),u(lt,w,null),e(w,fn),e(w,Da),e(Da,hn),e(w,gn),e(w,st),e(st,Ea),e(Ea,un),e(st,_n),e(st,wa),e(wa,vn),e(w,kn),u(Ae,w,null),e(w,bn),e(w,qe),u(it,qe,null),e(qe,$n),e(qe,xa),e(xa,yn),e(w,Dn),e(w,Ie),u(dt,Ie,null),e(Ie,En),e(Ie,Ca),e(Ca,wn),e(w,xn),e(w,Me),u(ct,Me,null),e(Me,Cn),e(Me,Ta),e(Ta,Tn),p(a,Co,m),p(a,ge,m),e(ge,Oe),e(Oe,Pa),u(mt,Pa,null),e(ge,Pn),e(ge,Sa),e(Sa,Sn),p(a,To,m),p(a,T,m),u(pt,T,null),e(T,Fn),e(T,Fa),e(Fa,Ln),e(T,zn),e(T,ft),e(ft,La),e(La,An),e(ft,qn),e(ft,za),e(za,In),e(T,Mn),e(T,J),u(ht,J,null),e(J,On),e(J,Aa),e(Aa,Nn),e(J,Wn),e(J,A),e(A,gt),e(gt,Vn),e(gt,qa),e(qa,jn),e(gt,Bn),e(A,Kn),e(A,ue),e(ue,Hn),e(ue,Ia),e(Ia,Un),e(ue,Rn),e(ue,Ma),e(Ma,Xn),e(ue,Jn),e(A,Gn),e(A,ut),e(ut,Qn),e(ut,Oa),e(Oa,Yn),e(ut,Zn),e(A,el),e(A,G),e(G,tl),e(G,Na),e(Na,al),e(G,ol),e(G,Wa),e(Wa,rl),e(G,nl),e(G,Va),e(Va,ll),e(A,sl),e(A,_e),e(_e,il),e(_e,ja),e(ja,dl),e(_e,cl),e(_e,Ba),e(Ba,ml),e(_e,pl),e(T,fl),e(T,Q),u(_t,Q,null),e(Q,hl),e(Q,Ka),e(Ka,gl),e(Q,ul),e(Q,q),e(q,vt),e(vt,_l),e(vt,Ha),e(Ha,vl),e(vt,kl),e(q,bl),e(q,ve),e(ve,$l),e(ve,Ua),e(Ua,yl),e(ve,Dl),e(ve,Ra),e(Ra,El),e(ve,wl),e(q,xl),e(q,kt),e(kt,Cl),e(kt,Xa),e(Xa,Tl),e(kt,Pl),e(q,Sl),e(q,Y),e(Y,Fl),e(Y,Ja),e(Ja,Ll),e(Y,zl),e(Y,Ga),e(Ga,Al),e(Y,ql),e(Y,Qa),e(Qa,Il),e(q,Ml),e(q,ke),e(ke,Ol),e(ke,Ya),e(Ya,Nl),e(ke,Wl),e(ke,Za),e(Za,Vl),e(ke,jl),e(T,Bl),e(T,Z),u(bt,Z,null),e(Z,Kl),e(Z,eo),e(eo,Hl),e(Z,Ul),e(Z,I),e(I,$t),e($t,Rl),e($t,to),e(to,Xl),e($t,Jl),e(I,Gl),e(I,be),e(be,Ql),e(be,ao),e(ao,Yl),e(be,Zl),e(be,oo),e(oo,es),e(be,ts),e(I,as),e(I,yt),e(yt,os),e(yt,ro),e(ro,rs),e(yt,ns),e(I,ls),e(I,ee),e(ee,ss),e(ee,no),e(no,is),e(ee,ds),e(ee,lo),e(lo,cs),e(ee,ms),e(ee,so),e(so,ps),e(I,fs),e(I,$e),e($e,hs),e($e,io),e(io,gs),e($e,us),e($e,co),e(co,_s),e($e,vs),Po=!0},p(a,[m]){const Dt={};m&2&&(Dt.$$scope={dirty:m,ctx:a}),Pe.$set(Dt);const mo={};m&2&&(mo.$$scope={dirty:m,ctx:a}),Ae.$set(mo)},i(a){Po||(_(y.$$.fragment,a),_(je.$$.fragment,a),_(Be.$$.fragment,a),_(He.$$.fragment,a),_(Ue.$$.fragment,a),_(Xe.$$.fragment,a),_(Je.$$.fragment,a),_(Ge.$$.fragment,a),_(Qe.$$.fragment,a),_(Ye.$$.fragment,a),_(Ze.$$.fragment,a),_(et.$$.fragment,a),_(tt.$$.fragment,a),_(Pe.$$.fragment,a),_(at.$$.fragment,a),_(ot.$$.fragment,a),_(rt.$$.fragment,a),_(nt.$$.fragment,a),_(lt.$$.fragment,a),_(Ae.$$.fragment,a),_(it.$$.fragment,a),_(dt.$$.fragment,a),_(ct.$$.fragment,a),_(mt.$$.fragment,a),_(pt.$$.fragment,a),_(ht.$$.fragment,a),_(_t.$$.fragment,a),_(bt.$$.fragment,a),Po=!0)},o(a){v(y.$$.fragment,a),v(je.$$.fragment,a),v(Be.$$.fragment,a),v(He.$$.fragment,a),v(Ue.$$.fragment,a),v(Xe.$$.fragment,a),v(Je.$$.fragment,a),v(Ge.$$.fragment,a),v(Qe.$$.fragment,a),v(Ye.$$.fragment,a),v(Ze.$$.fragment,a),v(et.$$.fragment,a),v(tt.$$.fragment,a),v(Pe.$$.fragment,a),v(at.$$.fragment,a),v(ot.$$.fragment,a),v(rt.$$.fragment,a),v(nt.$$.fragment,a),v(lt.$$.fragment,a),v(Ae.$$.fragment,a),v(it.$$.fragment,a),v(dt.$$.fragment,a),v(ct.$$.fragment,a),v(mt.$$.fragment,a),v(pt.$$.fragment,a),v(ht.$$.fragment,a),v(_t.$$.fragment,a),v(bt.$$.fragment,a),Po=!1},d(a){t(f),a&&t(z),a&&t(b),k(y),a&&t(V),a&&t(x),a&&t(Ve),a&&t(U),a&&t(po),a&&t(X),a&&t(fo),a&&t(ne),k(je),a&&t(ho),a&&t(W),k(Be),a&&t(go),a&&t(le),k(He),a&&t(uo),a&&t(L),k(Ue),a&&t(_o),a&&t(se),k(Xe),a&&t(vo),a&&t(ie),k(Je),a&&t(ko),a&&t(de),k(Ge),a&&t(bo),a&&t(ce),k(Qe),a&&t($o),a&&t(me),k(Ye),a&&t(yo),a&&t(pe),k(Ze),a&&t(Do),a&&t(fe),k(et),a&&t(Eo),a&&t(C),k(tt),k(Pe),k(at),k(ot),k(rt),a&&t(wo),a&&t(he),k(nt),a&&t(xo),a&&t(w),k(lt),k(Ae),k(it),k(dt),k(ct),a&&t(Co),a&&t(ge),k(mt),a&&t(To),a&&t(T),k(pt),k(ht),k(_t),k(bt)}}}const dd={local:"data-collator",sections:[{local:"transformers.default_data_collator",title:"Default data collator"},{local:"transformers.DefaultDataCollator",title:"DefaultDataCollator"},{local:"transformers.DataCollatorWithPadding",title:"DataCollatorWithPadding"},{local:"transformers.DataCollatorForTokenClassification",title:"DataCollatorForTokenClassification"},{local:"transformers.DataCollatorForSeq2Seq",title:"DataCollatorForSeq2Seq"},{local:"transformers.DataCollatorForLanguageModeling",title:"DataCollatorForLanguageModeling"},{local:"transformers.DataCollatorForWholeWordMask",title:"DataCollatorForWholeWordMask"},{local:"transformers.DataCollatorForPermutationLanguageModeling",title:"DataCollatorForPermutationLanguageModeling"}],title:"Data Collator"};function cd(We,f,z){let{fw:b}=f;return We.$$set=E=>{"fw"in E&&z(0,b=E.fw)},[b]}class gd extends ad{constructor(f){super();od(this,f,cd,id,rd,{fw:0})}}export{gd as default,dd as metadata};
