import{S as ad,i as od,s as rd,e as o,k as i,w as h,t as l,M as nd,c as r,d as t,m as d,a as n,x as g,h as s,b as c,G as e,g as p,y as u,q as _,o as v,B as b,v as ld}from"../../chunks/vendor-hf-doc-builder.js";import{T as td}from"../../chunks/Tip-hf-doc-builder.js";import{D as T}from"../../chunks/Docstring-hf-doc-builder.js";import{I as $e}from"../../chunks/IconCopyLink-hf-doc-builder.js";function sd(wt){let f,W,k,P,I,$,H,O,S,V,E,F,D,R;return{c(){f=o("p"),W=l(`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the `),k=o("code"),P=l('"special_tokens_mask"'),I=l(" key, as returned by a "),$=o("a"),H=l("PreTrainedTokenizer"),O=l(` or a
`),S=o("a"),V=l("PreTrainedTokenizerFast"),E=l(" with the argument "),F=o("code"),D=l("return_special_tokens_mask=True"),R=l("."),this.h()},l(j){f=r(j,"P",{});var y=n(f);W=s(y,`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the `),k=r(y,"CODE",{});var Ct=n(k);P=s(Ct,'"special_tokens_mask"'),Ct.forEach(t),I=s(y," key, as returned by a "),$=r(y,"A",{href:!0});var Tt=n($);H=s(Tt,"PreTrainedTokenizer"),Tt.forEach(t),O=s(y,` or a
`),S=r(y,"A",{href:!0});var We=n(S);V=s(We,"PreTrainedTokenizerFast"),We.forEach(t),E=s(y," with the argument "),F=r(y,"CODE",{});var U=n(F);D=s(U,"return_special_tokens_mask=True"),U.forEach(t),R=s(y,"."),y.forEach(t),this.h()},h(){c($,"href","/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),c(S,"href","/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast")},m(j,y){p(j,f,y),e(f,W),e(f,k),e(k,P),e(f,I),e(f,$),e($,H),e(f,O),e(f,S),e(S,V),e(f,E),e(f,F),e(F,D),e(f,R)},d(j){j&&t(f)}}}function id(wt){let f,W,k,P,I,$,H,O,S,V,E;return{c(){f=o("p"),W=l("This collator relies on details of the implementation of subword tokenization by "),k=o("a"),P=l("BertTokenizer"),I=l(`, specifically
that subword tokens are prefixed with `),$=o("em"),H=l("##"),O=l(`. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to `),S=o("code"),V=l(".DataCollatorForLanguageModeling"),E=l("."),this.h()},l(F){f=r(F,"P",{});var D=n(f);W=s(D,"This collator relies on details of the implementation of subword tokenization by "),k=r(D,"A",{href:!0});var R=n(k);P=s(R,"BertTokenizer"),R.forEach(t),I=s(D,`, specifically
that subword tokens are prefixed with `),$=r(D,"EM",{});var j=n($);H=s(j,"##"),j.forEach(t),O=s(D,`. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to `),S=r(D,"CODE",{});var y=n(S);V=s(y,".DataCollatorForLanguageModeling"),y.forEach(t),E=s(D,"."),D.forEach(t),this.h()},h(){c(k,"href","/docs/transformers/pr_19907/en/model_doc/bert#transformers.BertTokenizer")},m(F,D){p(F,f,D),e(f,W),e(f,k),e(k,P),e(f,I),e(f,$),e($,H),e(f,O),e(f,S),e(S,V),e(f,E)},d(F){F&&t(f)}}}function dd(wt){let f,W,k,P,I,$,H,O,S,V,E,F,D,R,j,y,Ct,Tt,We,U,rr,Pt,nr,lr,po,X,sr,St,ir,dr,Ft,cr,mr,fo,ne,De,Rt,Ve,pr,Xt,fr,ho,N,je,hr,Gt,gr,ur,Be,Lt,Jt,_r,vr,br,zt,Qt,kr,yr,$r,Yt,Dr,go,le,xe,Zt,Ke,xr,ea,Er,uo,L,He,wr,ta,Cr,Tr,Ue,At,aa,Pr,Sr,Fr,qt,oa,Lr,zr,Ar,ra,qr,Mr,na,Ir,_o,se,Ee,la,Re,Or,sa,Nr,vo,ie,Xe,Wr,ia,Vr,bo,de,we,da,Ge,jr,ca,Br,ko,ce,Je,Kr,ma,Hr,yo,me,Ce,pa,Qe,Ur,fa,Rr,$o,pe,Ye,Xr,ha,Gr,Do,fe,Te,ga,Ze,Jr,ua,Qr,xo,w,et,Yr,_a,Zr,en,Pe,tn,Se,tt,an,va,on,rn,Fe,at,nn,ba,ln,sn,Le,ot,dn,ka,cn,Eo,he,ze,ya,rt,mn,$a,pn,wo,x,nt,fn,Da,hn,gn,lt,xa,un,_n,Ea,vn,bn,Ae,kn,qe,st,yn,wa,$n,Dn,Me,it,xn,Ca,En,wn,Ie,dt,Cn,Ta,Tn,Co,ge,Oe,Pa,ct,Pn,Sa,Sn,To,C,mt,Fn,Fa,Ln,zn,pt,La,An,qn,za,Mn,In,G,ft,On,Aa,Nn,Wn,z,ht,Vn,qa,jn,Bn,Kn,ue,Hn,Ma,Un,Rn,Ia,Xn,Gn,Jn,gt,Qn,Oa,Yn,Zn,el,J,tl,Na,al,ol,Wa,rl,nl,Va,ll,sl,_e,il,ja,dl,cl,Ba,ml,pl,fl,Q,ut,hl,Ka,gl,ul,A,_t,_l,Ha,vl,bl,kl,ve,yl,Ua,$l,Dl,Ra,xl,El,wl,vt,Cl,Xa,Tl,Pl,Sl,Y,Fl,Ga,Ll,zl,Ja,Al,ql,Qa,Ml,Il,be,Ol,Ya,Nl,Wl,Za,Vl,jl,Bl,Z,bt,Kl,eo,Hl,Ul,q,kt,Rl,to,Xl,Gl,Jl,ke,Ql,ao,Yl,Zl,oo,es,ts,as,yt,os,ro,rs,ns,ls,ee,ss,no,is,ds,lo,cs,ms,so,ps,fs,ye,hs,io,gs,us,co,_s,vs,Po;return $=new $e({}),Ve=new $e({}),je=new T({props:{name:"transformers.default_data_collator",anchor:"transformers.default_data_collator",parameters:[{name:"features",val:": typing.List[InputDataClass]"},{name:"return_tensors",val:" = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L49"}}),Ke=new $e({}),He=new T({props:{name:"class transformers.DefaultDataCollator",anchor:"transformers.DefaultDataCollator",parameters:[{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DefaultDataCollator.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L75"}}),Re=new $e({}),Xe=new T({props:{name:"class transformers.DataCollatorWithPadding",anchor:"transformers.DataCollatorWithPadding",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorWithPadding.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorWithPadding.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_19907/en/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorWithPadding.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorWithPadding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorWithPadding.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L213"}}),Ge=new $e({}),Je=new T({props:{name:"class transformers.DataCollatorForTokenClassification",anchor:"transformers.DataCollatorForTokenClassification",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForTokenClassification.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForTokenClassification.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_19907/en/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code>: Pad to the longest sequence in the batch (or no padding if only a single sequence
is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code> (default): No padding (i.e., can output a batch with sequences of different
lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForTokenClassification.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForTokenClassification.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForTokenClassification.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForTokenClassification.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L264"}}),Qe=new $e({}),Ye=new T({props:{name:"class transformers.DataCollatorForSeq2Seq",anchor:"transformers.DataCollatorForSeq2Seq",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"model",val:": typing.Optional[typing.Any] = None"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForSeq2Seq.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForSeq2Seq.model",description:`<strong>model</strong> (<a href="/docs/transformers/pr_19907/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>) &#x2014;
The model that is being trained. If set and has the <em>prepare_decoder_input_ids_from_labels</em>, use it to
prepare the <em>decoder_input_ids</em></p>
<p>This is useful when using <em>label_smoothing</em> to avoid calculating loss twice.`,name:"model"},{anchor:"transformers.DataCollatorForSeq2Seq.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_19907/en/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code>: Pad to the longest sequence in the batch (or no padding if only a single sequence
is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code> (default): No padding (i.e., can output a batch with sequences of different
lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForSeq2Seq.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForSeq2Seq.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForSeq2Seq.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForSeq2Seq.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L514"}}),Ze=new $e({}),et=new T({props:{name:"class transformers.DataCollatorForLanguageModeling",anchor:"transformers.DataCollatorForLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForLanguageModeling.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_19907/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm",description:`<strong>mlm</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to use masked language modeling. If set to <code>False</code>, the labels are the same as the inputs
with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked
tokens and the value to predict for the masked token.`,name:"mlm"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm_probability",description:`<strong>mlm_probability</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) &#x2014;
The probability with which to (randomly) mask tokens in the input, when <code>mlm</code> is set to <code>True</code>.`,name:"mlm_probability"},{anchor:"transformers.DataCollatorForLanguageModeling.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForLanguageModeling.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L607"}}),Pe=new td({props:{$$slots:{default:[sd]},$$scope:{ctx:wt}}}),tt=new T({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L805"}}),at=new T({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"vocab_size",val:""},{name:"mask_token_id",val:""},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L659"}}),ot=new T({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L748"}}),rt=new $e({}),nt=new T({props:{name:"class transformers.DataCollatorForWholeWordMask",anchor:"transformers.DataCollatorForWholeWordMask",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L846"}}),Ae=new td({props:{$$slots:{default:[id]},$$scope:{ctx:wt}}}),st=new T({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L1074"}}),it=new T({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L1032"}}),dt=new T({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L992"}}),ct=new $e({}),mt=new T({props:{name:"class transformers.DataCollatorForPermutationLanguageModeling",anchor:"transformers.DataCollatorForPermutationLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"plm_probability",val:": float = 0.16666666666666666"},{name:"max_span_length",val:": int = 5"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L1201"}}),ft=new T({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L1444"}}),ut=new T({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L1334"}}),bt=new T({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/data/data_collator.py#L1235"}}),{c(){f=o("meta"),W=i(),k=o("h1"),P=o("a"),I=o("span"),h($.$$.fragment),H=i(),O=o("span"),S=l("Data Collator"),V=i(),E=o("p"),F=l(`Data collators are objects that will form a batch by using a list of dataset elements as input. These elements are of
the same type as the elements of `),D=o("code"),R=l("train_dataset"),j=l(" or "),y=o("code"),Ct=l("eval_dataset"),Tt=l("."),We=i(),U=o("p"),rr=l(`To be able to build batches, data collators may apply some processing (like padding). Some of them (like
`),Pt=o("a"),nr=l("DataCollatorForLanguageModeling"),lr=l(`) also apply some random data augmentation (like random masking)
on the formed batch.`),po=i(),X=o("p"),sr=l("Examples of use can be found in the "),St=o("a"),ir=l("example scripts"),dr=l(" or "),Ft=o("a"),cr=l("example notebooks"),mr=l("."),fo=i(),ne=o("h2"),De=o("a"),Rt=o("span"),h(Ve.$$.fragment),pr=i(),Xt=o("span"),fr=l("Default data collator"),ho=i(),N=o("div"),h(je.$$.fragment),hr=i(),Gt=o("p"),gr=l(`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),ur=i(),Be=o("ul"),Lt=o("li"),Jt=o("code"),_r=l("label"),vr=l(": handles a single value (int or float) per object"),br=i(),zt=o("li"),Qt=o("code"),kr=l("label_ids"),yr=l(": handles a list of values per object"),$r=i(),Yt=o("p"),Dr=l(`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),go=i(),le=o("h2"),xe=o("a"),Zt=o("span"),h(Ke.$$.fragment),xr=i(),ea=o("span"),Er=l("DefaultDataCollator"),uo=i(),L=o("div"),h(He.$$.fragment),wr=i(),ta=o("p"),Cr=l(`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),Tr=i(),Ue=o("ul"),At=o("li"),aa=o("code"),Pr=l("label"),Sr=l(": handles a single value (int or float) per object"),Fr=i(),qt=o("li"),oa=o("code"),Lr=l("label_ids"),zr=l(": handles a list of values per object"),Ar=i(),ra=o("p"),qr=l(`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),Mr=i(),na=o("p"),Ir=l(`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`),_o=i(),se=o("h2"),Ee=o("a"),la=o("span"),h(Re.$$.fragment),Or=i(),sa=o("span"),Nr=l("DataCollatorWithPadding"),vo=i(),ie=o("div"),h(Xe.$$.fragment),Wr=i(),ia=o("p"),Vr=l("Data collator that will dynamically pad the inputs received."),bo=i(),de=o("h2"),we=o("a"),da=o("span"),h(Ge.$$.fragment),jr=i(),ca=o("span"),Br=l("DataCollatorForTokenClassification"),ko=i(),ce=o("div"),h(Je.$$.fragment),Kr=i(),ma=o("p"),Hr=l("Data collator that will dynamically pad the inputs received, as well as the labels."),yo=i(),me=o("h2"),Ce=o("a"),pa=o("span"),h(Qe.$$.fragment),Ur=i(),fa=o("span"),Rr=l("DataCollatorForSeq2Seq"),$o=i(),pe=o("div"),h(Ye.$$.fragment),Xr=i(),ha=o("p"),Gr=l("Data collator that will dynamically pad the inputs received, as well as the labels."),Do=i(),fe=o("h2"),Te=o("a"),ga=o("span"),h(Ze.$$.fragment),Jr=i(),ua=o("span"),Qr=l("DataCollatorForLanguageModeling"),xo=i(),w=o("div"),h(et.$$.fragment),Yr=i(),_a=o("p"),Zr=l(`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`),en=i(),h(Pe.$$.fragment),tn=i(),Se=o("div"),h(tt.$$.fragment),an=i(),va=o("p"),on=l("Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),rn=i(),Fe=o("div"),h(at.$$.fragment),nn=i(),ba=o("p"),ln=l("Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),sn=i(),Le=o("div"),h(ot.$$.fragment),dn=i(),ka=o("p"),cn=l("Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),Eo=i(),he=o("h2"),ze=o("a"),ya=o("span"),h(rt.$$.fragment),mn=i(),$a=o("span"),pn=l("DataCollatorForWholeWordMask"),wo=i(),x=o("div"),h(nt.$$.fragment),fn=i(),Da=o("p"),hn=l("Data collator used for language modeling that masks entire words."),gn=i(),lt=o("ul"),xa=o("li"),un=l("collates batches of tensors, honoring their tokenizer\u2019s pad_token"),_n=i(),Ea=o("li"),vn=l("preprocesses batches for masked language modeling"),bn=i(),h(Ae.$$.fragment),kn=i(),qe=o("div"),h(st.$$.fragment),yn=i(),wa=o("p"),$n=l(`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),Dn=i(),Me=o("div"),h(it.$$.fragment),xn=i(),Ca=o("p"),En=l(`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),wn=i(),Ie=o("div"),h(dt.$$.fragment),Cn=i(),Ta=o("p"),Tn=l(`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),Co=i(),ge=o("h2"),Oe=o("a"),Pa=o("span"),h(ct.$$.fragment),Pn=i(),Sa=o("span"),Sn=l("DataCollatorForPermutationLanguageModeling"),To=i(),C=o("div"),h(mt.$$.fragment),Fn=i(),Fa=o("p"),Ln=l("Data collator used for permutation language modeling."),zn=i(),pt=o("ul"),La=o("li"),An=l("collates batches of tensors, honoring their tokenizer\u2019s pad_token"),qn=i(),za=o("li"),Mn=l("preprocesses batches for permutation language modeling with procedures specific to XLNet"),In=i(),G=o("div"),h(ft.$$.fragment),On=i(),Aa=o("p"),Nn=l("The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),Wn=i(),z=o("ol"),ht=o("li"),Vn=l("Start from the beginning of the sequence by setting "),qa=o("code"),jn=l("cur_len = 0"),Bn=l(" (number of tokens processed so far)."),Kn=i(),ue=o("li"),Hn=l("Sample a "),Ma=o("code"),Un=l("span_length"),Rn=l(" from the interval "),Ia=o("code"),Xn=l("[1, max_span_length]"),Gn=l(" (length of span of tokens to be masked)"),Jn=i(),gt=o("li"),Qn=l("Reserve a context of length "),Oa=o("code"),Yn=l("context_length = span_length / plm_probability"),Zn=l(` to surround span to be
masked`),el=i(),J=o("li"),tl=l("Sample a starting point "),Na=o("code"),al=l("start_index"),ol=l(" from the interval "),Wa=o("code"),rl=l("[cur_len, cur_len + context_length - span_length]"),nl=l(" and mask tokens "),Va=o("code"),ll=l("start_index:start_index + span_length"),sl=i(),_e=o("li"),il=l("Set "),ja=o("code"),dl=l("cur_len = cur_len + context_length"),cl=l(". If "),Ba=o("code"),ml=l("cur_len < max_len"),pl=l(` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),fl=i(),Q=o("div"),h(ut.$$.fragment),hl=i(),Ka=o("p"),gl=l("The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),ul=i(),A=o("ol"),_t=o("li"),_l=l("Start from the beginning of the sequence by setting "),Ha=o("code"),vl=l("cur_len = 0"),bl=l(" (number of tokens processed so far)."),kl=i(),ve=o("li"),yl=l("Sample a "),Ua=o("code"),$l=l("span_length"),Dl=l(" from the interval "),Ra=o("code"),xl=l("[1, max_span_length]"),El=l(" (length of span of tokens to be masked)"),wl=i(),vt=o("li"),Cl=l("Reserve a context of length "),Xa=o("code"),Tl=l("context_length = span_length / plm_probability"),Pl=l(` to surround span to be
masked`),Sl=i(),Y=o("li"),Fl=l("Sample a starting point "),Ga=o("code"),Ll=l("start_index"),zl=l(" from the interval "),Ja=o("code"),Al=l("[cur_len, cur_len + context_length - span_length]"),ql=l(" and mask tokens "),Qa=o("code"),Ml=l("start_index:start_index + span_length"),Il=i(),be=o("li"),Ol=l("Set "),Ya=o("code"),Nl=l("cur_len = cur_len + context_length"),Wl=l(". If "),Za=o("code"),Vl=l("cur_len < max_len"),jl=l(` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Bl=i(),Z=o("div"),h(bt.$$.fragment),Kl=i(),eo=o("p"),Hl=l("The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),Ul=i(),q=o("ol"),kt=o("li"),Rl=l("Start from the beginning of the sequence by setting "),to=o("code"),Xl=l("cur_len = 0"),Gl=l(" (number of tokens processed so far)."),Jl=i(),ke=o("li"),Ql=l("Sample a "),ao=o("code"),Yl=l("span_length"),Zl=l(" from the interval "),oo=o("code"),es=l("[1, max_span_length]"),ts=l(" (length of span of tokens to be masked)"),as=i(),yt=o("li"),os=l("Reserve a context of length "),ro=o("code"),rs=l("context_length = span_length / plm_probability"),ns=l(` to surround span to be
masked`),ls=i(),ee=o("li"),ss=l("Sample a starting point "),no=o("code"),is=l("start_index"),ds=l(" from the interval "),lo=o("code"),cs=l("[cur_len, cur_len + context_length - span_length]"),ms=l(" and mask tokens "),so=o("code"),ps=l("start_index:start_index + span_length"),fs=i(),ye=o("li"),hs=l("Set "),io=o("code"),gs=l("cur_len = cur_len + context_length"),us=l(". If "),co=o("code"),_s=l("cur_len < max_len"),vs=l(` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),this.h()},l(a){const m=nd('[data-svelte="svelte-1phssyn"]',document.head);f=r(m,"META",{name:!0,content:!0}),m.forEach(t),W=d(a),k=r(a,"H1",{class:!0});var $t=n(k);P=r($t,"A",{id:!0,class:!0,href:!0});var mo=n(P);I=r(mo,"SPAN",{});var Ds=n(I);g($.$$.fragment,Ds),Ds.forEach(t),mo.forEach(t),H=d($t),O=r($t,"SPAN",{});var xs=n(O);S=s(xs,"Data Collator"),xs.forEach(t),$t.forEach(t),V=d(a),E=r(a,"P",{});var Mt=n(E);F=s(Mt,`Data collators are objects that will form a batch by using a list of dataset elements as input. These elements are of
the same type as the elements of `),D=r(Mt,"CODE",{});var Es=n(D);R=s(Es,"train_dataset"),Es.forEach(t),j=s(Mt," or "),y=r(Mt,"CODE",{});var ws=n(y);Ct=s(ws,"eval_dataset"),ws.forEach(t),Tt=s(Mt,"."),Mt.forEach(t),We=d(a),U=r(a,"P",{});var So=n(U);rr=s(So,`To be able to build batches, data collators may apply some processing (like padding). Some of them (like
`),Pt=r(So,"A",{href:!0});var Cs=n(Pt);nr=s(Cs,"DataCollatorForLanguageModeling"),Cs.forEach(t),lr=s(So,`) also apply some random data augmentation (like random masking)
on the formed batch.`),So.forEach(t),po=d(a),X=r(a,"P",{});var It=n(X);sr=s(It,"Examples of use can be found in the "),St=r(It,"A",{href:!0});var Ts=n(St);ir=s(Ts,"example scripts"),Ts.forEach(t),dr=s(It," or "),Ft=r(It,"A",{href:!0});var Ps=n(Ft);cr=s(Ps,"example notebooks"),Ps.forEach(t),mr=s(It,"."),It.forEach(t),fo=d(a),ne=r(a,"H2",{class:!0});var Fo=n(ne);De=r(Fo,"A",{id:!0,class:!0,href:!0});var Ss=n(De);Rt=r(Ss,"SPAN",{});var Fs=n(Rt);g(Ve.$$.fragment,Fs),Fs.forEach(t),Ss.forEach(t),pr=d(Fo),Xt=r(Fo,"SPAN",{});var Ls=n(Xt);fr=s(Ls,"Default data collator"),Ls.forEach(t),Fo.forEach(t),ho=d(a),N=r(a,"DIV",{class:!0});var Ne=n(N);g(je.$$.fragment,Ne),hr=d(Ne),Gt=r(Ne,"P",{});var zs=n(Gt);gr=s(zs,`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),zs.forEach(t),ur=d(Ne),Be=r(Ne,"UL",{});var Lo=n(Be);Lt=r(Lo,"LI",{});var bs=n(Lt);Jt=r(bs,"CODE",{});var As=n(Jt);_r=s(As,"label"),As.forEach(t),vr=s(bs,": handles a single value (int or float) per object"),bs.forEach(t),br=d(Lo),zt=r(Lo,"LI",{});var ks=n(zt);Qt=r(ks,"CODE",{});var qs=n(Qt);kr=s(qs,"label_ids"),qs.forEach(t),yr=s(ks,": handles a list of values per object"),ks.forEach(t),Lo.forEach(t),$r=d(Ne),Yt=r(Ne,"P",{});var Ms=n(Yt);Dr=s(Ms,`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),Ms.forEach(t),Ne.forEach(t),go=d(a),le=r(a,"H2",{class:!0});var zo=n(le);xe=r(zo,"A",{id:!0,class:!0,href:!0});var Is=n(xe);Zt=r(Is,"SPAN",{});var Os=n(Zt);g(Ke.$$.fragment,Os),Os.forEach(t),Is.forEach(t),xr=d(zo),ea=r(zo,"SPAN",{});var Ns=n(ea);Er=s(Ns,"DefaultDataCollator"),Ns.forEach(t),zo.forEach(t),uo=d(a),L=r(a,"DIV",{class:!0});var te=n(L);g(He.$$.fragment,te),wr=d(te),ta=r(te,"P",{});var Ws=n(ta);Cr=s(Ws,`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`),Ws.forEach(t),Tr=d(te),Ue=r(te,"UL",{});var Ao=n(Ue);At=r(Ao,"LI",{});var ys=n(At);aa=r(ys,"CODE",{});var Vs=n(aa);Pr=s(Vs,"label"),Vs.forEach(t),Sr=s(ys,": handles a single value (int or float) per object"),ys.forEach(t),Fr=d(Ao),qt=r(Ao,"LI",{});var $s=n(qt);oa=r($s,"CODE",{});var js=n(oa);Lr=s(js,"label_ids"),js.forEach(t),zr=s($s,": handles a list of values per object"),$s.forEach(t),Ao.forEach(t),Ar=d(te),ra=r(te,"P",{});var Bs=n(ra);qr=s(Bs,`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it\u2019s useful.`),Bs.forEach(t),Mr=d(te),na=r(te,"P",{});var Ks=n(na);Ir=s(Ks,`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`),Ks.forEach(t),te.forEach(t),_o=d(a),se=r(a,"H2",{class:!0});var qo=n(se);Ee=r(qo,"A",{id:!0,class:!0,href:!0});var Hs=n(Ee);la=r(Hs,"SPAN",{});var Us=n(la);g(Re.$$.fragment,Us),Us.forEach(t),Hs.forEach(t),Or=d(qo),sa=r(qo,"SPAN",{});var Rs=n(sa);Nr=s(Rs,"DataCollatorWithPadding"),Rs.forEach(t),qo.forEach(t),vo=d(a),ie=r(a,"DIV",{class:!0});var Mo=n(ie);g(Xe.$$.fragment,Mo),Wr=d(Mo),ia=r(Mo,"P",{});var Xs=n(ia);Vr=s(Xs,"Data collator that will dynamically pad the inputs received."),Xs.forEach(t),Mo.forEach(t),bo=d(a),de=r(a,"H2",{class:!0});var Io=n(de);we=r(Io,"A",{id:!0,class:!0,href:!0});var Gs=n(we);da=r(Gs,"SPAN",{});var Js=n(da);g(Ge.$$.fragment,Js),Js.forEach(t),Gs.forEach(t),jr=d(Io),ca=r(Io,"SPAN",{});var Qs=n(ca);Br=s(Qs,"DataCollatorForTokenClassification"),Qs.forEach(t),Io.forEach(t),ko=d(a),ce=r(a,"DIV",{class:!0});var Oo=n(ce);g(Je.$$.fragment,Oo),Kr=d(Oo),ma=r(Oo,"P",{});var Ys=n(ma);Hr=s(Ys,"Data collator that will dynamically pad the inputs received, as well as the labels."),Ys.forEach(t),Oo.forEach(t),yo=d(a),me=r(a,"H2",{class:!0});var No=n(me);Ce=r(No,"A",{id:!0,class:!0,href:!0});var Zs=n(Ce);pa=r(Zs,"SPAN",{});var ei=n(pa);g(Qe.$$.fragment,ei),ei.forEach(t),Zs.forEach(t),Ur=d(No),fa=r(No,"SPAN",{});var ti=n(fa);Rr=s(ti,"DataCollatorForSeq2Seq"),ti.forEach(t),No.forEach(t),$o=d(a),pe=r(a,"DIV",{class:!0});var Wo=n(pe);g(Ye.$$.fragment,Wo),Xr=d(Wo),ha=r(Wo,"P",{});var ai=n(ha);Gr=s(ai,"Data collator that will dynamically pad the inputs received, as well as the labels."),ai.forEach(t),Wo.forEach(t),Do=d(a),fe=r(a,"H2",{class:!0});var Vo=n(fe);Te=r(Vo,"A",{id:!0,class:!0,href:!0});var oi=n(Te);ga=r(oi,"SPAN",{});var ri=n(ga);g(Ze.$$.fragment,ri),ri.forEach(t),oi.forEach(t),Jr=d(Vo),ua=r(Vo,"SPAN",{});var ni=n(ua);Qr=s(ni,"DataCollatorForLanguageModeling"),ni.forEach(t),Vo.forEach(t),xo=d(a),w=r(a,"DIV",{class:!0});var B=n(w);g(et.$$.fragment,B),Yr=d(B),_a=r(B,"P",{});var li=n(_a);Zr=s(li,`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`),li.forEach(t),en=d(B),g(Pe.$$.fragment,B),tn=d(B),Se=r(B,"DIV",{class:!0});var jo=n(Se);g(tt.$$.fragment,jo),an=d(jo),va=r(jo,"P",{});var si=n(va);on=s(si,"Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),si.forEach(t),jo.forEach(t),rn=d(B),Fe=r(B,"DIV",{class:!0});var Bo=n(Fe);g(at.$$.fragment,Bo),nn=d(Bo),ba=r(Bo,"P",{});var ii=n(ba);ln=s(ii,"Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),ii.forEach(t),Bo.forEach(t),sn=d(B),Le=r(B,"DIV",{class:!0});var Ko=n(Le);g(ot.$$.fragment,Ko),dn=d(Ko),ka=r(Ko,"P",{});var di=n(ka);cn=s(di,"Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original."),di.forEach(t),Ko.forEach(t),B.forEach(t),Eo=d(a),he=r(a,"H2",{class:!0});var Ho=n(he);ze=r(Ho,"A",{id:!0,class:!0,href:!0});var ci=n(ze);ya=r(ci,"SPAN",{});var mi=n(ya);g(rt.$$.fragment,mi),mi.forEach(t),ci.forEach(t),mn=d(Ho),$a=r(Ho,"SPAN",{});var pi=n($a);pn=s(pi,"DataCollatorForWholeWordMask"),pi.forEach(t),Ho.forEach(t),wo=d(a),x=r(a,"DIV",{class:!0});var M=n(x);g(nt.$$.fragment,M),fn=d(M),Da=r(M,"P",{});var fi=n(Da);hn=s(fi,"Data collator used for language modeling that masks entire words."),fi.forEach(t),gn=d(M),lt=r(M,"UL",{});var Uo=n(lt);xa=r(Uo,"LI",{});var hi=n(xa);un=s(hi,"collates batches of tensors, honoring their tokenizer\u2019s pad_token"),hi.forEach(t),_n=d(Uo),Ea=r(Uo,"LI",{});var gi=n(Ea);vn=s(gi,"preprocesses batches for masked language modeling"),gi.forEach(t),Uo.forEach(t),bn=d(M),g(Ae.$$.fragment,M),kn=d(M),qe=r(M,"DIV",{class:!0});var Ro=n(qe);g(st.$$.fragment,Ro),yn=d(Ro),wa=r(Ro,"P",{});var ui=n(wa);$n=s(ui,`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),ui.forEach(t),Ro.forEach(t),Dn=d(M),Me=r(M,"DIV",{class:!0});var Xo=n(Me);g(it.$$.fragment,Xo),xn=d(Xo),Ca=r(Xo,"P",{});var _i=n(Ca);En=s(_i,`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),_i.forEach(t),Xo.forEach(t),wn=d(M),Ie=r(M,"DIV",{class:!0});var Go=n(Ie);g(dt.$$.fragment,Go),Cn=d(Go),Ta=r(Go,"P",{});var vi=n(Ta);Tn=s(vi,`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
\u2018mask_labels\u2019 means we use whole word mask (wwm), we directly mask idxs according to it\u2019s ref.`),vi.forEach(t),Go.forEach(t),M.forEach(t),Co=d(a),ge=r(a,"H2",{class:!0});var Jo=n(ge);Oe=r(Jo,"A",{id:!0,class:!0,href:!0});var bi=n(Oe);Pa=r(bi,"SPAN",{});var ki=n(Pa);g(ct.$$.fragment,ki),ki.forEach(t),bi.forEach(t),Pn=d(Jo),Sa=r(Jo,"SPAN",{});var yi=n(Sa);Sn=s(yi,"DataCollatorForPermutationLanguageModeling"),yi.forEach(t),Jo.forEach(t),To=d(a),C=r(a,"DIV",{class:!0});var K=n(C);g(mt.$$.fragment,K),Fn=d(K),Fa=r(K,"P",{});var $i=n(Fa);Ln=s($i,"Data collator used for permutation language modeling."),$i.forEach(t),zn=d(K),pt=r(K,"UL",{});var Qo=n(pt);La=r(Qo,"LI",{});var Di=n(La);An=s(Di,"collates batches of tensors, honoring their tokenizer\u2019s pad_token"),Di.forEach(t),qn=d(Qo),za=r(Qo,"LI",{});var xi=n(za);Mn=s(xi,"preprocesses batches for permutation language modeling with procedures specific to XLNet"),xi.forEach(t),Qo.forEach(t),In=d(K),G=r(K,"DIV",{class:!0});var Ot=n(G);g(ft.$$.fragment,Ot),On=d(Ot),Aa=r(Ot,"P",{});var Ei=n(Aa);Nn=s(Ei,"The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),Ei.forEach(t),Wn=d(Ot),z=r(Ot,"OL",{start:!0});var ae=n(z);ht=r(ae,"LI",{});var Yo=n(ht);Vn=s(Yo,"Start from the beginning of the sequence by setting "),qa=r(Yo,"CODE",{});var wi=n(qa);jn=s(wi,"cur_len = 0"),wi.forEach(t),Bn=s(Yo," (number of tokens processed so far)."),Yo.forEach(t),Kn=d(ae),ue=r(ae,"LI",{});var Nt=n(ue);Hn=s(Nt,"Sample a "),Ma=r(Nt,"CODE",{});var Ci=n(Ma);Un=s(Ci,"span_length"),Ci.forEach(t),Rn=s(Nt," from the interval "),Ia=r(Nt,"CODE",{});var Ti=n(Ia);Xn=s(Ti,"[1, max_span_length]"),Ti.forEach(t),Gn=s(Nt," (length of span of tokens to be masked)"),Nt.forEach(t),Jn=d(ae),gt=r(ae,"LI",{});var Zo=n(gt);Qn=s(Zo,"Reserve a context of length "),Oa=r(Zo,"CODE",{});var Pi=n(Oa);Yn=s(Pi,"context_length = span_length / plm_probability"),Pi.forEach(t),Zn=s(Zo,` to surround span to be
masked`),Zo.forEach(t),el=d(ae),J=r(ae,"LI",{});var Dt=n(J);tl=s(Dt,"Sample a starting point "),Na=r(Dt,"CODE",{});var Si=n(Na);al=s(Si,"start_index"),Si.forEach(t),ol=s(Dt," from the interval "),Wa=r(Dt,"CODE",{});var Fi=n(Wa);rl=s(Fi,"[cur_len, cur_len + context_length - span_length]"),Fi.forEach(t),nl=s(Dt," and mask tokens "),Va=r(Dt,"CODE",{});var Li=n(Va);ll=s(Li,"start_index:start_index + span_length"),Li.forEach(t),Dt.forEach(t),sl=d(ae),_e=r(ae,"LI",{});var Wt=n(_e);il=s(Wt,"Set "),ja=r(Wt,"CODE",{});var zi=n(ja);dl=s(zi,"cur_len = cur_len + context_length"),zi.forEach(t),cl=s(Wt,". If "),Ba=r(Wt,"CODE",{});var Ai=n(Ba);ml=s(Ai,"cur_len < max_len"),Ai.forEach(t),pl=s(Wt,` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Wt.forEach(t),ae.forEach(t),Ot.forEach(t),fl=d(K),Q=r(K,"DIV",{class:!0});var Vt=n(Q);g(ut.$$.fragment,Vt),hl=d(Vt),Ka=r(Vt,"P",{});var qi=n(Ka);gl=s(qi,"The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),qi.forEach(t),ul=d(Vt),A=r(Vt,"OL",{start:!0});var oe=n(A);_t=r(oe,"LI",{});var er=n(_t);_l=s(er,"Start from the beginning of the sequence by setting "),Ha=r(er,"CODE",{});var Mi=n(Ha);vl=s(Mi,"cur_len = 0"),Mi.forEach(t),bl=s(er," (number of tokens processed so far)."),er.forEach(t),kl=d(oe),ve=r(oe,"LI",{});var jt=n(ve);yl=s(jt,"Sample a "),Ua=r(jt,"CODE",{});var Ii=n(Ua);$l=s(Ii,"span_length"),Ii.forEach(t),Dl=s(jt," from the interval "),Ra=r(jt,"CODE",{});var Oi=n(Ra);xl=s(Oi,"[1, max_span_length]"),Oi.forEach(t),El=s(jt," (length of span of tokens to be masked)"),jt.forEach(t),wl=d(oe),vt=r(oe,"LI",{});var tr=n(vt);Cl=s(tr,"Reserve a context of length "),Xa=r(tr,"CODE",{});var Ni=n(Xa);Tl=s(Ni,"context_length = span_length / plm_probability"),Ni.forEach(t),Pl=s(tr,` to surround span to be
masked`),tr.forEach(t),Sl=d(oe),Y=r(oe,"LI",{});var xt=n(Y);Fl=s(xt,"Sample a starting point "),Ga=r(xt,"CODE",{});var Wi=n(Ga);Ll=s(Wi,"start_index"),Wi.forEach(t),zl=s(xt," from the interval "),Ja=r(xt,"CODE",{});var Vi=n(Ja);Al=s(Vi,"[cur_len, cur_len + context_length - span_length]"),Vi.forEach(t),ql=s(xt," and mask tokens "),Qa=r(xt,"CODE",{});var ji=n(Qa);Ml=s(ji,"start_index:start_index + span_length"),ji.forEach(t),xt.forEach(t),Il=d(oe),be=r(oe,"LI",{});var Bt=n(be);Ol=s(Bt,"Set "),Ya=r(Bt,"CODE",{});var Bi=n(Ya);Nl=s(Bi,"cur_len = cur_len + context_length"),Bi.forEach(t),Wl=s(Bt,". If "),Za=r(Bt,"CODE",{});var Ki=n(Za);Vl=s(Ki,"cur_len < max_len"),Ki.forEach(t),jl=s(Bt,` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Bt.forEach(t),oe.forEach(t),Vt.forEach(t),Bl=d(K),Z=r(K,"DIV",{class:!0});var Kt=n(Z);g(bt.$$.fragment,Kt),Kl=d(Kt),eo=r(Kt,"P",{});var Hi=n(eo);Hl=s(Hi,"The masked tokens to be predicted for a particular sequence are determined by the following algorithm:"),Hi.forEach(t),Ul=d(Kt),q=r(Kt,"OL",{start:!0});var re=n(q);kt=r(re,"LI",{});var ar=n(kt);Rl=s(ar,"Start from the beginning of the sequence by setting "),to=r(ar,"CODE",{});var Ui=n(to);Xl=s(Ui,"cur_len = 0"),Ui.forEach(t),Gl=s(ar," (number of tokens processed so far)."),ar.forEach(t),Jl=d(re),ke=r(re,"LI",{});var Ht=n(ke);Ql=s(Ht,"Sample a "),ao=r(Ht,"CODE",{});var Ri=n(ao);Yl=s(Ri,"span_length"),Ri.forEach(t),Zl=s(Ht," from the interval "),oo=r(Ht,"CODE",{});var Xi=n(oo);es=s(Xi,"[1, max_span_length]"),Xi.forEach(t),ts=s(Ht," (length of span of tokens to be masked)"),Ht.forEach(t),as=d(re),yt=r(re,"LI",{});var or=n(yt);os=s(or,"Reserve a context of length "),ro=r(or,"CODE",{});var Gi=n(ro);rs=s(Gi,"context_length = span_length / plm_probability"),Gi.forEach(t),ns=s(or,` to surround span to be
masked`),or.forEach(t),ls=d(re),ee=r(re,"LI",{});var Et=n(ee);ss=s(Et,"Sample a starting point "),no=r(Et,"CODE",{});var Ji=n(no);is=s(Ji,"start_index"),Ji.forEach(t),ds=s(Et," from the interval "),lo=r(Et,"CODE",{});var Qi=n(lo);cs=s(Qi,"[cur_len, cur_len + context_length - span_length]"),Qi.forEach(t),ms=s(Et," and mask tokens "),so=r(Et,"CODE",{});var Yi=n(so);ps=s(Yi,"start_index:start_index + span_length"),Yi.forEach(t),Et.forEach(t),fs=d(re),ye=r(re,"LI",{});var Ut=n(ye);hs=s(Ut,"Set "),io=r(Ut,"CODE",{});var Zi=n(io);gs=s(Zi,"cur_len = cur_len + context_length"),Zi.forEach(t),us=s(Ut,". If "),co=r(Ut,"CODE",{});var ed=n(co);_s=s(ed,"cur_len < max_len"),ed.forEach(t),vs=s(Ut,` (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.`),Ut.forEach(t),re.forEach(t),Kt.forEach(t),K.forEach(t),this.h()},h(){c(f,"name","hf:doc:metadata"),c(f,"content",JSON.stringify(cd)),c(P,"id","data-collator"),c(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(P,"href","#data-collator"),c(k,"class","relative group"),c(Pt,"href","/docs/transformers/pr_19907/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling"),c(St,"href","../examples"),c(Ft,"href","../notebooks"),c(De,"id","transformers.default_data_collator"),c(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(De,"href","#transformers.default_data_collator"),c(ne,"class","relative group"),c(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(xe,"id","transformers.DefaultDataCollator"),c(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(xe,"href","#transformers.DefaultDataCollator"),c(le,"class","relative group"),c(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ee,"id","transformers.DataCollatorWithPadding"),c(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ee,"href","#transformers.DataCollatorWithPadding"),c(se,"class","relative group"),c(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(we,"id","transformers.DataCollatorForTokenClassification"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#transformers.DataCollatorForTokenClassification"),c(de,"class","relative group"),c(ce,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ce,"id","transformers.DataCollatorForSeq2Seq"),c(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ce,"href","#transformers.DataCollatorForSeq2Seq"),c(me,"class","relative group"),c(pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Te,"id","transformers.DataCollatorForLanguageModeling"),c(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Te,"href","#transformers.DataCollatorForLanguageModeling"),c(fe,"class","relative group"),c(Se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Fe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ze,"id","transformers.DataCollatorForWholeWordMask"),c(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ze,"href","#transformers.DataCollatorForWholeWordMask"),c(he,"class","relative group"),c(qe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Oe,"id","transformers.DataCollatorForPermutationLanguageModeling"),c(Oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Oe,"href","#transformers.DataCollatorForPermutationLanguageModeling"),c(ge,"class","relative group"),c(z,"start","0"),c(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(A,"start","0"),c(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(q,"start","0"),c(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(a,m){e(document.head,f),p(a,W,m),p(a,k,m),e(k,P),e(P,I),u($,I,null),e(k,H),e(k,O),e(O,S),p(a,V,m),p(a,E,m),e(E,F),e(E,D),e(D,R),e(E,j),e(E,y),e(y,Ct),e(E,Tt),p(a,We,m),p(a,U,m),e(U,rr),e(U,Pt),e(Pt,nr),e(U,lr),p(a,po,m),p(a,X,m),e(X,sr),e(X,St),e(St,ir),e(X,dr),e(X,Ft),e(Ft,cr),e(X,mr),p(a,fo,m),p(a,ne,m),e(ne,De),e(De,Rt),u(Ve,Rt,null),e(ne,pr),e(ne,Xt),e(Xt,fr),p(a,ho,m),p(a,N,m),u(je,N,null),e(N,hr),e(N,Gt),e(Gt,gr),e(N,ur),e(N,Be),e(Be,Lt),e(Lt,Jt),e(Jt,_r),e(Lt,vr),e(Be,br),e(Be,zt),e(zt,Qt),e(Qt,kr),e(zt,yr),e(N,$r),e(N,Yt),e(Yt,Dr),p(a,go,m),p(a,le,m),e(le,xe),e(xe,Zt),u(Ke,Zt,null),e(le,xr),e(le,ea),e(ea,Er),p(a,uo,m),p(a,L,m),u(He,L,null),e(L,wr),e(L,ta),e(ta,Cr),e(L,Tr),e(L,Ue),e(Ue,At),e(At,aa),e(aa,Pr),e(At,Sr),e(Ue,Fr),e(Ue,qt),e(qt,oa),e(oa,Lr),e(qt,zr),e(L,Ar),e(L,ra),e(ra,qr),e(L,Mr),e(L,na),e(na,Ir),p(a,_o,m),p(a,se,m),e(se,Ee),e(Ee,la),u(Re,la,null),e(se,Or),e(se,sa),e(sa,Nr),p(a,vo,m),p(a,ie,m),u(Xe,ie,null),e(ie,Wr),e(ie,ia),e(ia,Vr),p(a,bo,m),p(a,de,m),e(de,we),e(we,da),u(Ge,da,null),e(de,jr),e(de,ca),e(ca,Br),p(a,ko,m),p(a,ce,m),u(Je,ce,null),e(ce,Kr),e(ce,ma),e(ma,Hr),p(a,yo,m),p(a,me,m),e(me,Ce),e(Ce,pa),u(Qe,pa,null),e(me,Ur),e(me,fa),e(fa,Rr),p(a,$o,m),p(a,pe,m),u(Ye,pe,null),e(pe,Xr),e(pe,ha),e(ha,Gr),p(a,Do,m),p(a,fe,m),e(fe,Te),e(Te,ga),u(Ze,ga,null),e(fe,Jr),e(fe,ua),e(ua,Qr),p(a,xo,m),p(a,w,m),u(et,w,null),e(w,Yr),e(w,_a),e(_a,Zr),e(w,en),u(Pe,w,null),e(w,tn),e(w,Se),u(tt,Se,null),e(Se,an),e(Se,va),e(va,on),e(w,rn),e(w,Fe),u(at,Fe,null),e(Fe,nn),e(Fe,ba),e(ba,ln),e(w,sn),e(w,Le),u(ot,Le,null),e(Le,dn),e(Le,ka),e(ka,cn),p(a,Eo,m),p(a,he,m),e(he,ze),e(ze,ya),u(rt,ya,null),e(he,mn),e(he,$a),e($a,pn),p(a,wo,m),p(a,x,m),u(nt,x,null),e(x,fn),e(x,Da),e(Da,hn),e(x,gn),e(x,lt),e(lt,xa),e(xa,un),e(lt,_n),e(lt,Ea),e(Ea,vn),e(x,bn),u(Ae,x,null),e(x,kn),e(x,qe),u(st,qe,null),e(qe,yn),e(qe,wa),e(wa,$n),e(x,Dn),e(x,Me),u(it,Me,null),e(Me,xn),e(Me,Ca),e(Ca,En),e(x,wn),e(x,Ie),u(dt,Ie,null),e(Ie,Cn),e(Ie,Ta),e(Ta,Tn),p(a,Co,m),p(a,ge,m),e(ge,Oe),e(Oe,Pa),u(ct,Pa,null),e(ge,Pn),e(ge,Sa),e(Sa,Sn),p(a,To,m),p(a,C,m),u(mt,C,null),e(C,Fn),e(C,Fa),e(Fa,Ln),e(C,zn),e(C,pt),e(pt,La),e(La,An),e(pt,qn),e(pt,za),e(za,Mn),e(C,In),e(C,G),u(ft,G,null),e(G,On),e(G,Aa),e(Aa,Nn),e(G,Wn),e(G,z),e(z,ht),e(ht,Vn),e(ht,qa),e(qa,jn),e(ht,Bn),e(z,Kn),e(z,ue),e(ue,Hn),e(ue,Ma),e(Ma,Un),e(ue,Rn),e(ue,Ia),e(Ia,Xn),e(ue,Gn),e(z,Jn),e(z,gt),e(gt,Qn),e(gt,Oa),e(Oa,Yn),e(gt,Zn),e(z,el),e(z,J),e(J,tl),e(J,Na),e(Na,al),e(J,ol),e(J,Wa),e(Wa,rl),e(J,nl),e(J,Va),e(Va,ll),e(z,sl),e(z,_e),e(_e,il),e(_e,ja),e(ja,dl),e(_e,cl),e(_e,Ba),e(Ba,ml),e(_e,pl),e(C,fl),e(C,Q),u(ut,Q,null),e(Q,hl),e(Q,Ka),e(Ka,gl),e(Q,ul),e(Q,A),e(A,_t),e(_t,_l),e(_t,Ha),e(Ha,vl),e(_t,bl),e(A,kl),e(A,ve),e(ve,yl),e(ve,Ua),e(Ua,$l),e(ve,Dl),e(ve,Ra),e(Ra,xl),e(ve,El),e(A,wl),e(A,vt),e(vt,Cl),e(vt,Xa),e(Xa,Tl),e(vt,Pl),e(A,Sl),e(A,Y),e(Y,Fl),e(Y,Ga),e(Ga,Ll),e(Y,zl),e(Y,Ja),e(Ja,Al),e(Y,ql),e(Y,Qa),e(Qa,Ml),e(A,Il),e(A,be),e(be,Ol),e(be,Ya),e(Ya,Nl),e(be,Wl),e(be,Za),e(Za,Vl),e(be,jl),e(C,Bl),e(C,Z),u(bt,Z,null),e(Z,Kl),e(Z,eo),e(eo,Hl),e(Z,Ul),e(Z,q),e(q,kt),e(kt,Rl),e(kt,to),e(to,Xl),e(kt,Gl),e(q,Jl),e(q,ke),e(ke,Ql),e(ke,ao),e(ao,Yl),e(ke,Zl),e(ke,oo),e(oo,es),e(ke,ts),e(q,as),e(q,yt),e(yt,os),e(yt,ro),e(ro,rs),e(yt,ns),e(q,ls),e(q,ee),e(ee,ss),e(ee,no),e(no,is),e(ee,ds),e(ee,lo),e(lo,cs),e(ee,ms),e(ee,so),e(so,ps),e(q,fs),e(q,ye),e(ye,hs),e(ye,io),e(io,gs),e(ye,us),e(ye,co),e(co,_s),e(ye,vs),Po=!0},p(a,[m]){const $t={};m&2&&($t.$$scope={dirty:m,ctx:a}),Pe.$set($t);const mo={};m&2&&(mo.$$scope={dirty:m,ctx:a}),Ae.$set(mo)},i(a){Po||(_($.$$.fragment,a),_(Ve.$$.fragment,a),_(je.$$.fragment,a),_(Ke.$$.fragment,a),_(He.$$.fragment,a),_(Re.$$.fragment,a),_(Xe.$$.fragment,a),_(Ge.$$.fragment,a),_(Je.$$.fragment,a),_(Qe.$$.fragment,a),_(Ye.$$.fragment,a),_(Ze.$$.fragment,a),_(et.$$.fragment,a),_(Pe.$$.fragment,a),_(tt.$$.fragment,a),_(at.$$.fragment,a),_(ot.$$.fragment,a),_(rt.$$.fragment,a),_(nt.$$.fragment,a),_(Ae.$$.fragment,a),_(st.$$.fragment,a),_(it.$$.fragment,a),_(dt.$$.fragment,a),_(ct.$$.fragment,a),_(mt.$$.fragment,a),_(ft.$$.fragment,a),_(ut.$$.fragment,a),_(bt.$$.fragment,a),Po=!0)},o(a){v($.$$.fragment,a),v(Ve.$$.fragment,a),v(je.$$.fragment,a),v(Ke.$$.fragment,a),v(He.$$.fragment,a),v(Re.$$.fragment,a),v(Xe.$$.fragment,a),v(Ge.$$.fragment,a),v(Je.$$.fragment,a),v(Qe.$$.fragment,a),v(Ye.$$.fragment,a),v(Ze.$$.fragment,a),v(et.$$.fragment,a),v(Pe.$$.fragment,a),v(tt.$$.fragment,a),v(at.$$.fragment,a),v(ot.$$.fragment,a),v(rt.$$.fragment,a),v(nt.$$.fragment,a),v(Ae.$$.fragment,a),v(st.$$.fragment,a),v(it.$$.fragment,a),v(dt.$$.fragment,a),v(ct.$$.fragment,a),v(mt.$$.fragment,a),v(ft.$$.fragment,a),v(ut.$$.fragment,a),v(bt.$$.fragment,a),Po=!1},d(a){t(f),a&&t(W),a&&t(k),b($),a&&t(V),a&&t(E),a&&t(We),a&&t(U),a&&t(po),a&&t(X),a&&t(fo),a&&t(ne),b(Ve),a&&t(ho),a&&t(N),b(je),a&&t(go),a&&t(le),b(Ke),a&&t(uo),a&&t(L),b(He),a&&t(_o),a&&t(se),b(Re),a&&t(vo),a&&t(ie),b(Xe),a&&t(bo),a&&t(de),b(Ge),a&&t(ko),a&&t(ce),b(Je),a&&t(yo),a&&t(me),b(Qe),a&&t($o),a&&t(pe),b(Ye),a&&t(Do),a&&t(fe),b(Ze),a&&t(xo),a&&t(w),b(et),b(Pe),b(tt),b(at),b(ot),a&&t(Eo),a&&t(he),b(rt),a&&t(wo),a&&t(x),b(nt),b(Ae),b(st),b(it),b(dt),a&&t(Co),a&&t(ge),b(ct),a&&t(To),a&&t(C),b(mt),b(ft),b(ut),b(bt)}}}const cd={local:"data-collator",sections:[{local:"transformers.default_data_collator",title:"Default data collator"},{local:"transformers.DefaultDataCollator",title:"DefaultDataCollator"},{local:"transformers.DataCollatorWithPadding",title:"DataCollatorWithPadding"},{local:"transformers.DataCollatorForTokenClassification",title:"DataCollatorForTokenClassification"},{local:"transformers.DataCollatorForSeq2Seq",title:"DataCollatorForSeq2Seq"},{local:"transformers.DataCollatorForLanguageModeling",title:"DataCollatorForLanguageModeling"},{local:"transformers.DataCollatorForWholeWordMask",title:"DataCollatorForWholeWordMask"},{local:"transformers.DataCollatorForPermutationLanguageModeling",title:"DataCollatorForPermutationLanguageModeling"}],title:"Data Collator"};function md(wt){return ld(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ud extends ad{constructor(f){super();od(this,f,md,dd,rd,{})}}export{ud as default,cd as metadata};
