import{S as cd,i as fd,s as md,e as s,k as i,w as g,t as n,M as hd,c as a,d as r,m as p,a as o,x as v,h as l,b as f,G as t,g as c,y as b,q as $,o as x,B as y,v as ud,L as vi}from"../../chunks/vendor-hf-doc-builder.js";import{T as _i}from"../../chunks/Tip-hf-doc-builder.js";import{D as I}from"../../chunks/Docstring-hf-doc-builder.js";import{C as As}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Oe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as gi}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function _d(M){let h,E,_,u,P,m,w,z;return{c(){h=s("p"),E=n(`This class method is simply calling the feature extractor
`),_=s("a"),u=n("from_pretrained()"),P=n(` and the tokenizer
`),m=s("code"),w=n("from_pretrained"),z=n(` methods. Please refer to the docstrings of the
methods above for more information.`),this.h()},l(T){h=a(T,"P",{});var q=o(h);E=l(q,`This class method is simply calling the feature extractor
`),_=a(q,"A",{href:!0});var V=o(_);u=l(V,"from_pretrained()"),V.forEach(r),P=l(q,` and the tokenizer
`),m=a(q,"CODE",{});var K=o(m);w=l(K,"from_pretrained"),K.forEach(r),z=l(q,` methods. Please refer to the docstrings of the
methods above for more information.`),q.forEach(r),this.h()},h(){f(_,"href","/docs/transformers/pr_17548/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.from_pretrained")},m(T,q){c(T,h,q),t(h,E),t(h,_),t(_,u),t(h,P),t(h,m),t(m,w),t(h,z)},d(T){T&&r(h)}}}function gd(M){let h,E,_,u,P;return u=new As({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("bert-base-cased")

# Push the processor to your namespace with the name "my-finetuned-bert" and have a local clone in the
# *my-finetuned-bert* folder.
processor.push_to_hub("my-finetuned-bert")

# Push the processor to your namespace with the name "my-finetuned-bert" with no local clone.
processor.push_to_hub("my-finetuned-bert", use_temp_dir=True)

# Push the processor to an organization with the name "my-finetuned-bert" and have a local clone in the
# *my-finetuned-bert* folder.
processor.push_to_hub("my-finetuned-bert", organization="huggingface")

# Make a change to an existing repo that has been cloned locally in *my-finetuned-bert*.
processor.push_to_hub("my-finetuned-bert", repo_url="https://huggingface.co/sgugger/my-finetuned-bert")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)

<span class="hljs-comment"># Push the processor to your namespace with the name &quot;my-finetuned-bert&quot; and have a local clone in the</span>
<span class="hljs-comment"># *my-finetuned-bert* folder.</span>
processor.push_to_hub(<span class="hljs-string">&quot;my-finetuned-bert&quot;</span>)

<span class="hljs-comment"># Push the processor to your namespace with the name &quot;my-finetuned-bert&quot; with no local clone.</span>
processor.push_to_hub(<span class="hljs-string">&quot;my-finetuned-bert&quot;</span>, use_temp_dir=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Push the processor to an organization with the name &quot;my-finetuned-bert&quot; and have a local clone in the</span>
<span class="hljs-comment"># *my-finetuned-bert* folder.</span>
processor.push_to_hub(<span class="hljs-string">&quot;my-finetuned-bert&quot;</span>, organization=<span class="hljs-string">&quot;huggingface&quot;</span>)

<span class="hljs-comment"># Make a change to an existing repo that has been cloned locally in *my-finetuned-bert*.</span>
processor.push_to_hub(<span class="hljs-string">&quot;my-finetuned-bert&quot;</span>, repo_url=<span class="hljs-string">&quot;https://huggingface.co/sgugger/my-finetuned-bert&quot;</span>)`}}),{c(){h=s("p"),E=n("Examples:"),_=i(),g(u.$$.fragment)},l(m){h=a(m,"P",{});var w=o(h);E=l(w,"Examples:"),w.forEach(r),_=p(m),v(u.$$.fragment,m)},m(m,w){c(m,h,w),t(h,E),c(m,_,w),b(u,m,w),P=!0},p:vi,i(m){P||($(u.$$.fragment,m),P=!0)},o(m){x(u.$$.fragment,m),P=!1},d(m){m&&r(h),m&&r(_),y(u,m)}}}function vd(M){let h,E;return{c(){h=s("p"),E=n("This API is experimental and may have some slight breaking changes in the next releases.")},l(_){h=a(_,"P",{});var u=o(h);E=l(u,"This API is experimental and may have some slight breaking changes in the next releases."),u.forEach(r)},m(_,u){c(_,h,u),t(h,E)},d(_){_&&r(h)}}}function bd(M){let h,E,_,u,P,m,w,z;return{c(){h=s("p"),E=n("This class method is simply calling "),_=s("a"),u=n("save_pretrained()"),P=n(` and
`),m=s("code"),w=n("save_pretrained"),z=n(`. Please refer to the docstrings of the methods
above for more information.`),this.h()},l(T){h=a(T,"P",{});var q=o(h);E=l(q,"This class method is simply calling "),_=a(q,"A",{href:!0});var V=o(_);u=l(V,"save_pretrained()"),V.forEach(r),P=l(q,` and
`),m=a(q,"CODE",{});var K=o(m);w=l(K,"save_pretrained"),K.forEach(r),z=l(q,`. Please refer to the docstrings of the methods
above for more information.`),q.forEach(r),this.h()},h(){f(_,"href","/docs/transformers/pr_17548/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.save_pretrained")},m(T,q){c(T,h,q),t(h,E),t(h,_),t(_,u),t(h,P),t(h,m),t(m,w),t(h,z)},d(T){T&&r(h)}}}function $d(M){let h,E,_,u,P;return u=new As({props:{code:`import tensorflow_datasets as tfds

dataset = tfds.load("squad")

training_examples = get_examples_from_dataset(dataset, evaluate=False)
evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow_datasets <span class="hljs-keyword">as</span> tfds

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = tfds.load(<span class="hljs-string">&quot;squad&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>training_examples = get_examples_from_dataset(dataset, evaluate=<span class="hljs-literal">False</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>evaluation_examples = get_examples_from_dataset(dataset, evaluate=<span class="hljs-literal">True</span>)`}}),{c(){h=s("p"),E=n("Examples:"),_=i(),g(u.$$.fragment)},l(m){h=a(m,"P",{});var w=o(h);E=l(w,"Examples:"),w.forEach(r),_=p(m),v(u.$$.fragment,m)},m(m,w){c(m,h,w),t(h,E),c(m,_,w),b(u,m,w),P=!0},p:vi,i(m){P||($(u.$$.fragment,m),P=!0)},o(m){x(u.$$.fragment,m),P=!1},d(m){m&&r(h),m&&r(_),y(u,m)}}}function xd(M){let h,E,_,u,P;return u=new As({props:{code:`processor = SquadV2Processor()
examples = processor.get_dev_examples(data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=args.max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=args.max_query_length,
    is_training=not evaluate,
)`,highlighted:`processor = SquadV2Processor()
examples = processor.get_dev_examples(data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=args.max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=args.max_query_length,
    is_training=<span class="hljs-keyword">not</span> evaluate,
)`}}),{c(){h=s("p"),E=n("Example:"),_=i(),g(u.$$.fragment)},l(m){h=a(m,"P",{});var w=o(h);E=l(w,"Example:"),w.forEach(r),_=p(m),v(u.$$.fragment,m)},m(m,w){c(m,h,w),t(h,E),c(m,_,w),b(u,m,w),P=!0},p:vi,i(m){P||($(u.$$.fragment,m),P=!0)},o(m){x(u.$$.fragment,m),P=!1},d(m){m&&r(h),m&&r(_),y(u,m)}}}function yd(M){let h,E,_,u,P,m,w,z,T,q,V,K,Ls,ie,Y,Ya,Ot,Za,eo,Qt,to,ro,so,gr,ao,Ds,Z,pe,vr,Qe,oo,br,no,Ts,Ut,lo,Ns,Ft,io,Ms,L,Ue,po,$r,co,fo,G,Fe,mo,xr,ho,uo,de,_o,R,Ge,go,Re,vo,yr,bo,$o,xo,ce,yo,H,He,wo,We,Eo,wr,Po,qo,ko,fe,Io,W,Xe,So,Be,Ao,Gt,Lo,Do,To,me,js,ee,he,Er,Je,No,Pr,Mo,zs,j,jo,Rt,zo,Vo,Ht,Co,Oo,Wt,Qo,Uo,Xt,Fo,Go,Vs,S,Ke,Ro,qr,Ho,Wo,ue,Ye,Xo,Ze,Bo,Bt,Jo,Ko,Yo,_e,et,Zo,kr,en,tn,ge,tt,rn,Ir,sn,an,ve,rt,on,st,nn,Jt,ln,pn,dn,be,at,cn,ot,fn,Kt,mn,hn,un,$e,nt,_n,Sr,gn,Cs,O,lt,vn,Ar,bn,$n,xe,it,xn,Lr,yn,Os,Q,pt,wn,Dr,En,Pn,ye,dt,qn,Tr,kn,Qs,te,we,Nr,ct,In,Mr,Sn,Us,Ee,ft,An,Ln,mt,Dn,Fs,Yt,Tn,Gs,Zt,Nn,Rs,k,jr,zr,Mn,jn,Vr,Cr,zn,Vn,Or,Qr,Cn,On,Ur,Fr,Qn,Un,Gr,Rr,Fn,Gn,Hr,Wr,Rn,Hn,Xr,Br,Wn,Xn,Jr,Kr,Bn,Jn,Yr,Zr,Kn,Hs,Pe,Yn,er,Zn,el,Ws,re,ht,tl,tr,rl,es,sl,Xs,se,qe,ts,ut,al,rs,ol,Bs,ae,_t,nl,ll,gt,ss,il,pl,Js,vt,dl,bt,cl,Ks,rr,fl,Ys,sr,as,os,ml,Zs,ar,hl,ea,ke,ul,$t,_l,gl,ta,oe,Ie,ns,xt,vl,ls,bl,ra,U,yt,$l,xl,wt,yl,wl,Et,El,Pl,sa,or,ql,aa,ne,Se,is,Pt,kl,ps,Il,oa,nr,Sl,na,Ae,ds,cs,Al,Ll,fs,ms,Dl,la,qt,Tl,hs,Nl,ia,N,kt,Ml,us,jl,zl,Le,It,Vl,_s,Cl,Ol,X,St,Ql,At,Ul,gs,Fl,Gl,Rl,De,Hl,Te,Lt,Wl,vs,Xl,pa,Ne,Bl,bs,Jl,Kl,da,F,Dt,Yl,$s,Zl,ei,Me,ca,je,ti,xs,ri,si,fa,le,ze,ys,Tt,ai,ws,oi,ma,lr,ni,ha,Nt,ua,Ve,li,Es,ii,pi,_a,Mt,ga,Ce,di,jt,ci,fi,va;return m=new Oe({}),Qe=new Oe({}),Ue=new I({props:{name:"class transformers.ProcessorMixin",anchor:"transformers.ProcessorMixin",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/processing_utils.py#L43"}}),Fe=new I({props:{name:"from_pretrained",anchor:"transformers.ProcessorMixin.from_pretrained",parameters:[{name:"pretrained_model_name_or_path",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.ProcessorMixin.from_pretrained.pretrained_model_name_or_path",description:`<strong>pretrained_model_name_or_path</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
This can be either:</p>
<ul>
<li>a string, the <em>model id</em> of a pretrained feature_extractor hosted inside a model repo on
huggingface.co. Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or
namespaced under a user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>a path to a <em>directory</em> containing a feature extractor file saved using the
<a href="/docs/transformers/pr_17548/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.save_pretrained">save_pretrained()</a> method, e.g., <code>./my_model_directory/</code>.</li>
<li>a path or url to a saved feature extractor JSON <em>file</em>, e.g.,
<code>./my_model_directory/preprocessor_config.json</code>.
**kwargs &#x2014;
Additional keyword arguments passed along to both
<a href="/docs/transformers/pr_17548/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.from_pretrained">from_pretrained()</a> and
<code>from_pretrained</code>.</li>
</ul>`,name:"pretrained_model_name_or_path"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/processing_utils.py#L156"}}),de=new _i({props:{$$slots:{default:[_d]},$$scope:{ctx:M}}}),Ge=new I({props:{name:"push_to_hub",anchor:"transformers.ProcessorMixin.push_to_hub",parameters:[{name:"repo_path_or_name",val:": typing.Optional[str] = None"},{name:"repo_url",val:": typing.Optional[str] = None"},{name:"use_temp_dir",val:": bool = False"},{name:"commit_message",val:": typing.Optional[str] = None"},{name:"organization",val:": typing.Optional[str] = None"},{name:"private",val:": typing.Optional[bool] = None"},{name:"use_auth_token",val:": typing.Union[bool, str, NoneType] = None"},{name:"**model_card_kwargs",val:""}],parametersDescription:[{anchor:"transformers.ProcessorMixin.push_to_hub.repo_path_or_name",description:`<strong>repo_path_or_name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Can either be a repository name for your processor in the Hub or a path to a local folder (in which case
the repository will have the name of that local folder). If not specified, will default to the name
given by <code>repo_url</code> and a local directory with that name will be created.`,name:"repo_path_or_name"},{anchor:"transformers.ProcessorMixin.push_to_hub.repo_url",description:`<strong>repo_url</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Specify this in case you want to push to an existing repository in the hub. If unspecified, a new
repository will be created in your namespace (unless you specify an <code>organization</code>) with <code>repo_name</code>.`,name:"repo_url"},{anchor:"transformers.ProcessorMixin.push_to_hub.use_temp_dir",description:`<strong>use_temp_dir</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to clone the distant repo in a temporary directory or in <code>repo_path_or_name</code> inside the
current working directory. This will slow things down if you are making changes in an existing repo
since you will need to clone the repo before every push.`,name:"use_temp_dir"},{anchor:"transformers.ProcessorMixin.push_to_hub.commit_message",description:`<strong>commit_message</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Message to commit while pushing. Will default to <code>&quot;add processor&quot;</code>.`,name:"commit_message"},{anchor:"transformers.ProcessorMixin.push_to_hub.organization",description:`<strong>organization</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Organization in which you want to push your processor (you must be a member of this organization).`,name:"organization"},{anchor:"transformers.ProcessorMixin.push_to_hub.private",description:`<strong>private</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not the repository created should be private (requires a paying subscription).`,name:"private"},{anchor:"transformers.ProcessorMixin.push_to_hub.use_auth_token",description:`<strong>use_auth_token</strong> (<code>bool</code> or <code>str</code>, <em>optional</em>) &#x2014;
The token to use as HTTP bearer authorization for remote files. If <code>True</code>, will use the token generated
when running <code>transformers-cli login</code> (stored in <code>~/.huggingface</code>). Will default to <code>True</code> if
<code>repo_url</code> is not specified.`,name:"use_auth_token"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/utils/hub.py#L846",returnDescription:`
<p>The url of the commit of your processor in the given repository.</p>
`,returnType:`
<p><code>str</code></p>
`}}),ce=new gi({props:{anchor:"transformers.ProcessorMixin.push_to_hub.example",$$slots:{default:[gd]},$$scope:{ctx:M}}}),He=new I({props:{name:"register_for_auto_class",anchor:"transformers.ProcessorMixin.register_for_auto_class",parameters:[{name:"auto_class",val:" = 'AutoProcessor'"}],parametersDescription:[{anchor:"transformers.ProcessorMixin.register_for_auto_class.auto_class",description:`<strong>auto_class</strong> (<code>str</code> or <code>type</code>, <em>optional</em>, defaults to <code>&quot;AutoProcessor&quot;</code>) &#x2014;
The auto class to register this new feature extractor with.`,name:"auto_class"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/processing_utils.py#L189"}}),fe=new _i({props:{warning:!0,$$slots:{default:[vd]},$$scope:{ctx:M}}}),Xe=new I({props:{name:"save_pretrained",anchor:"transformers.ProcessorMixin.save_pretrained",parameters:[{name:"save_directory",val:""},{name:"push_to_hub",val:": bool = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.ProcessorMixin.save_pretrained.save_directory",description:`<strong>save_directory</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
Directory where the feature extractor JSON file and the tokenizer files will be saved (directory will
be created if it does not exist).`,name:"save_directory"},{anchor:"transformers.ProcessorMixin.save_pretrained.push_to_hub",description:`<strong>push_to_hub</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to push your processor to the Hugging Face model hub after saving it.</p>
<div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400">
						
<p>Using <code>push_to_hub=True</code> will synchronize the repository you are pushing to with <code>save_directory</code>,
which requires <code>save_directory</code> to be a local clone of the repo you are pushing to if it&#x2019;s an existing
folder. Pass along <code>temp_dir=True</code> to use a temporary directory instead.</p>

					</div>
<p>kwargs &#x2014;
Additional key word arguments passed along to the <a href="/docs/transformers/pr_17548/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.push_to_hub">push_to_hub()</a> method.`,name:"push_to_hub"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/processing_utils.py#L94"}}),me=new _i({props:{$$slots:{default:[bd]},$$scope:{ctx:M}}}),Je=new Oe({}),Ke=new I({props:{name:"class transformers.DataProcessor",anchor:"transformers.DataProcessor",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L80"}}),Ye=new I({props:{name:"get_dev_examples",anchor:"transformers.DataProcessor.get_dev_examples",parameters:[{name:"data_dir",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L97"}}),et=new I({props:{name:"get_example_from_tensor_dict",anchor:"transformers.DataProcessor.get_example_from_tensor_dict",parameters:[{name:"tensor_dict",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L83"}}),tt=new I({props:{name:"get_labels",anchor:"transformers.DataProcessor.get_labels",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L105"}}),rt=new I({props:{name:"get_test_examples",anchor:"transformers.DataProcessor.get_test_examples",parameters:[{name:"data_dir",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L101"}}),at=new I({props:{name:"get_train_examples",anchor:"transformers.DataProcessor.get_train_examples",parameters:[{name:"data_dir",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L93"}}),nt=new I({props:{name:"tfds_map",anchor:"transformers.DataProcessor.tfds_map",parameters:[{name:"example",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L109"}}),lt=new I({props:{name:"class transformers.InputExample",anchor:"transformers.InputExample",parameters:[{name:"guid",val:": str"},{name:"text_a",val:": str"},{name:"text_b",val:": typing.Optional[str] = None"},{name:"label",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L30"}}),it=new I({props:{name:"to_json_string",anchor:"transformers.InputExample.to_json_string",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L49"}}),pt=new I({props:{name:"class transformers.InputFeatures",anchor:"transformers.InputFeatures",parameters:[{name:"input_ids",val:": typing.List[int]"},{name:"attention_mask",val:": typing.Optional[typing.List[int]] = None"},{name:"token_type_ids",val:": typing.Optional[typing.List[int]] = None"},{name:"label",val:": typing.Union[int, float, NoneType] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L55"}}),dt=new I({props:{name:"to_json_string",anchor:"transformers.InputFeatures.to_json_string",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/utils.py#L75"}}),ct=new Oe({}),ht=new I({props:{name:"transformers.glue_convert_examples_to_features",anchor:"transformers.glue_convert_examples_to_features",parameters:[{name:"examples",val:": typing.Union[typing.List[transformers.data.processors.utils.InputExample], ForwardRef('tf.data.Dataset')]"},{name:"tokenizer",val:": PreTrainedTokenizer"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"task",val:" = None"},{name:"label_list",val:" = None"},{name:"output_mode",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/glue.py#L41",returnDescription:`
<p>If the <code>examples</code> input is a <code>tf.data.Dataset</code>, will return a <code>tf.data.Dataset</code> containing the task-specific
features. If the input is a list of <code>InputExamples</code>, will return a list of task-specific <code>InputFeatures</code> which
can be fed to the model.</p>
`}}),ut=new Oe({}),xt=new Oe({}),Pt=new Oe({}),kt=new I({props:{name:"class transformers.data.processors.squad.SquadProcessor",anchor:"transformers.data.processors.squad.SquadProcessor",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/squad.py#L542"}}),It=new I({props:{name:"get_dev_examples",anchor:"transformers.data.processors.squad.SquadProcessor.get_dev_examples",parameters:[{name:"data_dir",val:""},{name:"filename",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/squad.py#L630"}}),St=new I({props:{name:"get_examples_from_dataset",anchor:"transformers.data.processors.squad.SquadProcessor.get_examples_from_dataset",parameters:[{name:"dataset",val:""},{name:"evaluate",val:" = False"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/squad.py#L575",returnDescription:`
<p>List of SquadExample</p>
`}}),De=new gi({props:{anchor:"transformers.data.processors.squad.SquadProcessor.get_examples_from_dataset.example",$$slots:{default:[$d]},$$scope:{ctx:M}}}),Lt=new I({props:{name:"get_train_examples",anchor:"transformers.data.processors.squad.SquadProcessor.get_train_examples",parameters:[{name:"data_dir",val:""},{name:"filename",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/squad.py#L608"}}),Dt=new I({props:{name:"transformers.squad_convert_examples_to_features",anchor:"transformers.squad_convert_examples_to_features",parameters:[{name:"examples",val:""},{name:"tokenizer",val:""},{name:"max_seq_length",val:""},{name:"doc_stride",val:""},{name:"max_query_length",val:""},{name:"is_training",val:""},{name:"padding_strategy",val:" = 'max_length'"},{name:"return_dataset",val:" = False"},{name:"threads",val:" = 1"},{name:"tqdm_enabled",val:" = True"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/data/processors/squad.py#L317",returnDescription:`
<p>list of <code>SquadFeatures</code></p>
`}}),Me=new gi({props:{anchor:"transformers.squad_convert_examples_to_features.example",$$slots:{default:[xd]},$$scope:{ctx:M}}}),Tt=new Oe({}),Nt=new As({props:{code:`# Loading a V2 processor
processor = SquadV2Processor()
examples = processor.get_dev_examples(squad_v2_data_dir)

# Loading a V1 processor
processor = SquadV1Processor()
examples = processor.get_dev_examples(squad_v1_data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=not evaluate,
)`,highlighted:`<span class="hljs-comment"># Loading a V2 processor</span>
processor = SquadV2Processor()
examples = processor.get_dev_examples(squad_v2_data_dir)

<span class="hljs-comment"># Loading a V1 processor</span>
processor = SquadV1Processor()
examples = processor.get_dev_examples(squad_v1_data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=<span class="hljs-keyword">not</span> evaluate,
)`}}),Mt=new As({props:{code:`# tensorflow_datasets only handle Squad V1.
tfds_examples = tfds.load("squad")
examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=not evaluate,
)`,highlighted:`<span class="hljs-comment"># tensorflow_datasets only handle Squad V1.</span>
tfds_examples = tfds.load(<span class="hljs-string">&quot;squad&quot;</span>)
examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=<span class="hljs-keyword">not</span> evaluate,
)`}}),{c(){h=s("meta"),E=i(),_=s("h1"),u=s("a"),P=s("span"),g(m.$$.fragment),w=i(),z=s("span"),T=n("Processors"),q=i(),V=s("p"),K=n("Processors can mean two different things in the Transformers library:"),Ls=i(),ie=s("ul"),Y=s("li"),Ya=n("the objects that pre-process inputs for multi-modal models such as "),Ot=s("a"),Za=n("Wav2Vec2"),eo=n(` (speech and text)
or `),Qt=s("a"),to=n("CLIP"),ro=n(" (text and vision)"),so=i(),gr=s("li"),ao=n("deprecated objects that were used in older versions of the library to preprocess data for GLUE or SQUAD."),Ds=i(),Z=s("h2"),pe=s("a"),vr=s("span"),g(Qe.$$.fragment),oo=i(),br=s("span"),no=n("Multi-modal processors"),Ts=i(),Ut=s("p"),lo=n(`Any multi-modal model will require an object to encode or decode the data that groups several modalities (among text,
vision and audio). This is handled by objects called processors, which group tokenizers (for the text modality) and
feature extractors (for vision and audio).`),Ns=i(),Ft=s("p"),io=n("Those processors inherit from the following base class that implements the saving and loading functionality:"),Ms=i(),L=s("div"),g(Ue.$$.fragment),po=i(),$r=s("p"),co=n("This is a mixin used to provide saving/loading functionality for all processor classes."),fo=i(),G=s("div"),g(Fe.$$.fragment),mo=i(),xr=s("p"),ho=n("Instantiate a processor associated with a pretrained model."),uo=i(),g(de.$$.fragment),_o=i(),R=s("div"),g(Ge.$$.fragment),go=i(),Re=s("p"),vo=n(`Upload the processor files to the \u{1F917} Model Hub while synchronizing a local clone of the repo in
`),yr=s("code"),bo=n("repo_path_or_name"),$o=n("."),xo=i(),g(ce.$$.fragment),yo=i(),H=s("div"),g(He.$$.fragment),wo=i(),We=s("p"),Eo=n(`Register this class with a given auto class. This should only be used for custom feature extractors as the ones
in the library are already mapped with `),wr=s("code"),Po=n("AutoProcessor"),qo=n("."),ko=i(),g(fe.$$.fragment),Io=i(),W=s("div"),g(Xe.$$.fragment),So=i(),Be=s("p"),Ao=n(`Saves the attributes of this processor (feature extractor, tokenizer\u2026) in the specified directory so that it
can be reloaded using the `),Gt=s("a"),Lo=n("from_pretrained()"),Do=n(" method."),To=i(),g(me.$$.fragment),js=i(),ee=s("h2"),he=s("a"),Er=s("span"),g(Je.$$.fragment),No=i(),Pr=s("span"),Mo=n("Deprecated processors"),zs=i(),j=s("p"),jo=n(`All processors follow the same architecture which is that of the
`),Rt=s("a"),zo=n("DataProcessor"),Vo=n(`. The processor returns a list of
`),Ht=s("a"),Co=n("InputExample"),Oo=n(`. These
`),Wt=s("a"),Qo=n("InputExample"),Uo=n(` can be converted to
`),Xt=s("a"),Fo=n("InputFeatures"),Go=n(" in order to be fed to the model."),Vs=i(),S=s("div"),g(Ke.$$.fragment),Ro=i(),qr=s("p"),Ho=n("Base class for data converters for sequence classification data sets."),Wo=i(),ue=s("div"),g(Ye.$$.fragment),Xo=i(),Ze=s("p"),Bo=n("Gets a collection of "),Bt=s("a"),Jo=n("InputExample"),Ko=n(" for the dev set."),Yo=i(),_e=s("div"),g(et.$$.fragment),Zo=i(),kr=s("p"),en=n("Gets an example from a dict with tensorflow tensors."),tn=i(),ge=s("div"),g(tt.$$.fragment),rn=i(),Ir=s("p"),sn=n("Gets the list of labels for this data set."),an=i(),ve=s("div"),g(rt.$$.fragment),on=i(),st=s("p"),nn=n("Gets a collection of "),Jt=s("a"),ln=n("InputExample"),pn=n(" for the test set."),dn=i(),be=s("div"),g(at.$$.fragment),cn=i(),ot=s("p"),fn=n("Gets a collection of "),Kt=s("a"),mn=n("InputExample"),hn=n(" for the train set."),un=i(),$e=s("div"),g(nt.$$.fragment),_n=i(),Sr=s("p"),gn=n(`Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts
examples to the correct format.`),Cs=i(),O=s("div"),g(lt.$$.fragment),vn=i(),Ar=s("p"),bn=n("A single training/test example for simple sequence classification."),$n=i(),xe=s("div"),g(it.$$.fragment),xn=i(),Lr=s("p"),yn=n("Serializes this instance to a JSON string."),Os=i(),Q=s("div"),g(pt.$$.fragment),wn=i(),Dr=s("p"),En=n("A single set of features of data. Property names are the same names as the corresponding inputs to a model."),Pn=i(),ye=s("div"),g(dt.$$.fragment),qn=i(),Tr=s("p"),kn=n("Serializes this instance to a JSON string."),Qs=i(),te=s("h2"),we=s("a"),Nr=s("span"),g(ct.$$.fragment),In=i(),Mr=s("span"),Sn=n("GLUE"),Us=i(),Ee=s("p"),ft=s("a"),An=n("General Language Understanding Evaluation (GLUE)"),Ln=n(` is a benchmark that evaluates the
performance of models across a diverse set of existing NLU tasks. It was released together with the paper `),mt=s("a"),Dn=n(`GLUE: A
multi-task benchmark and analysis platform for natural language understanding`),Fs=i(),Yt=s("p"),Tn=n(`This library hosts a total of 10 processors for the following tasks: MRPC, MNLI, MNLI (mismatched), CoLA, SST2, STSB,
QQP, QNLI, RTE and WNLI.`),Gs=i(),Zt=s("p"),Nn=n("Those processors are:"),Rs=i(),k=s("ul"),jr=s("li"),zr=s("code"),Mn=n("MrpcProcessor"),jn=i(),Vr=s("li"),Cr=s("code"),zn=n("MnliProcessor"),Vn=i(),Or=s("li"),Qr=s("code"),Cn=n("MnliMismatchedProcessor"),On=i(),Ur=s("li"),Fr=s("code"),Qn=n("Sst2Processor"),Un=i(),Gr=s("li"),Rr=s("code"),Fn=n("StsbProcessor"),Gn=i(),Hr=s("li"),Wr=s("code"),Rn=n("QqpProcessor"),Hn=i(),Xr=s("li"),Br=s("code"),Wn=n("QnliProcessor"),Xn=i(),Jr=s("li"),Kr=s("code"),Bn=n("RteProcessor"),Jn=i(),Yr=s("li"),Zr=s("code"),Kn=n("WnliProcessor"),Hs=i(),Pe=s("p"),Yn=n(`Additionally, the following method can be used to load values from a data file and convert them to a list of
`),er=s("a"),Zn=n("InputExample"),el=n("."),Ws=i(),re=s("div"),g(ht.$$.fragment),tl=i(),tr=s("p"),rl=n("Loads a data file into a list of "),es=s("code"),sl=n("InputFeatures"),Xs=i(),se=s("h2"),qe=s("a"),ts=s("span"),g(ut.$$.fragment),al=i(),rs=s("span"),ol=n("XNLI"),Bs=i(),ae=s("p"),_t=s("a"),nl=n("The Cross-Lingual NLI Corpus (XNLI)"),ll=n(` is a benchmark that evaluates the
quality of cross-lingual text representations. XNLI is crowd-sourced dataset based on `),gt=s("a"),ss=s("em"),il=n("MultiNLI"),pl=n(`: pairs of text are labeled with textual entailment annotations for 15
different languages (including both high-resource language such as English and low-resource languages such as Swahili).`),Js=i(),vt=s("p"),dl=n("It was released together with the paper "),bt=s("a"),cl=n("XNLI: Evaluating Cross-lingual Sentence Representations"),Ks=i(),rr=s("p"),fl=n("This library hosts the processor to load the XNLI data:"),Ys=i(),sr=s("ul"),as=s("li"),os=s("code"),ml=n("XnliProcessor"),Zs=i(),ar=s("p"),hl=n("Please note that since the gold labels are available on the test set, evaluation is performed on the test set."),ea=i(),ke=s("p"),ul=n("An example using these processors is given in the "),$t=s("a"),_l=n("run_xnli.py"),gl=n(" script."),ta=i(),oe=s("h2"),Ie=s("a"),ns=s("span"),g(xt.$$.fragment),vl=i(),ls=s("span"),bl=n("SQuAD"),ra=i(),U=s("p"),yt=s("a"),$l=n("The Stanford Question Answering Dataset (SQuAD)"),xl=n(` is a benchmark that
evaluates the performance of models on question answering. Two versions are available, v1.1 and v2.0. The first version
(v1.1) was released together with the paper `),wt=s("a"),yl=n("SQuAD: 100,000+ Questions for Machine Comprehension of Text"),wl=n(". The second version (v2.0) was released alongside the paper "),Et=s("a"),El=n(`Know What You Don\u2019t
Know: Unanswerable Questions for SQuAD`),Pl=n("."),sa=i(),or=s("p"),ql=n("This library hosts a processor for each of the two versions:"),aa=i(),ne=s("h3"),Se=s("a"),is=s("span"),g(Pt.$$.fragment),kl=i(),ps=s("span"),Il=n("Processors"),oa=i(),nr=s("p"),Sl=n("Those processors are:"),na=i(),Ae=s("ul"),ds=s("li"),cs=s("code"),Al=n("SquadV1Processor"),Ll=i(),fs=s("li"),ms=s("code"),Dl=n("SquadV2Processor"),la=i(),qt=s("p"),Tl=n("They both inherit from the abstract class "),hs=s("code"),Nl=n("SquadProcessor"),ia=i(),N=s("div"),g(kt.$$.fragment),Ml=i(),us=s("p"),jl=n(`Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
version 2.0 of SQuAD, respectively.`),zl=i(),Le=s("div"),g(It.$$.fragment),Vl=i(),_s=s("p"),Cl=n("Returns the evaluation example from the data directory."),Ol=i(),X=s("div"),g(St.$$.fragment),Ql=i(),At=s("p"),Ul=n("Creates a list of "),gs=s("code"),Fl=n("SquadExample"),Gl=n(" using a TFDS dataset."),Rl=i(),g(De.$$.fragment),Hl=i(),Te=s("div"),g(Lt.$$.fragment),Wl=i(),vs=s("p"),Xl=n("Returns the training examples from the data directory."),pa=i(),Ne=s("p"),Bl=n(`Additionally, the following method can be used to convert SQuAD examples into
`),bs=s("code"),Jl=n("SquadFeatures"),Kl=n(" that can be used as model inputs."),da=i(),F=s("div"),g(Dt.$$.fragment),Yl=i(),$s=s("p"),Zl=n(`Converts a list of examples into a list of features that can be directly given as input to a model. It is
model-dependant and takes advantage of many of the tokenizer\u2019s features to create the model\u2019s inputs.`),ei=i(),g(Me.$$.fragment),ca=i(),je=s("p"),ti=n(`These processors as well as the aforementionned method can be used with files containing the data as well as with the
`),xs=s("em"),ri=n("tensorflow_datasets"),si=n(" package. Examples are given below."),fa=i(),le=s("h3"),ze=s("a"),ys=s("span"),g(Tt.$$.fragment),ai=i(),ws=s("span"),oi=n("Example usage"),ma=i(),lr=s("p"),ni=n("Here is an example using the processors as well as the conversion method using data files:"),ha=i(),g(Nt.$$.fragment),ua=i(),Ve=s("p"),li=n("Using "),Es=s("em"),ii=n("tensorflow_datasets"),pi=n(" is as easy as using a data file:"),_a=i(),g(Mt.$$.fragment),ga=i(),Ce=s("p"),di=n("Another example using these processors is given in the "),jt=s("a"),ci=n("run_squad.py"),fi=n(" script."),this.h()},l(e){const d=hd('[data-svelte="svelte-1phssyn"]',document.head);h=a(d,"META",{name:!0,content:!0}),d.forEach(r),E=p(e),_=a(e,"H1",{class:!0});var zt=o(_);u=a(zt,"A",{id:!0,class:!0,href:!0});var Ps=o(u);P=a(Ps,"SPAN",{});var qs=o(P);v(m.$$.fragment,qs),qs.forEach(r),Ps.forEach(r),w=p(zt),z=a(zt,"SPAN",{});var ks=o(z);T=l(ks,"Processors"),ks.forEach(r),zt.forEach(r),q=p(e),V=a(e,"P",{});var Is=o(V);K=l(Is,"Processors can mean two different things in the Transformers library:"),Is.forEach(r),Ls=p(e),ie=a(e,"UL",{});var Vt=o(ie);Y=a(Vt,"LI",{});var ir=o(Y);Ya=l(ir,"the objects that pre-process inputs for multi-modal models such as "),Ot=a(ir,"A",{href:!0});var bi=o(Ot);Za=l(bi,"Wav2Vec2"),bi.forEach(r),eo=l(ir,` (speech and text)
or `),Qt=a(ir,"A",{href:!0});var $i=o(Qt);to=l($i,"CLIP"),$i.forEach(r),ro=l(ir," (text and vision)"),ir.forEach(r),so=p(Vt),gr=a(Vt,"LI",{});var xi=o(gr);ao=l(xi,"deprecated objects that were used in older versions of the library to preprocess data for GLUE or SQUAD."),xi.forEach(r),Vt.forEach(r),Ds=p(e),Z=a(e,"H2",{class:!0});var ba=o(Z);pe=a(ba,"A",{id:!0,class:!0,href:!0});var yi=o(pe);vr=a(yi,"SPAN",{});var wi=o(vr);v(Qe.$$.fragment,wi),wi.forEach(r),yi.forEach(r),oo=p(ba),br=a(ba,"SPAN",{});var Ei=o(br);no=l(Ei,"Multi-modal processors"),Ei.forEach(r),ba.forEach(r),Ts=p(e),Ut=a(e,"P",{});var Pi=o(Ut);lo=l(Pi,`Any multi-modal model will require an object to encode or decode the data that groups several modalities (among text,
vision and audio). This is handled by objects called processors, which group tokenizers (for the text modality) and
feature extractors (for vision and audio).`),Pi.forEach(r),Ns=p(e),Ft=a(e,"P",{});var qi=o(Ft);io=l(qi,"Those processors inherit from the following base class that implements the saving and loading functionality:"),qi.forEach(r),Ms=p(e),L=a(e,"DIV",{class:!0});var C=o(L);v(Ue.$$.fragment,C),po=p(C),$r=a(C,"P",{});var ki=o($r);co=l(ki,"This is a mixin used to provide saving/loading functionality for all processor classes."),ki.forEach(r),fo=p(C),G=a(C,"DIV",{class:!0});var pr=o(G);v(Fe.$$.fragment,pr),mo=p(pr),xr=a(pr,"P",{});var Ii=o(xr);ho=l(Ii,"Instantiate a processor associated with a pretrained model."),Ii.forEach(r),uo=p(pr),v(de.$$.fragment,pr),pr.forEach(r),_o=p(C),R=a(C,"DIV",{class:!0});var dr=o(R);v(Ge.$$.fragment,dr),go=p(dr),Re=a(dr,"P",{});var $a=o(Re);vo=l($a,`Upload the processor files to the \u{1F917} Model Hub while synchronizing a local clone of the repo in
`),yr=a($a,"CODE",{});var Si=o(yr);bo=l(Si,"repo_path_or_name"),Si.forEach(r),$o=l($a,"."),$a.forEach(r),xo=p(dr),v(ce.$$.fragment,dr),dr.forEach(r),yo=p(C),H=a(C,"DIV",{class:!0});var cr=o(H);v(He.$$.fragment,cr),wo=p(cr),We=a(cr,"P",{});var xa=o(We);Eo=l(xa,`Register this class with a given auto class. This should only be used for custom feature extractors as the ones
in the library are already mapped with `),wr=a(xa,"CODE",{});var Ai=o(wr);Po=l(Ai,"AutoProcessor"),Ai.forEach(r),qo=l(xa,"."),xa.forEach(r),ko=p(cr),v(fe.$$.fragment,cr),cr.forEach(r),Io=p(C),W=a(C,"DIV",{class:!0});var fr=o(W);v(Xe.$$.fragment,fr),So=p(fr),Be=a(fr,"P",{});var ya=o(Be);Ao=l(ya,`Saves the attributes of this processor (feature extractor, tokenizer\u2026) in the specified directory so that it
can be reloaded using the `),Gt=a(ya,"A",{href:!0});var Li=o(Gt);Lo=l(Li,"from_pretrained()"),Li.forEach(r),Do=l(ya," method."),ya.forEach(r),To=p(fr),v(me.$$.fragment,fr),fr.forEach(r),C.forEach(r),js=p(e),ee=a(e,"H2",{class:!0});var wa=o(ee);he=a(wa,"A",{id:!0,class:!0,href:!0});var Di=o(he);Er=a(Di,"SPAN",{});var Ti=o(Er);v(Je.$$.fragment,Ti),Ti.forEach(r),Di.forEach(r),No=p(wa),Pr=a(wa,"SPAN",{});var Ni=o(Pr);Mo=l(Ni,"Deprecated processors"),Ni.forEach(r),wa.forEach(r),zs=p(e),j=a(e,"P",{});var B=o(j);jo=l(B,`All processors follow the same architecture which is that of the
`),Rt=a(B,"A",{href:!0});var Mi=o(Rt);zo=l(Mi,"DataProcessor"),Mi.forEach(r),Vo=l(B,`. The processor returns a list of
`),Ht=a(B,"A",{href:!0});var ji=o(Ht);Co=l(ji,"InputExample"),ji.forEach(r),Oo=l(B,`. These
`),Wt=a(B,"A",{href:!0});var zi=o(Wt);Qo=l(zi,"InputExample"),zi.forEach(r),Uo=l(B,` can be converted to
`),Xt=a(B,"A",{href:!0});var Vi=o(Xt);Fo=l(Vi,"InputFeatures"),Vi.forEach(r),Go=l(B," in order to be fed to the model."),B.forEach(r),Vs=p(e),S=a(e,"DIV",{class:!0});var D=o(S);v(Ke.$$.fragment,D),Ro=p(D),qr=a(D,"P",{});var Ci=o(qr);Ho=l(Ci,"Base class for data converters for sequence classification data sets."),Ci.forEach(r),Wo=p(D),ue=a(D,"DIV",{class:!0});var Ea=o(ue);v(Ye.$$.fragment,Ea),Xo=p(Ea),Ze=a(Ea,"P",{});var Pa=o(Ze);Bo=l(Pa,"Gets a collection of "),Bt=a(Pa,"A",{href:!0});var Oi=o(Bt);Jo=l(Oi,"InputExample"),Oi.forEach(r),Ko=l(Pa," for the dev set."),Pa.forEach(r),Ea.forEach(r),Yo=p(D),_e=a(D,"DIV",{class:!0});var qa=o(_e);v(et.$$.fragment,qa),Zo=p(qa),kr=a(qa,"P",{});var Qi=o(kr);en=l(Qi,"Gets an example from a dict with tensorflow tensors."),Qi.forEach(r),qa.forEach(r),tn=p(D),ge=a(D,"DIV",{class:!0});var ka=o(ge);v(tt.$$.fragment,ka),rn=p(ka),Ir=a(ka,"P",{});var Ui=o(Ir);sn=l(Ui,"Gets the list of labels for this data set."),Ui.forEach(r),ka.forEach(r),an=p(D),ve=a(D,"DIV",{class:!0});var Ia=o(ve);v(rt.$$.fragment,Ia),on=p(Ia),st=a(Ia,"P",{});var Sa=o(st);nn=l(Sa,"Gets a collection of "),Jt=a(Sa,"A",{href:!0});var Fi=o(Jt);ln=l(Fi,"InputExample"),Fi.forEach(r),pn=l(Sa," for the test set."),Sa.forEach(r),Ia.forEach(r),dn=p(D),be=a(D,"DIV",{class:!0});var Aa=o(be);v(at.$$.fragment,Aa),cn=p(Aa),ot=a(Aa,"P",{});var La=o(ot);fn=l(La,"Gets a collection of "),Kt=a(La,"A",{href:!0});var Gi=o(Kt);mn=l(Gi,"InputExample"),Gi.forEach(r),hn=l(La," for the train set."),La.forEach(r),Aa.forEach(r),un=p(D),$e=a(D,"DIV",{class:!0});var Da=o($e);v(nt.$$.fragment,Da),_n=p(Da),Sr=a(Da,"P",{});var Ri=o(Sr);gn=l(Ri,`Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts
examples to the correct format.`),Ri.forEach(r),Da.forEach(r),D.forEach(r),Cs=p(e),O=a(e,"DIV",{class:!0});var mr=o(O);v(lt.$$.fragment,mr),vn=p(mr),Ar=a(mr,"P",{});var Hi=o(Ar);bn=l(Hi,"A single training/test example for simple sequence classification."),Hi.forEach(r),$n=p(mr),xe=a(mr,"DIV",{class:!0});var Ta=o(xe);v(it.$$.fragment,Ta),xn=p(Ta),Lr=a(Ta,"P",{});var Wi=o(Lr);yn=l(Wi,"Serializes this instance to a JSON string."),Wi.forEach(r),Ta.forEach(r),mr.forEach(r),Os=p(e),Q=a(e,"DIV",{class:!0});var hr=o(Q);v(pt.$$.fragment,hr),wn=p(hr),Dr=a(hr,"P",{});var Xi=o(Dr);En=l(Xi,"A single set of features of data. Property names are the same names as the corresponding inputs to a model."),Xi.forEach(r),Pn=p(hr),ye=a(hr,"DIV",{class:!0});var Na=o(ye);v(dt.$$.fragment,Na),qn=p(Na),Tr=a(Na,"P",{});var Bi=o(Tr);kn=l(Bi,"Serializes this instance to a JSON string."),Bi.forEach(r),Na.forEach(r),hr.forEach(r),Qs=p(e),te=a(e,"H2",{class:!0});var Ma=o(te);we=a(Ma,"A",{id:!0,class:!0,href:!0});var Ji=o(we);Nr=a(Ji,"SPAN",{});var Ki=o(Nr);v(ct.$$.fragment,Ki),Ki.forEach(r),Ji.forEach(r),In=p(Ma),Mr=a(Ma,"SPAN",{});var Yi=o(Mr);Sn=l(Yi,"GLUE"),Yi.forEach(r),Ma.forEach(r),Us=p(e),Ee=a(e,"P",{});var ja=o(Ee);ft=a(ja,"A",{href:!0,rel:!0});var Zi=o(ft);An=l(Zi,"General Language Understanding Evaluation (GLUE)"),Zi.forEach(r),Ln=l(ja,` is a benchmark that evaluates the
performance of models across a diverse set of existing NLU tasks. It was released together with the paper `),mt=a(ja,"A",{href:!0,rel:!0});var ep=o(mt);Dn=l(ep,`GLUE: A
multi-task benchmark and analysis platform for natural language understanding`),ep.forEach(r),ja.forEach(r),Fs=p(e),Yt=a(e,"P",{});var tp=o(Yt);Tn=l(tp,`This library hosts a total of 10 processors for the following tasks: MRPC, MNLI, MNLI (mismatched), CoLA, SST2, STSB,
QQP, QNLI, RTE and WNLI.`),tp.forEach(r),Gs=p(e),Zt=a(e,"P",{});var rp=o(Zt);Nn=l(rp,"Those processors are:"),rp.forEach(r),Rs=p(e),k=a(e,"UL",{});var A=o(k);jr=a(A,"LI",{});var sp=o(jr);zr=a(sp,"CODE",{});var ap=o(zr);Mn=l(ap,"MrpcProcessor"),ap.forEach(r),sp.forEach(r),jn=p(A),Vr=a(A,"LI",{});var op=o(Vr);Cr=a(op,"CODE",{});var np=o(Cr);zn=l(np,"MnliProcessor"),np.forEach(r),op.forEach(r),Vn=p(A),Or=a(A,"LI",{});var lp=o(Or);Qr=a(lp,"CODE",{});var ip=o(Qr);Cn=l(ip,"MnliMismatchedProcessor"),ip.forEach(r),lp.forEach(r),On=p(A),Ur=a(A,"LI",{});var pp=o(Ur);Fr=a(pp,"CODE",{});var dp=o(Fr);Qn=l(dp,"Sst2Processor"),dp.forEach(r),pp.forEach(r),Un=p(A),Gr=a(A,"LI",{});var cp=o(Gr);Rr=a(cp,"CODE",{});var fp=o(Rr);Fn=l(fp,"StsbProcessor"),fp.forEach(r),cp.forEach(r),Gn=p(A),Hr=a(A,"LI",{});var mp=o(Hr);Wr=a(mp,"CODE",{});var hp=o(Wr);Rn=l(hp,"QqpProcessor"),hp.forEach(r),mp.forEach(r),Hn=p(A),Xr=a(A,"LI",{});var up=o(Xr);Br=a(up,"CODE",{});var _p=o(Br);Wn=l(_p,"QnliProcessor"),_p.forEach(r),up.forEach(r),Xn=p(A),Jr=a(A,"LI",{});var gp=o(Jr);Kr=a(gp,"CODE",{});var vp=o(Kr);Bn=l(vp,"RteProcessor"),vp.forEach(r),gp.forEach(r),Jn=p(A),Yr=a(A,"LI",{});var bp=o(Yr);Zr=a(bp,"CODE",{});var $p=o(Zr);Kn=l($p,"WnliProcessor"),$p.forEach(r),bp.forEach(r),A.forEach(r),Hs=p(e),Pe=a(e,"P",{});var za=o(Pe);Yn=l(za,`Additionally, the following method can be used to load values from a data file and convert them to a list of
`),er=a(za,"A",{href:!0});var xp=o(er);Zn=l(xp,"InputExample"),xp.forEach(r),el=l(za,"."),za.forEach(r),Ws=p(e),re=a(e,"DIV",{class:!0});var Va=o(re);v(ht.$$.fragment,Va),tl=p(Va),tr=a(Va,"P",{});var mi=o(tr);rl=l(mi,"Loads a data file into a list of "),es=a(mi,"CODE",{});var yp=o(es);sl=l(yp,"InputFeatures"),yp.forEach(r),mi.forEach(r),Va.forEach(r),Xs=p(e),se=a(e,"H2",{class:!0});var Ca=o(se);qe=a(Ca,"A",{id:!0,class:!0,href:!0});var wp=o(qe);ts=a(wp,"SPAN",{});var Ep=o(ts);v(ut.$$.fragment,Ep),Ep.forEach(r),wp.forEach(r),al=p(Ca),rs=a(Ca,"SPAN",{});var Pp=o(rs);ol=l(Pp,"XNLI"),Pp.forEach(r),Ca.forEach(r),Bs=p(e),ae=a(e,"P",{});var Ss=o(ae);_t=a(Ss,"A",{href:!0,rel:!0});var qp=o(_t);nl=l(qp,"The Cross-Lingual NLI Corpus (XNLI)"),qp.forEach(r),ll=l(Ss,` is a benchmark that evaluates the
quality of cross-lingual text representations. XNLI is crowd-sourced dataset based on `),gt=a(Ss,"A",{href:!0,rel:!0});var kp=o(gt);ss=a(kp,"EM",{});var Ip=o(ss);il=l(Ip,"MultiNLI"),Ip.forEach(r),kp.forEach(r),pl=l(Ss,`: pairs of text are labeled with textual entailment annotations for 15
different languages (including both high-resource language such as English and low-resource languages such as Swahili).`),Ss.forEach(r),Js=p(e),vt=a(e,"P",{});var hi=o(vt);dl=l(hi,"It was released together with the paper "),bt=a(hi,"A",{href:!0,rel:!0});var Sp=o(bt);cl=l(Sp,"XNLI: Evaluating Cross-lingual Sentence Representations"),Sp.forEach(r),hi.forEach(r),Ks=p(e),rr=a(e,"P",{});var Ap=o(rr);fl=l(Ap,"This library hosts the processor to load the XNLI data:"),Ap.forEach(r),Ys=p(e),sr=a(e,"UL",{});var Lp=o(sr);as=a(Lp,"LI",{});var Dp=o(as);os=a(Dp,"CODE",{});var Tp=o(os);ml=l(Tp,"XnliProcessor"),Tp.forEach(r),Dp.forEach(r),Lp.forEach(r),Zs=p(e),ar=a(e,"P",{});var Np=o(ar);hl=l(Np,"Please note that since the gold labels are available on the test set, evaluation is performed on the test set."),Np.forEach(r),ea=p(e),ke=a(e,"P",{});var Oa=o(ke);ul=l(Oa,"An example using these processors is given in the "),$t=a(Oa,"A",{href:!0,rel:!0});var Mp=o($t);_l=l(Mp,"run_xnli.py"),Mp.forEach(r),gl=l(Oa," script."),Oa.forEach(r),ta=p(e),oe=a(e,"H2",{class:!0});var Qa=o(oe);Ie=a(Qa,"A",{id:!0,class:!0,href:!0});var jp=o(Ie);ns=a(jp,"SPAN",{});var zp=o(ns);v(xt.$$.fragment,zp),zp.forEach(r),jp.forEach(r),vl=p(Qa),ls=a(Qa,"SPAN",{});var Vp=o(ls);bl=l(Vp,"SQuAD"),Vp.forEach(r),Qa.forEach(r),ra=p(e),U=a(e,"P",{});var Ct=o(U);yt=a(Ct,"A",{href:!0,rel:!0});var Cp=o(yt);$l=l(Cp,"The Stanford Question Answering Dataset (SQuAD)"),Cp.forEach(r),xl=l(Ct,` is a benchmark that
evaluates the performance of models on question answering. Two versions are available, v1.1 and v2.0. The first version
(v1.1) was released together with the paper `),wt=a(Ct,"A",{href:!0,rel:!0});var Op=o(wt);yl=l(Op,"SQuAD: 100,000+ Questions for Machine Comprehension of Text"),Op.forEach(r),wl=l(Ct,". The second version (v2.0) was released alongside the paper "),Et=a(Ct,"A",{href:!0,rel:!0});var Qp=o(Et);El=l(Qp,`Know What You Don\u2019t
Know: Unanswerable Questions for SQuAD`),Qp.forEach(r),Pl=l(Ct,"."),Ct.forEach(r),sa=p(e),or=a(e,"P",{});var Up=o(or);ql=l(Up,"This library hosts a processor for each of the two versions:"),Up.forEach(r),aa=p(e),ne=a(e,"H3",{class:!0});var Ua=o(ne);Se=a(Ua,"A",{id:!0,class:!0,href:!0});var Fp=o(Se);is=a(Fp,"SPAN",{});var Gp=o(is);v(Pt.$$.fragment,Gp),Gp.forEach(r),Fp.forEach(r),kl=p(Ua),ps=a(Ua,"SPAN",{});var Rp=o(ps);Il=l(Rp,"Processors"),Rp.forEach(r),Ua.forEach(r),oa=p(e),nr=a(e,"P",{});var Hp=o(nr);Sl=l(Hp,"Those processors are:"),Hp.forEach(r),na=p(e),Ae=a(e,"UL",{});var Fa=o(Ae);ds=a(Fa,"LI",{});var Wp=o(ds);cs=a(Wp,"CODE",{});var Xp=o(cs);Al=l(Xp,"SquadV1Processor"),Xp.forEach(r),Wp.forEach(r),Ll=p(Fa),fs=a(Fa,"LI",{});var Bp=o(fs);ms=a(Bp,"CODE",{});var Jp=o(ms);Dl=l(Jp,"SquadV2Processor"),Jp.forEach(r),Bp.forEach(r),Fa.forEach(r),la=p(e),qt=a(e,"P",{});var ui=o(qt);Tl=l(ui,"They both inherit from the abstract class "),hs=a(ui,"CODE",{});var Kp=o(hs);Nl=l(Kp,"SquadProcessor"),Kp.forEach(r),ui.forEach(r),ia=p(e),N=a(e,"DIV",{class:!0});var J=o(N);v(kt.$$.fragment,J),Ml=p(J),us=a(J,"P",{});var Yp=o(us);jl=l(Yp,`Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
version 2.0 of SQuAD, respectively.`),Yp.forEach(r),zl=p(J),Le=a(J,"DIV",{class:!0});var Ga=o(Le);v(It.$$.fragment,Ga),Vl=p(Ga),_s=a(Ga,"P",{});var Zp=o(_s);Cl=l(Zp,"Returns the evaluation example from the data directory."),Zp.forEach(r),Ga.forEach(r),Ol=p(J),X=a(J,"DIV",{class:!0});var ur=o(X);v(St.$$.fragment,ur),Ql=p(ur),At=a(ur,"P",{});var Ra=o(At);Ul=l(Ra,"Creates a list of "),gs=a(Ra,"CODE",{});var ed=o(gs);Fl=l(ed,"SquadExample"),ed.forEach(r),Gl=l(Ra," using a TFDS dataset."),Ra.forEach(r),Rl=p(ur),v(De.$$.fragment,ur),ur.forEach(r),Hl=p(J),Te=a(J,"DIV",{class:!0});var Ha=o(Te);v(Lt.$$.fragment,Ha),Wl=p(Ha),vs=a(Ha,"P",{});var td=o(vs);Xl=l(td,"Returns the training examples from the data directory."),td.forEach(r),Ha.forEach(r),J.forEach(r),pa=p(e),Ne=a(e,"P",{});var Wa=o(Ne);Bl=l(Wa,`Additionally, the following method can be used to convert SQuAD examples into
`),bs=a(Wa,"CODE",{});var rd=o(bs);Jl=l(rd,"SquadFeatures"),rd.forEach(r),Kl=l(Wa," that can be used as model inputs."),Wa.forEach(r),da=p(e),F=a(e,"DIV",{class:!0});var _r=o(F);v(Dt.$$.fragment,_r),Yl=p(_r),$s=a(_r,"P",{});var sd=o($s);Zl=l(sd,`Converts a list of examples into a list of features that can be directly given as input to a model. It is
model-dependant and takes advantage of many of the tokenizer\u2019s features to create the model\u2019s inputs.`),sd.forEach(r),ei=p(_r),v(Me.$$.fragment,_r),_r.forEach(r),ca=p(e),je=a(e,"P",{});var Xa=o(je);ti=l(Xa,`These processors as well as the aforementionned method can be used with files containing the data as well as with the
`),xs=a(Xa,"EM",{});var ad=o(xs);ri=l(ad,"tensorflow_datasets"),ad.forEach(r),si=l(Xa," package. Examples are given below."),Xa.forEach(r),fa=p(e),le=a(e,"H3",{class:!0});var Ba=o(le);ze=a(Ba,"A",{id:!0,class:!0,href:!0});var od=o(ze);ys=a(od,"SPAN",{});var nd=o(ys);v(Tt.$$.fragment,nd),nd.forEach(r),od.forEach(r),ai=p(Ba),ws=a(Ba,"SPAN",{});var ld=o(ws);oi=l(ld,"Example usage"),ld.forEach(r),Ba.forEach(r),ma=p(e),lr=a(e,"P",{});var id=o(lr);ni=l(id,"Here is an example using the processors as well as the conversion method using data files:"),id.forEach(r),ha=p(e),v(Nt.$$.fragment,e),ua=p(e),Ve=a(e,"P",{});var Ja=o(Ve);li=l(Ja,"Using "),Es=a(Ja,"EM",{});var pd=o(Es);ii=l(pd,"tensorflow_datasets"),pd.forEach(r),pi=l(Ja," is as easy as using a data file:"),Ja.forEach(r),_a=p(e),v(Mt.$$.fragment,e),ga=p(e),Ce=a(e,"P",{});var Ka=o(Ce);di=l(Ka,"Another example using these processors is given in the "),jt=a(Ka,"A",{href:!0,rel:!0});var dd=o(jt);ci=l(dd,"run_squad.py"),dd.forEach(r),fi=l(Ka," script."),Ka.forEach(r),this.h()},h(){f(h,"name","hf:doc:metadata"),f(h,"content",JSON.stringify(wd)),f(u,"id","processors"),f(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(u,"href","#processors"),f(_,"class","relative group"),f(Ot,"href","../model_doc/wav2vec2"),f(Qt,"href","../model_doc/clip"),f(pe,"id","transformers.ProcessorMixin"),f(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(pe,"href","#transformers.ProcessorMixin"),f(Z,"class","relative group"),f(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(Gt,"href","/docs/transformers/pr_17548/en/model_doc/speech_to_text#transformers.Speech2TextProcessor.from_pretrained"),f(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(he,"id","transformers.DataProcessor"),f(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(he,"href","#transformers.DataProcessor"),f(ee,"class","relative group"),f(Rt,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.DataProcessor"),f(Ht,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.InputExample"),f(Wt,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.InputExample"),f(Xt,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.InputFeatures"),f(Bt,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.InputExample"),f(ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(_e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(ge,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(Jt,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.InputExample"),f(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(Kt,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.InputExample"),f(be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(we,"id","transformers.glue_convert_examples_to_features"),f(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(we,"href","#transformers.glue_convert_examples_to_features"),f(te,"class","relative group"),f(ft,"href","https://gluebenchmark.com/"),f(ft,"rel","nofollow"),f(mt,"href","https://openreview.net/pdf?id=rJ4km2R5t7"),f(mt,"rel","nofollow"),f(er,"href","/docs/transformers/pr_17548/en/main_classes/processors#transformers.InputExample"),f(re,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(qe,"id","xnli"),f(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(qe,"href","#xnli"),f(se,"class","relative group"),f(_t,"href","https://www.nyu.edu/projects/bowman/xnli/"),f(_t,"rel","nofollow"),f(gt,"href","http://www.nyu.edu/projects/bowman/multinli/"),f(gt,"rel","nofollow"),f(bt,"href","https://arxiv.org/abs/1809.05053"),f(bt,"rel","nofollow"),f($t,"href","https://github.com/huggingface/transformers/tree/main/examples/legacy/text-classification/run_xnli.py"),f($t,"rel","nofollow"),f(Ie,"id","squad"),f(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ie,"href","#squad"),f(oe,"class","relative group"),f(yt,"href","https://rajpurkar.github.io/SQuAD-explorer//"),f(yt,"rel","nofollow"),f(wt,"href","https://arxiv.org/abs/1606.05250"),f(wt,"rel","nofollow"),f(Et,"href","https://arxiv.org/abs/1806.03822"),f(Et,"rel","nofollow"),f(Se,"id","transformers.data.processors.squad.SquadProcessor"),f(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Se,"href","#transformers.data.processors.squad.SquadProcessor"),f(ne,"class","relative group"),f(Le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(Te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(ze,"id","example-usage"),f(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ze,"href","#example-usage"),f(le,"class","relative group"),f(jt,"href","https://github.com/huggingface/transformers/tree/main/examples/legacy/question-answering/run_squad.py"),f(jt,"rel","nofollow")},m(e,d){t(document.head,h),c(e,E,d),c(e,_,d),t(_,u),t(u,P),b(m,P,null),t(_,w),t(_,z),t(z,T),c(e,q,d),c(e,V,d),t(V,K),c(e,Ls,d),c(e,ie,d),t(ie,Y),t(Y,Ya),t(Y,Ot),t(Ot,Za),t(Y,eo),t(Y,Qt),t(Qt,to),t(Y,ro),t(ie,so),t(ie,gr),t(gr,ao),c(e,Ds,d),c(e,Z,d),t(Z,pe),t(pe,vr),b(Qe,vr,null),t(Z,oo),t(Z,br),t(br,no),c(e,Ts,d),c(e,Ut,d),t(Ut,lo),c(e,Ns,d),c(e,Ft,d),t(Ft,io),c(e,Ms,d),c(e,L,d),b(Ue,L,null),t(L,po),t(L,$r),t($r,co),t(L,fo),t(L,G),b(Fe,G,null),t(G,mo),t(G,xr),t(xr,ho),t(G,uo),b(de,G,null),t(L,_o),t(L,R),b(Ge,R,null),t(R,go),t(R,Re),t(Re,vo),t(Re,yr),t(yr,bo),t(Re,$o),t(R,xo),b(ce,R,null),t(L,yo),t(L,H),b(He,H,null),t(H,wo),t(H,We),t(We,Eo),t(We,wr),t(wr,Po),t(We,qo),t(H,ko),b(fe,H,null),t(L,Io),t(L,W),b(Xe,W,null),t(W,So),t(W,Be),t(Be,Ao),t(Be,Gt),t(Gt,Lo),t(Be,Do),t(W,To),b(me,W,null),c(e,js,d),c(e,ee,d),t(ee,he),t(he,Er),b(Je,Er,null),t(ee,No),t(ee,Pr),t(Pr,Mo),c(e,zs,d),c(e,j,d),t(j,jo),t(j,Rt),t(Rt,zo),t(j,Vo),t(j,Ht),t(Ht,Co),t(j,Oo),t(j,Wt),t(Wt,Qo),t(j,Uo),t(j,Xt),t(Xt,Fo),t(j,Go),c(e,Vs,d),c(e,S,d),b(Ke,S,null),t(S,Ro),t(S,qr),t(qr,Ho),t(S,Wo),t(S,ue),b(Ye,ue,null),t(ue,Xo),t(ue,Ze),t(Ze,Bo),t(Ze,Bt),t(Bt,Jo),t(Ze,Ko),t(S,Yo),t(S,_e),b(et,_e,null),t(_e,Zo),t(_e,kr),t(kr,en),t(S,tn),t(S,ge),b(tt,ge,null),t(ge,rn),t(ge,Ir),t(Ir,sn),t(S,an),t(S,ve),b(rt,ve,null),t(ve,on),t(ve,st),t(st,nn),t(st,Jt),t(Jt,ln),t(st,pn),t(S,dn),t(S,be),b(at,be,null),t(be,cn),t(be,ot),t(ot,fn),t(ot,Kt),t(Kt,mn),t(ot,hn),t(S,un),t(S,$e),b(nt,$e,null),t($e,_n),t($e,Sr),t(Sr,gn),c(e,Cs,d),c(e,O,d),b(lt,O,null),t(O,vn),t(O,Ar),t(Ar,bn),t(O,$n),t(O,xe),b(it,xe,null),t(xe,xn),t(xe,Lr),t(Lr,yn),c(e,Os,d),c(e,Q,d),b(pt,Q,null),t(Q,wn),t(Q,Dr),t(Dr,En),t(Q,Pn),t(Q,ye),b(dt,ye,null),t(ye,qn),t(ye,Tr),t(Tr,kn),c(e,Qs,d),c(e,te,d),t(te,we),t(we,Nr),b(ct,Nr,null),t(te,In),t(te,Mr),t(Mr,Sn),c(e,Us,d),c(e,Ee,d),t(Ee,ft),t(ft,An),t(Ee,Ln),t(Ee,mt),t(mt,Dn),c(e,Fs,d),c(e,Yt,d),t(Yt,Tn),c(e,Gs,d),c(e,Zt,d),t(Zt,Nn),c(e,Rs,d),c(e,k,d),t(k,jr),t(jr,zr),t(zr,Mn),t(k,jn),t(k,Vr),t(Vr,Cr),t(Cr,zn),t(k,Vn),t(k,Or),t(Or,Qr),t(Qr,Cn),t(k,On),t(k,Ur),t(Ur,Fr),t(Fr,Qn),t(k,Un),t(k,Gr),t(Gr,Rr),t(Rr,Fn),t(k,Gn),t(k,Hr),t(Hr,Wr),t(Wr,Rn),t(k,Hn),t(k,Xr),t(Xr,Br),t(Br,Wn),t(k,Xn),t(k,Jr),t(Jr,Kr),t(Kr,Bn),t(k,Jn),t(k,Yr),t(Yr,Zr),t(Zr,Kn),c(e,Hs,d),c(e,Pe,d),t(Pe,Yn),t(Pe,er),t(er,Zn),t(Pe,el),c(e,Ws,d),c(e,re,d),b(ht,re,null),t(re,tl),t(re,tr),t(tr,rl),t(tr,es),t(es,sl),c(e,Xs,d),c(e,se,d),t(se,qe),t(qe,ts),b(ut,ts,null),t(se,al),t(se,rs),t(rs,ol),c(e,Bs,d),c(e,ae,d),t(ae,_t),t(_t,nl),t(ae,ll),t(ae,gt),t(gt,ss),t(ss,il),t(ae,pl),c(e,Js,d),c(e,vt,d),t(vt,dl),t(vt,bt),t(bt,cl),c(e,Ks,d),c(e,rr,d),t(rr,fl),c(e,Ys,d),c(e,sr,d),t(sr,as),t(as,os),t(os,ml),c(e,Zs,d),c(e,ar,d),t(ar,hl),c(e,ea,d),c(e,ke,d),t(ke,ul),t(ke,$t),t($t,_l),t(ke,gl),c(e,ta,d),c(e,oe,d),t(oe,Ie),t(Ie,ns),b(xt,ns,null),t(oe,vl),t(oe,ls),t(ls,bl),c(e,ra,d),c(e,U,d),t(U,yt),t(yt,$l),t(U,xl),t(U,wt),t(wt,yl),t(U,wl),t(U,Et),t(Et,El),t(U,Pl),c(e,sa,d),c(e,or,d),t(or,ql),c(e,aa,d),c(e,ne,d),t(ne,Se),t(Se,is),b(Pt,is,null),t(ne,kl),t(ne,ps),t(ps,Il),c(e,oa,d),c(e,nr,d),t(nr,Sl),c(e,na,d),c(e,Ae,d),t(Ae,ds),t(ds,cs),t(cs,Al),t(Ae,Ll),t(Ae,fs),t(fs,ms),t(ms,Dl),c(e,la,d),c(e,qt,d),t(qt,Tl),t(qt,hs),t(hs,Nl),c(e,ia,d),c(e,N,d),b(kt,N,null),t(N,Ml),t(N,us),t(us,jl),t(N,zl),t(N,Le),b(It,Le,null),t(Le,Vl),t(Le,_s),t(_s,Cl),t(N,Ol),t(N,X),b(St,X,null),t(X,Ql),t(X,At),t(At,Ul),t(At,gs),t(gs,Fl),t(At,Gl),t(X,Rl),b(De,X,null),t(N,Hl),t(N,Te),b(Lt,Te,null),t(Te,Wl),t(Te,vs),t(vs,Xl),c(e,pa,d),c(e,Ne,d),t(Ne,Bl),t(Ne,bs),t(bs,Jl),t(Ne,Kl),c(e,da,d),c(e,F,d),b(Dt,F,null),t(F,Yl),t(F,$s),t($s,Zl),t(F,ei),b(Me,F,null),c(e,ca,d),c(e,je,d),t(je,ti),t(je,xs),t(xs,ri),t(je,si),c(e,fa,d),c(e,le,d),t(le,ze),t(ze,ys),b(Tt,ys,null),t(le,ai),t(le,ws),t(ws,oi),c(e,ma,d),c(e,lr,d),t(lr,ni),c(e,ha,d),b(Nt,e,d),c(e,ua,d),c(e,Ve,d),t(Ve,li),t(Ve,Es),t(Es,ii),t(Ve,pi),c(e,_a,d),b(Mt,e,d),c(e,ga,d),c(e,Ce,d),t(Ce,di),t(Ce,jt),t(jt,ci),t(Ce,fi),va=!0},p(e,[d]){const zt={};d&2&&(zt.$$scope={dirty:d,ctx:e}),de.$set(zt);const Ps={};d&2&&(Ps.$$scope={dirty:d,ctx:e}),ce.$set(Ps);const qs={};d&2&&(qs.$$scope={dirty:d,ctx:e}),fe.$set(qs);const ks={};d&2&&(ks.$$scope={dirty:d,ctx:e}),me.$set(ks);const Is={};d&2&&(Is.$$scope={dirty:d,ctx:e}),De.$set(Is);const Vt={};d&2&&(Vt.$$scope={dirty:d,ctx:e}),Me.$set(Vt)},i(e){va||($(m.$$.fragment,e),$(Qe.$$.fragment,e),$(Ue.$$.fragment,e),$(Fe.$$.fragment,e),$(de.$$.fragment,e),$(Ge.$$.fragment,e),$(ce.$$.fragment,e),$(He.$$.fragment,e),$(fe.$$.fragment,e),$(Xe.$$.fragment,e),$(me.$$.fragment,e),$(Je.$$.fragment,e),$(Ke.$$.fragment,e),$(Ye.$$.fragment,e),$(et.$$.fragment,e),$(tt.$$.fragment,e),$(rt.$$.fragment,e),$(at.$$.fragment,e),$(nt.$$.fragment,e),$(lt.$$.fragment,e),$(it.$$.fragment,e),$(pt.$$.fragment,e),$(dt.$$.fragment,e),$(ct.$$.fragment,e),$(ht.$$.fragment,e),$(ut.$$.fragment,e),$(xt.$$.fragment,e),$(Pt.$$.fragment,e),$(kt.$$.fragment,e),$(It.$$.fragment,e),$(St.$$.fragment,e),$(De.$$.fragment,e),$(Lt.$$.fragment,e),$(Dt.$$.fragment,e),$(Me.$$.fragment,e),$(Tt.$$.fragment,e),$(Nt.$$.fragment,e),$(Mt.$$.fragment,e),va=!0)},o(e){x(m.$$.fragment,e),x(Qe.$$.fragment,e),x(Ue.$$.fragment,e),x(Fe.$$.fragment,e),x(de.$$.fragment,e),x(Ge.$$.fragment,e),x(ce.$$.fragment,e),x(He.$$.fragment,e),x(fe.$$.fragment,e),x(Xe.$$.fragment,e),x(me.$$.fragment,e),x(Je.$$.fragment,e),x(Ke.$$.fragment,e),x(Ye.$$.fragment,e),x(et.$$.fragment,e),x(tt.$$.fragment,e),x(rt.$$.fragment,e),x(at.$$.fragment,e),x(nt.$$.fragment,e),x(lt.$$.fragment,e),x(it.$$.fragment,e),x(pt.$$.fragment,e),x(dt.$$.fragment,e),x(ct.$$.fragment,e),x(ht.$$.fragment,e),x(ut.$$.fragment,e),x(xt.$$.fragment,e),x(Pt.$$.fragment,e),x(kt.$$.fragment,e),x(It.$$.fragment,e),x(St.$$.fragment,e),x(De.$$.fragment,e),x(Lt.$$.fragment,e),x(Dt.$$.fragment,e),x(Me.$$.fragment,e),x(Tt.$$.fragment,e),x(Nt.$$.fragment,e),x(Mt.$$.fragment,e),va=!1},d(e){r(h),e&&r(E),e&&r(_),y(m),e&&r(q),e&&r(V),e&&r(Ls),e&&r(ie),e&&r(Ds),e&&r(Z),y(Qe),e&&r(Ts),e&&r(Ut),e&&r(Ns),e&&r(Ft),e&&r(Ms),e&&r(L),y(Ue),y(Fe),y(de),y(Ge),y(ce),y(He),y(fe),y(Xe),y(me),e&&r(js),e&&r(ee),y(Je),e&&r(zs),e&&r(j),e&&r(Vs),e&&r(S),y(Ke),y(Ye),y(et),y(tt),y(rt),y(at),y(nt),e&&r(Cs),e&&r(O),y(lt),y(it),e&&r(Os),e&&r(Q),y(pt),y(dt),e&&r(Qs),e&&r(te),y(ct),e&&r(Us),e&&r(Ee),e&&r(Fs),e&&r(Yt),e&&r(Gs),e&&r(Zt),e&&r(Rs),e&&r(k),e&&r(Hs),e&&r(Pe),e&&r(Ws),e&&r(re),y(ht),e&&r(Xs),e&&r(se),y(ut),e&&r(Bs),e&&r(ae),e&&r(Js),e&&r(vt),e&&r(Ks),e&&r(rr),e&&r(Ys),e&&r(sr),e&&r(Zs),e&&r(ar),e&&r(ea),e&&r(ke),e&&r(ta),e&&r(oe),y(xt),e&&r(ra),e&&r(U),e&&r(sa),e&&r(or),e&&r(aa),e&&r(ne),y(Pt),e&&r(oa),e&&r(nr),e&&r(na),e&&r(Ae),e&&r(la),e&&r(qt),e&&r(ia),e&&r(N),y(kt),y(It),y(St),y(De),y(Lt),e&&r(pa),e&&r(Ne),e&&r(da),e&&r(F),y(Dt),y(Me),e&&r(ca),e&&r(je),e&&r(fa),e&&r(le),y(Tt),e&&r(ma),e&&r(lr),e&&r(ha),y(Nt,e),e&&r(ua),e&&r(Ve),e&&r(_a),y(Mt,e),e&&r(ga),e&&r(Ce)}}}const wd={local:"processors",sections:[{local:"transformers.ProcessorMixin",title:"Multi-modal processors"},{local:"transformers.DataProcessor",title:"Deprecated processors"},{local:"transformers.glue_convert_examples_to_features",title:"GLUE"},{local:"xnli",title:"XNLI"},{local:"squad",sections:[{local:"transformers.data.processors.squad.SquadProcessor",title:"Processors"},{local:"example-usage",title:"Example usage"}],title:"SQuAD"}],title:"Processors"};function Ed(M){return ud(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ld extends cd{constructor(h){super();fd(this,h,Ed,yd,md,{})}}export{Ld as default,wd as metadata};
