<meta charset="utf-8" /><meta http-equiv="content-security-policy" content=""><meta name="hf:doc:metadata" content="{&quot;local&quot;:&quot;&quot;,&quot;title&quot;:&quot;章末小测试&quot;}" data-svelte="svelte-1phssyn">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/assets/pages/__layout.svelte-hf-doc-builder.css">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/start-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/chunks/vendor-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/chunks/paths-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/pages/__layout.svelte-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/pages/chapter2/8.mdx-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/chunks/IconCopyLink-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/chunks/CodeBlock-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/chunks/Question-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/course/pr_212/zh-CN/_app/chunks/FrameworkSwitchCourse-hf-doc-builder.js"> 





<div class="bg-white leading-none border border-gray-100 rounded-lg flex p-0.5 w-56 text-sm mb-4"><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-l bg-red-50 dark:bg-transparent text-red-600" href="?fw=pt"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg>
			Pytorch
		</a><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-r text-gray-500 filter grayscale" href="?fw=tf"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg>
			TensorFlow
		</a></div>

<h1 id="">章末小测试</h1>
<h3 class="relative group"><a id="1.-语言建模-pipeline-的顺序是什么？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1.-语言建模-pipeline-的顺序是什么？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>1. 语言建模 Pipeline 的顺序是什么？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->首先是模型它处理文本并返回原始预测。然后，分词器会对这些预测进行解释，并在需要时将它们转换回文本<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->首先是 标记器(Tokenizer)，它处理文本并返回id。模型根据这些 id 并输出预测，可以是一些文本。<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->标记器(Tokenizer) 处理文本并返回 id。模型处理这些 id 并输出一个预测。然后可以再次使用 标记器(Tokenizer) 将这些预测转换回某些文本。<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="2.-transformer模型的输出有多少个维度，每个维度分别是什么？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2.-transformer模型的输出有多少个维度，每个维度分别是什么？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>2. Transformer模型的输出有多少个维度，每个维度分别是什么？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->2个维度，分别是: 序列长度(Sequence Length)和批次大小(Batch Size)<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->2个维度，分别是: 序列长度(Sequence Length)和隐藏层大小(Hidden Size)<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->3个维度，分别是: 序列长度(Sequence Length)、批次大小(Batch Size)和隐藏层大小(Hidden Size)<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="3.下列哪一个是subword标记(tokenization)的例子（从分词的颗粒度来划分）？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3.下列哪一个是subword标记(tokenization)的例子（从分词的颗粒度来划分）？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>3.下列哪一个是Subword标记(Tokenization)的例子（从分词的颗粒度来划分）？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->WordPiece<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->基于单个字符的标记(Tokenization)<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->基于空格和标点符号的分割<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="3">
				<!-- HTML_TAG_START -->BPE(Byte Pair Encoding)<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="4">
				<!-- HTML_TAG_START -->Unigram<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="5">
				<!-- HTML_TAG_START -->以上都不是<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="4.什么是模型的head层？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#4.什么是模型的head层？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>4.什么是模型的Head层？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->原始Transformer网络的一种组件，将张量(Tensors)重定向到它们的正确层<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->也称为自注意力(self-attention)机制，它根据序列中的其他标记(Token)来调整一个标记(Token)<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->一个附加组件，通常由一个或几个层组成，用于将Transformer的预测转换为特定于任务的输出<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="5.什么是automodel？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#5.什么是automodel？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>5.什么是AutoModel？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->根据您的数据自动进行训练的模型<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->一个根据Checkpoint(检查点)返回模型体系结构的对象<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->一种可以自动检测输入语言来加载正确权重的模型<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="6.当将不同长度的序列批处理在一起时，需要进行哪些处理？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#6.当将不同长度的序列批处理在一起时，需要进行哪些处理？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>6.当将不同长度的序列批处理在一起时，需要进行哪些处理？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->截短<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->返回Tensors<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->填充<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="3">
				<!-- HTML_TAG_START -->注意力遮蔽(Attention masking)<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="7.将-softmax激活函数应用于序列分类(sequence-classification)模型的-logits-输出有什么意义？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#7.将-softmax激活函数应用于序列分类(sequence-classification)模型的-logits-输出有什么意义？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>7.将 SoftMax激活函数应用于序列分类(Sequence Classification)模型的 logits 输出有什么意义？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->它软化了logits输出，使结果更可靠。<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->它限定了上下界，使模型的输出结果可以被解释。<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->输出的和是1，从而产生一个可能的概率解释。<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="8.大多数标记器(tokenizer)的api以什么方法为核心？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#8.大多数标记器(tokenizer)的api以什么方法为核心？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>8.大多数标记器(Tokenizer)的API以什么方法为核心？
	</span></h3>

<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START --><code>编码</code> ，因为它可以将文本编码为id，将预测的id解码为文本<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->直接调用标记器(Tokenizer)对象。<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START --><code>pad</code>(填充)<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="3">
				<!-- HTML_TAG_START --><code>tokenize</code>(标记)<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="9.这个代码示例中的<code>result</code>变量包含什么？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#9.这个代码示例中的<code>result</code>变量包含什么？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>9.这个代码示例中的<code>result</code>变量包含什么？
	</span></h3>


	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
result = tokenizer.tokenize(<span class="hljs-string">&quot;Hello!&quot;</span>)<!-- HTML_TAG_END --></pre></div>
<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->字符串列表，每个字符串都是一个标记(Token)<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->一个ID的列表<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->包含所有标记(Token)的字符串<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>
<h3 class="relative group"><a id="10.下面的代码有什么错误吗？" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#10.下面的代码有什么错误吗？"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>10.下面的代码有什么错误吗？
	</span></h3>


	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
model = AutoModel.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)

encoded = tokenizer(<span class="hljs-string">&quot;Hey!&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
result = model(**encoded)<!-- HTML_TAG_END --></pre></div>
<div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0">
				<!-- HTML_TAG_START -->不，看起来是对的。<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1">
				<!-- HTML_TAG_START -->Tokenizer(标记器)和模型应该来自相同的Checkpoint(检查点)。<!-- HTML_TAG_END --></label>
			<label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2">
				<!-- HTML_TAG_START -->由于每个输入都是一个Batch，因此可以使用标记器(Tokenizer)对其进行平移和截断来改善这段代码。<!-- HTML_TAG_END --></label>
			
		<div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit </button>

			</div></form></div>


		<script type="module" data-hydrate="1jbbaf1">
		import { start } from "/docs/course/pr_212/zh-CN/_app/start-hf-doc-builder.js";
		start({
			target: document.querySelector('[data-hydrate="1jbbaf1"]').parentNode,
			paths: {"base":"/docs/course/pr_212/zh-CN","assets":"/docs/course/pr_212/zh-CN"},
			session: {},
			route: false,
			spa: false,
			trailing_slash: "never",
			hydrate: {
				status: 200,
				error: null,
				nodes: [
					import("/docs/course/pr_212/zh-CN/_app/pages/__layout.svelte-hf-doc-builder.js"),
						import("/docs/course/pr_212/zh-CN/_app/pages/chapter2/8.mdx-hf-doc-builder.js")
				],
				params: {}
			}
		});
	</script>
