<meta charset="utf-8" /><meta http-equiv="content-security-policy" content=""><meta name="hf:doc:metadata" content="{&quot;local&quot;:&quot;migrating-from-previous-packages&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;migrating-from-transformers-v3x-to-v4x&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;1-autotokenizers-and-pipelines-now-use-fast-rust-tokenizers-by-default&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;how-to-obtain-the-same-behavior-as-v3x-in-v4x&quot;,&quot;title&quot;:&quot;How to obtain the same behavior as v3.x in v4.x&quot;}],&quot;title&quot;:&quot;1. AutoTokenizers and pipelines now use fast (rust) tokenizers by default.&quot;},{&quot;local&quot;:&quot;2-sentencepiece-is-removed-from-the-required-dependencies&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;how-to-obtain-the-same-behavior-as-v3x-in-v4x&quot;,&quot;title&quot;:&quot;How to obtain the same behavior as v3.x in v4.x&quot;}],&quot;title&quot;:&quot;2. SentencePiece is removed from the required dependencies&quot;},{&quot;local&quot;:&quot;3-the-architecture-of-the-repo-has-been-updated-so-that-each-model-resides-in-its-folder&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;how-to-obtain-the-same-behavior-as-v3x-in-v4x&quot;,&quot;title&quot;:&quot;How to obtain the same behavior as v3.x in v4.x&quot;}],&quot;title&quot;:&quot;3. The architecture of the repo has been updated so that each model resides in its folder&quot;},{&quot;local&quot;:&quot;4-switching-the-returndict-argument-to-true-by-default&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;how-to-obtain-the-same-behavior-as-v3x-in-v4x&quot;,&quot;title&quot;:&quot;How to obtain the same behavior as v3.x in v4.x&quot;}],&quot;title&quot;:&quot;4. Switching the `return_dict` argument to `True` by default&quot;},{&quot;local&quot;:&quot;5-removed-some-deprecated-attributes&quot;,&quot;title&quot;:&quot;5. Removed some deprecated attributes&quot;}],&quot;title&quot;:&quot;Migrating from transformers `v3.x` to `v4.x`&quot;},{&quot;local&quot;:&quot;migrating-from-pytorchtransformers-to-transformers&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;positional-order-of-some-models-keywords-inputs-attentionmask-tokentypeids-changed&quot;,&quot;title&quot;:&quot;Positional order of some models&#39; keywords inputs (`attention_mask`, `token_type_ids`...) changed&quot;}],&quot;title&quot;:&quot;Migrating from pytorch-transformers to 🤗 Transformers&quot;},{&quot;local&quot;:&quot;migrating-from-pytorchpretrainedbert&quot;,&quot;sections&quot;:[{&quot;local&quot;:&quot;models-always-output-tuples&quot;,&quot;title&quot;:&quot;Models always output `tuples`&quot;},{&quot;local&quot;:&quot;serialization&quot;,&quot;title&quot;:&quot;Serialization&quot;},{&quot;local&quot;:&quot;optimizers-bertadam-openaiadam-are-now-adamw-schedules-are-standard-pytorch-schedules&quot;,&quot;title&quot;:&quot;Optimizers: BertAdam &amp; OpenAIAdam are now AdamW, schedules are standard PyTorch schedules&quot;}],&quot;title&quot;:&quot;Migrating from pytorch-pretrained-bert&quot;}],&quot;title&quot;:&quot;Migrating from previous packages&quot;}" data-svelte="svelte-1phssyn">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/assets/pages/__layout.svelte-hf-doc-builder.css">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/start-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/chunks/vendor-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/chunks/paths-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/pages/__layout.svelte-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/pages/migration.mdx-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/chunks/IconCopyLink-hf-doc-builder.js">
	<link rel="modulepreload" href="/docs/transformers/pr_19043/en/_app/chunks/CodeBlock-hf-doc-builder.js"> 






<h1 class="relative group"><a id="migrating-from-previous-packages" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#migrating-from-previous-packages"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Migrating from previous packages
	</span></h1>

<h2 class="relative group"><a id="migrating-from-transformers-v3x-to-v4x" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#migrating-from-transformers-v3x-to-v4x"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Migrating from transformers <code>v3.x</code> to <code>v4.x</code></span></h2>

<p>A couple of changes were introduced when the switch from version 3 to version 4 was done. Below is a summary of the
expected changes:</p>
<h4 class="relative group"><a id="1-autotokenizers-and-pipelines-now-use-fast-rust-tokenizers-by-default" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-autotokenizers-and-pipelines-now-use-fast-rust-tokenizers-by-default"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>1. AutoTokenizers and pipelines now use fast (rust) tokenizers by default.
	</span></h4>

<p>The python and rust tokenizers have roughly the same API, but the rust tokenizers have a more complete feature set.</p>
<p>This introduces two breaking changes:</p>
<ul><li>The handling of overflowing tokens between the python and rust tokenizers is different.</li>
<li>The rust tokenizers do not accept integers in the encoding methods.</li></ul>
<h5 class="relative group"><a id="how-to-obtain-the-same-behavior-as-v3x-in-v4x" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-to-obtain-the-same-behavior-as-v3x-in-v4x"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>How to obtain the same behavior as v3.x in v4.x
	</span></h5>

<ul><li>The pipelines now contain additional features out of the box. See the <a href="main_classes/pipelines#transformers.TokenClassificationPipeline">token-classification pipeline with the <code>grouped_entities</code> flag</a>.</li>
<li>The auto-tokenizers now return rust tokenizers. In order to obtain the python tokenizers instead, the user may use the <code>use_fast</code> flag by setting it to <code>False</code>:</li></ul>
<p>In version <code>v3.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)<!-- HTML_TAG_END --></pre></div>
<p>to obtain the same in version <code>v4.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, use_fast=<span class="hljs-literal">False</span>)<!-- HTML_TAG_END --></pre></div>
<h4 class="relative group"><a id="2-sentencepiece-is-removed-from-the-required-dependencies" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-sentencepiece-is-removed-from-the-required-dependencies"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>2. SentencePiece is removed from the required dependencies
	</span></h4>

<p>The requirement on the SentencePiece dependency has been lifted from the <code>setup.py</code>. This is done so that we may have a channel on anaconda cloud without relying on <code>conda-forge</code>. This means that the tokenizers that depend on the SentencePiece library will not be available with a standard <code>transformers</code> installation.</p>
<p>This includes the <strong>slow</strong> versions of:</p>
<ul><li><code>XLNetTokenizer</code></li>
<li><code>AlbertTokenizer</code></li>
<li><code>CamembertTokenizer</code></li>
<li><code>MBartTokenizer</code></li>
<li><code>PegasusTokenizer</code></li>
<li><code>T5Tokenizer</code></li>
<li><code>ReformerTokenizer</code></li>
<li><code>XLMRobertaTokenizer</code></li></ul>
<h5 class="relative group"><a id="how-to-obtain-the-same-behavior-as-v3x-in-v4x" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-to-obtain-the-same-behavior-as-v3x-in-v4x"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>How to obtain the same behavior as v3.x in v4.x
	</span></h5>

<p>In order to obtain the same behavior as version <code>v3.x</code>, you should install <code>sentencepiece</code> additionally:</p>
<p>In version <code>v3.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->pip install transformers<!-- HTML_TAG_END --></pre></div>
<p>to obtain the same in version <code>v4.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->pip install transformers[sentencepiece]<!-- HTML_TAG_END --></pre></div>
<p>or</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->pip install transformers sentencepiece<!-- HTML_TAG_END --></pre></div>
<h4 class="relative group"><a id="3-the-architecture-of-the-repo-has-been-updated-so-that-each-model-resides-in-its-folder" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-the-architecture-of-the-repo-has-been-updated-so-that-each-model-resides-in-its-folder"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>3. The architecture of the repo has been updated so that each model resides in its folder
	</span></h4>

<p>The past and foreseeable addition of new models means that the number of files in the directory <code>src/transformers</code> keeps growing and becomes harder to navigate and understand. We made the choice to put each model and the files accompanying it in their own sub-directories.</p>
<p>This is a breaking change as importing intermediary layers using a model’s module directly needs to be done via a different path.</p>
<h5 class="relative group"><a id="how-to-obtain-the-same-behavior-as-v3x-in-v4x" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-to-obtain-the-same-behavior-as-v3x-in-v4x"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>How to obtain the same behavior as v3.x in v4.x
	</span></h5>

<p>In order to obtain the same behavior as version <code>v3.x</code>, you should update the path used to access the layers.</p>
<p>In version <code>v3.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->from transformers.modeling_bert import BertLayer<!-- HTML_TAG_END --></pre></div>
<p>to obtain the same in version <code>v4.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->from transformers.models.bert.modeling_bert import BertLayer<!-- HTML_TAG_END --></pre></div>
<h4 class="relative group"><a id="4-switching-the-returndict-argument-to-true-by-default" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#4-switching-the-returndict-argument-to-true-by-default"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>4. Switching the <code>return_dict</code> argument to <code>True</code> by default
	</span></h4>

<p>The <a href="main_classes/output"><code>return_dict</code> argument</a> enables the return of dict-like python objects containing the model outputs, instead of the standard tuples. This object is self-documented as keys can be used to retrieve values, while also behaving as a tuple as users may retrieve objects by index or by slice.</p>
<p>This is a breaking change as the limitation of that tuple is that it cannot be unpacked: <code>value0, value1 = outputs</code> will not work.</p>
<h5 class="relative group"><a id="how-to-obtain-the-same-behavior-as-v3x-in-v4x" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-to-obtain-the-same-behavior-as-v3x-in-v4x"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>How to obtain the same behavior as v3.x in v4.x
	</span></h5>

<p>In order to obtain the same behavior as version <code>v3.x</code>, you should specify the <code>return_dict</code> argument to <code>False</code>, either in the model configuration or during the forward pass.</p>
<p>In version <code>v3.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->model = BertModel.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
outputs = model(**inputs)<!-- HTML_TAG_END --></pre></div>
<p>to obtain the same in version <code>v4.x</code>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->model = BertModel.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
outputs = model(**inputs, return_dict=False)<!-- HTML_TAG_END --></pre></div>
<p>or</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START -->model = BertModel.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, return_dict=False)
outputs = model(**inputs)<!-- HTML_TAG_END --></pre></div>
<h4 class="relative group"><a id="5-removed-some-deprecated-attributes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#5-removed-some-deprecated-attributes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>5. Removed some deprecated attributes
	</span></h4>

<p>Attributes that were deprecated have been removed if they had been deprecated for at least a month. The full list of deprecated attributes can be found in <a href="https://github.com/huggingface/transformers/pull/8604" rel="nofollow">#8604</a>.</p>
<p>Here is a list of these attributes/methods/arguments and what their replacements should be:</p>
<p>In several models, the labels become consistent with the other models:</p>
<ul><li><code>masked_lm_labels</code> becomes <code>labels</code> in <code>AlbertForMaskedLM</code> and <code>AlbertForPreTraining</code>.</li>
<li><code>masked_lm_labels</code> becomes <code>labels</code> in <code>BertForMaskedLM</code> and <code>BertForPreTraining</code>.</li>
<li><code>masked_lm_labels</code> becomes <code>labels</code> in <code>DistilBertForMaskedLM</code>.</li>
<li><code>masked_lm_labels</code> becomes <code>labels</code> in <code>ElectraForMaskedLM</code>.</li>
<li><code>masked_lm_labels</code> becomes <code>labels</code> in <code>LongformerForMaskedLM</code>.</li>
<li><code>masked_lm_labels</code> becomes <code>labels</code> in <code>MobileBertForMaskedLM</code>.</li>
<li><code>masked_lm_labels</code> becomes <code>labels</code> in <code>RobertaForMaskedLM</code>.</li>
<li><code>lm_labels</code> becomes <code>labels</code> in <code>BartForConditionalGeneration</code>.</li>
<li><code>lm_labels</code> becomes <code>labels</code> in <code>GPT2DoubleHeadsModel</code>.</li>
<li><code>lm_labels</code> becomes <code>labels</code> in <code>OpenAIGPTDoubleHeadsModel</code>.</li>
<li><code>lm_labels</code> becomes <code>labels</code> in <code>T5ForConditionalGeneration</code>.</li></ul>
<p>In several models, the caching mechanism becomes consistent with the other models:</p>
<ul><li><code>decoder_cached_states</code> becomes <code>past_key_values</code> in all BART-like, FSMT and T5 models.</li>
<li><code>decoder_past_key_values</code> becomes <code>past_key_values</code> in all BART-like, FSMT and T5 models.</li>
<li><code>past</code> becomes <code>past_key_values</code> in all CTRL models.</li>
<li><code>past</code> becomes <code>past_key_values</code> in all GPT-2 models.</li></ul>
<p>Regarding the tokenizer classes:</p>
<ul><li>The tokenizer attribute <code>max_len</code> becomes <code>model_max_length</code>.</li>
<li>The tokenizer attribute <code>return_lengths</code> becomes <code>return_length</code>.</li>
<li>The tokenizer encoding argument <code>is_pretokenized</code> becomes <code>is_split_into_words</code>.</li></ul>
<p>Regarding the <code>Trainer</code> class:</p>
<ul><li>The <code>Trainer</code> argument <code>tb_writer</code> is removed in favor of the callback <code>TensorBoardCallback(tb_writer=...)</code>.</li>
<li>The <code>Trainer</code> argument <code>prediction_loss_only</code> is removed in favor of the class argument <code>args.prediction_loss_only</code>.</li>
<li>The <code>Trainer</code> attribute <code>data_collator</code> should be a callable.</li>
<li>The <code>Trainer</code> method <code>_log</code> is deprecated in favor of <code>log</code>.</li>
<li>The <code>Trainer</code> method <code>_training_step</code> is deprecated in favor of <code>training_step</code>.</li>
<li>The <code>Trainer</code> method <code>_prediction_loop</code> is deprecated in favor of <code>prediction_loop</code>.</li>
<li>The <code>Trainer</code> method <code>is_local_master</code> is deprecated in favor of <code>is_local_process_zero</code>.</li>
<li>The <code>Trainer</code> method <code>is_world_master</code> is deprecated in favor of <code>is_world_process_zero</code>.</li></ul>
<p>Regarding the <code>TFTrainer</code> class:</p>
<ul><li>The <code>TFTrainer</code> argument <code>prediction_loss_only</code> is removed in favor of the class argument <code>args.prediction_loss_only</code>.</li>
<li>The <code>Trainer</code> method <code>_log</code> is deprecated in favor of <code>log</code>.</li>
<li>The <code>TFTrainer</code> method <code>_prediction_loop</code> is deprecated in favor of <code>prediction_loop</code>.</li>
<li>The <code>TFTrainer</code> method <code>_setup_wandb</code> is deprecated in favor of <code>setup_wandb</code>.</li>
<li>The <code>TFTrainer</code> method <code>_run_model</code> is deprecated in favor of <code>run_model</code>.</li></ul>
<p>Regarding the <code>TrainingArguments</code> class:</p>
<ul><li>The <code>TrainingArguments</code> argument <code>evaluate_during_training</code> is deprecated in favor of <code>evaluation_strategy</code>.</li></ul>
<p>Regarding the Transfo-XL model:</p>
<ul><li>The Transfo-XL configuration attribute <code>tie_weight</code> becomes <code>tie_words_embeddings</code>.</li>
<li>The Transfo-XL modeling method <code>reset_length</code> becomes <code>reset_memory_length</code>.</li></ul>
<p>Regarding pipelines:</p>
<ul><li>The <code>FillMaskPipeline</code> argument <code>topk</code> becomes <code>top_k</code>.</li></ul>
<h2 class="relative group"><a id="migrating-from-pytorchtransformers-to-transformers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#migrating-from-pytorchtransformers-to-transformers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Migrating from pytorch-transformers to 🤗 Transformers
	</span></h2>

<p>Here is a quick summary of what you should take care of when migrating from <code>pytorch-transformers</code> to 🤗 Transformers.</p>
<h3 class="relative group"><a id="positional-order-of-some-models-keywords-inputs-attentionmask-tokentypeids-changed" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#positional-order-of-some-models-keywords-inputs-attentionmask-tokentypeids-changed"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Positional order of some models&#39; keywords inputs (<code>attention_mask</code>, <code>token_type_ids</code>...) changed
	</span></h3>

<p>To be able to use Torchscript (see #1010, #1204 and #1195) the specific order of some models <strong>keywords inputs</strong> (<code>attention_mask</code>, <code>token_type_ids</code>…) has been changed.</p>
<p>If you used to call the models with keyword names for keyword arguments, e.g. <code>model(inputs_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)</code>, this should not cause any change.</p>
<p>If you used to call the models with positional inputs for keyword arguments, e.g. <code>model(inputs_ids, attention_mask, token_type_ids)</code>, you may have to double check the exact order of input arguments.</p>
<h2 class="relative group"><a id="migrating-from-pytorchpretrainedbert" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#migrating-from-pytorchpretrainedbert"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Migrating from pytorch-pretrained-bert
	</span></h2>

<p>Here is a quick summary of what you should take care of when migrating from <code>pytorch-pretrained-bert</code> to 🤗 Transformers</p>
<h3 class="relative group"><a id="models-always-output-tuples" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#models-always-output-tuples"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Models always output <code>tuples</code></span></h3>

<p>The main breaking change when migrating from <code>pytorch-pretrained-bert</code> to 🤗 Transformers is that the models forward method always outputs a <code>tuple</code> with various elements depending on the model and the configuration parameters.</p>
<p>The exact content of the tuples for each model are detailed in the models’ docstrings and the <a href="https://huggingface.co/transformers/" rel="nofollow">documentation</a>.</p>
<p>In pretty much every case, you will be fine by taking the first element of the output as the output you previously used in <code>pytorch-pretrained-bert</code>.</p>
<p>Here is a <code>pytorch-pretrained-bert</code> to 🤗 Transformers conversion example for a <code>BertForSequenceClassification</code> classification model:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-comment"># Let&#x27;s load our model</span>
model = BertForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)

<span class="hljs-comment"># If you used to have this line in pytorch-pretrained-bert:</span>
loss = model(input_ids, labels=labels)

<span class="hljs-comment"># Now just use this line in 🤗 Transformers to extract the loss from the output tuple:</span>
outputs = model(input_ids, labels=labels)
loss = outputs[<span class="hljs-number">0</span>]

<span class="hljs-comment"># In 🤗 Transformers you can also have access to the logits:</span>
loss, logits = outputs[:<span class="hljs-number">2</span>]

<span class="hljs-comment"># And even the attention weights if you configure the model to output them (and other outputs too, see the docstrings and documentation)</span>
model = BertForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>, output_attentions=<span class="hljs-literal">True</span>)
outputs = model(input_ids, labels=labels)
loss, logits, attentions = outputs<!-- HTML_TAG_END --></pre></div>
<h3 class="relative group"><a id="serialization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#serialization"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Serialization
	</span></h3>

<p>Breaking change in the <code>from_pretrained()</code>method:</p>
<ol><li><p>Models are now set in evaluation mode by default when instantiated with the <code>from_pretrained()</code> method. To train them don’t forget to set them back in training mode (<code>model.train()</code>) to activate the dropout modules.</p></li>
<li><p>The additional <code>*inputs</code> and <code>**kwargs</code> arguments supplied to the <code>from_pretrained()</code> method used to be directly passed to the underlying model’s class <code>__init__()</code> method. They are now used to update the model configuration attribute first which can break derived model classes build based on the previous <code>BertForSequenceClassification</code> examples. More precisely, the positional arguments <code>*inputs</code> provided to <code>from_pretrained()</code> are directly forwarded the model <code>__init__()</code> method while the keyword arguments <code>**kwargs</code> (i) which match configuration class attributes are used to update said attributes (ii) which don’t match any configuration class attributes are forwarded to the model <code>__init__()</code> method.</p></li></ol>
<p>Also, while not a breaking change, the serialization methods have been standardized and you probably should switch to the new method <code>save_pretrained(save_directory)</code> if you were using any other serialization method before.</p>
<p>Here is an example:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-comment">### Let&#x27;s load a model and tokenizer</span>
model = BertForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
tokenizer = BertTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)

<span class="hljs-comment">### Do some stuff to our model and tokenizer</span>
<span class="hljs-comment"># Ex: add new tokens to the vocabulary and embeddings of our model</span>
tokenizer.add_tokens([<span class="hljs-string">&quot;[SPECIAL_TOKEN_1]&quot;</span>, <span class="hljs-string">&quot;[SPECIAL_TOKEN_2]&quot;</span>])
model.resize_token_embeddings(<span class="hljs-built_in">len</span>(tokenizer))
<span class="hljs-comment"># Train our model</span>
train(model)

<span class="hljs-comment">### Now let&#x27;s save our model and tokenizer to a directory</span>
model.save_pretrained(<span class="hljs-string">&quot;./my_saved_model_directory/&quot;</span>)
tokenizer.save_pretrained(<span class="hljs-string">&quot;./my_saved_model_directory/&quot;</span>)

<span class="hljs-comment">### Reload the model and the tokenizer</span>
model = BertForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;./my_saved_model_directory/&quot;</span>)
tokenizer = BertTokenizer.from_pretrained(<span class="hljs-string">&quot;./my_saved_model_directory/&quot;</span>)<!-- HTML_TAG_END --></pre></div>
<h3 class="relative group"><a id="optimizers-bertadam-openaiadam-are-now-adamw-schedules-are-standard-pytorch-schedules" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#optimizers-bertadam-openaiadam-are-now-adamw-schedules-are-standard-pytorch-schedules"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Optimizers: BertAdam &amp; OpenAIAdam are now AdamW, schedules are standard PyTorch schedules
	</span></h3>

<p>The two optimizers previously included, <code>BertAdam</code> and <code>OpenAIAdam</code>, have been replaced by a single <code>AdamW</code> optimizer which has a few differences:</p>
<ul><li>it only implements weights decay correction,</li>
<li>schedules are now externals (see below),</li>
<li>gradient clipping is now also external (see below).</li></ul>
<p>The new optimizer <code>AdamW</code> matches PyTorch <code>Adam</code> optimizer API and let you use standard PyTorch or apex methods for the schedule and clipping.</p>
<p>The schedules are now standard <a href="https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate" rel="nofollow">PyTorch learning rate schedulers</a> and not part of the optimizer anymore.</p>
<p>Here is a conversion examples from <code>BertAdam</code> with a linear warmup and decay schedule to <code>AdamW</code> and the same schedule:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-comment"># Parameters:</span>
lr = <span class="hljs-number">1e-3</span>
max_grad_norm = <span class="hljs-number">1.0</span>
num_training_steps = <span class="hljs-number">1000</span>
num_warmup_steps = <span class="hljs-number">100</span>
warmup_proportion = <span class="hljs-built_in">float</span>(num_warmup_steps) / <span class="hljs-built_in">float</span>(num_training_steps)  <span class="hljs-comment"># 0.1</span>

<span class="hljs-comment">### Previously BertAdam optimizer was instantiated like this:</span>
optimizer = BertAdam(
    model.parameters(),
    lr=lr,
    schedule=<span class="hljs-string">&quot;warmup_linear&quot;</span>,
    warmup=warmup_proportion,
    num_training_steps=num_training_steps,
)
<span class="hljs-comment">### and used like this:</span>
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_data:
    loss = model(batch)
    loss.backward()
    optimizer.step()

<span class="hljs-comment">### In 🤗 Transformers, optimizer and schedules are split and instantiated like this:</span>
optimizer = AdamW(
    model.parameters(), lr=lr, correct_bias=<span class="hljs-literal">False</span>
)  <span class="hljs-comment"># To reproduce BertAdam specific behavior set correct_bias=False</span>
scheduler = get_linear_schedule_with_warmup(
    optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps
)  <span class="hljs-comment"># PyTorch scheduler</span>
<span class="hljs-comment">### and used like this:</span>
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_data:
    loss = model(batch)
    loss.backward()
    torch.nn.utils.clip_grad_norm_(
        model.parameters(), max_grad_norm
    )  <span class="hljs-comment"># Gradient clipping is not in AdamW anymore (so you can use amp without issue)</span>
    optimizer.step()
    scheduler.step()<!-- HTML_TAG_END --></pre></div>


		<script type="module" data-hydrate="1kh1foc">
		import { start } from "/docs/transformers/pr_19043/en/_app/start-hf-doc-builder.js";
		start({
			target: document.querySelector('[data-hydrate="1kh1foc"]').parentNode,
			paths: {"base":"/docs/transformers/pr_19043/en","assets":"/docs/transformers/pr_19043/en"},
			session: {},
			route: false,
			spa: false,
			trailing_slash: "never",
			hydrate: {
				status: 200,
				error: null,
				nodes: [
					import("/docs/transformers/pr_19043/en/_app/pages/__layout.svelte-hf-doc-builder.js"),
						import("/docs/transformers/pr_19043/en/_app/pages/migration.mdx-hf-doc-builder.js")
				],
				params: {}
			}
		});
	</script>
