import{S as at,i as et,s as tt,e as p,k as r,w as R,t as n,Y as _,M as nt,c as o,d as a,m as h,a as m,x as J,h as l,Z as b,b as c,N as re,G as e,g as i,y as U,L as lt,q as Y,o as Z,B as K,v as it}from"../chunks/vendor-hf-doc-builder.js";import{I as he}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as ce}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as pt}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";function ot(de){let k,ks,P,T,os,A,ya,ms,xa,Ps,S,zs,$,_a,Q,ba,ka,qs,w,Pa,Ts,Xe='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>X</mi><mo>=</mo><mo stretchy="false">(</mo><msub><mi>x</mi><mn>0</mn></msub><mo separator="true">,</mo><msub><mi>x</mi><mn>1</mn></msub><mo separator="true">,</mo><mo>\u2026</mo><mo separator="true">,</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">X = (x_0, x_1, \\dots, x_t)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal" style="margin-right:0.07847em;">X</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">0</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">\u2026</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span>',$s,Es,Re='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>X</mi></mrow><annotation encoding="application/x-tex">X</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal" style="margin-right:0.07847em;">X</span></span></span></span>',Ms,js,Je='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mtext>PPL</mtext><mo stretchy="false">(</mo><mi>X</mi><mo stretchy="false">)</mo><mo>=</mo><mi>exp</mi><mo>\u2061</mo><mrow><mo fence="true">{</mo><mrow><mo>\u2212</mo><mfrac><mn>1</mn><mi>t</mi></mfrac><munderover><mo>\u2211</mo><mi>i</mi><mi>t</mi></munderover><mi>log</mi><mo>\u2061</mo><msub><mi>p</mi><mi>\u03B8</mi></msub><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mi mathvariant="normal">\u2223</mi><msub><mi>x</mi><mrow><mo>&lt;</mo><mi>i</mi></mrow></msub><mo stretchy="false">)</mo></mrow><mo fence="true">}</mo></mrow></mrow><annotation encoding="application/x-tex">\\text{PPL}(X) = \\exp \\left\\{ {-\\frac{1}{t}\\sum_i^t \\log p_\\theta (x_i|x_{&lt;i}) } \\right\\}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord text"><span class="mord">PPL</span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.07847em;">X</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:3.0582em;vertical-align:-1.2777em;"></span><span class="mop">exp</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size4">{</span></span><span class="mord"><span class="mord">\u2212</span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">t</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.7806em;"><span style="top:-1.8723em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">\u2211</span></span></span><span style="top:-4.3em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:1.2777em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop">lo<span style="margin-right:0.01389em;">g</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">p</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">\u03B8</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">\u2223</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mrel mtight">&lt;</span><span class="mord mathnormal mtight">i</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.1774em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size4">}</span></span></span></span></span></span></span>',Ls,y,za,Cs,Ue='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>log</mi><mo>\u2061</mo><msub><mi>p</mi><mi>\u03B8</mi></msub><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mi mathvariant="normal">\u2223</mi><msub><mi>x</mi><mrow><mo>&lt;</mo><mi>i</mi></mrow></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\log p_\\theta (x_i|x_{&lt;i})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mop">lo<span style="margin-right:0.01389em;">g</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">p</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">\u03B8</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">\u2223</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mrel mtight">&lt;</span><span class="mord mathnormal mtight">i</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.1774em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span>',Gs,Is,Ye='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>x</mi><mrow><mo>&lt;</mo><mi>i</mi></mrow></msub></mrow><annotation encoding="application/x-tex">x_{&lt;i}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6079em;vertical-align:-0.1774em;"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mrel mtight">&lt;</span><span class="mord mathnormal mtight">i</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.1774em;"><span></span></span></span></span></span></span></span></span></span>',Ws,As,E,qa,B,Ta,$a,Ss,z,M,rs,D,Ea,hs,Ma,Bs,V,ja,Ds,j,ge,Ns,u,La,ss,Ca,Ga,Os,Ze='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>p</mi><mi>\u03B8</mi></msub><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mi mathvariant="normal">\u2223</mi><msub><mi>x</mi><mrow><mo>&lt;</mo><mi>t</mi></mrow></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">p_\\theta(x_t|x_{&lt;t})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal">p</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">\u03B8</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">\u2223</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mrel mtight">&lt;</span><span class="mord mathnormal mtight">t</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.1774em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span>',Fs,Hs,Ke='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>t</mi></mrow><annotation encoding="application/x-tex">t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6151em;"></span><span class="mord mathnormal">t</span></span></span></span>',Xs,Rs,f,Ia,Js,Qe='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>k</mi></mrow><annotation encoding="application/x-tex">k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span></span></span></span>',Us,Ys,Ve='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>x</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">x_t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2806em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Zs,Ks,st='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>k</mi><mo>\u2212</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">k-1</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.7778em;vertical-align:-0.0833em;"></span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\u2212</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">1</span></span></span></span>',Qs,Vs,L,ue,sa,as,Wa,aa,es,Aa,ea,C,fe,ta,ts,Sa,na,q,G,cs,N,Ba,ds,Da,la,ns,Na,ia,O,pa,ls,Oa,oa,F,ma,g,Fa,gs,Ha,Xa,us,Ra,Ja,fs,Ua,Ya,vs,Za,Ka,ra,H,ha,is,Qa,ca,d,Va,ws,se,ae,ys,ee,te,xs,ne,le,_s,ie,pe,bs,oe,me,da;return A=new he({}),S=new pt({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/perplexity.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/perplexity.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/perplexity.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/perplexity.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/perplexity.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/perplexity.ipynb"}]}}),D=new he({}),N=new he({}),O=new ce({props:{code:`from transformers import GPT2LMHeadModel, GPT2TokenizerFast

device = "cuda"
model_id = "gpt2-large"
model = GPT2LMHeadModel.from_pretrained(model_id).to(device)
tokenizer = GPT2TokenizerFast.from_pretrained(model_id)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPT2LMHeadModel, GPT2TokenizerFast

device = <span class="hljs-string">&quot;cuda&quot;</span>
model_id = <span class="hljs-string">&quot;gpt2-large&quot;</span>
model = GPT2LMHeadModel.from_pretrained(model_id).to(device)
tokenizer = GPT2TokenizerFast.from_pretrained(model_id)`}}),F=new ce({props:{code:`from datasets import load_dataset

test = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")
encodings = tokenizer("\\n\\n".join(test["text"]), return_tensors="pt")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

test = load_dataset(<span class="hljs-string">&quot;wikitext&quot;</span>, <span class="hljs-string">&quot;wikitext-2-raw-v1&quot;</span>, split=<span class="hljs-string">&quot;test&quot;</span>)
encodings = tokenizer(<span class="hljs-string">&quot;\\n\\n&quot;</span>.join(test[<span class="hljs-string">&quot;text&quot;</span>]), return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`}}),H=new ce({props:{code:`import torch
from tqdm import tqdm

max_length = model.config.n_positions
stride = 512
seq_len = encodings.input_ids.size(1)

nlls = []
prev_end_loc = 0
for begin_loc in tqdm(range(0, seq_len, stride)):
    end_loc = min(begin_loc + max_length, seq_len)
    trg_len = end_loc - prev_end_loc  # may be different from stride on last loop
    input_ids = encodings.input_ids[:, begin_loc:end_loc].to(device)
    target_ids = input_ids.clone()
    target_ids[:, :-trg_len] = -100

    with torch.no_grad():
        outputs = model(input_ids, labels=target_ids)

        # loss is calculated using CrossEntropyLoss which averages over input tokens.
        # Multiply it with trg_len to get the summation instead of average.
        # We will take average over all the tokens to get the true average
        # in the last step of this example.
        neg_log_likelihood = outputs.loss * trg_len

    nlls.append(neg_log_likelihood)

    prev_end_loc = end_loc
    if end_loc == seq_len:
        break

ppl = torch.exp(torch.stack(nlls).sum() / end_loc)`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm

max_length = model.config.n_positions
stride = <span class="hljs-number">512</span>
seq_len = encodings.input_ids.size(<span class="hljs-number">1</span>)

nlls = []
prev_end_loc = <span class="hljs-number">0</span>
<span class="hljs-keyword">for</span> begin_loc <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, seq_len, stride)):
    end_loc = <span class="hljs-built_in">min</span>(begin_loc + max_length, seq_len)
    trg_len = end_loc - prev_end_loc  <span class="hljs-comment"># may be different from stride on last loop</span>
    input_ids = encodings.input_ids[:, begin_loc:end_loc].to(device)
    target_ids = input_ids.clone()
    target_ids[:, :-trg_len] = -<span class="hljs-number">100</span>

    <span class="hljs-keyword">with</span> torch.no_grad():
        outputs = model(input_ids, labels=target_ids)

        <span class="hljs-comment"># loss is calculated using CrossEntropyLoss which averages over input tokens.</span>
        <span class="hljs-comment"># Multiply it with trg_len to get the summation instead of average.</span>
        <span class="hljs-comment"># We will take average over all the tokens to get the true average</span>
        <span class="hljs-comment"># in the last step of this example.</span>
        neg_log_likelihood = outputs.loss * trg_len

    nlls.append(neg_log_likelihood)

    prev_end_loc = end_loc
    <span class="hljs-keyword">if</span> end_loc == seq_len:
        <span class="hljs-keyword">break</span>

ppl = torch.exp(torch.stack(nlls).<span class="hljs-built_in">sum</span>() / end_loc)`}}),{c(){k=p("meta"),ks=r(),P=p("h1"),T=p("a"),os=p("span"),R(A.$$.fragment),ya=r(),ms=p("span"),xa=n("Perplexity of fixed-length models"),Ps=r(),R(S.$$.fragment),zs=r(),$=p("p"),_a=n(`Perplexity (PPL) is one of the most common metrics for evaluating language models. Before diving in, we should note
that the metric applies specifically to classical language models (sometimes called autoregressive or causal language
models) and is not well defined for masked language models like BERT (see `),Q=p("a"),ba=n("summary of the models"),ka=n(")."),qs=r(),w=p("p"),Pa=n(`Perplexity is defined as the exponentiated average negative log-likelihood of a sequence. If we have a tokenized
sequence `),Ts=new _,$s=n(", then the perplexity of "),Es=new _,Ms=n(` is,
`),js=new _,Ls=r(),y=p("p"),za=n("where "),Cs=new _,Gs=n(" is the log-likelihood of the ith token conditioned on the preceding tokens "),Is=new _,Ws=n(" according to our model. Intuitively, it can be thought of as an evaluation of the model\u2019s ability to predict uniformly among the set of specified tokens in a corpus. Importantly, this means that the tokenization procedure has a direct impact on a model\u2019s perplexity which should always be taken into consideration when comparing different models."),As=r(),E=p("p"),qa=n(`This is also equivalent to the exponentiation of the cross-entropy between the data and model predictions. For more
intuition about perplexity and its relationship to Bits Per Character (BPC) and data compression, check out this
`),B=p("a"),Ta=n("fantastic blog post on The Gradient"),$a=n("."),Ss=r(),z=p("h2"),M=p("a"),rs=p("span"),R(D.$$.fragment),Ea=r(),hs=p("span"),Ma=n("Calculating PPL with fixed-length models"),Bs=r(),V=p("p"),ja=n(`If we weren\u2019t limited by a model\u2019s context size, we would evaluate the model\u2019s perplexity by autoregressively
factorizing a sequence and conditioning on the entire preceding subsequence at each step, as shown below.`),Ds=r(),j=p("img"),Ns=r(),u=p("p"),La=n(`When working with approximate models, however, we typically have a constraint on the number of tokens the model can
process. The largest version of `),ss=p("a"),Ca=n("GPT-2"),Ga=n(`, for example, has a fixed length of 1024 tokens, so we
cannot calculate `),Os=new _,Fs=n(" directly when "),Hs=new _,Xs=n(" is greater than 1024."),Rs=r(),f=p("p"),Ia=n(`Instead, the sequence is typically broken into subsequences equal to the model\u2019s maximum input size. If a model\u2019s max
input size is `),Js=new _,Us=n(", we then approximate the likelihood of a token "),Ys=new _,Zs=n(` by conditioning only on the
`),Ks=new _,Qs=n(` tokens that precede it rather than the entire context. When evaluating the model\u2019s perplexity of a
sequence, a tempting but suboptimal approach is to break the sequence into disjoint chunks and add up the decomposed
log-likelihoods of each segment independently.`),Vs=r(),L=p("img"),sa=r(),as=p("p"),Wa=n(`This is quick to compute since the perplexity of each segment can be computed in one forward pass, but serves as a poor
approximation of the fully-factorized perplexity and will typically yield a higher (worse) PPL because the model will
have less context at most of the prediction steps.`),aa=r(),es=p("p"),Aa=n(`Instead, the PPL of fixed-length models should be evaluated with a sliding-window strategy. This involves repeatedly
sliding the context window so that the model has more context when making each prediction.`),ea=r(),C=p("img"),ta=r(),ts=p("p"),Sa=n(`This is a closer approximation to the true decomposition of the sequence probability and will typically yield a more
favorable score. The downside is that it requires a separate forward pass for each token in the corpus. A good
practical compromise is to employ a strided sliding window, moving the context by larger strides rather than sliding by
1 token a time. This allows computation to proceed much faster while still giving the model a large context to make
predictions at each step.`),na=r(),q=p("h2"),G=p("a"),cs=p("span"),R(N.$$.fragment),Ba=r(),ds=p("span"),Da=n("Example: Calculating perplexity with GPT-2 in \u{1F917} Transformers"),la=r(),ns=p("p"),Na=n("Let\u2019s demonstrate this process with GPT-2."),ia=r(),R(O.$$.fragment),pa=r(),ls=p("p"),Oa=n(`We\u2019ll load in the WikiText-2 dataset and evaluate the perplexity using a few different sliding-window strategies. Since
this dataset is small and we\u2019re just doing one forward pass over the set, we can just load and encode the entire
dataset in memory.`),oa=r(),R(F.$$.fragment),ma=r(),g=p("p"),Fa=n("With \u{1F917} Transformers, we can simply pass the "),gs=p("code"),Ha=n("input_ids"),Xa=n(" as the "),us=p("code"),Ra=n("labels"),Ja=n(` to our model, and the average negative
log-likelihood for each token is returned as the loss. With our sliding window approach, however, there is overlap in
the tokens we pass to the model at each iteration. We don\u2019t want the log-likelihood for the tokens we\u2019re just treating
as context to be included in our loss, so we can set these targets to `),fs=p("code"),Ua=n("-100"),Ya=n(` so that they are ignored. The following
is an example of how we could do this with a stride of `),vs=p("code"),Za=n("512"),Ka=n(`. This means that the model will have at least 512 tokens
for context when calculating the conditional likelihood of any one token (provided there are 512 preceding tokens
available to condition on).`),ra=r(),R(H.$$.fragment),ha=r(),is=p("p"),Qa=n(`Running this with the stride length equal to the max input length is equivalent to the suboptimal, non-sliding-window
strategy we discussed above. The smaller the stride, the more context the model will have in making each prediction,
and the better the reported perplexity will typically be.`),ca=r(),d=p("p"),Va=n("When we run the above with "),ws=p("code"),se=n("stride = 1024"),ae=n(", i.e. no overlap, the resulting PPL is "),ys=p("code"),ee=n("19.64"),te=n(`, which is about the same
as the `),xs=p("code"),ne=n("19.93"),le=n(" reported in the GPT-2 paper. By using "),_s=p("code"),ie=n("stride = 512"),pe=n(` and thereby employing our striding window
strategy, this jumps down to `),bs=p("code"),oe=n("16.44"),me=n(`. This is not only a more favorable score, but is calculated in a way that is
closer to the true autoregressive decomposition of a sequence likelihood.`),this.h()},l(s){const t=nt('[data-svelte="svelte-1phssyn"]',document.head);k=o(t,"META",{name:!0,content:!0}),t.forEach(a),ks=h(s),P=o(s,"H1",{class:!0});var ga=m(P);T=o(ga,"A",{id:!0,class:!0,href:!0});var ve=m(T);os=o(ve,"SPAN",{});var we=m(os);J(A.$$.fragment,we),we.forEach(a),ve.forEach(a),ya=h(ga),ms=o(ga,"SPAN",{});var ye=m(ms);xa=l(ye,"Perplexity of fixed-length models"),ye.forEach(a),ga.forEach(a),Ps=h(s),J(S.$$.fragment,s),zs=h(s),$=o(s,"P",{});var ua=m($);_a=l(ua,`Perplexity (PPL) is one of the most common metrics for evaluating language models. Before diving in, we should note
that the metric applies specifically to classical language models (sometimes called autoregressive or causal language
models) and is not well defined for masked language models like BERT (see `),Q=o(ua,"A",{href:!0});var xe=m(Q);ba=l(xe,"summary of the models"),xe.forEach(a),ka=l(ua,")."),ua.forEach(a),qs=h(s),w=o(s,"P",{});var X=m(w);Pa=l(X,`Perplexity is defined as the exponentiated average negative log-likelihood of a sequence. If we have a tokenized
sequence `),Ts=b(X),$s=l(X,", then the perplexity of "),Es=b(X),Ms=l(X,` is,
`),js=b(X),X.forEach(a),Ls=h(s),y=o(s,"P",{});var ps=m(y);za=l(ps,"where "),Cs=b(ps),Gs=l(ps," is the log-likelihood of the ith token conditioned on the preceding tokens "),Is=b(ps),Ws=l(ps," according to our model. Intuitively, it can be thought of as an evaluation of the model\u2019s ability to predict uniformly among the set of specified tokens in a corpus. Importantly, this means that the tokenization procedure has a direct impact on a model\u2019s perplexity which should always be taken into consideration when comparing different models."),ps.forEach(a),As=h(s),E=o(s,"P",{});var fa=m(E);qa=l(fa,`This is also equivalent to the exponentiation of the cross-entropy between the data and model predictions. For more
intuition about perplexity and its relationship to Bits Per Character (BPC) and data compression, check out this
`),B=o(fa,"A",{href:!0,rel:!0});var _e=m(B);Ta=l(_e,"fantastic blog post on The Gradient"),_e.forEach(a),$a=l(fa,"."),fa.forEach(a),Ss=h(s),z=o(s,"H2",{class:!0});var va=m(z);M=o(va,"A",{id:!0,class:!0,href:!0});var be=m(M);rs=o(be,"SPAN",{});var ke=m(rs);J(D.$$.fragment,ke),ke.forEach(a),be.forEach(a),Ea=h(va),hs=o(va,"SPAN",{});var Pe=m(hs);Ma=l(Pe,"Calculating PPL with fixed-length models"),Pe.forEach(a),va.forEach(a),Bs=h(s),V=o(s,"P",{});var ze=m(V);ja=l(ze,`If we weren\u2019t limited by a model\u2019s context size, we would evaluate the model\u2019s perplexity by autoregressively
factorizing a sequence and conditioning on the entire preceding subsequence at each step, as shown below.`),ze.forEach(a),Ds=h(s),j=o(s,"IMG",{width:!0,alt:!0,src:!0}),Ns=h(s),u=o(s,"P",{});var I=m(u);La=l(I,`When working with approximate models, however, we typically have a constraint on the number of tokens the model can
process. The largest version of `),ss=o(I,"A",{href:!0});var qe=m(ss);Ca=l(qe,"GPT-2"),qe.forEach(a),Ga=l(I,`, for example, has a fixed length of 1024 tokens, so we
cannot calculate `),Os=b(I),Fs=l(I," directly when "),Hs=b(I),Xs=l(I," is greater than 1024."),I.forEach(a),Rs=h(s),f=o(s,"P",{});var W=m(f);Ia=l(W,`Instead, the sequence is typically broken into subsequences equal to the model\u2019s maximum input size. If a model\u2019s max
input size is `),Js=b(W),Us=l(W,", we then approximate the likelihood of a token "),Ys=b(W),Zs=l(W,` by conditioning only on the
`),Ks=b(W),Qs=l(W,` tokens that precede it rather than the entire context. When evaluating the model\u2019s perplexity of a
sequence, a tempting but suboptimal approach is to break the sequence into disjoint chunks and add up the decomposed
log-likelihoods of each segment independently.`),W.forEach(a),Vs=h(s),L=o(s,"IMG",{width:!0,alt:!0,src:!0}),sa=h(s),as=o(s,"P",{});var Te=m(as);Wa=l(Te,`This is quick to compute since the perplexity of each segment can be computed in one forward pass, but serves as a poor
approximation of the fully-factorized perplexity and will typically yield a higher (worse) PPL because the model will
have less context at most of the prediction steps.`),Te.forEach(a),aa=h(s),es=o(s,"P",{});var $e=m(es);Aa=l($e,`Instead, the PPL of fixed-length models should be evaluated with a sliding-window strategy. This involves repeatedly
sliding the context window so that the model has more context when making each prediction.`),$e.forEach(a),ea=h(s),C=o(s,"IMG",{width:!0,alt:!0,src:!0}),ta=h(s),ts=o(s,"P",{});var Ee=m(ts);Sa=l(Ee,`This is a closer approximation to the true decomposition of the sequence probability and will typically yield a more
favorable score. The downside is that it requires a separate forward pass for each token in the corpus. A good
practical compromise is to employ a strided sliding window, moving the context by larger strides rather than sliding by
1 token a time. This allows computation to proceed much faster while still giving the model a large context to make
predictions at each step.`),Ee.forEach(a),na=h(s),q=o(s,"H2",{class:!0});var wa=m(q);G=o(wa,"A",{id:!0,class:!0,href:!0});var Me=m(G);cs=o(Me,"SPAN",{});var je=m(cs);J(N.$$.fragment,je),je.forEach(a),Me.forEach(a),Ba=h(wa),ds=o(wa,"SPAN",{});var Le=m(ds);Da=l(Le,"Example: Calculating perplexity with GPT-2 in \u{1F917} Transformers"),Le.forEach(a),wa.forEach(a),la=h(s),ns=o(s,"P",{});var Ce=m(ns);Na=l(Ce,"Let\u2019s demonstrate this process with GPT-2."),Ce.forEach(a),ia=h(s),J(O.$$.fragment,s),pa=h(s),ls=o(s,"P",{});var Ge=m(ls);Oa=l(Ge,`We\u2019ll load in the WikiText-2 dataset and evaluate the perplexity using a few different sliding-window strategies. Since
this dataset is small and we\u2019re just doing one forward pass over the set, we can just load and encode the entire
dataset in memory.`),Ge.forEach(a),oa=h(s),J(F.$$.fragment,s),ma=h(s),g=o(s,"P",{});var x=m(g);Fa=l(x,"With \u{1F917} Transformers, we can simply pass the "),gs=o(x,"CODE",{});var Ie=m(gs);Ha=l(Ie,"input_ids"),Ie.forEach(a),Xa=l(x," as the "),us=o(x,"CODE",{});var We=m(us);Ra=l(We,"labels"),We.forEach(a),Ja=l(x,` to our model, and the average negative
log-likelihood for each token is returned as the loss. With our sliding window approach, however, there is overlap in
the tokens we pass to the model at each iteration. We don\u2019t want the log-likelihood for the tokens we\u2019re just treating
as context to be included in our loss, so we can set these targets to `),fs=o(x,"CODE",{});var Ae=m(fs);Ua=l(Ae,"-100"),Ae.forEach(a),Ya=l(x,` so that they are ignored. The following
is an example of how we could do this with a stride of `),vs=o(x,"CODE",{});var Se=m(vs);Za=l(Se,"512"),Se.forEach(a),Ka=l(x,`. This means that the model will have at least 512 tokens
for context when calculating the conditional likelihood of any one token (provided there are 512 preceding tokens
available to condition on).`),x.forEach(a),ra=h(s),J(H.$$.fragment,s),ha=h(s),is=o(s,"P",{});var Be=m(is);Qa=l(Be,`Running this with the stride length equal to the max input length is equivalent to the suboptimal, non-sliding-window
strategy we discussed above. The smaller the stride, the more context the model will have in making each prediction,
and the better the reported perplexity will typically be.`),Be.forEach(a),ca=h(s),d=o(s,"P",{});var v=m(d);Va=l(v,"When we run the above with "),ws=o(v,"CODE",{});var De=m(ws);se=l(De,"stride = 1024"),De.forEach(a),ae=l(v,", i.e. no overlap, the resulting PPL is "),ys=o(v,"CODE",{});var Ne=m(ys);ee=l(Ne,"19.64"),Ne.forEach(a),te=l(v,`, which is about the same
as the `),xs=o(v,"CODE",{});var Oe=m(xs);ne=l(Oe,"19.93"),Oe.forEach(a),le=l(v," reported in the GPT-2 paper. By using "),_s=o(v,"CODE",{});var Fe=m(_s);ie=l(Fe,"stride = 512"),Fe.forEach(a),pe=l(v,` and thereby employing our striding window
strategy, this jumps down to `),bs=o(v,"CODE",{});var He=m(bs);oe=l(He,"16.44"),He.forEach(a),me=l(v,`. This is not only a more favorable score, but is calculated in a way that is
closer to the true autoregressive decomposition of a sequence likelihood.`),v.forEach(a),this.h()},h(){c(k,"name","hf:doc:metadata"),c(k,"content",JSON.stringify(mt)),c(T,"id","perplexity-of-fixedlength-models"),c(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(T,"href","#perplexity-of-fixedlength-models"),c(P,"class","relative group"),c(Q,"href","model_summary"),Ts.a=$s,Es.a=Ms,js.a=null,Cs.a=Gs,Is.a=Ws,c(B,"href","https://thegradient.pub/understanding-evaluation-metrics-for-language-models/"),c(B,"rel","nofollow"),c(M,"id","calculating-ppl-with-fixedlength-models"),c(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(M,"href","#calculating-ppl-with-fixedlength-models"),c(z,"class","relative group"),c(j,"width","600"),c(j,"alt","Full decomposition of a sequence with unlimited context length"),re(j.src,ge="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/ppl_full.gif")||c(j,"src",ge),c(ss,"href","model_doc/gpt2"),Os.a=Fs,Hs.a=Xs,Js.a=Us,Ys.a=Zs,Ks.a=Qs,c(L,"width","600"),c(L,"alt","Suboptimal PPL not taking advantage of full available context"),re(L.src,ue="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/ppl_chunked.gif")||c(L,"src",ue),c(C,"width","600"),c(C,"alt","Sliding window PPL taking advantage of all available context"),re(C.src,fe="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/ppl_sliding.gif")||c(C,"src",fe),c(G,"id","example-calculating-perplexity-with-gpt2-in-transformers"),c(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(G,"href","#example-calculating-perplexity-with-gpt2-in-transformers"),c(q,"class","relative group")},m(s,t){e(document.head,k),i(s,ks,t),i(s,P,t),e(P,T),e(T,os),U(A,os,null),e(P,ya),e(P,ms),e(ms,xa),i(s,Ps,t),U(S,s,t),i(s,zs,t),i(s,$,t),e($,_a),e($,Q),e(Q,ba),e($,ka),i(s,qs,t),i(s,w,t),e(w,Pa),Ts.m(Xe,w),e(w,$s),Es.m(Re,w),e(w,Ms),js.m(Je,w),i(s,Ls,t),i(s,y,t),e(y,za),Cs.m(Ue,y),e(y,Gs),Is.m(Ye,y),e(y,Ws),i(s,As,t),i(s,E,t),e(E,qa),e(E,B),e(B,Ta),e(E,$a),i(s,Ss,t),i(s,z,t),e(z,M),e(M,rs),U(D,rs,null),e(z,Ea),e(z,hs),e(hs,Ma),i(s,Bs,t),i(s,V,t),e(V,ja),i(s,Ds,t),i(s,j,t),i(s,Ns,t),i(s,u,t),e(u,La),e(u,ss),e(ss,Ca),e(u,Ga),Os.m(Ze,u),e(u,Fs),Hs.m(Ke,u),e(u,Xs),i(s,Rs,t),i(s,f,t),e(f,Ia),Js.m(Qe,f),e(f,Us),Ys.m(Ve,f),e(f,Zs),Ks.m(st,f),e(f,Qs),i(s,Vs,t),i(s,L,t),i(s,sa,t),i(s,as,t),e(as,Wa),i(s,aa,t),i(s,es,t),e(es,Aa),i(s,ea,t),i(s,C,t),i(s,ta,t),i(s,ts,t),e(ts,Sa),i(s,na,t),i(s,q,t),e(q,G),e(G,cs),U(N,cs,null),e(q,Ba),e(q,ds),e(ds,Da),i(s,la,t),i(s,ns,t),e(ns,Na),i(s,ia,t),U(O,s,t),i(s,pa,t),i(s,ls,t),e(ls,Oa),i(s,oa,t),U(F,s,t),i(s,ma,t),i(s,g,t),e(g,Fa),e(g,gs),e(gs,Ha),e(g,Xa),e(g,us),e(us,Ra),e(g,Ja),e(g,fs),e(fs,Ua),e(g,Ya),e(g,vs),e(vs,Za),e(g,Ka),i(s,ra,t),U(H,s,t),i(s,ha,t),i(s,is,t),e(is,Qa),i(s,ca,t),i(s,d,t),e(d,Va),e(d,ws),e(ws,se),e(d,ae),e(d,ys),e(ys,ee),e(d,te),e(d,xs),e(xs,ne),e(d,le),e(d,_s),e(_s,ie),e(d,pe),e(d,bs),e(bs,oe),e(d,me),da=!0},p:lt,i(s){da||(Y(A.$$.fragment,s),Y(S.$$.fragment,s),Y(D.$$.fragment,s),Y(N.$$.fragment,s),Y(O.$$.fragment,s),Y(F.$$.fragment,s),Y(H.$$.fragment,s),da=!0)},o(s){Z(A.$$.fragment,s),Z(S.$$.fragment,s),Z(D.$$.fragment,s),Z(N.$$.fragment,s),Z(O.$$.fragment,s),Z(F.$$.fragment,s),Z(H.$$.fragment,s),da=!1},d(s){a(k),s&&a(ks),s&&a(P),K(A),s&&a(Ps),K(S,s),s&&a(zs),s&&a($),s&&a(qs),s&&a(w),s&&a(Ls),s&&a(y),s&&a(As),s&&a(E),s&&a(Ss),s&&a(z),K(D),s&&a(Bs),s&&a(V),s&&a(Ds),s&&a(j),s&&a(Ns),s&&a(u),s&&a(Rs),s&&a(f),s&&a(Vs),s&&a(L),s&&a(sa),s&&a(as),s&&a(aa),s&&a(es),s&&a(ea),s&&a(C),s&&a(ta),s&&a(ts),s&&a(na),s&&a(q),K(N),s&&a(la),s&&a(ns),s&&a(ia),K(O,s),s&&a(pa),s&&a(ls),s&&a(oa),K(F,s),s&&a(ma),s&&a(g),s&&a(ra),K(H,s),s&&a(ha),s&&a(is),s&&a(ca),s&&a(d)}}}const mt={local:"perplexity-of-fixedlength-models",sections:[{local:"calculating-ppl-with-fixedlength-models",title:"Calculating PPL with fixed-length models"},{local:"example-calculating-perplexity-with-gpt2-in-transformers",title:"Example: Calculating perplexity with GPT-2 in \u{1F917} Transformers"}],title:"Perplexity of fixed-length models"};function rt(de){return it(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ut extends at{constructor(k){super();et(this,k,rt,ot,tt,{})}}export{ut as default,mt as metadata};
