import{s as lt,o as st}from"../chunks/scheduler.37c15a92.js";import{S as nt,i as at,g as b,s as o,r as u,A as ot,h,f as l,c as i,j as Le,u as m,x as w,k as Ke,y as it,a as n,v as M,t as r,b as Oe,d as c,w as d,p as et}from"../chunks/index.2bf4358c.js";import{C as J}from"../chunks/CodeBlock.4e987730.js";import{C as tt}from"../chunks/CourseFloatingBanner.15ba07e6.js";import{F as pt}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as Xe,E as rt}from"../chunks/index.fb305b5e.js";function ct(T){let a,p;return a=new tt({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"}]}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function ut(T){let a,p;return a=new tt({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"}]}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function mt(T){let a,p;return a=new J({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBc2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoKip0b2tlbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function Mt(T){let a,p;return a=new J({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function dt(T){let a,p,s,y,g,se,Z,ne,f,j,ee,k,Ne="In the last few sections, we’ve been trying our best to do most of the work by hand. We’ve explored how tokenizers work and looked at tokenization, conversion to input IDs, padding, truncation, and attention masks.",ae,V,ze="However, as we saw in section 2, the 🤗 Transformers API can handle all of this for us with a high-level function that we’ll dive into here. When you call your <code>tokenizer</code> directly on the sentence, you get back inputs that are ready to pass through your model:",oe,$,ie,W,ve="Here, the <code>model_inputs</code> variable contains everything that’s necessary for a model to operate well. For DistilBERT, that includes the input IDs as well as the attention mask. Other models that accept additional inputs will also have those output by the <code>tokenizer</code> object.",pe,B,xe="As we’ll see in some examples below, this method is very powerful. First, it can tokenize a single sequence:",re,G,ce,X,_e="It also handles multiple sequences at a time, with no change in the API:",ue,N,me,z,Se="It can pad according to several objectives:",Me,v,de,x,qe="It can also truncate sequences:",be,_,he,S,Ee="The <code>tokenizer</code> object can handle the conversion to specific framework tensors, which can then be directly sent to the model. For example, in the following code sample we are prompting the tokenizer to return tensors from the different frameworks — <code>&quot;pt&quot;</code> returns PyTorch tensors, <code>&quot;tf&quot;</code> returns TensorFlow tensors, and <code>&quot;np&quot;</code> returns NumPy arrays:",ye,q,we,E,Je,R,Re="If we take a look at the input IDs returned by the tokenizer, we will see they are a tiny bit different from what we had earlier:",Te,C,fe,H,je,Y,Ce="One token ID was added at the beginning, and one at the end. Let’s decode the two sequences of IDs above to see what this is about:",Ie,Q,Ue,F,ge,A,He="The tokenizer added the special word <code>[CLS]</code> at the beginning and the special word <code>[SEP]</code> at the end. This is because the model was pretrained with those, so to get the same results for inference we need to add them as well. Note that some models don’t add special words, or add different ones; models may also add these special words only at the beginning, or only at the end. In any case, the tokenizer knows which ones are expected and will deal with this for you.",Ze,D,ke,P,Ye="Now that we’ve seen all the individual steps the <code>tokenizer</code> object uses when applied on texts, let’s see one final time how it can handle multiple sequences (padding!), very long sequences (truncation!), and multiple types of tensors with its main API:",Ve,I,U,te,L,$e,le,We;g=new pt({props:{fw:T[0]}}),Z=new Xe({props:{title:"Putting it all together",local:"putting-it-all-together",headingTag:"h1"}});const Qe=[ut,ct],K=[];function Fe(e,t){return e[0]==="pt"?0:1}f=Fe(T),j=K[f]=Qe[f](T),$=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),G=new J({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),N=new J({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),v=new J({props:{code:"JTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtYXhpbXVtJTIwc2VxdWVuY2UlMjBsZW5ndGglMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMmxvbmdlc3QlMjIpJTBBJTBBJTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtb2RlbCUyMG1heCUyMGxlbmd0aCUwQSUyMyUyMCg1MTIlMjBmb3IlMjBCRVJUJTIwb3IlMjBEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiklMEElMEElMjMlMjBXaWxsJTIwcGFkJTIwdGhlJTIwc2VxdWVuY2VzJTIwdXAlMjB0byUyMHRoZSUyMHNwZWNpZmllZCUyMG1heCUyMGxlbmd0aCUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiUyQyUyMG1heF9sZW5ndGglM0Q4KQ==",highlighted:`<span class="hljs-comment"># Will pad the sequences up to the maximum sequence length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the specified max length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),_=new J({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBXaWxsJTIwdHJ1bmNhdGUlMjB0aGUlMjBzZXF1ZW5jZXMlMjB0aGF0JTIwYXJlJTIwbG9uZ2VyJTIwdGhhbiUyMHRoZSUyMG1vZGVsJTIwbWF4JTIwbGVuZ3RoJTBBJTIzJTIwKDUxMiUyMGZvciUyMEJFUlQlMjBvciUyMERpc3RpbEJFUlQpJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQSUwQSUyMyUyMFdpbGwlMjB0cnVuY2F0ZSUyMHRoZSUyMHNlcXVlbmNlcyUyMHRoYXQlMjBhcmUlMjBsb25nZXIlMjB0aGFuJTIwdGhlJTIwc3BlY2lmaWVkJTIwbWF4JTIwbGVuZ3RoJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMG1heF9sZW5ndGglM0Q4JTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Will truncate the sequences that are longer than the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Will truncate the sequences that are longer than the specified max length</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),q=new J({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBSZXR1cm5zJTIwUHlUb3JjaCUyMHRlbnNvcnMlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMFJldHVybnMlMjBUZW5zb3JGbG93JTIwdGVuc29ycyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBJTBBJTIzJTIwUmV0dXJucyUyME51bVB5JTIwYXJyYXlzJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJucCUyMik=",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Returns PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Returns TensorFlow tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Returns NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),E=new Xe({props:{title:"Special tokens",local:"special-tokens",headingTag:"h2"}}),C=new J({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),H=new J({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),Q=new J({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),F=new J({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),D=new Xe({props:{title:"Wrapping up: From tokenizer to model",local:"wrapping-up-from-tokenizer-to-model",headingTag:"h2"}});const Ae=[Mt,mt],O=[];function De(e,t){return e[0]==="pt"?0:1}return I=De(T),U=O[I]=Ae[I](T),L=new rt({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter2/6.mdx"}}),{c(){a=b("meta"),p=o(),s=b("p"),y=o(),u(g.$$.fragment),se=o(),u(Z.$$.fragment),ne=o(),j.c(),ee=o(),k=b("p"),k.textContent=Ne,ae=o(),V=b("p"),V.innerHTML=ze,oe=o(),u($.$$.fragment),ie=o(),W=b("p"),W.innerHTML=ve,pe=o(),B=b("p"),B.textContent=xe,re=o(),u(G.$$.fragment),ce=o(),X=b("p"),X.textContent=_e,ue=o(),u(N.$$.fragment),me=o(),z=b("p"),z.textContent=Se,Me=o(),u(v.$$.fragment),de=o(),x=b("p"),x.textContent=qe,be=o(),u(_.$$.fragment),he=o(),S=b("p"),S.innerHTML=Ee,ye=o(),u(q.$$.fragment),we=o(),u(E.$$.fragment),Je=o(),R=b("p"),R.textContent=Re,Te=o(),u(C.$$.fragment),fe=o(),u(H.$$.fragment),je=o(),Y=b("p"),Y.textContent=Ce,Ie=o(),u(Q.$$.fragment),Ue=o(),u(F.$$.fragment),ge=o(),A=b("p"),A.innerHTML=He,Ze=o(),u(D.$$.fragment),ke=o(),P=b("p"),P.innerHTML=Ye,Ve=o(),U.c(),te=o(),u(L.$$.fragment),$e=o(),le=b("p"),this.h()},l(e){const t=ot("svelte-u9bgzb",document.head);a=h(t,"META",{name:!0,content:!0}),t.forEach(l),p=i(e),s=h(e,"P",{}),Le(s).forEach(l),y=i(e),m(g.$$.fragment,e),se=i(e),m(Z.$$.fragment,e),ne=i(e),j.l(e),ee=i(e),k=h(e,"P",{"data-svelte-h":!0}),w(k)!=="svelte-8x3t43"&&(k.textContent=Ne),ae=i(e),V=h(e,"P",{"data-svelte-h":!0}),w(V)!=="svelte-1m0etnl"&&(V.innerHTML=ze),oe=i(e),m($.$$.fragment,e),ie=i(e),W=h(e,"P",{"data-svelte-h":!0}),w(W)!=="svelte-1ppe5qh"&&(W.innerHTML=ve),pe=i(e),B=h(e,"P",{"data-svelte-h":!0}),w(B)!=="svelte-y34ype"&&(B.textContent=xe),re=i(e),m(G.$$.fragment,e),ce=i(e),X=h(e,"P",{"data-svelte-h":!0}),w(X)!=="svelte-164017d"&&(X.textContent=_e),ue=i(e),m(N.$$.fragment,e),me=i(e),z=h(e,"P",{"data-svelte-h":!0}),w(z)!=="svelte-lvhia5"&&(z.textContent=Se),Me=i(e),m(v.$$.fragment,e),de=i(e),x=h(e,"P",{"data-svelte-h":!0}),w(x)!=="svelte-rx5s8"&&(x.textContent=qe),be=i(e),m(_.$$.fragment,e),he=i(e),S=h(e,"P",{"data-svelte-h":!0}),w(S)!=="svelte-114spo5"&&(S.innerHTML=Ee),ye=i(e),m(q.$$.fragment,e),we=i(e),m(E.$$.fragment,e),Je=i(e),R=h(e,"P",{"data-svelte-h":!0}),w(R)!=="svelte-1tipyqt"&&(R.textContent=Re),Te=i(e),m(C.$$.fragment,e),fe=i(e),m(H.$$.fragment,e),je=i(e),Y=h(e,"P",{"data-svelte-h":!0}),w(Y)!=="svelte-s6ie2x"&&(Y.textContent=Ce),Ie=i(e),m(Q.$$.fragment,e),Ue=i(e),m(F.$$.fragment,e),ge=i(e),A=h(e,"P",{"data-svelte-h":!0}),w(A)!=="svelte-yfjwtr"&&(A.innerHTML=He),Ze=i(e),m(D.$$.fragment,e),ke=i(e),P=h(e,"P",{"data-svelte-h":!0}),w(P)!=="svelte-mudtzg"&&(P.innerHTML=Ye),Ve=i(e),U.l(e),te=i(e),m(L.$$.fragment,e),$e=i(e),le=h(e,"P",{}),Le(le).forEach(l),this.h()},h(){Ke(a,"name","hf:doc:metadata"),Ke(a,"content",bt)},m(e,t){it(document.head,a),n(e,p,t),n(e,s,t),n(e,y,t),M(g,e,t),n(e,se,t),M(Z,e,t),n(e,ne,t),K[f].m(e,t),n(e,ee,t),n(e,k,t),n(e,ae,t),n(e,V,t),n(e,oe,t),M($,e,t),n(e,ie,t),n(e,W,t),n(e,pe,t),n(e,B,t),n(e,re,t),M(G,e,t),n(e,ce,t),n(e,X,t),n(e,ue,t),M(N,e,t),n(e,me,t),n(e,z,t),n(e,Me,t),M(v,e,t),n(e,de,t),n(e,x,t),n(e,be,t),M(_,e,t),n(e,he,t),n(e,S,t),n(e,ye,t),M(q,e,t),n(e,we,t),M(E,e,t),n(e,Je,t),n(e,R,t),n(e,Te,t),M(C,e,t),n(e,fe,t),M(H,e,t),n(e,je,t),n(e,Y,t),n(e,Ie,t),M(Q,e,t),n(e,Ue,t),M(F,e,t),n(e,ge,t),n(e,A,t),n(e,Ze,t),M(D,e,t),n(e,ke,t),n(e,P,t),n(e,Ve,t),O[I].m(e,t),n(e,te,t),M(L,e,t),n(e,$e,t),n(e,le,t),We=!0},p(e,[t]){const Pe={};t&1&&(Pe.fw=e[0]),g.$set(Pe);let Be=f;f=Fe(e),f!==Be&&(et(),r(K[Be],1,1,()=>{K[Be]=null}),Oe(),j=K[f],j||(j=K[f]=Qe[f](e),j.c()),c(j,1),j.m(ee.parentNode,ee));let Ge=I;I=De(e),I!==Ge&&(et(),r(O[Ge],1,1,()=>{O[Ge]=null}),Oe(),U=O[I],U||(U=O[I]=Ae[I](e),U.c()),c(U,1),U.m(te.parentNode,te))},i(e){We||(c(g.$$.fragment,e),c(Z.$$.fragment,e),c(j),c($.$$.fragment,e),c(G.$$.fragment,e),c(N.$$.fragment,e),c(v.$$.fragment,e),c(_.$$.fragment,e),c(q.$$.fragment,e),c(E.$$.fragment,e),c(C.$$.fragment,e),c(H.$$.fragment,e),c(Q.$$.fragment,e),c(F.$$.fragment,e),c(D.$$.fragment,e),c(U),c(L.$$.fragment,e),We=!0)},o(e){r(g.$$.fragment,e),r(Z.$$.fragment,e),r(j),r($.$$.fragment,e),r(G.$$.fragment,e),r(N.$$.fragment,e),r(v.$$.fragment,e),r(_.$$.fragment,e),r(q.$$.fragment,e),r(E.$$.fragment,e),r(C.$$.fragment,e),r(H.$$.fragment,e),r(Q.$$.fragment,e),r(F.$$.fragment,e),r(D.$$.fragment,e),r(U),r(L.$$.fragment,e),We=!1},d(e){e&&(l(p),l(s),l(y),l(se),l(ne),l(ee),l(k),l(ae),l(V),l(oe),l(ie),l(W),l(pe),l(B),l(re),l(ce),l(X),l(ue),l(me),l(z),l(Me),l(de),l(x),l(be),l(he),l(S),l(ye),l(we),l(Je),l(R),l(Te),l(fe),l(je),l(Y),l(Ie),l(Ue),l(ge),l(A),l(Ze),l(ke),l(P),l(Ve),l(te),l($e),l(le)),l(a),d(g,e),d(Z,e),K[f].d(e),d($,e),d(G,e),d(N,e),d(v,e),d(_,e),d(q,e),d(E,e),d(C,e),d(H,e),d(Q,e),d(F,e),d(D,e),O[I].d(e),d(L,e)}}}const bt='{"title":"Putting it all together","local":"putting-it-all-together","sections":[{"title":"Special tokens","local":"special-tokens","sections":[],"depth":2},{"title":"Wrapping up: From tokenizer to model","local":"wrapping-up-from-tokenizer-to-model","sections":[],"depth":2}],"depth":1}';function ht(T,a,p){let s="pt";return st(()=>{const y=new URLSearchParams(window.location.search);p(0,s=y.get("fw")||"pt")}),[s]}class It extends nt{constructor(a){super(),at(this,a,ht,dt,lt,{})}}export{It as component};
