import{s as Ra,o as Xa,n as Ka}from"../chunks/scheduler.9bc65507.js";import{S as Ga,i as Ja,g as l,s as a,r as m,A as Qa,h as s,f as n,c as o,j as v,u as p,x as d,k as _,y as t,a as i,v as c,d as g,t as f,w as u}from"../chunks/index.24549926.js";import{T as Ua}from"../chunks/Tip.2b78d0ae.js";import{D as C}from"../chunks/Docstring.faf778e5.js";import{H}from"../chunks/Heading.cc20227a.js";function Ya(De){let h,L=`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the <code>&quot;special_tokens_mask&quot;</code> key, as returned by a <a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or a
<a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> with the argument <code>return_special_tokens_mask=True</code>.`;return{c(){h=l("p"),h.innerHTML=L},l($){h=s($,"P",{"data-svelte-h":!0}),d(h)!=="svelte-1gijru4"&&(h.innerHTML=L)},m($,V){i($,h,V)},p:Ka,d($){$&&n(h)}}}function Za(De){let h,L=`This collator relies on details of the implementation of subword tokenization by <code>BertTokenizer</code>, specifically
that subword tokens are prefixed with <em>##</em>. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to <code>.DataCollatorForLanguageModeling</code>.`;return{c(){h=l("p"),h.innerHTML=L},l($){h=s($,"P",{"data-svelte-h":!0}),d(h)!=="svelte-3v0y1u"&&(h.innerHTML=L)},m($,V){i($,h,V)},p:Ka,d($){$&&n(h)}}}function eo(De){let h,L,$,V,X,tt,G,ua="Data collators是一个对象，通过使用数据集元素列表作为输入来形成一个批次。这些元素与 <code>train_dataset</code> 或 <code>eval_dataset</code> 的元素类型相同。",at,J,ha='为了能够构建批次，Data collators可能会应用一些预处理（比如填充）。其中一些（比如<a href="/docs/transformers/main/zh/main_classes/data_collator#transformers.DataCollatorForLanguageModeling">DataCollatorForLanguageModeling</a>）还会在形成的批次上应用一些随机数据增强（比如随机掩码）。',ot,Q,_a='在<a href="../examples">示例脚本</a>或<a href="../notebooks">示例notebooks</a>中可以找到使用的示例。',nt,Y,rt,D,Z,Lt,we,ba=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,qt,Pe,va="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",Mt,ze,$a=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,lt,ee,st,y,te,St,Fe,ka=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,It,Le,xa="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",At,qe,Ca=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,Wt,Me,ya=`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`,it,ae,dt,I,oe,Ht,Se,Ta="Data collator that will dynamically pad the inputs received.",mt,ne,pt,A,re,Vt,Ie,Da="Data collator that will dynamically pad the inputs received, as well as the labels.",ct,le,gt,W,se,Et,Ae,wa="Data collator that will dynamically pad the inputs received, as well as the labels.",ft,ie,ut,k,de,jt,We,Pa=`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`,Nt,E,Ot,j,me,Bt,He,za="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Ut,N,pe,Kt,Ve,Fa="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Rt,O,ce,Xt,Ee,La="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",ht,ge,_t,b,fe,Gt,je,qa="Data collator used for language modeling that masks entire words.",Jt,Ne,Ma="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for masked language modeling</li>",Qt,B,Yt,U,ue,Zt,Oe,Sa=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,ea,K,he,ta,Be,Ia=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,aa,R,_e,oa,Ue,Aa=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,bt,be,vt,x,ve,na,Ke,Wa="Data collator used for permutation language modeling.",ra,Re,Ha="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for permutation language modeling with procedures specific to XLNet</li>",la,q,$e,sa,Xe,Va="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",ia,ke,Ea=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,da,M,xe,ma,Ge,ja="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",pa,Ce,Na=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,ca,S,ye,ga,Je,Oa="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",fa,Te,Ba=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,$t,et,kt;return X=new H({props:{title:"Data Collator",local:"data-collator",headingTag:"h1"}}),Y=new H({props:{title:"Default data collator",local:"transformers.default_data_collator",headingTag:"h2"}}),Z=new C({props:{name:"transformers.default_data_collator",anchor:"transformers.default_data_collator",parameters:[{name:"features",val:": typing.List[InputDataClass]"},{name:"return_tensors",val:" = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L52"}}),ee=new H({props:{title:"DefaultDataCollator",local:"transformers.DefaultDataCollator",headingTag:"h2"}}),te=new C({props:{name:"class transformers.DefaultDataCollator",anchor:"transformers.DefaultDataCollator",parameters:[{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DefaultDataCollator.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L78"}}),ae=new H({props:{title:"DataCollatorWithPadding",local:"transformers.DataCollatorWithPadding",headingTag:"h2"}}),oe=new C({props:{name:"class transformers.DataCollatorWithPadding",anchor:"transformers.DataCollatorWithPadding",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorWithPadding.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorWithPadding.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <code>PaddingStrategy</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorWithPadding.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorWithPadding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorWithPadding.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L215"}}),ne=new H({props:{title:"DataCollatorForTokenClassification",local:"transformers.DataCollatorForTokenClassification",headingTag:"h2"}}),re=new C({props:{name:"class transformers.DataCollatorForTokenClassification",anchor:"transformers.DataCollatorForTokenClassification",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForTokenClassification.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForTokenClassification.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <code>PaddingStrategy</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForTokenClassification.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForTokenClassification.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForTokenClassification.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForTokenClassification.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L266"}}),le=new H({props:{title:"DataCollatorForSeq2Seq",local:"transformers.DataCollatorForSeq2Seq",headingTag:"h2"}}),se=new C({props:{name:"class transformers.DataCollatorForSeq2Seq",anchor:"transformers.DataCollatorForSeq2Seq",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"model",val:": typing.Optional[typing.Any] = None"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForSeq2Seq.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForSeq2Seq.model",description:`<strong>model</strong> (<a href="/docs/transformers/main/zh/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>, <em>optional</em>) &#x2014;
The model that is being trained. If set and has the <em>prepare_decoder_input_ids_from_labels</em>, use it to
prepare the <em>decoder_input_ids</em></p>
<p>This is useful when using <em>label_smoothing</em> to avoid calculating loss twice.`,name:"model"},{anchor:"transformers.DataCollatorForSeq2Seq.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <code>PaddingStrategy</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForSeq2Seq.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForSeq2Seq.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForSeq2Seq.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForSeq2Seq.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L517"}}),ie=new H({props:{title:"DataCollatorForLanguageModeling",local:"transformers.DataCollatorForLanguageModeling",headingTag:"h2"}}),de=new C({props:{name:"class transformers.DataCollatorForLanguageModeling",anchor:"transformers.DataCollatorForLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForLanguageModeling.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/zh/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm",description:`<strong>mlm</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to use masked language modeling. If set to <code>False</code>, the labels are the same as the inputs
with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked
tokens and the value to predict for the masked token.`,name:"mlm"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm_probability",description:`<strong>mlm_probability</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) &#x2014;
The probability with which to (randomly) mask tokens in the input, when <code>mlm</code> is set to <code>True</code>.`,name:"mlm_probability"},{anchor:"transformers.DataCollatorForLanguageModeling.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForLanguageModeling.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L607"}}),E=new Ua({props:{$$slots:{default:[Ya]},$$scope:{ctx:De}}}),me=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L806"}}),pe=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"vocab_size",val:""},{name:"mask_token_id",val:""},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L659"}}),ce=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L751"}}),ge=new H({props:{title:"DataCollatorForWholeWordMask",local:"transformers.DataCollatorForWholeWordMask",headingTag:"h2"}}),fe=new C({props:{name:"class transformers.DataCollatorForWholeWordMask",anchor:"transformers.DataCollatorForWholeWordMask",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L845"}}),B=new Ua({props:{$$slots:{default:[Za]},$$scope:{ctx:De}}}),ue=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1075"}}),he=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1033"}}),_e=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L993"}}),be=new H({props:{title:"DataCollatorForPermutationLanguageModeling",local:"transformers.DataCollatorForPermutationLanguageModeling",headingTag:"h2"}}),ve=new C({props:{name:"class transformers.DataCollatorForPermutationLanguageModeling",anchor:"transformers.DataCollatorForPermutationLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"plm_probability",val:": float = 0.16666666666666666"},{name:"max_span_length",val:": int = 5"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1200"}}),$e=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1440"}}),xe=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1333"}}),ye=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1234"}}),{c(){h=l("meta"),L=a(),$=l("p"),V=a(),m(X.$$.fragment),tt=a(),G=l("p"),G.innerHTML=ua,at=a(),J=l("p"),J.innerHTML=ha,ot=a(),Q=l("p"),Q.innerHTML=_a,nt=a(),m(Y.$$.fragment),rt=a(),D=l("div"),m(Z.$$.fragment),Lt=a(),we=l("p"),we.textContent=ba,qt=a(),Pe=l("ul"),Pe.innerHTML=va,Mt=a(),ze=l("p"),ze.textContent=$a,lt=a(),m(ee.$$.fragment),st=a(),y=l("div"),m(te.$$.fragment),St=a(),Fe=l("p"),Fe.textContent=ka,It=a(),Le=l("ul"),Le.innerHTML=xa,At=a(),qe=l("p"),qe.textContent=Ca,Wt=a(),Me=l("p"),Me.textContent=ya,it=a(),m(ae.$$.fragment),dt=a(),I=l("div"),m(oe.$$.fragment),Ht=a(),Se=l("p"),Se.textContent=Ta,mt=a(),m(ne.$$.fragment),pt=a(),A=l("div"),m(re.$$.fragment),Vt=a(),Ie=l("p"),Ie.textContent=Da,ct=a(),m(le.$$.fragment),gt=a(),W=l("div"),m(se.$$.fragment),Et=a(),Ae=l("p"),Ae.textContent=wa,ft=a(),m(ie.$$.fragment),ut=a(),k=l("div"),m(de.$$.fragment),jt=a(),We=l("p"),We.textContent=Pa,Nt=a(),m(E.$$.fragment),Ot=a(),j=l("div"),m(me.$$.fragment),Bt=a(),He=l("p"),He.textContent=za,Ut=a(),N=l("div"),m(pe.$$.fragment),Kt=a(),Ve=l("p"),Ve.textContent=Fa,Rt=a(),O=l("div"),m(ce.$$.fragment),Xt=a(),Ee=l("p"),Ee.textContent=La,ht=a(),m(ge.$$.fragment),_t=a(),b=l("div"),m(fe.$$.fragment),Gt=a(),je=l("p"),je.textContent=qa,Jt=a(),Ne=l("ul"),Ne.innerHTML=Ma,Qt=a(),m(B.$$.fragment),Yt=a(),U=l("div"),m(ue.$$.fragment),Zt=a(),Oe=l("p"),Oe.textContent=Sa,ea=a(),K=l("div"),m(he.$$.fragment),ta=a(),Be=l("p"),Be.textContent=Ia,aa=a(),R=l("div"),m(_e.$$.fragment),oa=a(),Ue=l("p"),Ue.textContent=Aa,bt=a(),m(be.$$.fragment),vt=a(),x=l("div"),m(ve.$$.fragment),na=a(),Ke=l("p"),Ke.textContent=Wa,ra=a(),Re=l("ul"),Re.innerHTML=Ha,la=a(),q=l("div"),m($e.$$.fragment),sa=a(),Xe=l("p"),Xe.textContent=Va,ia=a(),ke=l("ol"),ke.innerHTML=Ea,da=a(),M=l("div"),m(xe.$$.fragment),ma=a(),Ge=l("p"),Ge.textContent=ja,pa=a(),Ce=l("ol"),Ce.innerHTML=Na,ca=a(),S=l("div"),m(ye.$$.fragment),ga=a(),Je=l("p"),Je.textContent=Oa,fa=a(),Te=l("ol"),Te.innerHTML=Ba,$t=a(),et=l("p"),this.h()},l(e){const r=Qa("svelte-u9bgzb",document.head);h=s(r,"META",{name:!0,content:!0}),r.forEach(n),L=o(e),$=s(e,"P",{}),v($).forEach(n),V=o(e),p(X.$$.fragment,e),tt=o(e),G=s(e,"P",{"data-svelte-h":!0}),d(G)!=="svelte-1t1zoyx"&&(G.innerHTML=ua),at=o(e),J=s(e,"P",{"data-svelte-h":!0}),d(J)!=="svelte-aec1vj"&&(J.innerHTML=ha),ot=o(e),Q=s(e,"P",{"data-svelte-h":!0}),d(Q)!=="svelte-54nsx2"&&(Q.innerHTML=_a),nt=o(e),p(Y.$$.fragment,e),rt=o(e),D=s(e,"DIV",{class:!0});var F=v(D);p(Z.$$.fragment,F),Lt=o(F),we=s(F,"P",{"data-svelte-h":!0}),d(we)!=="svelte-1hmsgsg"&&(we.textContent=ba),qt=o(F),Pe=s(F,"UL",{"data-svelte-h":!0}),d(Pe)!=="svelte-1pq8qks"&&(Pe.innerHTML=va),Mt=o(F),ze=s(F,"P",{"data-svelte-h":!0}),d(ze)!=="svelte-1vi1gug"&&(ze.textContent=$a),F.forEach(n),lt=o(e),p(ee.$$.fragment,e),st=o(e),y=s(e,"DIV",{class:!0});var w=v(y);p(te.$$.fragment,w),St=o(w),Fe=s(w,"P",{"data-svelte-h":!0}),d(Fe)!=="svelte-1hmsgsg"&&(Fe.textContent=ka),It=o(w),Le=s(w,"UL",{"data-svelte-h":!0}),d(Le)!=="svelte-1pq8qks"&&(Le.innerHTML=xa),At=o(w),qe=s(w,"P",{"data-svelte-h":!0}),d(qe)!=="svelte-1vi1gug"&&(qe.textContent=Ca),Wt=o(w),Me=s(w,"P",{"data-svelte-h":!0}),d(Me)!=="svelte-vfvbwr"&&(Me.textContent=ya),w.forEach(n),it=o(e),p(ae.$$.fragment,e),dt=o(e),I=s(e,"DIV",{class:!0});var xt=v(I);p(oe.$$.fragment,xt),Ht=o(xt),Se=s(xt,"P",{"data-svelte-h":!0}),d(Se)!=="svelte-1iebpai"&&(Se.textContent=Ta),xt.forEach(n),mt=o(e),p(ne.$$.fragment,e),pt=o(e),A=s(e,"DIV",{class:!0});var Ct=v(A);p(re.$$.fragment,Ct),Vt=o(Ct),Ie=s(Ct,"P",{"data-svelte-h":!0}),d(Ie)!=="svelte-4uvw1w"&&(Ie.textContent=Da),Ct.forEach(n),ct=o(e),p(le.$$.fragment,e),gt=o(e),W=s(e,"DIV",{class:!0});var yt=v(W);p(se.$$.fragment,yt),Et=o(yt),Ae=s(yt,"P",{"data-svelte-h":!0}),d(Ae)!=="svelte-4uvw1w"&&(Ae.textContent=wa),yt.forEach(n),ft=o(e),p(ie.$$.fragment,e),ut=o(e),k=s(e,"DIV",{class:!0});var P=v(k);p(de.$$.fragment,P),jt=o(P),We=s(P,"P",{"data-svelte-h":!0}),d(We)!=="svelte-10km8jk"&&(We.textContent=Pa),Nt=o(P),p(E.$$.fragment,P),Ot=o(P),j=s(P,"DIV",{class:!0});var Tt=v(j);p(me.$$.fragment,Tt),Bt=o(Tt),He=s(Tt,"P",{"data-svelte-h":!0}),d(He)!=="svelte-iv4xqf"&&(He.textContent=za),Tt.forEach(n),Ut=o(P),N=s(P,"DIV",{class:!0});var Dt=v(N);p(pe.$$.fragment,Dt),Kt=o(Dt),Ve=s(Dt,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-iv4xqf"&&(Ve.textContent=Fa),Dt.forEach(n),Rt=o(P),O=s(P,"DIV",{class:!0});var wt=v(O);p(ce.$$.fragment,wt),Xt=o(wt),Ee=s(wt,"P",{"data-svelte-h":!0}),d(Ee)!=="svelte-iv4xqf"&&(Ee.textContent=La),wt.forEach(n),P.forEach(n),ht=o(e),p(ge.$$.fragment,e),_t=o(e),b=s(e,"DIV",{class:!0});var T=v(b);p(fe.$$.fragment,T),Gt=o(T),je=s(T,"P",{"data-svelte-h":!0}),d(je)!=="svelte-sjqk55"&&(je.textContent=qa),Jt=o(T),Ne=s(T,"UL",{"data-svelte-h":!0}),d(Ne)!=="svelte-d2ozxj"&&(Ne.innerHTML=Ma),Qt=o(T),p(B.$$.fragment,T),Yt=o(T),U=s(T,"DIV",{class:!0});var Pt=v(U);p(ue.$$.fragment,Pt),Zt=o(Pt),Oe=s(Pt,"P",{"data-svelte-h":!0}),d(Oe)!=="svelte-10msjh1"&&(Oe.textContent=Sa),Pt.forEach(n),ea=o(T),K=s(T,"DIV",{class:!0});var zt=v(K);p(he.$$.fragment,zt),ta=o(zt),Be=s(zt,"P",{"data-svelte-h":!0}),d(Be)!=="svelte-10msjh1"&&(Be.textContent=Ia),zt.forEach(n),aa=o(T),R=s(T,"DIV",{class:!0});var Ft=v(R);p(_e.$$.fragment,Ft),oa=o(Ft),Ue=s(Ft,"P",{"data-svelte-h":!0}),d(Ue)!=="svelte-10msjh1"&&(Ue.textContent=Aa),Ft.forEach(n),T.forEach(n),bt=o(e),p(be.$$.fragment,e),vt=o(e),x=s(e,"DIV",{class:!0});var z=v(x);p(ve.$$.fragment,z),na=o(z),Ke=s(z,"P",{"data-svelte-h":!0}),d(Ke)!=="svelte-12inif1"&&(Ke.textContent=Wa),ra=o(z),Re=s(z,"UL",{"data-svelte-h":!0}),d(Re)!=="svelte-1t4qh7k"&&(Re.innerHTML=Ha),la=o(z),q=s(z,"DIV",{class:!0});var Qe=v(q);p($e.$$.fragment,Qe),sa=o(Qe),Xe=s(Qe,"P",{"data-svelte-h":!0}),d(Xe)!=="svelte-1u70phx"&&(Xe.textContent=Va),ia=o(Qe),ke=s(Qe,"OL",{start:!0,"data-svelte-h":!0}),d(ke)!=="svelte-1r6jrbi"&&(ke.innerHTML=Ea),Qe.forEach(n),da=o(z),M=s(z,"DIV",{class:!0});var Ye=v(M);p(xe.$$.fragment,Ye),ma=o(Ye),Ge=s(Ye,"P",{"data-svelte-h":!0}),d(Ge)!=="svelte-1u70phx"&&(Ge.textContent=ja),pa=o(Ye),Ce=s(Ye,"OL",{start:!0,"data-svelte-h":!0}),d(Ce)!=="svelte-1r6jrbi"&&(Ce.innerHTML=Na),Ye.forEach(n),ca=o(z),S=s(z,"DIV",{class:!0});var Ze=v(S);p(ye.$$.fragment,Ze),ga=o(Ze),Je=s(Ze,"P",{"data-svelte-h":!0}),d(Je)!=="svelte-1u70phx"&&(Je.textContent=Oa),fa=o(Ze),Te=s(Ze,"OL",{start:!0,"data-svelte-h":!0}),d(Te)!=="svelte-1r6jrbi"&&(Te.innerHTML=Ba),Ze.forEach(n),z.forEach(n),$t=o(e),et=s(e,"P",{}),v(et).forEach(n),this.h()},h(){_(h,"name","hf:doc:metadata"),_(h,"content",to),_(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(ke,"start","0"),_(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Ce,"start","0"),_(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Te,"start","0"),_(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){t(document.head,h),i(e,L,r),i(e,$,r),i(e,V,r),c(X,e,r),i(e,tt,r),i(e,G,r),i(e,at,r),i(e,J,r),i(e,ot,r),i(e,Q,r),i(e,nt,r),c(Y,e,r),i(e,rt,r),i(e,D,r),c(Z,D,null),t(D,Lt),t(D,we),t(D,qt),t(D,Pe),t(D,Mt),t(D,ze),i(e,lt,r),c(ee,e,r),i(e,st,r),i(e,y,r),c(te,y,null),t(y,St),t(y,Fe),t(y,It),t(y,Le),t(y,At),t(y,qe),t(y,Wt),t(y,Me),i(e,it,r),c(ae,e,r),i(e,dt,r),i(e,I,r),c(oe,I,null),t(I,Ht),t(I,Se),i(e,mt,r),c(ne,e,r),i(e,pt,r),i(e,A,r),c(re,A,null),t(A,Vt),t(A,Ie),i(e,ct,r),c(le,e,r),i(e,gt,r),i(e,W,r),c(se,W,null),t(W,Et),t(W,Ae),i(e,ft,r),c(ie,e,r),i(e,ut,r),i(e,k,r),c(de,k,null),t(k,jt),t(k,We),t(k,Nt),c(E,k,null),t(k,Ot),t(k,j),c(me,j,null),t(j,Bt),t(j,He),t(k,Ut),t(k,N),c(pe,N,null),t(N,Kt),t(N,Ve),t(k,Rt),t(k,O),c(ce,O,null),t(O,Xt),t(O,Ee),i(e,ht,r),c(ge,e,r),i(e,_t,r),i(e,b,r),c(fe,b,null),t(b,Gt),t(b,je),t(b,Jt),t(b,Ne),t(b,Qt),c(B,b,null),t(b,Yt),t(b,U),c(ue,U,null),t(U,Zt),t(U,Oe),t(b,ea),t(b,K),c(he,K,null),t(K,ta),t(K,Be),t(b,aa),t(b,R),c(_e,R,null),t(R,oa),t(R,Ue),i(e,bt,r),c(be,e,r),i(e,vt,r),i(e,x,r),c(ve,x,null),t(x,na),t(x,Ke),t(x,ra),t(x,Re),t(x,la),t(x,q),c($e,q,null),t(q,sa),t(q,Xe),t(q,ia),t(q,ke),t(x,da),t(x,M),c(xe,M,null),t(M,ma),t(M,Ge),t(M,pa),t(M,Ce),t(x,ca),t(x,S),c(ye,S,null),t(S,ga),t(S,Je),t(S,fa),t(S,Te),i(e,$t,r),i(e,et,r),kt=!0},p(e,[r]){const F={};r&2&&(F.$$scope={dirty:r,ctx:e}),E.$set(F);const w={};r&2&&(w.$$scope={dirty:r,ctx:e}),B.$set(w)},i(e){kt||(g(X.$$.fragment,e),g(Y.$$.fragment,e),g(Z.$$.fragment,e),g(ee.$$.fragment,e),g(te.$$.fragment,e),g(ae.$$.fragment,e),g(oe.$$.fragment,e),g(ne.$$.fragment,e),g(re.$$.fragment,e),g(le.$$.fragment,e),g(se.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(E.$$.fragment,e),g(me.$$.fragment,e),g(pe.$$.fragment,e),g(ce.$$.fragment,e),g(ge.$$.fragment,e),g(fe.$$.fragment,e),g(B.$$.fragment,e),g(ue.$$.fragment,e),g(he.$$.fragment,e),g(_e.$$.fragment,e),g(be.$$.fragment,e),g(ve.$$.fragment,e),g($e.$$.fragment,e),g(xe.$$.fragment,e),g(ye.$$.fragment,e),kt=!0)},o(e){f(X.$$.fragment,e),f(Y.$$.fragment,e),f(Z.$$.fragment,e),f(ee.$$.fragment,e),f(te.$$.fragment,e),f(ae.$$.fragment,e),f(oe.$$.fragment,e),f(ne.$$.fragment,e),f(re.$$.fragment,e),f(le.$$.fragment,e),f(se.$$.fragment,e),f(ie.$$.fragment,e),f(de.$$.fragment,e),f(E.$$.fragment,e),f(me.$$.fragment,e),f(pe.$$.fragment,e),f(ce.$$.fragment,e),f(ge.$$.fragment,e),f(fe.$$.fragment,e),f(B.$$.fragment,e),f(ue.$$.fragment,e),f(he.$$.fragment,e),f(_e.$$.fragment,e),f(be.$$.fragment,e),f(ve.$$.fragment,e),f($e.$$.fragment,e),f(xe.$$.fragment,e),f(ye.$$.fragment,e),kt=!1},d(e){e&&(n(L),n($),n(V),n(tt),n(G),n(at),n(J),n(ot),n(Q),n(nt),n(rt),n(D),n(lt),n(st),n(y),n(it),n(dt),n(I),n(mt),n(pt),n(A),n(ct),n(gt),n(W),n(ft),n(ut),n(k),n(ht),n(_t),n(b),n(bt),n(vt),n(x),n($t),n(et)),n(h),u(X,e),u(Y,e),u(Z),u(ee,e),u(te),u(ae,e),u(oe),u(ne,e),u(re),u(le,e),u(se),u(ie,e),u(de),u(E),u(me),u(pe),u(ce),u(ge,e),u(fe),u(B),u(ue),u(he),u(_e),u(be,e),u(ve),u($e),u(xe),u(ye)}}}const to='{"title":"Data Collator","local":"data-collator","sections":[{"title":"Default data collator","local":"transformers.default_data_collator","sections":[],"depth":2},{"title":"DefaultDataCollator","local":"transformers.DefaultDataCollator","sections":[],"depth":2},{"title":"DataCollatorWithPadding","local":"transformers.DataCollatorWithPadding","sections":[],"depth":2},{"title":"DataCollatorForTokenClassification","local":"transformers.DataCollatorForTokenClassification","sections":[],"depth":2},{"title":"DataCollatorForSeq2Seq","local":"transformers.DataCollatorForSeq2Seq","sections":[],"depth":2},{"title":"DataCollatorForLanguageModeling","local":"transformers.DataCollatorForLanguageModeling","sections":[],"depth":2},{"title":"DataCollatorForWholeWordMask","local":"transformers.DataCollatorForWholeWordMask","sections":[],"depth":2},{"title":"DataCollatorForPermutationLanguageModeling","local":"transformers.DataCollatorForPermutationLanguageModeling","sections":[],"depth":2}],"depth":1}';function ao(De){return Xa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class io extends Ga{constructor(h){super(),Ja(this,h,ao,eo,Ra,{})}}export{io as component};
