import{s as Me,n as se,o as ee}from"../chunks/scheduler.1f027d5a.js";import{S as te,i as ae,e as y,s as t,c as J,h as ye,a as n,d as s,b as a,f as Os,g as T,j as U,k as le,l as ne,m as e,n as j,t as w,o as p,p as C}from"../chunks/index.6117a65d.js";import{C as Ue}from"../chunks/CopyLLMTxtMenu.610b106b.js";import{D as Je,C as I}from"../chunks/DocNotebookDropdown.4ae0d470.js";import{H as Ql,E as Te}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.bd18ae98.js";function je(ns){let m,Sl,Fl,Nl,i,gl,A,Yl,o,Hl,c,Us='<em>作者: <a href="https://github.com/MKhalusova" rel="nofollow">Maria Khalusova</a></em>',vl,d,Js="公开发布的代码 LLM，如 Codex、StarCoder 和 Code Llama，在生成遵循通用编程原则和语法的代码方面表现出色，但它们可能不符合组织的内部惯例，或者不了解某些特定的库。",xl,r,Ts="在这个 notebook 中，我们将展示如何微调代码 LLM 来更好的理解你们公司或组织的代码风格和习惯。由于代码 LLM 非常大，按照传统的微调方式可能会消耗大量资源。但不用担心！我们会教你一些技巧，让你只用单个 GPU 就能完成微调工作。",zl,b,$l,f,js='对于这个例子，我们选择了 GitHub 上 Hugging Face 的前 10 个公共仓库。我们已经排除了非代码文件，如图片、音频文件、演示文稿等。对于 Jupyter notebook，我们只保留了包含代码的单元格。生成的代码被存储为一个数据集，你可以在 Hugging Face Hub 上找到，位于 <a href="https://huggingface.co/datasets/smangrul/hf-stack-v1" rel="nofollow"><code>smangrul/hf-stack-v1</code></a>。它包含仓库 id、文件路径和文件内容。',Ll,u,Dl,h,ws='我们将微调 <a href="https://huggingface.co/bigcode/starcoderbase-1b" rel="nofollow"><code>bigcode/starcoderbase-1b</code></a> 模型，这是一个在 80 多种编程语言上训练的 10 亿参数模型。这是一个需要权限的模型，所以如果你计划使用这个确切模型运行这个 notebook，你需要在其模型页面上获得访问权限。登录你的 Hugging Face 帐户以执行此操作：',ql,Z,Pl,B,ps="为了开始，首先让我们安装所有必要的库。正如你所看到的，除了 <code>transformers</code> 和 <code>datasets</code>，我们还将使用 <code>peft</code>、<code>bitsandbytes</code> 和 <code>flash-attn</code> 来优化训练过程。",Kl,_,Cs="通过采用参数高效的训练技术，我们可以在一张 A100 高内存 GPU 上运行这个 Notebook。",Ol,G,lM,E,Is="现在让我们定义一些变量。请随意调整这些变量。",MM,R,sM,k,eM,X,tM,V,ms="首先加载数据。由于数据集可能相当大，请确保启用流模式。流模式允许我们在遍历数据集时逐步加载数据，而不是一次性下载数据集的整个内容。",aM,Q,is="我们将前 4000 个示例作为验证集，其余的全部作为训练数据。",yM,F,nM,W,As="在这一步，数据集仍然包含任意长度的原始数据。为了训练，我们需要固定长度的输入。让我们创建一个可迭代的数据集，它可以从文本文件流中返回固定长度的 token 块。",UM,S,os="首先，让我们估计数据集中每个 token 的平均字符数，这将帮助我们稍后估计文本缓冲区中的 token 数量。默认情况下，我们只从数据集中取 400 个示例（<code>nb_examples</code>）。只使用整个数据集的一个子集可以减少计算成本，同时仍然提供了对整体字符到 token 比的合理估计。",JM,N,TM,g,cs=`The character to token ratio of the dataset is: 2.43
`,jM,Y,ds="字符到 token 的比也可以用作文本标记质量的一个指标。例如，字符到 token 的比为 1.0 意味着每个字符都由一个 token 表示，这并没有太多意义。表明标记化做得不好。在标准的英文文本中，一个 token 通常相当于大约四个字符，这意味着字符到 token 的比率大约是 4.0。我们可以预见在代码数据集中的比率会更低，但一般来说，2.0 到 3.5 之间的数字可以认为是足够好的。",wM,H,rs=`<strong>可选的 FIM 变换</strong>
自回归语言模型通常是从左到右生成序列的。通过应用 FIM 变换，模型也可以学习填充文本。详细信息可以看<a href="https://arxiv.org/pdf/2207.14255.pdf" rel="nofollow">“Efficient Training of Language Models to Fill in the Middle” 这篇论文</a>了解这种技术。`,pM,v,bs="我们将在下面定义 FIM 变换，并在创建可迭代数据集时使用它们。然而，如果你想省略变换步骤，请将 <code>fim_rate</code> 设置为 0。",CM,x,IM,z,fs="让我们定义 <code>ConstantLengthDataset</code>，这是一个可迭代的数据集，它将返回固定长度的 token 块。为此，我们将从原始数据集中读取文本缓冲区，直到达到大小限制，然后应用分词器将原始文本转换为 token 后的输入。可选项，我们可以在一些序列上执行 FIM 变换（受影响的序列比例由 <code>fim_rate</code> 控制）。",mM,$,us="定义好后，我们可以从训练和验证数据中创建 <code>ConstantLengthDataset</code> 的实例。",iM,L,AM,D,oM,q,hs="现在数据已经准备好了，是时候加载模型了！我们将加载量化的模型。",cM,P,Zs="因为量化使用更少的位来表示数据，所以会减少内存使用。我们将使用 <code>bitsandbytes</code> 库来量化模型，因为它与 <code>transformers</code> 有很好的集成。我们需要做的只是定义一个 <code>bitsandbytes</code> 配置，然后在加载模型时使用它。",dM,K,Bs="4 比特位量化有不同的变体，但通常我们推荐使用 NF4 量化以获得更好的性能（<code>bnb_4bit_quant_type=&quot;nf4&quot;</code>）。",rM,O,_s="<code>bnb_4bit_use_double_quant</code> 选项在第一次量化后添加第二次量化，以节省每个参数额外的 0.4 位。",bM,ll,Gs='要了解更多关于量化的信息，请查看 <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">“利用 bitsandbytes、4 比特位量化和 QLoRA 让 LLMs 更易于访问” 的博客</a>。',fM,Ml,Es="定义好后，将配置传递给 <code>from_pretrained</code> 方法以加载量化的模型。",uM,sl,hM,el,Rs="当使用量化模型进行训练时，你需要调用 <code>prepare_model_for_kbit_training()</code> 函数来预处理量化模型以进行训练。",ZM,tl,BM,al,ks="现在量化模型已经准备好了，我们可以设置一个 LoRA 配置。LoRA 通过大幅减少可训练参数的数量，使得微调更加高效。",_M,yl,Xs="要使用 LoRA 技术训练模型，我们需要将基础模型包装为 <code>PeftModel</code>。这涉及到使用 <code>LoraConfig</code> 定义 LoRA 配置，并使用 <code>get_peft_model()</code> 和 <code>LoraConfig</code> 包装原始模型。",GM,nl,Vs='要了解更多关于 LoRA 及其参数的信息，请参考 <a href="https://huggingface.co/docs/peft/main/en/conceptual_guides/lora" rel="nofollow">PEFT 文档</a>。',EM,Ul,RM,Jl,Qs=`trainable params: 5,554,176 || all params: 1,142,761,472 || trainable%: 0.4860310866343243
`,kM,Tl,Fs="可以看到，通过应用 LoRA 技术，我们现在只需要训练不到 1% 的参数。",XM,jl,VM,wl,Ws="现在我们已经准备好了数据，并且优化了模型，我们可以将所有东西整合在一起开始训练。",QM,pl,Ss="要实例化一个 <code>Trainer</code>，你需要定义训练配置。最重要的是 <code>TrainingArguments</code>，这是一个包含所有用于配置训练的属性的类。",FM,Cl,Ns="这些与你可能运行的任何其他类型的模型训练相似，所以我们这里不会详细说明。",WM,Il,SM,ml,gs="最后一步，实例化 <code>Trainer</code> 并调用 <code>train</code> 方法。",NM,il,gM,Al,Ys=`Training...
`,YM,ol,Hs="最后，你可以将微调好的模型推送到你的 Hub 仓库中，并分享给你的团队。",HM,cl,vM,dl,xM,rl,vs="一旦模型被上传到 Hub，我们就可以使用它进行推理。为此，我们首先初始化原始的基础模型及其分词器。接下来，我们需要将微调后的权重与基础模型合并。",zM,bl,$M,fl,xs="现在我们可以使用合并后的模型进行推理。为了方便起见，我们将定义一个 <code>get_code_completion</code> 函数 - 请随意尝试文本生成参数！",LM,ul,DM,hl,zs="现在，为了获得代码补全，我们只需要调用 <code>get_code_complete</code> 函数，并将我们希望补全的前几行作为前缀传递，以及一个空字符串作为后缀。",qM,Zl,PM,Bl,$s=`from peft import LoraConfig, TaskType, get_peft_model
from transformers import AutoModelForCausalLM
peft_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=8,
    lora_alpha=32,
    target_modules=["q_proj", "v_proj"],
    lora_dropout=0.1,
    bias="none",
    modules_to_save=["q_proj", "v_proj"],
    inference_mode=False,
)
model = AutoModelForCausalLM.from_pretrained("gpt2")
model = get_peft_model(model, peft_config)
model.print_trainable_parameters()
`,KM,_l,Ls="作为刚刚在这个 notebook 中使用过 PEFT 库的人，你可以看到创建为 <code>LoraConfig</code> 函数的生成结果相当不错！",OM,Gl,Ds="如果你回到我们为推理实例化模型的单元格，并注释掉我们合并微调权重的行，你可以看到原始模型对于完全相同的前缀会生成什么内容：",ls,El,Ms,Rl,qs=`from peft import LoraConfig, TaskType, get_peft_model
from transformers import AutoModelForCausalLM
peft_config = LoraConfig(
    model_name_or_path="facebook/wav2vec2-base-960h",
    num_labels=1,
    num_features=1,
    num_hidden_layers=1,
    num_attention_heads=1,
    num_hidden_layers_per_attention_head=1,
    num_attention_heads_per_hidden_layer=1,
    hidden_size=1024,
    hidden_dropout_prob=0.1,
    hidden_act="gelu",
    hidden_act_dropout_prob=0.1,
    hidden
`,ss,kl,Ps="尽管这是 Python 语法，但你可以看到原始模型并不理解 <code>LoraConfig</code> 应该做什么。",es,Xl,Ks='要了解这种高效参数微调与完全微调的比较，以及如何通过推理端点在 VS Code 中使用这样的模型作为你的编程助手(copilot)，或者在本地使用，请查看<a href="https://huggingface.co/blog/personal-copilot" rel="nofollow">“个人编程助手(copilot)：训练你自己的编码助手”博客</a>。这个 notebook 补充了原始博客内容。',ts,Vl,as,Wl,ys;return i=new Je({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/main/notebooks/zh-CN/fine_tuning_code_llm_on_single_gpu.ipynb"}]}}),A=new Ue({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),o=new Ql({props:{title:"在单个 GPU 上针对自定义代码微调代码 LLM",local:"在单个-gpu-上针对自定义代码微调代码-llm",headingTag:"h1"}}),b=new Ql({props:{title:"数据集",local:"数据集",headingTag:"h2"}}),u=new Ql({props:{title:"模型",local:"模型",headingTag:"h2"}}),Z=new I({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),G=new I({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtcSUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwcGVmdCUyMGJpdHNhbmRieXRlcyUyMGZsYXNoLWF0dG4=",highlighted:"!pip install -q transformers datasets peft bitsandbytes flash-attn",wrap:!1}}),R=new I({props:{code:"TU9ERUwlM0QlMjJiaWdjb2RlJTJGc3RhcmNvZGVyYmFzZS0xYiUyMiUyMCUyMyUyME1vZGVsJTIwY2hlY2twb2ludCUyMG9uJTIwdGhlJTIwSHVnZ2luZyUyMEZhY2UlMjBIdWIlMEFEQVRBU0VUJTNEJTIyc21hbmdydWwlMkZoZi1zdGFjay12MSUyMiUyMCUyMCUyMCUyMyUyMERhdGFzZXQlMjBvbiUyMHRoZSUyMEh1Z2dpbmclMjBGYWNlJTIwSHViJTBBREFUQV9DT0xVTU4lM0QlMjJjb250ZW50JTIyJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQ29sdW1uJTIwbmFtZSUyMGNvbnRhaW5pbmclMjB0aGUlMjBjb2RlJTIwY29udGVudCUwQSUwQVNFUV9MRU5HVEglM0QyMDQ4JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwU2VxdWVuY2UlMjBsZW5ndGglMEElMEElMjMlMjBUcmFpbmluZyUyMGFyZ3VtZW50cyUwQU1BWF9TVEVQUyUzRDIwMDAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBtYXhfc3RlcHMlMEFCQVRDSF9TSVpFJTNEMTYlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBiYXRjaF9zaXplJTBBR1JfQUNDX1NURVBTJTNEMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUwQUxSJTNENWUtNCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGxlYXJuaW5nX3JhdGUlMEFMUl9TQ0hFRFVMRVJfVFlQRSUzRCUyMmNvc2luZSUyMiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGxyX3NjaGVkdWxlcl90eXBlJTBBV0VJR0hUX0RFQ0FZJTNEMC4wMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMHdlaWdodF9kZWNheSUwQU5VTV9XQVJNVVBfU1RFUFMlM0QzMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMG51bV93YXJtdXBfc3RlcHMlMEFFVkFMX0ZSRVElM0QxMDAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBldmFsX2ZyZXElMEFTQVZFX0ZSRVElM0QxMDAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBzYXZlX2ZyZXElMEFMT0dfRlJFUSUzRDI1JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwbG9nX2ZyZXElMEFPVVRQVVRfRElSJTNEJTIycGVmdC1zdGFyY29kZXItbG9yYS1hMTAwJTIyJTIwJTIzJTIwb3V0cHV0X2RpciUwQUJGMTYlM0RUcnVlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwYmYxNiUwQUZQMTYlM0RGYWxzZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMG5vX2ZwMTYlMEElMEElMjMlMjBGSU0lMjB0cmFzZm9ybWF0aW9ucyUyMGFyZ3VtZW50cyUwQUZJTV9SQVRFJTNEMC41JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwZmltX3JhdGUlMEFGSU1fU1BNX1JBVEUlM0QwLjUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBmaW1fc3BtX3JhdGUlMEElMEElMjMlMjBMT1JBJTBBTE9SQV9SJTNEOCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGxvcmFfciUwQUxPUkFfQUxQSEElM0QzMiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGxvcmFfYWxwaGElMEFMT1JBX0RST1BPVVQlM0QwLjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBsb3JhX2Ryb3BvdXQlMEFMT1JBX1RBUkdFVF9NT0RVTEVTJTNEJTIyY19wcm9qJTJDY19hdHRuJTJDcV9hdHRuJTJDY19mYyUyQ2NfcHJvaiUyMiUyMCUyMCUyMCUyMCUyMyUyMGxvcmFfdGFyZ2V0X21vZHVsZXMlMEElMEElMjMlMjBiaXRzYW5kYnl0ZXMlMjBjb25maWclMEFVU0VfTkVTVEVEX1FVQU5UJTNEVHJ1ZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMHVzZV9uZXN0ZWRfcXVhbnQlMEFCTkJfNEJJVF9DT01QVVRFX0RUWVBFJTNEJTIyYmZsb2F0MTYlMjIlMjMlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTBBJTBBU0VFRCUzRDA=",highlighted:`MODEL=<span class="hljs-string">&quot;bigcode/starcoderbase-1b&quot;</span> <span class="hljs-comment"># Model checkpoint on the Hugging Face Hub</span>
DATASET=<span class="hljs-string">&quot;smangrul/hf-stack-v1&quot;</span>   <span class="hljs-comment"># Dataset on the Hugging Face Hub</span>
DATA_COLUMN=<span class="hljs-string">&quot;content&quot;</span>            <span class="hljs-comment"># Column name containing the code content</span>

SEQ_LENGTH=<span class="hljs-number">2048</span>                  <span class="hljs-comment"># Sequence length</span>

<span class="hljs-comment"># Training arguments</span>
MAX_STEPS=<span class="hljs-number">2000</span>                   <span class="hljs-comment"># max_steps</span>
BATCH_SIZE=<span class="hljs-number">16</span>                    <span class="hljs-comment"># batch_size</span>
GR_ACC_STEPS=<span class="hljs-number">1</span>                   <span class="hljs-comment"># gradient_accumulation_steps</span>
LR=<span class="hljs-number">5e-4</span>                          <span class="hljs-comment"># learning_rate</span>
LR_SCHEDULER_TYPE=<span class="hljs-string">&quot;cosine&quot;</span>       <span class="hljs-comment"># lr_scheduler_type</span>
WEIGHT_DECAY=<span class="hljs-number">0.01</span>                <span class="hljs-comment"># weight_decay</span>
NUM_WARMUP_STEPS=<span class="hljs-number">30</span>              <span class="hljs-comment"># num_warmup_steps</span>
EVAL_FREQ=<span class="hljs-number">100</span>                    <span class="hljs-comment"># eval_freq</span>
SAVE_FREQ=<span class="hljs-number">100</span>                    <span class="hljs-comment"># save_freq</span>
LOG_FREQ=<span class="hljs-number">25</span>                      <span class="hljs-comment"># log_freq</span>
OUTPUT_DIR=<span class="hljs-string">&quot;peft-starcoder-lora-a100&quot;</span> <span class="hljs-comment"># output_dir</span>
BF16=<span class="hljs-literal">True</span>                        <span class="hljs-comment"># bf16</span>
FP16=<span class="hljs-literal">False</span>                       <span class="hljs-comment"># no_fp16</span>

<span class="hljs-comment"># FIM trasformations arguments</span>
FIM_RATE=<span class="hljs-number">0.5</span>                     <span class="hljs-comment"># fim_rate</span>
FIM_SPM_RATE=<span class="hljs-number">0.5</span>                 <span class="hljs-comment"># fim_spm_rate</span>

<span class="hljs-comment"># LORA</span>
LORA_R=<span class="hljs-number">8</span>                         <span class="hljs-comment"># lora_r</span>
LORA_ALPHA=<span class="hljs-number">32</span>                    <span class="hljs-comment"># lora_alpha</span>
LORA_DROPOUT=<span class="hljs-number">0.0</span>                 <span class="hljs-comment"># lora_dropout</span>
LORA_TARGET_MODULES=<span class="hljs-string">&quot;c_proj,c_attn,q_attn,c_fc,c_proj&quot;</span>    <span class="hljs-comment"># lora_target_modules</span>

<span class="hljs-comment"># bitsandbytes config</span>
USE_NESTED_QUANT=<span class="hljs-literal">True</span>            <span class="hljs-comment"># use_nested_quant</span>
BNB_4BIT_COMPUTE_DTYPE=<span class="hljs-string">&quot;bfloat16&quot;</span><span class="hljs-comment"># bnb_4bit_compute_dtype</span>

SEED=<span class="hljs-number">0</span>`,wrap:!1}}),k=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUwQSUyMCUyMCUyMCUyMEF1dG9Ub2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBUcmFpbmVyJTJDJTBBJTIwJTIwJTIwJTIwVHJhaW5pbmdBcmd1bWVudHMlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nJTJDJTBBJTIwJTIwJTIwJTIwc2V0X3NlZWQlMkMlMEElMjAlMjAlMjAlMjBCaXRzQW5kQnl0ZXNDb25maWclMkMlMEEpJTBBJTBBc2V0X3NlZWQoU0VFRCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
    AutoModelForCausalLM,
    AutoTokenizer,
    Trainer,
    TrainingArguments,
    logging,
    set_seed,
    BitsAndBytesConfig,
)

set_seed(SEED)`,wrap:!1}}),X=new Ql({props:{title:"准备数据",local:"准备数据",headingTag:"h2"}}),F=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHFkbSUyMGltcG9ydCUyMHRxZG0lMEElMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMERBVEFTRVQlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2RpciUzRCUyMmRhdGElMjIlMkMlMEElMjAlMjAlMjAlMjBzcGxpdCUzRCUyMnRyYWluJTIyJTJDJTBBJTIwJTIwJTIwJTIwc3RyZWFtaW5nJTNEVHJ1ZSUyQyUwQSklMEElMEF2YWxpZF9kYXRhJTIwJTNEJTIwZGF0YXNldC50YWtlKDQwMDApJTBBdHJhaW5fZGF0YSUyMCUzRCUyMGRhdGFzZXQuc2tpcCg0MDAwKSUwQXRyYWluX2RhdGElMjAlM0QlMjB0cmFpbl9kYXRhLnNodWZmbGUoYnVmZmVyX3NpemUlM0Q1MDAwJTJDJTIwc2VlZCUzRFNFRUQp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm


dataset = load_dataset(
    DATASET,
    data_dir=<span class="hljs-string">&quot;data&quot;</span>,
    split=<span class="hljs-string">&quot;train&quot;</span>,
    streaming=<span class="hljs-literal">True</span>,
)

valid_data = dataset.take(<span class="hljs-number">4000</span>)
train_data = dataset.skip(<span class="hljs-number">4000</span>)
train_data = train_data.shuffle(buffer_size=<span class="hljs-number">5000</span>, seed=SEED)`,wrap:!1}}),N=new I({props:{code:"dG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoTU9ERUwlMkMlMjB0cnVzdF9yZW1vdGVfY29kZSUzRFRydWUpJTBBJTBBZGVmJTIwY2hhcnNfdG9rZW5fcmF0aW8oZGF0YXNldCUyQyUyMHRva2VuaXplciUyQyUyMGRhdGFfY29sdW1uJTJDJTIwbmJfZXhhbXBsZXMlM0Q0MDApJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwRXN0aW1hdGUlMjB0aGUlMjBhdmVyYWdlJTIwbnVtYmVyJTIwb2YlMjBjaGFyYWN0ZXJzJTIwcGVyJTIwdG9rZW4lMjBpbiUyMHRoZSUyMGRhdGFzZXQuJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwdG90YWxfY2hhcmFjdGVycyUyQyUyMHRvdGFsX3Rva2VucyUyMCUzRCUyMDAlMkMlMjAwJTBBJTIwJTIwJTIwJTIwZm9yJTIwXyUyQyUyMGV4YW1wbGUlMjBpbiUyMHRxZG0oemlwKHJhbmdlKG5iX2V4YW1wbGVzKSUyQyUyMGl0ZXIoZGF0YXNldCkpJTJDJTIwdG90YWwlM0RuYl9leGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3RhbF9jaGFyYWN0ZXJzJTIwJTJCJTNEJTIwbGVuKGV4YW1wbGUlNUJkYXRhX2NvbHVtbiU1RCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3RhbF90b2tlbnMlMjAlMkIlM0QlMjBsZW4odG9rZW5pemVyKGV4YW1wbGUlNUJkYXRhX2NvbHVtbiU1RCkudG9rZW5zKCkpJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwdG90YWxfY2hhcmFjdGVycyUyMCUyRiUyMHRvdGFsX3Rva2VucyUwQSUwQSUwQWNoYXJzX3Blcl90b2tlbiUyMCUzRCUyMGNoYXJzX3Rva2VuX3JhdGlvKHRyYWluX2RhdGElMkMlMjB0b2tlbml6ZXIlMkMlMjBEQVRBX0NPTFVNTiklMEFwcmludChmJTIyVGhlJTIwY2hhcmFjdGVyJTIwdG8lMjB0b2tlbiUyMHJhdGlvJTIwb2YlMjB0aGUlMjBkYXRhc2V0JTIwaXMlM0ElMjAlN0JjaGFyc19wZXJfdG9rZW4lM0EuMmYlN0QlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=<span class="hljs-literal">True</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">chars_token_ratio</span>(<span class="hljs-params">dataset, tokenizer, data_column, nb_examples=<span class="hljs-number">400</span></span>):
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;&quot;&quot;
<span class="hljs-meta">... </span>    Estimate the average number of characters per token in the dataset.
<span class="hljs-meta">... </span>    &quot;&quot;&quot;</span>

<span class="hljs-meta">... </span>    total_characters, total_tokens = <span class="hljs-number">0</span>, <span class="hljs-number">0</span>
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> _, example <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">zip</span>(<span class="hljs-built_in">range</span>(nb_examples), <span class="hljs-built_in">iter</span>(dataset)), total=nb_examples):
<span class="hljs-meta">... </span>        total_characters += <span class="hljs-built_in">len</span>(example[data_column])
<span class="hljs-meta">... </span>        total_tokens += <span class="hljs-built_in">len</span>(tokenizer(example[data_column]).tokens())

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> total_characters / total_tokens


<span class="hljs-meta">&gt;&gt;&gt; </span>chars_per_token = chars_token_ratio(train_data, tokenizer, DATA_COLUMN)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;The character to token ratio of the dataset is: <span class="hljs-subst">{chars_per_token:<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),x=new I({props:{code:" ",highlighted:`<span class="hljs-keyword">import</span> functools
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-comment"># Helper function to get token ids of the special tokens for prefix, suffix and middle for FIM transformations.</span>
<span class="hljs-meta">@functools.lru_cache(<span class="hljs-params">maxsize=<span class="hljs-literal">None</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_fim_token_ids</span>(<span class="hljs-params">tokenizer</span>):
    <span class="hljs-keyword">try</span>:
        FIM_PREFIX, FIM_MIDDLE, FIM_SUFFIX, FIM_PAD = tokenizer.special_tokens_map[<span class="hljs-string">&quot;additional_special_tokens&quot;</span>][<span class="hljs-number">1</span>:<span class="hljs-number">5</span>]
        suffix_tok_id, prefix_tok_id, middle_tok_id, pad_tok_id = (
            tokenizer.vocab[tok] <span class="hljs-keyword">for</span> tok <span class="hljs-keyword">in</span> [FIM_SUFFIX, FIM_PREFIX, FIM_MIDDLE, FIM_PAD]
        )
    <span class="hljs-keyword">except</span> KeyError:
        suffix_tok_id, prefix_tok_id, middle_tok_id, pad_tok_id = <span class="hljs-literal">None</span>, <span class="hljs-literal">None</span>, <span class="hljs-literal">None</span>, <span class="hljs-literal">None</span>
    <span class="hljs-keyword">return</span> suffix_tok_id, prefix_tok_id, middle_tok_id, pad_tok_id


<span class="hljs-comment">## Adapted from https://github.com/bigcode-project/Megatron-LM/blob/6c4bf908df8fd86b4977f54bf5b8bd4b521003d1/megatron/data/gpt_dataset.py</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">permute</span>(<span class="hljs-params">
    sample,
    np_rng,
    suffix_tok_id,
    prefix_tok_id,
    middle_tok_id,
    pad_tok_id,
    fim_rate=<span class="hljs-number">0.5</span>,
    fim_spm_rate=<span class="hljs-number">0.5</span>,
    truncate_or_pad=<span class="hljs-literal">False</span>,
</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Take in a sample (list of tokens) and perform a FIM transformation on it with a probability of fim_rate, using two FIM modes:
    PSM and SPM (with a probability of fim_spm_rate).
    &quot;&quot;&quot;</span>

    <span class="hljs-comment"># The if condition will trigger with the probability of fim_rate</span>
    <span class="hljs-comment"># This means FIM transformations will apply to samples with a probability of fim_rate</span>
    <span class="hljs-keyword">if</span> np_rng.binomial(<span class="hljs-number">1</span>, fim_rate):

        <span class="hljs-comment"># Split the sample into prefix, middle, and suffix, based on randomly generated indices stored in the boundaries list.</span>
        boundaries = <span class="hljs-built_in">list</span>(np_rng.randint(low=<span class="hljs-number">0</span>, high=<span class="hljs-built_in">len</span>(sample) + <span class="hljs-number">1</span>, size=<span class="hljs-number">2</span>))
        boundaries.sort()

        prefix = np.array(sample[: boundaries[<span class="hljs-number">0</span>]], dtype=np.int64)
        middle = np.array(sample[boundaries[<span class="hljs-number">0</span>] : boundaries[<span class="hljs-number">1</span>]], dtype=np.int64)
        suffix = np.array(sample[boundaries[<span class="hljs-number">1</span>] :], dtype=np.int64)

        <span class="hljs-keyword">if</span> truncate_or_pad:
            <span class="hljs-comment"># calculate the new total length of the sample, taking into account tokens indicating prefix, middle, and suffix</span>
            new_length = suffix.shape[<span class="hljs-number">0</span>] + prefix.shape[<span class="hljs-number">0</span>] + middle.shape[<span class="hljs-number">0</span>] + <span class="hljs-number">3</span>
            diff = new_length - <span class="hljs-built_in">len</span>(sample)

            <span class="hljs-comment"># trancate or pad if there&#x27;s a difference in length between the new length and the original</span>
            <span class="hljs-keyword">if</span> diff &gt; <span class="hljs-number">0</span>:
                <span class="hljs-keyword">if</span> suffix.shape[<span class="hljs-number">0</span>] &lt;= diff:
                    <span class="hljs-keyword">return</span> sample, np_rng
                suffix = suffix[: suffix.shape[<span class="hljs-number">0</span>] - diff]
            <span class="hljs-keyword">elif</span> diff &lt; <span class="hljs-number">0</span>:
                suffix = np.concatenate([suffix, np.full((-<span class="hljs-number">1</span> * diff), pad_tok_id)])

        <span class="hljs-comment"># With the probability of fim_spm_rateapply SPM variant of FIM transformations</span>
        <span class="hljs-comment"># SPM: suffix, prefix, middle</span>
        <span class="hljs-keyword">if</span> np_rng.binomial(<span class="hljs-number">1</span>, fim_spm_rate):
            new_sample = np.concatenate(
                [
                    [prefix_tok_id, suffix_tok_id],
                    suffix,
                    [middle_tok_id],
                    prefix,
                    middle,
                ]
            )
        <span class="hljs-comment"># Otherwise, apply the PSM variant of FIM transformations</span>
        <span class="hljs-comment"># PSM: prefix, suffix, middle</span>
        <span class="hljs-keyword">else</span>:

            new_sample = np.concatenate(
                [
                    [prefix_tok_id],
                    prefix,
                    [suffix_tok_id],
                    suffix,
                    [middle_tok_id],
                    middle,
                ]
            )
    <span class="hljs-keyword">else</span>:
        <span class="hljs-comment"># don&#x27;t apply FIM transformations</span>
        new_sample = sample

    <span class="hljs-keyword">return</span> <span class="hljs-built_in">list</span>(new_sample), np_rng`,wrap:!1}}),L=new I({props:{code:" ",highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> IterableDataset
<span class="hljs-keyword">from</span> torch.utils.data.dataloader <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-keyword">import</span> random

<span class="hljs-comment"># Create an Iterable dataset that returns constant-length chunks of tokens from a stream of text files.</span>

<span class="hljs-keyword">class</span> <span class="hljs-title class_">ConstantLengthDataset</span>(<span class="hljs-title class_ inherited__">IterableDataset</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Iterable dataset that returns constant length chunks of tokens from stream of text files.
        Args:
            tokenizer (Tokenizer): The processor used for proccessing the data.
            dataset (dataset.Dataset): Dataset with text files.
            infinite (bool): If True the iterator is reset after dataset reaches end else stops.
            seq_length (int): Length of token sequences to return.
            num_of_sequences (int): Number of token sequences to keep in buffer.
            chars_per_token (int): Number of characters per token used to estimate number of tokens in text buffer.
            fim_rate (float): Rate (0.0 to 1.0) that sample will be permuted with FIM.
            fim_spm_rate (float): Rate (0.0 to 1.0) of FIM permuations that will use SPM.
            seed (int): Seed for random number generator.
    &quot;&quot;&quot;</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">
        self,
        tokenizer,
        dataset,
        infinite=<span class="hljs-literal">False</span>,
        seq_length=<span class="hljs-number">1024</span>,
        num_of_sequences=<span class="hljs-number">1024</span>,
        chars_per_token=<span class="hljs-number">3.6</span>,
        content_field=<span class="hljs-string">&quot;content&quot;</span>,
        fim_rate=<span class="hljs-number">0.5</span>,
        fim_spm_rate=<span class="hljs-number">0.5</span>,
        seed=<span class="hljs-number">0</span>,
    </span>):
        self.tokenizer = tokenizer
        self.concat_token_id = tokenizer.eos_token_id
        self.dataset = dataset
        self.seq_length = seq_length
        self.infinite = infinite
        self.current_size = <span class="hljs-number">0</span>
        self.max_buffer_size = seq_length * chars_per_token * num_of_sequences
        self.content_field = content_field
        self.fim_rate = fim_rate
        self.fim_spm_rate = fim_spm_rate
        self.seed = seed

        (
            self.suffix_tok_id,
            self.prefix_tok_id,
            self.middle_tok_id,
            self.pad_tok_id,
        ) = get_fim_token_ids(self.tokenizer)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> self.suffix_tok_id <span class="hljs-keyword">and</span> self.fim_rate &gt; <span class="hljs-number">0</span>:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;FIM is not supported by tokenizer, disabling FIM&quot;</span>)
            self.fim_rate = <span class="hljs-number">0</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__iter__</span>(<span class="hljs-params">self</span>):
        iterator = <span class="hljs-built_in">iter</span>(self.dataset)
        more_examples = <span class="hljs-literal">True</span>
        np_rng = np.random.RandomState(seed=self.seed)
        <span class="hljs-keyword">while</span> more_examples:
            buffer, buffer_len = [], <span class="hljs-number">0</span>
            <span class="hljs-keyword">while</span> <span class="hljs-literal">True</span>:
                <span class="hljs-keyword">if</span> buffer_len &gt;= self.max_buffer_size:
                    <span class="hljs-keyword">break</span>
                <span class="hljs-keyword">try</span>:
                    buffer.append(<span class="hljs-built_in">next</span>(iterator)[self.content_field])
                    buffer_len += <span class="hljs-built_in">len</span>(buffer[-<span class="hljs-number">1</span>])
                <span class="hljs-keyword">except</span> StopIteration:
                    <span class="hljs-keyword">if</span> self.infinite:
                        iterator = <span class="hljs-built_in">iter</span>(self.dataset)
                    <span class="hljs-keyword">else</span>:
                        more_examples = <span class="hljs-literal">False</span>
                        <span class="hljs-keyword">break</span>
            tokenized_inputs = self.tokenizer(buffer, truncation=<span class="hljs-literal">False</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
            all_token_ids = []

            <span class="hljs-keyword">for</span> tokenized_input <span class="hljs-keyword">in</span> tokenized_inputs:
                <span class="hljs-comment"># optionally do FIM permutations</span>
                <span class="hljs-keyword">if</span> self.fim_rate &gt; <span class="hljs-number">0</span>:
                    tokenized_input, np_rng = permute(
                        tokenized_input,
                        np_rng,
                        self.suffix_tok_id,
                        self.prefix_tok_id,
                        self.middle_tok_id,
                        self.pad_tok_id,
                        fim_rate=self.fim_rate,
                        fim_spm_rate=self.fim_spm_rate,
                        truncate_or_pad=<span class="hljs-literal">False</span>,
                    )

                all_token_ids.extend(tokenized_input + [self.concat_token_id])
            examples = []
            <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(all_token_ids), self.seq_length):
                input_ids = all_token_ids[i : i + self.seq_length]
                <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(input_ids) == self.seq_length:
                    examples.append(input_ids)
            random.shuffle(examples)
            <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> examples:
                self.current_size += <span class="hljs-number">1</span>
                <span class="hljs-keyword">yield</span> {
                    <span class="hljs-string">&quot;input_ids&quot;</span>: torch.LongTensor(example),
                    <span class="hljs-string">&quot;labels&quot;</span>: torch.LongTensor(example),
                }


train_dataset = ConstantLengthDataset(
        tokenizer,
        train_data,
        infinite=<span class="hljs-literal">True</span>,
        seq_length=SEQ_LENGTH,
        chars_per_token=chars_per_token,
        content_field=DATA_COLUMN,
        fim_rate=FIM_RATE,
        fim_spm_rate=FIM_SPM_RATE,
        seed=SEED,
)
eval_dataset = ConstantLengthDataset(
        tokenizer,
        valid_data,
        infinite=<span class="hljs-literal">False</span>,
        seq_length=SEQ_LENGTH,
        chars_per_token=chars_per_token,
        content_field=DATA_COLUMN,
        fim_rate=FIM_RATE,
        fim_spm_rate=FIM_SPM_RATE,
        seed=SEED,
)`,wrap:!1}}),D=new Ql({props:{title:"准备模型",local:"准备模型",headingTag:"h2"}}),sl=new I({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTJDJTIwZ2V0X3BlZnRfbW9kZWwlMkMlMjBwcmVwYXJlX21vZGVsX2Zvcl9rYml0X3RyYWluaW5nJTBBZnJvbSUyMHBlZnQudHVuZXJzLmxvcmElMjBpbXBvcnQlMjBMb3JhTGF5ZXIlMEElMEFsb2FkX2luXzhiaXQlMjAlM0QlMjBGYWxzZSUwQSUwQSUyMyUyMDQtYml0JTIwcXVhbnRpemF0aW9uJTBBY29tcHV0ZV9kdHlwZSUyMCUzRCUyMGdldGF0dHIodG9yY2glMkMlMjBCTkJfNEJJVF9DT01QVVRFX0RUWVBFKSUwQSUwQWJuYl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTNEY29tcHV0ZV9kdHlwZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RVU0VfTkVTVEVEX1FVQU5UJTJDJTBBKSUwQSUwQWRldmljZV9tYXAlMjAlM0QlMjAlN0IlMjIlMjIlM0ElMjAwJTdEJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTU9ERUwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb2FkX2luXzhiaXQlM0Rsb2FkX2luXzhiaXQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0RkZXZpY2VfbWFwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdXNlX2NhY2hlJTNERmFsc2UlMkMlMjAlMjAlMjMlMjBXZSUyMHdpbGwlMjBiZSUyMHVzaW5nJTIwZ3JhZGllbnQlMjBjaGVja3BvaW50aW5nJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ1c3RfcmVtb3RlX2NvZGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdXNlX2ZsYXNoX2F0dGVudGlvbl8yJTNEVHJ1ZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, get_peft_model, prepare_model_for_kbit_training
<span class="hljs-keyword">from</span> peft.tuners.lora <span class="hljs-keyword">import</span> LoraLayer

load_in_8bit = <span class="hljs-literal">False</span>

<span class="hljs-comment"># 4-bit quantization</span>
compute_dtype = <span class="hljs-built_in">getattr</span>(torch, BNB_4BIT_COMPUTE_DTYPE)

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=compute_dtype,
    bnb_4bit_use_double_quant=USE_NESTED_QUANT,
)

device_map = {<span class="hljs-string">&quot;&quot;</span>: <span class="hljs-number">0</span>}

model = AutoModelForCausalLM.from_pretrained(
        MODEL,
        load_in_8bit=load_in_8bit,
        quantization_config=bnb_config,
        device_map=device_map,
        use_cache=<span class="hljs-literal">False</span>,  <span class="hljs-comment"># We will be using gradient checkpointing</span>
        trust_remote_code=<span class="hljs-literal">True</span>,
        use_flash_attention_2=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),tl=new I({props:{code:"bW9kZWwlMjAlM0QlMjBwcmVwYXJlX21vZGVsX2Zvcl9rYml0X3RyYWluaW5nKG1vZGVsKQ==",highlighted:"model = prepare_model_for_kbit_training(model)",wrap:!1}}),Ul=new I({props:{code:"JTIzJTIwU2V0JTIwdXAlMjBsb3JhJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0RMT1JBX0FMUEhBJTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9kcm9wb3V0JTNETE9SQV9EUk9QT1VUJTJDJTBBJTIwJTIwJTIwJTIwciUzRExPUkFfUiUyQyUwQSUyMCUyMCUyMCUyMGJpYXMlM0QlMjJub25lJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0RMT1JBX1RBUkdFVF9NT0RVTEVTLnNwbGl0KCUyMiUyQyUyMiklMkMlMEEpJTBBJTBBbW9kZWwlMjAlM0QlMjBnZXRfcGVmdF9tb2RlbChtb2RlbCUyQyUyMHBlZnRfY29uZmlnKSUwQW1vZGVsLnByaW50X3RyYWluYWJsZV9wYXJhbWV0ZXJzKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Set up lora</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>peft_config = LoraConfig(
<span class="hljs-meta">... </span>    lora_alpha=LORA_ALPHA,
<span class="hljs-meta">... </span>    lora_dropout=LORA_DROPOUT,
<span class="hljs-meta">... </span>    r=LORA_R,
<span class="hljs-meta">... </span>    bias=<span class="hljs-string">&quot;none&quot;</span>,
<span class="hljs-meta">... </span>    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
<span class="hljs-meta">... </span>    target_modules=LORA_TARGET_MODULES.split(<span class="hljs-string">&quot;,&quot;</span>),
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = get_peft_model(model, peft_config)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.print_trainable_parameters()`,wrap:!1}}),jl=new Ql({props:{title:"训练模型",local:"训练模型",headingTag:"h2"}}),Il=new I({props:{code:"dHJhaW5fZGF0YS5zdGFydF9pdGVyYXRpb24lMjAlM0QlMjAwJTBBJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0RmJTIyWW91cl9IRl91c2VybmFtZSUyRiU3Qk9VVFBVVF9ESVIlN0QlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhbG9hZGVyX2Ryb3BfbGFzdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBldmFsdWF0aW9uX3N0cmF0ZWd5JTNEJTIyc3RlcHMlMjIlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0cmF0ZWd5JTNEJTIyc3RlcHMlMjIlMkMlMEElMjAlMjAlMjAlMjBtYXhfc3RlcHMlM0RNQVhfU1RFUFMlMkMlMEElMjAlMjAlMjAlMjBldmFsX3N0ZXBzJTNERVZBTF9GUkVRJTJDJTBBJTIwJTIwJTIwJTIwc2F2ZV9zdGVwcyUzRFNBVkVfRlJFUSUyQyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0RMT0dfRlJFUSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzREJBVENIX1NJWkUlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX2V2YWxfYmF0Y2hfc2l6ZSUzREJBVENIX1NJWkUlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNETFIlMkMlMEElMjAlMjAlMjAlMjBscl9zY2hlZHVsZXJfdHlwZSUzRExSX1NDSEVEVUxFUl9UWVBFJTJDJTBBJTIwJTIwJTIwJTIwd2FybXVwX3N0ZXBzJTNETlVNX1dBUk1VUF9TVEVQUyUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzREdSX0FDQ19TVEVQUyUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2NoZWNrcG9pbnRpbmclM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZnAxNiUzREZQMTYlMkMlMEElMjAlMjAlMjAlMjBiZjE2JTNEQkYxNiUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRFdFSUdIVF9ERUNBWSUyQyUwQSUyMCUyMCUyMCUyMHB1c2hfdG9faHViJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGluY2x1ZGVfdG9rZW5zX3Blcl9zZWNvbmQlM0RUcnVlJTJDJTBBKQ==",highlighted:`train_data.start_iteration = <span class="hljs-number">0</span>


training_args = TrainingArguments(
    output_dir=<span class="hljs-string">f&quot;Your_HF_username/<span class="hljs-subst">{OUTPUT_DIR}</span>&quot;</span>,
    dataloader_drop_last=<span class="hljs-literal">True</span>,
    evaluation_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    save_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    max_steps=MAX_STEPS,
    eval_steps=EVAL_FREQ,
    save_steps=SAVE_FREQ,
    logging_steps=LOG_FREQ,
    per_device_train_batch_size=BATCH_SIZE,
    per_device_eval_batch_size=BATCH_SIZE,
    learning_rate=LR,
    lr_scheduler_type=LR_SCHEDULER_TYPE,
    warmup_steps=NUM_WARMUP_STEPS,
    gradient_accumulation_steps=GR_ACC_STEPS,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    fp16=FP16,
    bf16=BF16,
    weight_decay=WEIGHT_DECAY,
    push_to_hub=<span class="hljs-literal">True</span>,
    include_tokens_per_second=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),il=new I({props:{code:"dHJhaW5lciUyMCUzRCUyMFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTIwdHJhaW5fZGF0YXNldCUzRHRyYWluX2RhdGFzZXQlMkMlMjBldmFsX2RhdGFzZXQlM0RldmFsX2RhdGFzZXQlMEEpJTBBJTBBcHJpbnQoJTIyVHJhaW5pbmcuLi4lMjIpJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Training...&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),cl=new I({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:"trainer.push_to_hub()",wrap:!1}}),dl=new Ql({props:{title:"推理",local:"推理",headingTag:"h2"}}),bl=new I({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBQZWZ0TW9kZWwlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQSUyMyUyMGxvYWQlMjB0aGUlMjBvcmlnaW5hbCUyMG1vZGVsJTIwZmlyc3QlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChNT0RFTCUyQyUyMHRydXN0X3JlbW90ZV9jb2RlJTNEVHJ1ZSklMEFiYXNlX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyME1PREVMJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRE5vbmUlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNETm9uZSUyQyUwQSUyMCUyMCUyMCUyMHRydXN0X3JlbW90ZV9jb2RlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEpLmN1ZGEoKSUwQSUwQSUyMyUyMG1lcmdlJTIwZmluZS10dW5lZCUyMHdlaWdodHMlMjB3aXRoJTIwdGhlJTIwYmFzZSUyMG1vZGVsJTBBcGVmdF9tb2RlbF9pZCUyMCUzRCUyMGYlMjJZb3VyX0hGX3VzZXJuYW1lJTJGJTdCT1VUUFVUX0RJUiU3RCUyMiUwQW1vZGVsJTIwJTNEJTIwUGVmdE1vZGVsLmZyb21fcHJldHJhaW5lZChiYXNlX21vZGVsJTJDJTIwcGVmdF9tb2RlbF9pZCklMEFtb2RlbC5tZXJnZV9hbmRfdW5sb2FkKCk=",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PeftModel
<span class="hljs-keyword">import</span> torch

<span class="hljs-comment"># load the original model first</span>
tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=<span class="hljs-literal">True</span>)
base_model = AutoModelForCausalLM.from_pretrained(
    MODEL,
    quantization_config=<span class="hljs-literal">None</span>,
    device_map=<span class="hljs-literal">None</span>,
    trust_remote_code=<span class="hljs-literal">True</span>,
    torch_dtype=torch.bfloat16,
).cuda()

<span class="hljs-comment"># merge fine-tuned weights with the base model</span>
peft_model_id = <span class="hljs-string">f&quot;Your_HF_username/<span class="hljs-subst">{OUTPUT_DIR}</span>&quot;</span>
model = PeftModel.from_pretrained(base_model, peft_model_id)
model.merge_and_unload()`,wrap:!1}}),ul=new I({props:{code:"ZGVmJTIwZ2V0X2NvZGVfY29tcGxldGlvbihwcmVmaXglMkMlMjBzdWZmaXgpJTNBJTBBJTIwJTIwJTIwJTIwdGV4dCUyMCUzRCUyMHByb21wdCUyMCUzRCUyMGYlMjIlMjIlMjIlN0JwcmVmaXglN0QlN0JzdWZmaXglN0QlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBtb2RlbC5ldmFsKCklMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5wdXRfaWRzJTNEdG9rZW5pemVyKHRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS5pbnB1dF9pZHMuY3VkYSgpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0QxMjglMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzRDAuMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcF9rJTNENTAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3BfcCUzRDAuOTUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_code_completion</span>(<span class="hljs-params">prefix, suffix</span>):
    text = prompt = <span class="hljs-string">f&quot;&quot;&quot;<span class="hljs-subst">{prefix}</span><span class="hljs-subst">{suffix}</span>&quot;&quot;&quot;</span>
    model.<span class="hljs-built_in">eval</span>()
    outputs = model.generate(
        input_ids=tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids.cuda(),
        max_new_tokens=<span class="hljs-number">128</span>,
        temperature=<span class="hljs-number">0.2</span>,
        top_k=<span class="hljs-number">50</span>,
        top_p=<span class="hljs-number">0.95</span>,
        do_sample=<span class="hljs-literal">True</span>,
        repetition_penalty=<span class="hljs-number">1.0</span>,
    )
    <span class="hljs-keyword">return</span> tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]`,wrap:!1}}),Zl=new I({props:{code:"cHJlZml4JTIwJTNEJTIwJTIyJTIyJTIyZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTJDJTIwVGFza1R5cGUlMkMlMjBnZXRfcGVmdF9tb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQXBlZnRfY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjIlMjIlMjIlMEFzdWZmaXglMjAlM0QlMjIlMjIlMjIlMjIlMjIlMjIlMEElMEFwcmludChnZXRfY29kZV9jb21wbGV0aW9uKHByZWZpeCUyQyUyMHN1ZmZpeCkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>prefix = <span class="hljs-string">&quot;&quot;&quot;from peft import LoraConfig, TaskType, get_peft_model
<span class="hljs-meta">... </span>from transformers import AutoModelForCausalLM
<span class="hljs-meta">... </span>peft_config = LoraConfig(
<span class="hljs-meta">... </span>&quot;&quot;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>suffix =<span class="hljs-string">&quot;&quot;&quot;&quot;&quot;&quot;</span>

<span class="hljs-meta">... </span><span class="hljs-built_in">print</span>(get_code_completion(prefix, suffix))`,wrap:!1}}),El=new I({props:{code:"cHJlZml4JTIwJTNEJTIwJTIyJTIyJTIyZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTJDJTIwVGFza1R5cGUlMkMlMjBnZXRfcGVmdF9tb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQXBlZnRfY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjIlMjIlMjIlMEFzdWZmaXglMjAlM0QlMjIlMjIlMjIlMjIlMjIlMjIlMEElMEFwcmludChnZXRfY29kZV9jb21wbGV0aW9uKHByZWZpeCUyQyUyMHN1ZmZpeCkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>prefix = <span class="hljs-string">&quot;&quot;&quot;from peft import LoraConfig, TaskType, get_peft_model
<span class="hljs-meta">... </span>from transformers import AutoModelForCausalLM
<span class="hljs-meta">... </span>peft_config = LoraConfig(
<span class="hljs-meta">... </span>&quot;&quot;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>suffix =<span class="hljs-string">&quot;&quot;&quot;&quot;&quot;&quot;</span>

<span class="hljs-meta">... </span><span class="hljs-built_in">print</span>(get_code_completion(prefix, suffix))`,wrap:!1}}),Vl=new Te({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/zh-CN/fine_tuning_code_llm_on_single_gpu.md"}}),{c(){m=y("meta"),Sl=t(),Fl=y("p"),Nl=t(),J(i.$$.fragment),gl=t(),J(A.$$.fragment),Yl=t(),J(o.$$.fragment),Hl=t(),c=y("p"),c.innerHTML=Us,vl=t(),d=y("p"),d.textContent=Js,xl=t(),r=y("p"),r.textContent=Ts,zl=t(),J(b.$$.fragment),$l=t(),f=y("p"),f.innerHTML=js,Ll=t(),J(u.$$.fragment),Dl=t(),h=y("p"),h.innerHTML=ws,ql=t(),J(Z.$$.fragment),Pl=t(),B=y("p"),B.innerHTML=ps,Kl=t(),_=y("p"),_.textContent=Cs,Ol=t(),J(G.$$.fragment),lM=t(),E=y("p"),E.textContent=Is,MM=t(),J(R.$$.fragment),sM=t(),J(k.$$.fragment),eM=t(),J(X.$$.fragment),tM=t(),V=y("p"),V.textContent=ms,aM=t(),Q=y("p"),Q.textContent=is,yM=t(),J(F.$$.fragment),nM=t(),W=y("p"),W.textContent=As,UM=t(),S=y("p"),S.innerHTML=os,JM=t(),J(N.$$.fragment),TM=t(),g=y("pre"),g.textContent=cs,jM=t(),Y=y("p"),Y.textContent=ds,wM=t(),H=y("p"),H.innerHTML=rs,pM=t(),v=y("p"),v.innerHTML=bs,CM=t(),J(x.$$.fragment),IM=t(),z=y("p"),z.innerHTML=fs,mM=t(),$=y("p"),$.innerHTML=us,iM=t(),J(L.$$.fragment),AM=t(),J(D.$$.fragment),oM=t(),q=y("p"),q.textContent=hs,cM=t(),P=y("p"),P.innerHTML=Zs,dM=t(),K=y("p"),K.innerHTML=Bs,rM=t(),O=y("p"),O.innerHTML=_s,bM=t(),ll=y("p"),ll.innerHTML=Gs,fM=t(),Ml=y("p"),Ml.innerHTML=Es,uM=t(),J(sl.$$.fragment),hM=t(),el=y("p"),el.innerHTML=Rs,ZM=t(),J(tl.$$.fragment),BM=t(),al=y("p"),al.textContent=ks,_M=t(),yl=y("p"),yl.innerHTML=Xs,GM=t(),nl=y("p"),nl.innerHTML=Vs,EM=t(),J(Ul.$$.fragment),RM=t(),Jl=y("pre"),Jl.textContent=Qs,kM=t(),Tl=y("p"),Tl.textContent=Fs,XM=t(),J(jl.$$.fragment),VM=t(),wl=y("p"),wl.textContent=Ws,QM=t(),pl=y("p"),pl.innerHTML=Ss,FM=t(),Cl=y("p"),Cl.textContent=Ns,WM=t(),J(Il.$$.fragment),SM=t(),ml=y("p"),ml.innerHTML=gs,NM=t(),J(il.$$.fragment),gM=t(),Al=y("pre"),Al.textContent=Ys,YM=t(),ol=y("p"),ol.textContent=Hs,HM=t(),J(cl.$$.fragment),vM=t(),J(dl.$$.fragment),xM=t(),rl=y("p"),rl.textContent=vs,zM=t(),J(bl.$$.fragment),$M=t(),fl=y("p"),fl.innerHTML=xs,LM=t(),J(ul.$$.fragment),DM=t(),hl=y("p"),hl.innerHTML=zs,qM=t(),J(Zl.$$.fragment),PM=t(),Bl=y("pre"),Bl.textContent=$s,KM=t(),_l=y("p"),_l.innerHTML=Ls,OM=t(),Gl=y("p"),Gl.textContent=Ds,ls=t(),J(El.$$.fragment),Ms=t(),Rl=y("pre"),Rl.textContent=qs,ss=t(),kl=y("p"),kl.innerHTML=Ps,es=t(),Xl=y("p"),Xl.innerHTML=Ks,ts=t(),J(Vl.$$.fragment),as=t(),Wl=y("p"),this.h()},l(l){const M=ye("svelte-u9bgzb",document.head);m=n(M,"META",{name:!0,content:!0}),M.forEach(s),Sl=a(l),Fl=n(l,"P",{}),Os(Fl).forEach(s),Nl=a(l),T(i.$$.fragment,l),gl=a(l),T(A.$$.fragment,l),Yl=a(l),T(o.$$.fragment,l),Hl=a(l),c=n(l,"P",{"data-svelte-h":!0}),U(c)!=="svelte-1dsumsc"&&(c.innerHTML=Us),vl=a(l),d=n(l,"P",{"data-svelte-h":!0}),U(d)!=="svelte-q659zy"&&(d.textContent=Js),xl=a(l),r=n(l,"P",{"data-svelte-h":!0}),U(r)!=="svelte-7d3r9t"&&(r.textContent=Ts),zl=a(l),T(b.$$.fragment,l),$l=a(l),f=n(l,"P",{"data-svelte-h":!0}),U(f)!=="svelte-16d0rk8"&&(f.innerHTML=js),Ll=a(l),T(u.$$.fragment,l),Dl=a(l),h=n(l,"P",{"data-svelte-h":!0}),U(h)!=="svelte-1lmoyae"&&(h.innerHTML=ws),ql=a(l),T(Z.$$.fragment,l),Pl=a(l),B=n(l,"P",{"data-svelte-h":!0}),U(B)!=="svelte-17gpr1g"&&(B.innerHTML=ps),Kl=a(l),_=n(l,"P",{"data-svelte-h":!0}),U(_)!=="svelte-bpmgpr"&&(_.textContent=Cs),Ol=a(l),T(G.$$.fragment,l),lM=a(l),E=n(l,"P",{"data-svelte-h":!0}),U(E)!=="svelte-q7qesu"&&(E.textContent=Is),MM=a(l),T(R.$$.fragment,l),sM=a(l),T(k.$$.fragment,l),eM=a(l),T(X.$$.fragment,l),tM=a(l),V=n(l,"P",{"data-svelte-h":!0}),U(V)!=="svelte-nbrbdm"&&(V.textContent=ms),aM=a(l),Q=n(l,"P",{"data-svelte-h":!0}),U(Q)!=="svelte-156cg1h"&&(Q.textContent=is),yM=a(l),T(F.$$.fragment,l),nM=a(l),W=n(l,"P",{"data-svelte-h":!0}),U(W)!=="svelte-w6f819"&&(W.textContent=As),UM=a(l),S=n(l,"P",{"data-svelte-h":!0}),U(S)!=="svelte-11jtxw5"&&(S.innerHTML=os),JM=a(l),T(N.$$.fragment,l),TM=a(l),g=n(l,"PRE",{"data-svelte-h":!0}),U(g)!=="svelte-ko6dme"&&(g.textContent=cs),jM=a(l),Y=n(l,"P",{"data-svelte-h":!0}),U(Y)!=="svelte-qkug4j"&&(Y.textContent=ds),wM=a(l),H=n(l,"P",{"data-svelte-h":!0}),U(H)!=="svelte-10u5js8"&&(H.innerHTML=rs),pM=a(l),v=n(l,"P",{"data-svelte-h":!0}),U(v)!=="svelte-a6uov0"&&(v.innerHTML=bs),CM=a(l),T(x.$$.fragment,l),IM=a(l),z=n(l,"P",{"data-svelte-h":!0}),U(z)!=="svelte-19bf5mc"&&(z.innerHTML=fs),mM=a(l),$=n(l,"P",{"data-svelte-h":!0}),U($)!=="svelte-1kw27m4"&&($.innerHTML=us),iM=a(l),T(L.$$.fragment,l),AM=a(l),T(D.$$.fragment,l),oM=a(l),q=n(l,"P",{"data-svelte-h":!0}),U(q)!=="svelte-1r3du7w"&&(q.textContent=hs),cM=a(l),P=n(l,"P",{"data-svelte-h":!0}),U(P)!=="svelte-2im68f"&&(P.innerHTML=Zs),dM=a(l),K=n(l,"P",{"data-svelte-h":!0}),U(K)!=="svelte-12xf50n"&&(K.innerHTML=Bs),rM=a(l),O=n(l,"P",{"data-svelte-h":!0}),U(O)!=="svelte-n8dwtj"&&(O.innerHTML=_s),bM=a(l),ll=n(l,"P",{"data-svelte-h":!0}),U(ll)!=="svelte-1frw6v8"&&(ll.innerHTML=Gs),fM=a(l),Ml=n(l,"P",{"data-svelte-h":!0}),U(Ml)!=="svelte-d91ns3"&&(Ml.innerHTML=Es),uM=a(l),T(sl.$$.fragment,l),hM=a(l),el=n(l,"P",{"data-svelte-h":!0}),U(el)!=="svelte-1xl6sjo"&&(el.innerHTML=Rs),ZM=a(l),T(tl.$$.fragment,l),BM=a(l),al=n(l,"P",{"data-svelte-h":!0}),U(al)!=="svelte-1eyziea"&&(al.textContent=ks),_M=a(l),yl=n(l,"P",{"data-svelte-h":!0}),U(yl)!=="svelte-m5kbu0"&&(yl.innerHTML=Xs),GM=a(l),nl=n(l,"P",{"data-svelte-h":!0}),U(nl)!=="svelte-16uefdy"&&(nl.innerHTML=Vs),EM=a(l),T(Ul.$$.fragment,l),RM=a(l),Jl=n(l,"PRE",{"data-svelte-h":!0}),U(Jl)!=="svelte-hpf32x"&&(Jl.textContent=Qs),kM=a(l),Tl=n(l,"P",{"data-svelte-h":!0}),U(Tl)!=="svelte-23hvvu"&&(Tl.textContent=Fs),XM=a(l),T(jl.$$.fragment,l),VM=a(l),wl=n(l,"P",{"data-svelte-h":!0}),U(wl)!=="svelte-tja9vl"&&(wl.textContent=Ws),QM=a(l),pl=n(l,"P",{"data-svelte-h":!0}),U(pl)!=="svelte-yjq99r"&&(pl.innerHTML=Ss),FM=a(l),Cl=n(l,"P",{"data-svelte-h":!0}),U(Cl)!=="svelte-12swt94"&&(Cl.textContent=Ns),WM=a(l),T(Il.$$.fragment,l),SM=a(l),ml=n(l,"P",{"data-svelte-h":!0}),U(ml)!=="svelte-emgn9w"&&(ml.innerHTML=gs),NM=a(l),T(il.$$.fragment,l),gM=a(l),Al=n(l,"PRE",{"data-svelte-h":!0}),U(Al)!=="svelte-9xzew6"&&(Al.textContent=Ys),YM=a(l),ol=n(l,"P",{"data-svelte-h":!0}),U(ol)!=="svelte-e3wwxu"&&(ol.textContent=Hs),HM=a(l),T(cl.$$.fragment,l),vM=a(l),T(dl.$$.fragment,l),xM=a(l),rl=n(l,"P",{"data-svelte-h":!0}),U(rl)!=="svelte-1rt49jd"&&(rl.textContent=vs),zM=a(l),T(bl.$$.fragment,l),$M=a(l),fl=n(l,"P",{"data-svelte-h":!0}),U(fl)!=="svelte-1tevq0n"&&(fl.innerHTML=xs),LM=a(l),T(ul.$$.fragment,l),DM=a(l),hl=n(l,"P",{"data-svelte-h":!0}),U(hl)!=="svelte-vmzo1"&&(hl.innerHTML=zs),qM=a(l),T(Zl.$$.fragment,l),PM=a(l),Bl=n(l,"PRE",{"data-svelte-h":!0}),U(Bl)!=="svelte-11ixuz2"&&(Bl.textContent=$s),KM=a(l),_l=n(l,"P",{"data-svelte-h":!0}),U(_l)!=="svelte-1s5jsz7"&&(_l.innerHTML=Ls),OM=a(l),Gl=n(l,"P",{"data-svelte-h":!0}),U(Gl)!=="svelte-10itto7"&&(Gl.textContent=Ds),ls=a(l),T(El.$$.fragment,l),Ms=a(l),Rl=n(l,"PRE",{"data-svelte-h":!0}),U(Rl)!=="svelte-1ud779o"&&(Rl.textContent=qs),ss=a(l),kl=n(l,"P",{"data-svelte-h":!0}),U(kl)!=="svelte-3s10dd"&&(kl.innerHTML=Ps),es=a(l),Xl=n(l,"P",{"data-svelte-h":!0}),U(Xl)!=="svelte-1x2bp5"&&(Xl.innerHTML=Ks),ts=a(l),T(Vl.$$.fragment,l),as=a(l),Wl=n(l,"P",{}),Os(Wl).forEach(s),this.h()},h(){le(m,"name","hf:doc:metadata"),le(m,"content",we)},m(l,M){ne(document.head,m),e(l,Sl,M),e(l,Fl,M),e(l,Nl,M),j(i,l,M),e(l,gl,M),j(A,l,M),e(l,Yl,M),j(o,l,M),e(l,Hl,M),e(l,c,M),e(l,vl,M),e(l,d,M),e(l,xl,M),e(l,r,M),e(l,zl,M),j(b,l,M),e(l,$l,M),e(l,f,M),e(l,Ll,M),j(u,l,M),e(l,Dl,M),e(l,h,M),e(l,ql,M),j(Z,l,M),e(l,Pl,M),e(l,B,M),e(l,Kl,M),e(l,_,M),e(l,Ol,M),j(G,l,M),e(l,lM,M),e(l,E,M),e(l,MM,M),j(R,l,M),e(l,sM,M),j(k,l,M),e(l,eM,M),j(X,l,M),e(l,tM,M),e(l,V,M),e(l,aM,M),e(l,Q,M),e(l,yM,M),j(F,l,M),e(l,nM,M),e(l,W,M),e(l,UM,M),e(l,S,M),e(l,JM,M),j(N,l,M),e(l,TM,M),e(l,g,M),e(l,jM,M),e(l,Y,M),e(l,wM,M),e(l,H,M),e(l,pM,M),e(l,v,M),e(l,CM,M),j(x,l,M),e(l,IM,M),e(l,z,M),e(l,mM,M),e(l,$,M),e(l,iM,M),j(L,l,M),e(l,AM,M),j(D,l,M),e(l,oM,M),e(l,q,M),e(l,cM,M),e(l,P,M),e(l,dM,M),e(l,K,M),e(l,rM,M),e(l,O,M),e(l,bM,M),e(l,ll,M),e(l,fM,M),e(l,Ml,M),e(l,uM,M),j(sl,l,M),e(l,hM,M),e(l,el,M),e(l,ZM,M),j(tl,l,M),e(l,BM,M),e(l,al,M),e(l,_M,M),e(l,yl,M),e(l,GM,M),e(l,nl,M),e(l,EM,M),j(Ul,l,M),e(l,RM,M),e(l,Jl,M),e(l,kM,M),e(l,Tl,M),e(l,XM,M),j(jl,l,M),e(l,VM,M),e(l,wl,M),e(l,QM,M),e(l,pl,M),e(l,FM,M),e(l,Cl,M),e(l,WM,M),j(Il,l,M),e(l,SM,M),e(l,ml,M),e(l,NM,M),j(il,l,M),e(l,gM,M),e(l,Al,M),e(l,YM,M),e(l,ol,M),e(l,HM,M),j(cl,l,M),e(l,vM,M),j(dl,l,M),e(l,xM,M),e(l,rl,M),e(l,zM,M),j(bl,l,M),e(l,$M,M),e(l,fl,M),e(l,LM,M),j(ul,l,M),e(l,DM,M),e(l,hl,M),e(l,qM,M),j(Zl,l,M),e(l,PM,M),e(l,Bl,M),e(l,KM,M),e(l,_l,M),e(l,OM,M),e(l,Gl,M),e(l,ls,M),j(El,l,M),e(l,Ms,M),e(l,Rl,M),e(l,ss,M),e(l,kl,M),e(l,es,M),e(l,Xl,M),e(l,ts,M),j(Vl,l,M),e(l,as,M),e(l,Wl,M),ys=!0},p:se,i(l){ys||(w(i.$$.fragment,l),w(A.$$.fragment,l),w(o.$$.fragment,l),w(b.$$.fragment,l),w(u.$$.fragment,l),w(Z.$$.fragment,l),w(G.$$.fragment,l),w(R.$$.fragment,l),w(k.$$.fragment,l),w(X.$$.fragment,l),w(F.$$.fragment,l),w(N.$$.fragment,l),w(x.$$.fragment,l),w(L.$$.fragment,l),w(D.$$.fragment,l),w(sl.$$.fragment,l),w(tl.$$.fragment,l),w(Ul.$$.fragment,l),w(jl.$$.fragment,l),w(Il.$$.fragment,l),w(il.$$.fragment,l),w(cl.$$.fragment,l),w(dl.$$.fragment,l),w(bl.$$.fragment,l),w(ul.$$.fragment,l),w(Zl.$$.fragment,l),w(El.$$.fragment,l),w(Vl.$$.fragment,l),ys=!0)},o(l){p(i.$$.fragment,l),p(A.$$.fragment,l),p(o.$$.fragment,l),p(b.$$.fragment,l),p(u.$$.fragment,l),p(Z.$$.fragment,l),p(G.$$.fragment,l),p(R.$$.fragment,l),p(k.$$.fragment,l),p(X.$$.fragment,l),p(F.$$.fragment,l),p(N.$$.fragment,l),p(x.$$.fragment,l),p(L.$$.fragment,l),p(D.$$.fragment,l),p(sl.$$.fragment,l),p(tl.$$.fragment,l),p(Ul.$$.fragment,l),p(jl.$$.fragment,l),p(Il.$$.fragment,l),p(il.$$.fragment,l),p(cl.$$.fragment,l),p(dl.$$.fragment,l),p(bl.$$.fragment,l),p(ul.$$.fragment,l),p(Zl.$$.fragment,l),p(El.$$.fragment,l),p(Vl.$$.fragment,l),ys=!1},d(l){l&&(s(Sl),s(Fl),s(Nl),s(gl),s(Yl),s(Hl),s(c),s(vl),s(d),s(xl),s(r),s(zl),s($l),s(f),s(Ll),s(Dl),s(h),s(ql),s(Pl),s(B),s(Kl),s(_),s(Ol),s(lM),s(E),s(MM),s(sM),s(eM),s(tM),s(V),s(aM),s(Q),s(yM),s(nM),s(W),s(UM),s(S),s(JM),s(TM),s(g),s(jM),s(Y),s(wM),s(H),s(pM),s(v),s(CM),s(IM),s(z),s(mM),s($),s(iM),s(AM),s(oM),s(q),s(cM),s(P),s(dM),s(K),s(rM),s(O),s(bM),s(ll),s(fM),s(Ml),s(uM),s(hM),s(el),s(ZM),s(BM),s(al),s(_M),s(yl),s(GM),s(nl),s(EM),s(RM),s(Jl),s(kM),s(Tl),s(XM),s(VM),s(wl),s(QM),s(pl),s(FM),s(Cl),s(WM),s(SM),s(ml),s(NM),s(gM),s(Al),s(YM),s(ol),s(HM),s(vM),s(xM),s(rl),s(zM),s($M),s(fl),s(LM),s(DM),s(hl),s(qM),s(PM),s(Bl),s(KM),s(_l),s(OM),s(Gl),s(ls),s(Ms),s(Rl),s(ss),s(kl),s(es),s(Xl),s(ts),s(as),s(Wl)),s(m),C(i,l),C(A,l),C(o,l),C(b,l),C(u,l),C(Z,l),C(G,l),C(R,l),C(k,l),C(X,l),C(F,l),C(N,l),C(x,l),C(L,l),C(D,l),C(sl,l),C(tl,l),C(Ul,l),C(jl,l),C(Il,l),C(il,l),C(cl,l),C(dl,l),C(bl,l),C(ul,l),C(Zl,l),C(El,l),C(Vl,l)}}}const we='{"title":"在单个 GPU 上针对自定义代码微调代码 LLM","local":"在单个-gpu-上针对自定义代码微调代码-llm","sections":[{"title":"数据集","local":"数据集","sections":[],"depth":2},{"title":"模型","local":"模型","sections":[],"depth":2},{"title":"准备数据","local":"准备数据","sections":[],"depth":2},{"title":"准备模型","local":"准备模型","sections":[],"depth":2},{"title":"训练模型","local":"训练模型","sections":[],"depth":2},{"title":"推理","local":"推理","sections":[],"depth":2}],"depth":1}';function pe(ns){return ee(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oe extends te{constructor(m){super(),ae(this,m,pe,je,Me,{})}}export{oe as component};
