import{s as We,n as Ge,o as _e}from"../chunks/scheduler.defa9a21.js";import{S as ke,i as Ze,g as a,s as o,r as c,A as xe,h as s,f as l,c as n,j as Ce,u as m,x as r,k as He,y as Re,a as i,v as p,d,t as u,w as h}from"../chunks/index.fe795e71.js";import{C as Be}from"../chunks/CodeBlock.204b6c34.js";import{H as P,E as Ve}from"../chunks/getInferenceSnippets.2234a8dd.js";function Pe(he){let f,q,z,Y,g,A,y,E,M,fe="Pytorch 2.0 introduced <code>torch.compile</code>, a powerful feature that makes PyTorch code run faster by JIT-compiling PyTorch code into optimized kernels. Key features of <code>torch.compile</code> include:",F,T,ge="<li><strong>Performance Improvement</strong>: Significantly speeds up model execution by optimizing the computation graph.</li> <li><strong>Ease of Use</strong>: Requires minimal code changes to implement, making it highly accessible.</li> <li><strong>Compatibility</strong>: Works seamlessly with existing PyTorch code and models.</li>",S,w,ye="When used with Accelerate, <code>torch.compile</code> integrates smoothly into distributed training workflows, allowing you to benefit from both distributed execution and compilation optimizations simultaneously.",X,b,Me="The first execution of compiled code typically takes longer as it includes the compilation time, but subsequent runs are significantly faster. For optimal performance in different scenarios, <code>torch.compile</code> offers various modes like <code>&quot;default&quot;</code>, <code>&quot;reduce-overhead&quot;</code> (which uses CUDA graphs to further reduce overhead), and <code>&quot;max-autotune&quot;</code> (which performs extensive autotuning to find the best kernels for your model).",Q,$,N,J,Te="Accelerate provides <code>TorchDynamoPlugin</code> for easy and seemless integration of <code>torch.compile</code> into your training scripts.",D,v,O,j,we="It is compatible with all other features and plugins of Accelerate, including mixed precision, distributed training (DDP, FSDP, Deepspeed), etc.",K,U,ee,I,be="Instead of trying to compile the whole model, which usually has a big problem space for optimization. Regional compilation targets repeated blocks of the same class and compiles them sequentially to hit the compiler’s cache. For example, in <code>GPT2LMHeadModel</code>, the repeated block/class is <code>GPT2Block</code>, and can be accessed as <code>model.transformer.h[0]</code>. The rest of the model (e.g model.lm_head) is compiled separately.",te,C,$e=`This allows us to speed up the compilation overhead / cold start of models like LLMs and Transformers in general.
See <a href="https://pytorch.org/tutorials/recipes/regional_compilation.html" rel="nofollow">https://pytorch.org/tutorials/recipes/regional_compilation.html</a> for more details.`,le,H,ie,B,Je="It can be enabled by setting <code>use_regional_compilation=True</code> in the <code>TorchDynamoPlugin</code> configuration:",oe,W,ne,G,ve="You could also use the <code>accelerate.utils.compile_regions</code> utility directly the same way you would use <code>torch.compile</code>.",ae,_,se,k,je='We have conducted extensive benchmarks comparing full compilation and regional compilation using the <code>torch.compile</code> feature in PyTorch. The full results are available in the <a href="https://github.com/huggingface/accelerate/tree/main/benchmarks/torch.compile/regional_compilation" rel="nofollow">accelerate repository</a>. The key findings from our benchmarks are:',re,Z,Ue="<li><strong>Comparable Performance</strong>: Regional compilation delivers performance speedups similar to full compilation, especially for larger models.</li> <li><strong>Faster Compilation</strong>: Regional compilation significantly reduces the time taken to compile models, making it a more efficient choice for deployment.</li> <li><strong>Batch Size Impact</strong>: The performance difference between compilation strategies diminishes with larger batch sizes, indicating that the overhead of compilation is less impactful in those scenarios.</li> <li><strong>Model Size Consideration</strong>: The benefits of regional compilation are more pronounced in larger models, where the compilation time savings can be substantial.</li> <li><strong>Practical Application</strong>: For real-world applications, regional compilation is a practical choice for optimizing training cold start times, especially when working with large models.</li>",ce,x,me,R,Ie="Both full and regional compilation can significantly speed up your models. Regional compilation offers a practical balance between compilation time and runtime performance, especially for training large models with substantial batch sizes.",pe,V,de,L,ue;return g=new P({props:{title:"Compilation",local:"compilation",headingTag:"h1"}}),y=new P({props:{title:"Overview",local:"overview",headingTag:"h2"}}),$=new P({props:{title:"Using torch.compile with Accelerate",local:"using-torchcompile-with-accelerate",headingTag:"h2"}}),v=new Be({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwVG9yY2hEeW5hbW9QbHVnaW4lMEElMEElMjMlMjBDb25maWd1cmUlMjB0aGUlMjBjb21waWxhdGlvbiUyMGJhY2tlbmQlMEFkeW5hbW9fcGx1Z2luJTIwJTNEJTIwVG9yY2hEeW5hbW9QbHVnaW4oJTBBJTIwJTIwJTIwJTIwYmFja2VuZCUzRCUyMmluZHVjdG9yJTIyJTJDJTIwJTIwJTIzJTIwT3B0aW9ucyUzQSUyMCUyMmluZHVjdG9yJTIyJTJDJTIwJTIyYW90X2VhZ2VyJTIyJTJDJTIwJTIyYW90X252ZnVzZXIlMjIlMkMlMjBldGMuJTBBJTIwJTIwJTIwJTIwbW9kZSUzRCUyMmRlZmF1bHQlMjIlMkMlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBPcHRpb25zJTNBJTIwJTIyZGVmYXVsdCUyMiUyQyUyMCUyMnJlZHVjZS1vdmVyaGVhZCUyMiUyQyUyMCUyMm1heC1hdXRvdHVuZSUyMiUwQSUyMCUyMCUyMCUyMGZ1bGxncmFwaCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBkeW5hbWljJTNERmFsc2UlMEEpJTBBJTBBJTIzJTIwSW5pdGlhbGl6ZSUyMGFjY2VsZXJhdG9yJTIwd2l0aCUyMHRoZSUyMHBsdWdpbiUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZHluYW1vX3BsdWdpbiUzRGR5bmFtb19wbHVnaW4pJTBBJTIzJTIwVGhpcyUyMHdpbGwlMjBhcHBseSUyMHRvcmNoLmNvbXBpbGUlMjB0byUyMHlvdXIlMjBtb2RlbCUwQW1vZGVsJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZShtb2RlbCk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator
<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> TorchDynamoPlugin

<span class="hljs-comment"># Configure the compilation backend</span>
dynamo_plugin = TorchDynamoPlugin(
    backend=<span class="hljs-string">&quot;inductor&quot;</span>,  <span class="hljs-comment"># Options: &quot;inductor&quot;, &quot;aot_eager&quot;, &quot;aot_nvfuser&quot;, etc.</span>
    mode=<span class="hljs-string">&quot;default&quot;</span>,      <span class="hljs-comment"># Options: &quot;default&quot;, &quot;reduce-overhead&quot;, &quot;max-autotune&quot;</span>
    fullgraph=<span class="hljs-literal">True</span>,
    dynamic=<span class="hljs-literal">False</span>
)

<span class="hljs-comment"># Initialize accelerator with the plugin</span>
accelerator = Accelerator(dynamo_plugin=dynamo_plugin)
<span class="hljs-comment"># This will apply torch.compile to your model</span>
model = accelerator.prepare(model)`,wrap:!1}}),U=new P({props:{title:"Regional Compilation",local:"regional-compilation",headingTag:"h2"}}),H=new P({props:{title:"How to Use Regional Compilation",local:"how-to-use-regional-compilation",headingTag:"h3"}}),W=new Be({props:{code:"JTIzJTIwQ29uZmlndXJlJTIwdGhlJTIwY29tcGlsYXRpb24lMjBiYWNrZW5kJTBBZHluYW1vX3BsdWdpbiUyMCUzRCUyMFRvcmNoRHluYW1vUGx1Z2luKCUwQSUyMCUyMCUyMCUyMHVzZV9yZWdpb25hbF9jb21waWxhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAuLi4lMjAlMjMlMjBvdGhlciUyMHBhcmFtZXRlcnMlMEEpJTBBJTIzJTIwSW5pdGlhbGl6ZSUyMGFjY2VsZXJhdG9yJTIwd2l0aCUyMHRoZSUyMHBsdWdpbiUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZHluYW1vX3BsdWdpbiUzRGR5bmFtb19wbHVnaW4pJTBBJTIzJTIwVGhpcyUyMHdpbGwlMjBhcHBseSUyMGNvbXBpbGVfcmVnaW9ucyUyMHRvJTIweW91ciUyMG1vZGVsJTBBbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKG1vZGVsKQ==",highlighted:`<span class="hljs-comment"># Configure the compilation backend</span>
dynamo_plugin = TorchDynamoPlugin(
    use_regional_compilation=<span class="hljs-literal">True</span>,
    ... <span class="hljs-comment"># other parameters</span>
)
<span class="hljs-comment"># Initialize accelerator with the plugin</span>
accelerator = Accelerator(dynamo_plugin=dynamo_plugin)
<span class="hljs-comment"># This will apply compile_regions to your model</span>
model = accelerator.prepare(model)`,wrap:!1}}),_=new P({props:{title:"Benefits of Regional Compilation",local:"benefits-of-regional-compilation",headingTag:"h3"}}),x=new P({props:{title:"Conclusion",local:"conclusion",headingTag:"h2"}}),V=new Ve({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/compilation.md"}}),{c(){f=a("meta"),q=o(),z=a("p"),Y=o(),c(g.$$.fragment),A=o(),c(y.$$.fragment),E=o(),M=a("p"),M.innerHTML=fe,F=o(),T=a("ul"),T.innerHTML=ge,S=o(),w=a("p"),w.innerHTML=ye,X=o(),b=a("p"),b.innerHTML=Me,Q=o(),c($.$$.fragment),N=o(),J=a("p"),J.innerHTML=Te,D=o(),c(v.$$.fragment),O=o(),j=a("p"),j.textContent=we,K=o(),c(U.$$.fragment),ee=o(),I=a("p"),I.innerHTML=be,te=o(),C=a("p"),C.innerHTML=$e,le=o(),c(H.$$.fragment),ie=o(),B=a("p"),B.innerHTML=Je,oe=o(),c(W.$$.fragment),ne=o(),G=a("p"),G.innerHTML=ve,ae=o(),c(_.$$.fragment),se=o(),k=a("p"),k.innerHTML=je,re=o(),Z=a("ol"),Z.innerHTML=Ue,ce=o(),c(x.$$.fragment),me=o(),R=a("p"),R.textContent=Ie,pe=o(),c(V.$$.fragment),de=o(),L=a("p"),this.h()},l(e){const t=xe("svelte-u9bgzb",document.head);f=s(t,"META",{name:!0,content:!0}),t.forEach(l),q=n(e),z=s(e,"P",{}),Ce(z).forEach(l),Y=n(e),m(g.$$.fragment,e),A=n(e),m(y.$$.fragment,e),E=n(e),M=s(e,"P",{"data-svelte-h":!0}),r(M)!=="svelte-az9dn0"&&(M.innerHTML=fe),F=n(e),T=s(e,"UL",{"data-svelte-h":!0}),r(T)!=="svelte-pkkxsd"&&(T.innerHTML=ge),S=n(e),w=s(e,"P",{"data-svelte-h":!0}),r(w)!=="svelte-gd5iye"&&(w.innerHTML=ye),X=n(e),b=s(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-1qamxj"&&(b.innerHTML=Me),Q=n(e),m($.$$.fragment,e),N=n(e),J=s(e,"P",{"data-svelte-h":!0}),r(J)!=="svelte-w2rs5o"&&(J.innerHTML=Te),D=n(e),m(v.$$.fragment,e),O=n(e),j=s(e,"P",{"data-svelte-h":!0}),r(j)!=="svelte-5uylqs"&&(j.textContent=we),K=n(e),m(U.$$.fragment,e),ee=n(e),I=s(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-1fkldr7"&&(I.innerHTML=be),te=n(e),C=s(e,"P",{"data-svelte-h":!0}),r(C)!=="svelte-cs154s"&&(C.innerHTML=$e),le=n(e),m(H.$$.fragment,e),ie=n(e),B=s(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-50g4qf"&&(B.innerHTML=Je),oe=n(e),m(W.$$.fragment,e),ne=n(e),G=s(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-j2ilao"&&(G.innerHTML=ve),ae=n(e),m(_.$$.fragment,e),se=n(e),k=s(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-w1r301"&&(k.innerHTML=je),re=n(e),Z=s(e,"OL",{"data-svelte-h":!0}),r(Z)!=="svelte-qsfo8r"&&(Z.innerHTML=Ue),ce=n(e),m(x.$$.fragment,e),me=n(e),R=s(e,"P",{"data-svelte-h":!0}),r(R)!=="svelte-1iudzer"&&(R.textContent=Ie),pe=n(e),m(V.$$.fragment,e),de=n(e),L=s(e,"P",{}),Ce(L).forEach(l),this.h()},h(){He(f,"name","hf:doc:metadata"),He(f,"content",ze)},m(e,t){Re(document.head,f),i(e,q,t),i(e,z,t),i(e,Y,t),p(g,e,t),i(e,A,t),p(y,e,t),i(e,E,t),i(e,M,t),i(e,F,t),i(e,T,t),i(e,S,t),i(e,w,t),i(e,X,t),i(e,b,t),i(e,Q,t),p($,e,t),i(e,N,t),i(e,J,t),i(e,D,t),p(v,e,t),i(e,O,t),i(e,j,t),i(e,K,t),p(U,e,t),i(e,ee,t),i(e,I,t),i(e,te,t),i(e,C,t),i(e,le,t),p(H,e,t),i(e,ie,t),i(e,B,t),i(e,oe,t),p(W,e,t),i(e,ne,t),i(e,G,t),i(e,ae,t),p(_,e,t),i(e,se,t),i(e,k,t),i(e,re,t),i(e,Z,t),i(e,ce,t),p(x,e,t),i(e,me,t),i(e,R,t),i(e,pe,t),p(V,e,t),i(e,de,t),i(e,L,t),ue=!0},p:Ge,i(e){ue||(d(g.$$.fragment,e),d(y.$$.fragment,e),d($.$$.fragment,e),d(v.$$.fragment,e),d(U.$$.fragment,e),d(H.$$.fragment,e),d(W.$$.fragment,e),d(_.$$.fragment,e),d(x.$$.fragment,e),d(V.$$.fragment,e),ue=!0)},o(e){u(g.$$.fragment,e),u(y.$$.fragment,e),u($.$$.fragment,e),u(v.$$.fragment,e),u(U.$$.fragment,e),u(H.$$.fragment,e),u(W.$$.fragment,e),u(_.$$.fragment,e),u(x.$$.fragment,e),u(V.$$.fragment,e),ue=!1},d(e){e&&(l(q),l(z),l(Y),l(A),l(E),l(M),l(F),l(T),l(S),l(w),l(X),l(b),l(Q),l(N),l(J),l(D),l(O),l(j),l(K),l(ee),l(I),l(te),l(C),l(le),l(ie),l(B),l(oe),l(ne),l(G),l(ae),l(se),l(k),l(re),l(Z),l(ce),l(me),l(R),l(pe),l(de),l(L)),l(f),h(g,e),h(y,e),h($,e),h(v,e),h(U,e),h(H,e),h(W,e),h(_,e),h(x,e),h(V,e)}}}const ze='{"title":"Compilation","local":"compilation","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Using torch.compile with Accelerate","local":"using-torchcompile-with-accelerate","sections":[],"depth":2},{"title":"Regional Compilation","local":"regional-compilation","sections":[{"title":"How to Use Regional Compilation","local":"how-to-use-regional-compilation","sections":[],"depth":3},{"title":"Benefits of Regional Compilation","local":"benefits-of-regional-compilation","sections":[],"depth":3}],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2}],"depth":1}';function Le(he){return _e(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fe extends ke{constructor(f){super(),Ze(this,f,Le,Pe,We,{})}}export{Fe as component};
