import{s as Gl,o as gl,n as Zl}from"../chunks/scheduler.8c3d61f6.js";import{S as Bl,i as Il,g as y,s as p,r as b,A as Wl,h as d,f as t,c as m,j as Ul,u as h,x as f,k as pl,y as _l,a,v as T,d as w,t as U,w as j}from"../chunks/index.da70eac4.js";import{C as il}from"../chunks/CodeBlock.a9c4becf.js";import{H as ml,E as vl}from"../chunks/getInferenceSnippets.cb466318.js";import{H as Xl,a as jl}from"../chunks/HfOption.6c3b4e77.js";function Vl(G){let s,u='<a href="./memory#model-offloading">Model CPU offloading</a> moves an individual pipeline component, like the transformer model, to the GPU when it is needed for computation. Otherwise, it is offloaded to the CPU.',c,i,n;return i=new il({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRGlmZnVzaW9uUGlwZWxpbmUlMEFmcm9tJTIwZGlmZnVzZXJzLnF1YW50aXplcnMlMjBpbXBvcnQlMjBQaXBlbGluZVF1YW50aXphdGlvbkNvbmZpZyUwQSUwQXRvcmNoLl9keW5hbW8uY29uZmlnLmNhY2hlX3NpemVfbGltaXQlMjAlM0QlMjAxMDAwJTBBdG9yY2guX2R5bmFtby5jb25maWcuY2FwdHVyZV9keW5hbWljX291dHB1dF9zaGFwZV9vcHMlMjAlM0QlMjBUcnVlJTBBJTBBJTIzJTIwcXVhbnRpemUlMEFwaXBlbGluZV9xdWFudF9jb25maWclMjAlM0QlMjBQaXBlbGluZVF1YW50aXphdGlvbkNvbmZpZyglMEElMjAlMjAlMjAlMjBxdWFudF9iYWNrZW5kJTNEJTIyYml0c2FuZGJ5dGVzXzRiaXQlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudF9rd2FyZ3MlM0QlN0IlMjJsb2FkX2luXzRiaXQlMjIlM0ElMjBUcnVlJTJDJTIwJTIyYm5iXzRiaXRfcXVhbnRfdHlwZSUyMiUzQSUyMCUyMm5mNCUyMiUyQyUyMCUyMmJuYl80Yml0X2NvbXB1dGVfZHR5cGUlMjIlM0ElMjB0b3JjaC5iZmxvYXQxNiU3RCUyQyUwQSUyMCUyMCUyMCUyMGNvbXBvbmVudHNfdG9fcXVhbnRpemUlM0QlNUIlMjJ0cmFuc2Zvcm1lciUyMiUyQyUyMCUyMnRleHRfZW5jb2Rlcl8yJTIyJTVEJTJDJTBBKSUwQXBpcGVsaW5lJTIwJTNEJTIwRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RwaXBlbGluZV9xdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKS50byglMjJjdWRhJTIyKSUwQSUwQSUyMyUyMG1vZGVsJTIwQ1BVJTIwb2ZmbG9hZGluZyUwQXBpcGVsaW5lLmVuYWJsZV9tb2RlbF9jcHVfb2ZmbG9hZCgpJTBBJTBBJTIzJTIwY29tcGlsZSUwQXBpcGVsaW5lLnRyYW5zZm9ybWVyLmNvbXBpbGUoKSUwQXBpcGVsaW5lKCUwQSUyMCUyMCUyMCUyMCUyMmNpbmVtYXRpYyUyMGZpbG0lMjBzdGlsbCUyMG9mJTIwYSUyMGNhdCUyMHNpcHBpbmclMjBhJTIwbWFyZ2FyaXRhJTIwaW4lMjBhJTIwcG9vbCUyMGluJTIwUGFsbSUyMFNwcmluZ3MlMkMlMjBDYWxpZm9ybmlhJTJDJTIwaGlnaGx5JTIwZGV0YWlsZWQlMkMlMjBoaWdoJTIwYnVkZ2V0JTIwaG9sbHl3b29kJTIwbW92aWUlMkMlMjBjaW5lbWFzY29wZSUyQyUyMG1vb2R5JTJDJTIwZXBpYyUyQyUyMGdvcmdlb3VzJTJDJTIwZmlsbSUyMGdyYWluJTIyJTBBKS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig

torch._dynamo.config.cache_size_limit = <span class="hljs-number">1000</span>
torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    quantization_config=pipeline_quant_config,
    torch_dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># model CPU offloading</span>
pipeline.enable_model_cpu_offload()

<span class="hljs-comment"># compile</span>
pipeline.transformer.<span class="hljs-built_in">compile</span>()
pipeline(
    <span class="hljs-string">&quot;cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain&quot;</span>
).images[<span class="hljs-number">0</span>]`,wrap:!1}}),{c(){s=y("p"),s.innerHTML=u,c=p(),b(i.$$.fragment)},l(o){s=d(o,"P",{"data-svelte-h":!0}),f(s)!=="svelte-2htktv"&&(s.innerHTML=u),c=m(o),h(i.$$.fragment,o)},m(o,M){a(o,s,M),a(o,c,M),T(i,o,M),n=!0},p:Zl,i(o){n||(w(i.$$.fragment,o),n=!0)},o(o){U(i.$$.fragment,o),n=!1},d(o){o&&(t(s),t(c)),j(i,o)}}}function Cl(G){let s,u='<a href="./memory#group-offloading">Group offloading</a> moves the internal layers of an individual pipeline component, like the transformer model, to the GPU for computation and offloads it when it’s not required. At the same time, it uses the <a href="./memory#cuda-stream">CUDA stream</a> feature to prefetch the next layer for execution.',c,i,n="By overlapping computation and data transfer, it is faster than model CPU offloading while also saving memory.",o,M,Z;return M=new il({props:{code:" ",highlighted:`<span class="hljs-comment"># pip install ftfy</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.hooks <span class="hljs-keyword">import</span> apply_group_offloading
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel

torch._dynamo.config.cache_size_limit = <span class="hljs-number">1000</span>
torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder&quot;</span>],
)

text_encoder = UMT5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;text_encoder&quot;</span>, torch_dtype=torch.bfloat16
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>,
    quantization_config=pipeline_quant_config,
    torch_dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># group offloading</span>
onload_device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
offload_device = torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)

pipeline.transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>,
    non_blocking=<span class="hljs-literal">True</span>
)
pipeline.vae.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>,
    non_blocking=<span class="hljs-literal">True</span>
)
apply_group_offloading(
    pipeline.text_encoder,
    onload_device=onload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>,
    non_blocking=<span class="hljs-literal">True</span>
)

<span class="hljs-comment"># compile</span>
pipeline.transformer.<span class="hljs-built_in">compile</span>()

prompt = <span class="hljs-string">&quot;&quot;&quot;
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
&quot;&quot;&quot;</span>
negative_prompt = <span class="hljs-string">&quot;&quot;&quot;
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
&quot;&quot;&quot;</span>

output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=<span class="hljs-number">81</span>,
    guidance_scale=<span class="hljs-number">5.0</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(output, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">16</span>)`,wrap:!1}}),{c(){s=y("p"),s.innerHTML=u,c=p(),i=y("p"),i.textContent=n,o=p(),b(M.$$.fragment)},l(r){s=d(r,"P",{"data-svelte-h":!0}),f(s)!=="svelte-h38yli"&&(s.innerHTML=u),c=m(r),i=d(r,"P",{"data-svelte-h":!0}),f(i)!=="svelte-1f0mtjr"&&(i.textContent=n),o=m(r),h(M.$$.fragment,r)},m(r,J){a(r,s,J),a(r,c,J),a(r,i,J),a(r,o,J),T(M,r,J),Z=!0},p:Zl,i(r){Z||(w(M.$$.fragment,r),Z=!0)},o(r){U(M.$$.fragment,r),Z=!1},d(r){r&&(t(s),t(c),t(i),t(o)),j(M,r)}}}function Yl(G){let s,u,c,i;return s=new jl({props:{id:"offloading",option:"model CPU offloading",$$slots:{default:[Vl]},$$scope:{ctx:G}}}),c=new jl({props:{id:"offloading",option:"group offloading",$$slots:{default:[Cl]},$$scope:{ctx:G}}}),{c(){b(s.$$.fragment),u=p(),b(c.$$.fragment)},l(n){h(s.$$.fragment,n),u=m(n),h(c.$$.fragment,n)},m(n,o){T(s,n,o),a(n,u,o),T(c,n,o),i=!0},p(n,o){const M={};o&2&&(M.$$scope={dirty:o,ctx:n}),s.$set(M);const Z={};o&2&&(Z.$$scope={dirty:o,ctx:n}),c.$set(Z)},i(n){i||(w(s.$$.fragment,n),w(c.$$.fragment,n),i=!0)},o(n){U(s.$$.fragment,n),U(c.$$.fragment,n),i=!1},d(n){n&&t(u),j(s,n),j(c,n)}}}function Rl(G){let s,u,c,i,n,o,M,Z='Optimizing models often involves trade-offs between <a href="./fp16">inference speed</a> and <a href="./memory">memory-usage</a>. For instance, while <a href="./cache">caching</a> can boost inference speed, it also increases memory consumption since it needs to store the outputs of intermediate attention layers. A more balanced optimization strategy combines quantizing a model, <a href="./fp16#torchcompile">torch.compile</a> and various <a href="./memory#offloading">offloading methods</a>.',r,J,cl='<p>Check the <a href="./fp16#torchcompile">torch.compile</a> guide to learn more about compilation and how they can be applied here. For example, regional compilation can significantly reduce compilation time without giving up any speedups.</p>',H,B,rl='For image generation, combining quantization and <a href="./memory#model-offloading">model offloading</a> can often give the best trade-off between quality, speed, and memory. Group offloading is not as effective for image generation because it is usually not possible to <em>fully</em> overlap data transfer if the compute kernel finishes faster. This results in some communication overhead between the CPU and GPU.',x,I,Ml='For video generation, combining quantization and <a href="./memory#group-offloading">group-offloading</a> tends to be better because video models are more compute-bound.',N,W,yl="The table below provides a comparison of optimization strategy combinations and their impact on latency and memory-usage for Flux.",$,_,dl="<thead><tr><th>combination</th> <th>latency (s)</th> <th>memory-usage (GB)</th></tr></thead> <tbody><tr><td>quantization</td> <td>32.602</td> <td>14.9453</td></tr> <tr><td>quantization, torch.compile</td> <td>25.847</td> <td>14.9448</td></tr> <tr><td>quantization, torch.compile, model CPU offloading</td> <td>32.312</td> <td>12.2369</td></tr></tbody>",S,v,ul='These results are benchmarked on Flux with a RTX 4090. The transformer and text_encoder components are quantized. Refer to the <a href="https://gist.github.com/sayakpaul/0db9d8eeeb3d2a0e5ed7cf0d9ca19b7d">benchmarking script</a> if you&#39;re interested in evaluating your own model.',A,X,fl='This guide will show you how to compile and offload a quantized model with <a href="../quantization/bitsandbytes#torchcompile">bitsandbytes</a>. Make sure you are using <a href="https://pytorch.org/get-started/locally/" rel="nofollow">PyTorch nightly</a> and the latest version of bitsandbytes.',L,V,D,C,P,Y,Jl='Start by <a href="../quantization/overview">quantizing</a> a model to reduce the memory required for storage and <a href="./fp16#torchcompile">compiling</a> it to accelerate inference.',K,R,bl='Configure the <a href="https://docs.pytorch.org/docs/stable/torch.compiler_dynamo_overview.html" rel="nofollow">Dynamo</a> <code>capture_dynamic_output_shape_ops = True</code> to handle dynamic outputs when compiling bitsandbytes models.',O,k,ll,q,el,Q,hl="In addition to quantization and torch.compile, try offloading if you need to reduce memory-usage further. Offloading moves various layers or model components from the CPU to the GPU as needed for computations.",tl,F,Tl='Configure the <a href="https://docs.pytorch.org/docs/stable/torch.compiler_dynamo_overview.html" rel="nofollow">Dynamo</a> <code>cache_size_limit</code> during offloading to avoid excessive recompilation and set <code>capture_dynamic_output_shape_ops = True</code> to handle dynamic outputs when compiling bitsandbytes models.',al,g,nl,z,sl,E,ol;return n=new ml({props:{title:"Compiling and offloading quantized models",local:"compiling-and-offloading-quantized-models",headingTag:"h1"}}),V=new il({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwYml0c2FuZGJ5dGVz",highlighted:"pip install -U bitsandbytes",wrap:!1}}),C=new ml({props:{title:"Quantization and torch.compile",local:"quantization-and-torchcompile",headingTag:"h2"}}),k=new il({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRGlmZnVzaW9uUGlwZWxpbmUlMEFmcm9tJTIwZGlmZnVzZXJzLnF1YW50aXplcnMlMjBpbXBvcnQlMjBQaXBlbGluZVF1YW50aXphdGlvbkNvbmZpZyUwQSUwQXRvcmNoLl9keW5hbW8uY29uZmlnLmNhcHR1cmVfZHluYW1pY19vdXRwdXRfc2hhcGVfb3BzJTIwJTNEJTIwVHJ1ZSUwQSUwQSUyMyUyMHF1YW50aXplJTBBcGlwZWxpbmVfcXVhbnRfY29uZmlnJTIwJTNEJTIwUGlwZWxpbmVRdWFudGl6YXRpb25Db25maWcoJTBBJTIwJTIwJTIwJTIwcXVhbnRfYmFja2VuZCUzRCUyMmJpdHNhbmRieXRlc180Yml0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRfa3dhcmdzJTNEJTdCJTIybG9hZF9pbl80Yml0JTIyJTNBJTIwVHJ1ZSUyQyUyMCUyMmJuYl80Yml0X3F1YW50X3R5cGUlMjIlM0ElMjAlMjJuZjQlMjIlMkMlMjAlMjJibmJfNGJpdF9jb21wdXRlX2R0eXBlJTIyJTNBJTIwdG9yY2guYmZsb2F0MTYlN0QlMkMlMEElMjAlMjAlMjAlMjBjb21wb25lbnRzX3RvX3F1YW50aXplJTNEJTVCJTIydHJhbnNmb3JtZXIlMjIlMkMlMjAlMjJ0ZXh0X2VuY29kZXJfMiUyMiU1RCUyQyUwQSklMEFwaXBlbGluZSUyMCUzRCUyMERpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcGlwZWxpbmVfcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSkudG8oJTIyY3VkYSUyMiklMEElMEElMjMlMjBjb21waWxlJTBBcGlwZWxpbmUudHJhbnNmb3JtZXIudG8obWVtb3J5X2Zvcm1hdCUzRHRvcmNoLmNoYW5uZWxzX2xhc3QpJTBBcGlwZWxpbmUudHJhbnNmb3JtZXIuY29tcGlsZShtb2RlJTNEJTIybWF4LWF1dG90dW5lJTIyJTJDJTIwZnVsbGdyYXBoJTNEVHJ1ZSklMEFwaXBlbGluZSglMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBjaW5lbWF0aWMlMjBmaWxtJTIwc3RpbGwlMjBvZiUyMGElMjBjYXQlMjBzaXBwaW5nJTIwYSUyMG1hcmdhcml0YSUyMGluJTIwYSUyMHBvb2wlMjBpbiUyMFBhbG0lMjBTcHJpbmdzJTJDJTIwQ2FsaWZvcm5pYSUwQSUyMCUyMCUyMCUyMGhpZ2hseSUyMGRldGFpbGVkJTJDJTIwaGlnaCUyMGJ1ZGdldCUyMGhvbGx5d29vZCUyMG1vdmllJTJDJTIwY2luZW1hc2NvcGUlMkMlMjBtb29keSUyQyUyMGVwaWMlMkMlMjBnb3JnZW91cyUyQyUyMGZpbG0lMjBncmFpbiUwQSUyMiUyMiUyMiUwQSkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig

torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    quantization_config=pipeline_quant_config,
    torch_dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># compile</span>
pipeline.transformer.to(memory_format=torch.channels_last)
pipeline.transformer.<span class="hljs-built_in">compile</span>(mode=<span class="hljs-string">&quot;max-autotune&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
pipeline(<span class="hljs-string">&quot;&quot;&quot;
    cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California
    highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain
&quot;&quot;&quot;</span>
).images[<span class="hljs-number">0</span>]`,wrap:!1}}),q=new ml({props:{title:"Quantization, torch.compile, and offloading",local:"quantization-torchcompile-and-offloading",headingTag:"h2"}}),g=new Xl({props:{id:"offloading",options:["model CPU offloading","group offloading"],$$slots:{default:[Yl]},$$scope:{ctx:G}}}),z=new vl({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/speed-memory-optims.md"}}),{c(){s=y("meta"),u=p(),c=y("p"),i=p(),b(n.$$.fragment),o=p(),M=y("p"),M.innerHTML=Z,r=p(),J=y("blockquote"),J.innerHTML=cl,H=p(),B=y("p"),B.innerHTML=rl,x=p(),I=y("p"),I.innerHTML=Ml,N=p(),W=y("p"),W.textContent=yl,$=p(),_=y("table"),_.innerHTML=dl,S=p(),v=y("small"),v.innerHTML=ul,A=p(),X=y("p"),X.innerHTML=fl,L=p(),b(V.$$.fragment),D=p(),b(C.$$.fragment),P=p(),Y=y("p"),Y.innerHTML=Jl,K=p(),R=y("p"),R.innerHTML=bl,O=p(),b(k.$$.fragment),ll=p(),b(q.$$.fragment),el=p(),Q=y("p"),Q.textContent=hl,tl=p(),F=y("p"),F.innerHTML=Tl,al=p(),b(g.$$.fragment),nl=p(),b(z.$$.fragment),sl=p(),E=y("p"),this.h()},l(l){const e=Wl("svelte-u9bgzb",document.head);s=d(e,"META",{name:!0,content:!0}),e.forEach(t),u=m(l),c=d(l,"P",{}),Ul(c).forEach(t),i=m(l),h(n.$$.fragment,l),o=m(l),M=d(l,"P",{"data-svelte-h":!0}),f(M)!=="svelte-xssjno"&&(M.innerHTML=Z),r=m(l),J=d(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),f(J)!=="svelte-neoyiw"&&(J.innerHTML=cl),H=m(l),B=d(l,"P",{"data-svelte-h":!0}),f(B)!=="svelte-1k9tg08"&&(B.innerHTML=rl),x=m(l),I=d(l,"P",{"data-svelte-h":!0}),f(I)!=="svelte-1gtsmte"&&(I.innerHTML=Ml),N=m(l),W=d(l,"P",{"data-svelte-h":!0}),f(W)!=="svelte-1hx16oz"&&(W.textContent=yl),$=m(l),_=d(l,"TABLE",{"data-svelte-h":!0}),f(_)!=="svelte-4braeo"&&(_.innerHTML=dl),S=m(l),v=d(l,"SMALL",{"data-svelte-h":!0}),f(v)!=="svelte-t67k1s"&&(v.innerHTML=ul),A=m(l),X=d(l,"P",{"data-svelte-h":!0}),f(X)!=="svelte-97u2df"&&(X.innerHTML=fl),L=m(l),h(V.$$.fragment,l),D=m(l),h(C.$$.fragment,l),P=m(l),Y=d(l,"P",{"data-svelte-h":!0}),f(Y)!=="svelte-1blxhao"&&(Y.innerHTML=Jl),K=m(l),R=d(l,"P",{"data-svelte-h":!0}),f(R)!=="svelte-xwvm0e"&&(R.innerHTML=bl),O=m(l),h(k.$$.fragment,l),ll=m(l),h(q.$$.fragment,l),el=m(l),Q=d(l,"P",{"data-svelte-h":!0}),f(Q)!=="svelte-2st498"&&(Q.textContent=hl),tl=m(l),F=d(l,"P",{"data-svelte-h":!0}),f(F)!=="svelte-136cwt1"&&(F.innerHTML=Tl),al=m(l),h(g.$$.fragment,l),nl=m(l),h(z.$$.fragment,l),sl=m(l),E=d(l,"P",{}),Ul(E).forEach(t),this.h()},h(){pl(s,"name","hf:doc:metadata"),pl(s,"content",kl),pl(J,"class","tip")},m(l,e){_l(document.head,s),a(l,u,e),a(l,c,e),a(l,i,e),T(n,l,e),a(l,o,e),a(l,M,e),a(l,r,e),a(l,J,e),a(l,H,e),a(l,B,e),a(l,x,e),a(l,I,e),a(l,N,e),a(l,W,e),a(l,$,e),a(l,_,e),a(l,S,e),a(l,v,e),a(l,A,e),a(l,X,e),a(l,L,e),T(V,l,e),a(l,D,e),T(C,l,e),a(l,P,e),a(l,Y,e),a(l,K,e),a(l,R,e),a(l,O,e),T(k,l,e),a(l,ll,e),T(q,l,e),a(l,el,e),a(l,Q,e),a(l,tl,e),a(l,F,e),a(l,al,e),T(g,l,e),a(l,nl,e),T(z,l,e),a(l,sl,e),a(l,E,e),ol=!0},p(l,[e]){const wl={};e&2&&(wl.$$scope={dirty:e,ctx:l}),g.$set(wl)},i(l){ol||(w(n.$$.fragment,l),w(V.$$.fragment,l),w(C.$$.fragment,l),w(k.$$.fragment,l),w(q.$$.fragment,l),w(g.$$.fragment,l),w(z.$$.fragment,l),ol=!0)},o(l){U(n.$$.fragment,l),U(V.$$.fragment,l),U(C.$$.fragment,l),U(k.$$.fragment,l),U(q.$$.fragment,l),U(g.$$.fragment,l),U(z.$$.fragment,l),ol=!1},d(l){l&&(t(u),t(c),t(i),t(o),t(M),t(r),t(J),t(H),t(B),t(x),t(I),t(N),t(W),t($),t(_),t(S),t(v),t(A),t(X),t(L),t(D),t(P),t(Y),t(K),t(R),t(O),t(ll),t(el),t(Q),t(tl),t(F),t(al),t(nl),t(sl),t(E)),t(s),j(n,l),j(V,l),j(C,l),j(k,l),j(q,l),j(g,l),j(z,l)}}}const kl='{"title":"Compiling and offloading quantized models","local":"compiling-and-offloading-quantized-models","sections":[{"title":"Quantization and torch.compile","local":"quantization-and-torchcompile","sections":[],"depth":2},{"title":"Quantization, torch.compile, and offloading","local":"quantization-torchcompile-and-offloading","sections":[],"depth":2}],"depth":1}';function ql(G){return gl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xl extends Bl{constructor(s){super(),Il(this,s,ql,Rl,Gl,{})}}export{xl as component};
