import{s as Lt,o as St,n as Ie}from"../chunks/scheduler.8c3d61f6.js";import{S as Dt,i as Pt,g as d,s as o,r as y,A as Kt,h as f,f as l,c as r,j as Yt,u as b,x as u,k as jt,y as Ot,a as s,v as J,d as h,t as T,w as U}from"../chunks/index.da70eac4.js";import{T as Qe}from"../chunks/Tip.1d9b8c37.js";import{C as W}from"../chunks/CodeBlock.a9c4becf.js";import{H as D,E as el}from"../chunks/getInferenceSnippets.725ed3d4.js";import{H as xt,a as gt}from"../chunks/HfOption.6c3b4e77.js";function tl(C){let a,M='The <code>CLIPTextModel</code> and <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> aren’t quantized because they’re already small in size and because <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> only has a few <code>torch.nn.Linear</code> layers.';return{c(){a=d("p"),a.innerHTML=M},l(i){a=f(i,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1bd21r0"&&(a.innerHTML=M)},m(i,c){s(i,a,c)},p:Ie,d(i){i&&l(a)}}}function ll(C){let a,M="Quantizing a model in 8-bit halves the memory-usage:",i,c,m=`bitsandbytes is supported in both Transformers and Diffusers, so you can quantize both the
<a href="/docs/diffusers/main/en/api/models/flux_transformer#diffusers.FluxTransformer2DModel">FluxTransformer2DModel</a> and <a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a>.`,w,Z,Q="For Ada and higher-series GPUs. we recommend changing <code>torch_dtype</code> to <code>torch.bfloat16</code>.",q,g,L,B,E,G,P="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter.",V,X,F,$,te="Let’s generate an image using our quantized models.",A,_,K=`Setting <code>device_map=&quot;auto&quot;</code> automatically fills all available space on the GPU(s) first, then the
CPU, and finally, the hard drive (the absolute slowest option) if there is still not enough memory.`,I,z,N,j,O='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/8bit.png"/>',k,v,Y='When there is enough memory, you can also directly move the pipeline to the GPU with <code>.to(&quot;cuda&quot;)</code> and apply <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to optimize GPU memory usage.',S,R,x='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights. You can also save the serialized 8-bit models locally with <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',H;return g=new Qe({props:{warning:!1,$$slots:{default:[tl]},$$scope:{ctx:C}}}),B=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1RW5jb2Rlck1vZGVsJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUlMkMpJTBBJTBBdGV4dF9lbmNvZGVyXzJfOGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUlMkMpJTBBJTBBdHJhbnNmb3JtZXJfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>,)

text_encoder_2_8bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>,)

transformer_8bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),X=new W({props:{code:"dHJhbnNmb3JtZXJfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTJCJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyJTJDJTBBKQ==",highlighted:`transformer_8bit = AutoModel.from_pretrained(
    &quot;black-forest-labs/FLUX.1-dev&quot;,
    subfolder=&quot;transformer&quot;,
    quantization_config=quant_config,
<span class="hljs-addition">+   torch_dtype=torch.float32,</span>
)`,wrap:!1}}),z=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhQaXBlbGluZSUwQSUwQXBpcGUlMjAlM0QlMjBGbHV4UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXJfOGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRleHRfZW5jb2Rlcl8yJTNEdGV4dF9lbmNvZGVyXzJfOGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQXBpcGVfa3dhcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycHJvbXB0JTIyJTNBJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmhlaWdodCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJ3aWR0aCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJndWlkYW5jZV9zY2FsZSUyMiUzQSUyMDMuNSUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV9pbmZlcmVuY2Vfc3RlcHMlMjIlM0ElMjA1MCUyQyUwQSUyMCUyMCUyMCUyMCUyMm1heF9zZXF1ZW5jZV9sZW5ndGglMjIlM0ElMjA1MTIlMkMlMEElN0QlMEElMEFpbWFnZSUyMCUzRCUyMHBpcGUoKipwaXBlX2t3YXJncyUyQyUyMGdlbmVyYXRvciUzRHRvcmNoLm1hbnVhbF9zZWVkKDApJTJDKS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer_8bit,
    text_encoder_2=text_encoder_2_8bit,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)

pipe_kwargs = {
    <span class="hljs-string">&quot;prompt&quot;</span>: <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
    <span class="hljs-string">&quot;height&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;width&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;guidance_scale&quot;</span>: <span class="hljs-number">3.5</span>,
    <span class="hljs-string">&quot;num_inference_steps&quot;</span>: <span class="hljs-number">50</span>,
    <span class="hljs-string">&quot;max_sequence_length&quot;</span>: <span class="hljs-number">512</span>,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(<span class="hljs-number">0</span>),).images[<span class="hljs-number">0</span>]`,wrap:!1}}),{c(){a=d("p"),a.textContent=M,i=o(),c=d("p"),c.innerHTML=m,w=o(),Z=d("p"),Z.innerHTML=Q,q=o(),y(g.$$.fragment),L=o(),y(B.$$.fragment),E=o(),G=d("p"),G.innerHTML=P,V=o(),y(X.$$.fragment),F=o(),$=d("p"),$.textContent=te,A=o(),_=d("p"),_.innerHTML=K,I=o(),y(z.$$.fragment),N=o(),j=d("div"),j.innerHTML=O,k=o(),v=d("p"),v.innerHTML=Y,S=o(),R=d("p"),R.innerHTML=x,this.h()},l(t){a=f(t,"P",{"data-svelte-h":!0}),u(a)!=="svelte-4djpqq"&&(a.textContent=M),i=r(t),c=f(t,"P",{"data-svelte-h":!0}),u(c)!=="svelte-8t0085"&&(c.innerHTML=m),w=r(t),Z=f(t,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-mdqrzc"&&(Z.innerHTML=Q),q=r(t),b(g.$$.fragment,t),L=r(t),b(B.$$.fragment,t),E=r(t),G=f(t,"P",{"data-svelte-h":!0}),u(G)!=="svelte-1qozsrk"&&(G.innerHTML=P),V=r(t),b(X.$$.fragment,t),F=r(t),$=f(t,"P",{"data-svelte-h":!0}),u($)!=="svelte-2bde4h"&&($.textContent=te),A=r(t),_=f(t,"P",{"data-svelte-h":!0}),u(_)!=="svelte-1eyrcy7"&&(_.innerHTML=K),I=r(t),b(z.$$.fragment,t),N=r(t),j=f(t,"DIV",{class:!0,"data-svelte-h":!0}),u(j)!=="svelte-14afenp"&&(j.innerHTML=O),k=r(t),v=f(t,"P",{"data-svelte-h":!0}),u(v)!=="svelte-uz8m1k"&&(v.innerHTML=Y),S=r(t),R=f(t,"P",{"data-svelte-h":!0}),u(R)!=="svelte-13z5cfr"&&(R.innerHTML=x),this.h()},h(){jt(j,"class","flex justify-center")},m(t,p){s(t,a,p),s(t,i,p),s(t,c,p),s(t,w,p),s(t,Z,p),s(t,q,p),J(g,t,p),s(t,L,p),J(B,t,p),s(t,E,p),s(t,G,p),s(t,V,p),J(X,t,p),s(t,F,p),s(t,$,p),s(t,A,p),s(t,_,p),s(t,I,p),J(z,t,p),s(t,N,p),s(t,j,p),s(t,k,p),s(t,v,p),s(t,S,p),s(t,R,p),H=!0},p(t,p){const ee={};p&2&&(ee.$$scope={dirty:p,ctx:t}),g.$set(ee)},i(t){H||(h(g.$$.fragment,t),h(B.$$.fragment,t),h(X.$$.fragment,t),h(z.$$.fragment,t),H=!0)},o(t){T(g.$$.fragment,t),T(B.$$.fragment,t),T(X.$$.fragment,t),T(z.$$.fragment,t),H=!1},d(t){t&&(l(a),l(i),l(c),l(w),l(Z),l(q),l(L),l(E),l(G),l(V),l(F),l($),l(A),l(_),l(I),l(N),l(j),l(k),l(v),l(S),l(R)),U(g,t),U(B,t),U(X,t),U(z,t)}}}function sl(C){let a,M='The <code>CLIPTextModel</code> and <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> aren’t quantized because they’re already small in size and because <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> only has a few <code>torch.nn.Linear</code> layers.';return{c(){a=d("p"),a.innerHTML=M},l(i){a=f(i,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1bd21r0"&&(a.innerHTML=M)},m(i,c){s(i,a,c)},p:Ie,d(i){i&&l(a)}}}function nl(C){let a,M="Quantizing a model in 4-bit reduces your memory-usage by 4x:",i,c,m=`bitsandbytes is supported in both Transformers and Diffusers, so you can can quantize both the
<a href="/docs/diffusers/main/en/api/models/flux_transformer#diffusers.FluxTransformer2DModel">FluxTransformer2DModel</a> and <a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a>.`,w,Z,Q="For Ada and higher-series GPUs. we recommend changing <code>torch_dtype</code> to <code>torch.bfloat16</code>.",q,g,L,B,E,G,P="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter.",V,X,F,$,te="Let’s generate an image using our quantized models.",A,_,K="Setting <code>device_map=&quot;auto&quot;</code> automatically fills all available space on the GPU(s) first, then the CPU, and finally, the hard drive (the absolute slowest option) if there is still not enough memory.",I,z,N,j,O='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/4bit.png"/>',k,v,Y='When there is enough memory, you can also directly move the pipeline to the GPU with <code>.to(&quot;cuda&quot;)</code> and apply <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to optimize GPU memory usage.',S,R,x='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights. You can also save the serialized 4-bit models locally with <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',H;return g=new Qe({props:{warning:!1,$$slots:{default:[sl]},$$scope:{ctx:C}}}),B=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1RW5jb2Rlck1vZGVsJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUlMkMpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUlMkMpJTBBJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>,)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>,)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),X=new W({props:{code:"dHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTJCJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyJTJDJTBBKQ==",highlighted:`transformer_4bit = AutoModel.from_pretrained(
    &quot;black-forest-labs/FLUX.1-dev&quot;,
    subfolder=&quot;transformer&quot;,
    quantization_config=quant_config,
<span class="hljs-addition">+   torch_dtype=torch.float32,</span>
)`,wrap:!1}}),z=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhQaXBlbGluZSUwQSUwQXBpcGUlMjAlM0QlMjBGbHV4UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXJfNGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRleHRfZW5jb2Rlcl8yJTNEdGV4dF9lbmNvZGVyXzJfNGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQXBpcGVfa3dhcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycHJvbXB0JTIyJTNBJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmhlaWdodCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJ3aWR0aCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJndWlkYW5jZV9zY2FsZSUyMiUzQSUyMDMuNSUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV9pbmZlcmVuY2Vfc3RlcHMlMjIlM0ElMjA1MCUyQyUwQSUyMCUyMCUyMCUyMCUyMm1heF9zZXF1ZW5jZV9sZW5ndGglMjIlM0ElMjA1MTIlMkMlMEElN0QlMEElMEFpbWFnZSUyMCUzRCUyMHBpcGUoKipwaXBlX2t3YXJncyUyQyUyMGdlbmVyYXRvciUzRHRvcmNoLm1hbnVhbF9zZWVkKDApJTJDKS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer_4bit,
    text_encoder_2=text_encoder_2_4bit,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)

pipe_kwargs = {
    <span class="hljs-string">&quot;prompt&quot;</span>: <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
    <span class="hljs-string">&quot;height&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;width&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;guidance_scale&quot;</span>: <span class="hljs-number">3.5</span>,
    <span class="hljs-string">&quot;num_inference_steps&quot;</span>: <span class="hljs-number">50</span>,
    <span class="hljs-string">&quot;max_sequence_length&quot;</span>: <span class="hljs-number">512</span>,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(<span class="hljs-number">0</span>),).images[<span class="hljs-number">0</span>]`,wrap:!1}}),{c(){a=d("p"),a.textContent=M,i=o(),c=d("p"),c.innerHTML=m,w=o(),Z=d("p"),Z.innerHTML=Q,q=o(),y(g.$$.fragment),L=o(),y(B.$$.fragment),E=o(),G=d("p"),G.innerHTML=P,V=o(),y(X.$$.fragment),F=o(),$=d("p"),$.textContent=te,A=o(),_=d("p"),_.innerHTML=K,I=o(),y(z.$$.fragment),N=o(),j=d("div"),j.innerHTML=O,k=o(),v=d("p"),v.innerHTML=Y,S=o(),R=d("p"),R.innerHTML=x,this.h()},l(t){a=f(t,"P",{"data-svelte-h":!0}),u(a)!=="svelte-i07c71"&&(a.textContent=M),i=r(t),c=f(t,"P",{"data-svelte-h":!0}),u(c)!=="svelte-q3mf8r"&&(c.innerHTML=m),w=r(t),Z=f(t,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-mdqrzc"&&(Z.innerHTML=Q),q=r(t),b(g.$$.fragment,t),L=r(t),b(B.$$.fragment,t),E=r(t),G=f(t,"P",{"data-svelte-h":!0}),u(G)!=="svelte-1qozsrk"&&(G.innerHTML=P),V=r(t),b(X.$$.fragment,t),F=r(t),$=f(t,"P",{"data-svelte-h":!0}),u($)!=="svelte-2bde4h"&&($.textContent=te),A=r(t),_=f(t,"P",{"data-svelte-h":!0}),u(_)!=="svelte-6ilppp"&&(_.innerHTML=K),I=r(t),b(z.$$.fragment,t),N=r(t),j=f(t,"DIV",{class:!0,"data-svelte-h":!0}),u(j)!=="svelte-qev6y1"&&(j.innerHTML=O),k=r(t),v=f(t,"P",{"data-svelte-h":!0}),u(v)!=="svelte-uz8m1k"&&(v.innerHTML=Y),S=r(t),R=f(t,"P",{"data-svelte-h":!0}),u(R)!=="svelte-1wxqg1v"&&(R.innerHTML=x),this.h()},h(){jt(j,"class","flex justify-center")},m(t,p){s(t,a,p),s(t,i,p),s(t,c,p),s(t,w,p),s(t,Z,p),s(t,q,p),J(g,t,p),s(t,L,p),J(B,t,p),s(t,E,p),s(t,G,p),s(t,V,p),J(X,t,p),s(t,F,p),s(t,$,p),s(t,A,p),s(t,_,p),s(t,I,p),J(z,t,p),s(t,N,p),s(t,j,p),s(t,k,p),s(t,v,p),s(t,S,p),s(t,R,p),H=!0},p(t,p){const ee={};p&2&&(ee.$$scope={dirty:p,ctx:t}),g.$set(ee)},i(t){H||(h(g.$$.fragment,t),h(B.$$.fragment,t),h(X.$$.fragment,t),h(z.$$.fragment,t),H=!0)},o(t){T(g.$$.fragment,t),T(B.$$.fragment,t),T(X.$$.fragment,t),T(z.$$.fragment,t),H=!1},d(t){t&&(l(a),l(i),l(c),l(w),l(Z),l(q),l(L),l(E),l(G),l(V),l(F),l($),l(A),l(_),l(I),l(N),l(j),l(k),l(v),l(S),l(R)),U(g,t),U(B,t),U(X,t),U(z,t)}}}function al(C){let a,M,i,c;return a=new gt({props:{id:"bnb",option:"8-bit",$$slots:{default:[ll]},$$scope:{ctx:C}}}),i=new gt({props:{id:"bnb",option:"4-bit",$$slots:{default:[nl]},$$scope:{ctx:C}}}),{c(){y(a.$$.fragment),M=o(),y(i.$$.fragment)},l(m){b(a.$$.fragment,m),M=r(m),b(i.$$.fragment,m)},m(m,w){J(a,m,w),s(m,M,w),J(i,m,w),c=!0},p(m,w){const Z={};w&2&&(Z.$$scope={dirty:w,ctx:m}),a.$set(Z);const Q={};w&2&&(Q.$$scope={dirty:w,ctx:m}),i.$set(Q)},i(m){c||(h(a.$$.fragment,m),h(i.$$.fragment,m),c=!0)},o(m){T(a.$$.fragment,m),T(i.$$.fragment,m),c=!1},d(m){m&&l(M),U(a,m),U(i,m)}}}function il(C){let a,M="Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.";return{c(){a=d("p"),a.innerHTML=M},l(i){a=f(i,"P",{"data-svelte-h":!0}),u(a)!=="svelte-of9sym"&&(a.innerHTML=M)},m(i,c){s(i,a,c)},p:Ie,d(i){i&&l(a)}}}function ol(C){let a,M='Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!';return{c(){a=d("p"),a.innerHTML=M},l(i){a=f(i,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1bb05fp"&&(a.innerHTML=M)},m(i,c){s(i,a,c)},p:Ie,d(i){i&&l(a)}}}function rl(C){let a,M='Learn more about its details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.';return{c(){a=d("p"),a.innerHTML=M},l(i){a=f(i,"P",{"data-svelte-h":!0}),u(a)!=="svelte-kpdzjq"&&(a.innerHTML=M)},m(i,c){s(i,a,c)},p:Ie,d(i){i&&l(a)}}}function pl(C){let a,M;return a=new W({props:{code:"dG9yY2guX2R5bmFtby5jb25maWcuY2FwdHVyZV9keW5hbWljX291dHB1dF9zaGFwZV9vcHMlMjAlM0QlMjBUcnVlJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKSUwQXRyYW5zZm9ybWVyXzRiaXQuY29tcGlsZShmdWxsZ3JhcGglM0RUcnVlKQ==",highlighted:`torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)
transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)
transformer_4bit.<span class="hljs-built_in">compile</span>(fullgraph=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){y(a.$$.fragment)},l(i){b(a.$$.fragment,i)},m(i,c){J(a,i,c),M=!0},p:Ie,i(i){M||(h(a.$$.fragment,i),M=!0)},o(i){T(a.$$.fragment,i),M=!1},d(i){U(a,i)}}}function ml(C){let a,M;return a=new W({props:{code:"cXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKSUwQXRyYW5zZm9ybWVyXzRiaXQuY29tcGlsZShmdWxsZ3JhcGglM0RUcnVlKQ==",highlighted:`quant_config = DiffusersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)
transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)
transformer_4bit.<span class="hljs-built_in">compile</span>(fullgraph=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){y(a.$$.fragment)},l(i){b(a.$$.fragment,i)},m(i,c){J(a,i,c),M=!0},p:Ie,i(i){M||(h(a.$$.fragment,i),M=!0)},o(i){T(a.$$.fragment,i),M=!1},d(i){U(a,i)}}}function dl(C){let a,M,i,c;return a=new gt({props:{id:"bnb",option:"8-bit",$$slots:{default:[pl]},$$scope:{ctx:C}}}),i=new gt({props:{id:"bnb",option:"4-bit",$$slots:{default:[ml]},$$scope:{ctx:C}}}),{c(){y(a.$$.fragment),M=o(),y(i.$$.fragment)},l(m){b(a.$$.fragment,m),M=r(m),b(i.$$.fragment,m)},m(m,w){J(a,m,w),s(m,M,w),J(i,m,w),c=!0},p(m,w){const Z={};w&2&&(Z.$$scope={dirty:w,ctx:m}),a.$set(Z);const Q={};w&2&&(Q.$$scope={dirty:w,ctx:m}),i.$set(Q)},i(m){c||(h(a.$$.fragment,m),h(i.$$.fragment,m),c=!0)},o(m){T(a.$$.fragment,m),T(i.$$.fragment,m),c=!1},d(m){m&&l(M),U(a,m),U(i,m)}}}function fl(C){let a,M,i,c,m,w,Z,Q='<a href="https://huggingface.co/docs/bitsandbytes/index" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance.',q,g,L='4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',B,E,G=`This guide demonstrates how quantization can enable running
<a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" rel="nofollow">FLUX.1-dev</a>
on less than 16GB of VRAM and even on a free Google
Colab instance.`,P,V,X='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/comparison.png" alt="comparison image"/>',F,$,te="To use bitsandbytes, make sure you have the following libraries installed:",A,_,K,I,z='Now you can quantize a model by passing a <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a> to <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. This works for any model in any modality, as long as it supports loading with <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> and contains <code>torch.nn.Linear</code> layers.',N,j,O,k,v,Y,S="Check your memory footprint with the <code>get_memory_footprint</code> method:",R,x,H,t,p="Note that this only tells you the memory footprint of the model params and does <em>not</em> estimate the inference memory requirements.",ee,ae,Ct='Quantized models can be loaded from the <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a> method without needing to specify the <code>quantization_config</code> parameters:',qe,ie,Fe,oe,Ae,le,Ne,re,$t="This section explores some of the specific features of 8-bit models, such as outlier thresholds and skipping module conversion.",He,pe,Ye,me,_t="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",xe,de,Bt='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Le,fe,Se,ue,De,Me,vt='For some models, you don’t need to quantize every module to 8-bit which can actually cause instability. For example, for diffusion models like <a href="../api/pipelines/stable_diffusion/stable_diffusion_3">Stable Diffusion 3</a>, the <code>proj_out</code> module can be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Pe,ce,Ke,ye,Oe,se,et,be,Rt="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",tt,Je,lt,he,Gt='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',st,Te,nt,Ue,at,we,Xt='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',it,Ze,ot,je,zt="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",rt,ge,pt,Ce,kt="Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an additional 0.4 bits/parameter.",mt,$e,dt,_e,ft,Be,Et="Once quantized, you can dequantize a model to its original precision, but this might result in a small loss of quality. Make sure you have enough GPU RAM to fit the dequantized model.",ut,ve,Mt,Re,ct,Ge,Vt='Speed up inference with <code>torch.compile</code>. Make sure you have the latest <code>bitsandbytes</code> installed and we also recommend installing <a href="https://pytorch.org/get-started/locally/" rel="nofollow">PyTorch nightly</a>.',yt,ne,bt,Xe,It="On an RTX 4090 with compilation, 4-bit Flux generation completed in 25.809 seconds versus 32.570 seconds without.",Jt,ze,Wt='Check out the <a href="https://gist.github.com/sayakpaul/0db9d8eeeb3d2a0e5ed7cf0d9ca19b7d" rel="nofollow">benchmarking script</a> for more details.',ht,ke,Tt,Ee,Qt='<li><a href="https://gist.github.com/sayakpaul/c76bd845b48759e11687ac550b99d8b4" rel="nofollow">End-to-end notebook showing Flux.1 Dev inference in a free-tier Colab</a></li> <li><a href="https://github.com/huggingface/diffusers/blob/8c661ea586bf11cb2440da740dd3c4cf84679b85/examples/dreambooth/README_hidream.md#using-quantization" rel="nofollow">Training</a></li>',Ut,Ve,wt,We,Zt;return m=new D({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),_=new W({props:{code:"cGlwJTIwaW5zdGFsbCUyMGRpZmZ1c2VycyUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMjAtVQ==",highlighted:"pip install diffusers transformers accelerate bitsandbytes -U",wrap:!1}}),j=new xt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[al]},$$scope:{ctx:C}}}),k=new Qe({props:{warning:!0,$$slots:{default:[il]},$$scope:{ctx:C}}}),x=new W({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),ie=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl80Yml0JTNEVHJ1ZSklMEElMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJoZi1pbnRlcm5hbC10ZXN0aW5nJTJGZmx1eC4xLWRldi1uZjQtcGtnJTIyJTJDJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;hf-internal-testing/flux.1-dev-nf4-pkg&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>
)`,wrap:!1}}),oe=new D({props:{title:"8-bit (LLM.int8() algorithm)",local:"8-bit-llmint8-algorithm",headingTag:"h2"}}),le=new Qe({props:{$$slots:{default:[ol]},$$scope:{ctx:C}}}),pe=new D({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),fe=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3RocmVzaG9sZCUzRDEwJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),ue=new D({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),ce=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3NraXBfbW9kdWxlcyUzRCU1QiUyMnByb2pfb3V0JTIyJTVEJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBTRDNUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi0zLW1lZGl1bS1kaWZmdXNlcnMlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> SD3Transformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_skip_modules=[<span class="hljs-string">&quot;proj_out&quot;</span>],
)

model_8bit = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),ye=new D({props:{title:"4-bit (QLoRA algorithm)",local:"4-bit-qlora-algorithm",headingTag:"h2"}}),se=new Qe({props:{$$slots:{default:[rl]},$$scope:{ctx:C}}}),Je=new D({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),Te=new W({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),Ue=new D({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),Ze=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSklMEElMEF0ZXh0X2VuY29kZXJfMl80Yml0JTIwJTNEJTIwVDVFbmNvZGVyTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRleHRfZW5jb2Rlcl8yJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50X2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSklMEElMEFxdWFudF9jb25maWclMjAlM0QlMjBEaWZmdXNlcnNCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),ge=new D({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),$e=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfdXNlX2RvdWJsZV9xdWFudCUzRFRydWUlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSklMEElMEF0cmFuc2Zvcm1lcl80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),_e=new D({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),ve=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfdXNlX2RvdWJsZV9xdWFudCUzRFRydWUlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSklMEElMEF0cmFuc2Zvcm1lcl80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdC5kZXF1YW50aXplKCklMEF0cmFuc2Zvcm1lcl80Yml0LmRlcXVhbnRpemUoKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

text_encoder_2_4bit.dequantize()
transformer_4bit.dequantize()`,wrap:!1}}),Re=new D({props:{title:"torch.compile",local:"torchcompile",headingTag:"h2"}}),ne=new xt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[dl]},$$scope:{ctx:C}}}),ke=new D({props:{title:"Resources",local:"resources",headingTag:"h2"}}),Ve=new el({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){a=d("meta"),M=o(),i=d("p"),c=o(),y(m.$$.fragment),w=o(),Z=d("p"),Z.innerHTML=Q,q=o(),g=d("p"),g.innerHTML=L,B=o(),E=d("p"),E.innerHTML=G,P=o(),V=d("p"),V.innerHTML=X,F=o(),$=d("p"),$.textContent=te,A=o(),y(_.$$.fragment),K=o(),I=d("p"),I.innerHTML=z,N=o(),y(j.$$.fragment),O=o(),y(k.$$.fragment),v=o(),Y=d("p"),Y.innerHTML=S,R=o(),y(x.$$.fragment),H=o(),t=d("p"),t.innerHTML=p,ee=o(),ae=d("p"),ae.innerHTML=Ct,qe=o(),y(ie.$$.fragment),Fe=o(),y(oe.$$.fragment),Ae=o(),y(le.$$.fragment),Ne=o(),re=d("p"),re.textContent=$t,He=o(),y(pe.$$.fragment),Ye=o(),me=d("p"),me.textContent=_t,xe=o(),de=d("p"),de.innerHTML=Bt,Le=o(),y(fe.$$.fragment),Se=o(),y(ue.$$.fragment),De=o(),Me=d("p"),Me.innerHTML=vt,Pe=o(),y(ce.$$.fragment),Ke=o(),y(ye.$$.fragment),Oe=o(),y(se.$$.fragment),et=o(),be=d("p"),be.textContent=Rt,tt=o(),y(Je.$$.fragment),lt=o(),he=d("p"),he.innerHTML=Gt,st=o(),y(Te.$$.fragment),nt=o(),y(Ue.$$.fragment),at=o(),we=d("p"),we.innerHTML=Xt,it=o(),y(Ze.$$.fragment),ot=o(),je=d("p"),je.innerHTML=zt,rt=o(),y(ge.$$.fragment),pt=o(),Ce=d("p"),Ce.textContent=kt,mt=o(),y($e.$$.fragment),dt=o(),y(_e.$$.fragment),ft=o(),Be=d("p"),Be.textContent=Et,ut=o(),y(ve.$$.fragment),Mt=o(),y(Re.$$.fragment),ct=o(),Ge=d("p"),Ge.innerHTML=Vt,yt=o(),y(ne.$$.fragment),bt=o(),Xe=d("p"),Xe.textContent=It,Jt=o(),ze=d("p"),ze.innerHTML=Wt,ht=o(),y(ke.$$.fragment),Tt=o(),Ee=d("ul"),Ee.innerHTML=Qt,Ut=o(),y(Ve.$$.fragment),wt=o(),We=d("p"),this.h()},l(e){const n=Kt("svelte-u9bgzb",document.head);a=f(n,"META",{name:!0,content:!0}),n.forEach(l),M=r(e),i=f(e,"P",{}),Yt(i).forEach(l),c=r(e),b(m.$$.fragment,e),w=r(e),Z=f(e,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-1m50ob7"&&(Z.innerHTML=Q),q=r(e),g=f(e,"P",{"data-svelte-h":!0}),u(g)!=="svelte-11pyf03"&&(g.innerHTML=L),B=r(e),E=f(e,"P",{"data-svelte-h":!0}),u(E)!=="svelte-1erhi5i"&&(E.innerHTML=G),P=r(e),V=f(e,"P",{"data-svelte-h":!0}),u(V)!=="svelte-ryfe7d"&&(V.innerHTML=X),F=r(e),$=f(e,"P",{"data-svelte-h":!0}),u($)!=="svelte-gf36q7"&&($.textContent=te),A=r(e),b(_.$$.fragment,e),K=r(e),I=f(e,"P",{"data-svelte-h":!0}),u(I)!=="svelte-17fqaf"&&(I.innerHTML=z),N=r(e),b(j.$$.fragment,e),O=r(e),b(k.$$.fragment,e),v=r(e),Y=f(e,"P",{"data-svelte-h":!0}),u(Y)!=="svelte-139tok6"&&(Y.innerHTML=S),R=r(e),b(x.$$.fragment,e),H=r(e),t=f(e,"P",{"data-svelte-h":!0}),u(t)!=="svelte-18sn6ad"&&(t.innerHTML=p),ee=r(e),ae=f(e,"P",{"data-svelte-h":!0}),u(ae)!=="svelte-10wvd51"&&(ae.innerHTML=Ct),qe=r(e),b(ie.$$.fragment,e),Fe=r(e),b(oe.$$.fragment,e),Ae=r(e),b(le.$$.fragment,e),Ne=r(e),re=f(e,"P",{"data-svelte-h":!0}),u(re)!=="svelte-1myadau"&&(re.textContent=$t),He=r(e),b(pe.$$.fragment,e),Ye=r(e),me=f(e,"P",{"data-svelte-h":!0}),u(me)!=="svelte-ur5rgd"&&(me.textContent=_t),xe=r(e),de=f(e,"P",{"data-svelte-h":!0}),u(de)!=="svelte-1pta66b"&&(de.innerHTML=Bt),Le=r(e),b(fe.$$.fragment,e),Se=r(e),b(ue.$$.fragment,e),De=r(e),Me=f(e,"P",{"data-svelte-h":!0}),u(Me)!=="svelte-1rff4ou"&&(Me.innerHTML=vt),Pe=r(e),b(ce.$$.fragment,e),Ke=r(e),b(ye.$$.fragment,e),Oe=r(e),b(se.$$.fragment,e),et=r(e),be=f(e,"P",{"data-svelte-h":!0}),u(be)!=="svelte-7ob7j"&&(be.textContent=Rt),tt=r(e),b(Je.$$.fragment,e),lt=r(e),he=f(e,"P",{"data-svelte-h":!0}),u(he)!=="svelte-19knmj2"&&(he.innerHTML=Gt),st=r(e),b(Te.$$.fragment,e),nt=r(e),b(Ue.$$.fragment,e),at=r(e),we=f(e,"P",{"data-svelte-h":!0}),u(we)!=="svelte-6pxdrm"&&(we.innerHTML=Xt),it=r(e),b(Ze.$$.fragment,e),ot=r(e),je=f(e,"P",{"data-svelte-h":!0}),u(je)!=="svelte-1qoc2ct"&&(je.innerHTML=zt),rt=r(e),b(ge.$$.fragment,e),pt=r(e),Ce=f(e,"P",{"data-svelte-h":!0}),u(Ce)!=="svelte-ep1hhf"&&(Ce.textContent=kt),mt=r(e),b($e.$$.fragment,e),dt=r(e),b(_e.$$.fragment,e),ft=r(e),Be=f(e,"P",{"data-svelte-h":!0}),u(Be)!=="svelte-lubruw"&&(Be.textContent=Et),ut=r(e),b(ve.$$.fragment,e),Mt=r(e),b(Re.$$.fragment,e),ct=r(e),Ge=f(e,"P",{"data-svelte-h":!0}),u(Ge)!=="svelte-7ubrrr"&&(Ge.innerHTML=Vt),yt=r(e),b(ne.$$.fragment,e),bt=r(e),Xe=f(e,"P",{"data-svelte-h":!0}),u(Xe)!=="svelte-7dnnek"&&(Xe.textContent=It),Jt=r(e),ze=f(e,"P",{"data-svelte-h":!0}),u(ze)!=="svelte-j15c34"&&(ze.innerHTML=Wt),ht=r(e),b(ke.$$.fragment,e),Tt=r(e),Ee=f(e,"UL",{"data-svelte-h":!0}),u(Ee)!=="svelte-1pkgul5"&&(Ee.innerHTML=Qt),Ut=r(e),b(Ve.$$.fragment,e),wt=r(e),We=f(e,"P",{}),Yt(We).forEach(l),this.h()},h(){jt(a,"name","hf:doc:metadata"),jt(a,"content",ul)},m(e,n){Ot(document.head,a),s(e,M,n),s(e,i,n),s(e,c,n),J(m,e,n),s(e,w,n),s(e,Z,n),s(e,q,n),s(e,g,n),s(e,B,n),s(e,E,n),s(e,P,n),s(e,V,n),s(e,F,n),s(e,$,n),s(e,A,n),J(_,e,n),s(e,K,n),s(e,I,n),s(e,N,n),J(j,e,n),s(e,O,n),J(k,e,n),s(e,v,n),s(e,Y,n),s(e,R,n),J(x,e,n),s(e,H,n),s(e,t,n),s(e,ee,n),s(e,ae,n),s(e,qe,n),J(ie,e,n),s(e,Fe,n),J(oe,e,n),s(e,Ae,n),J(le,e,n),s(e,Ne,n),s(e,re,n),s(e,He,n),J(pe,e,n),s(e,Ye,n),s(e,me,n),s(e,xe,n),s(e,de,n),s(e,Le,n),J(fe,e,n),s(e,Se,n),J(ue,e,n),s(e,De,n),s(e,Me,n),s(e,Pe,n),J(ce,e,n),s(e,Ke,n),J(ye,e,n),s(e,Oe,n),J(se,e,n),s(e,et,n),s(e,be,n),s(e,tt,n),J(Je,e,n),s(e,lt,n),s(e,he,n),s(e,st,n),J(Te,e,n),s(e,nt,n),J(Ue,e,n),s(e,at,n),s(e,we,n),s(e,it,n),J(Ze,e,n),s(e,ot,n),s(e,je,n),s(e,rt,n),J(ge,e,n),s(e,pt,n),s(e,Ce,n),s(e,mt,n),J($e,e,n),s(e,dt,n),J(_e,e,n),s(e,ft,n),s(e,Be,n),s(e,ut,n),J(ve,e,n),s(e,Mt,n),J(Re,e,n),s(e,ct,n),s(e,Ge,n),s(e,yt,n),J(ne,e,n),s(e,bt,n),s(e,Xe,n),s(e,Jt,n),s(e,ze,n),s(e,ht,n),J(ke,e,n),s(e,Tt,n),s(e,Ee,n),s(e,Ut,n),J(Ve,e,n),s(e,wt,n),s(e,We,n),Zt=!0},p(e,[n]){const qt={};n&2&&(qt.$$scope={dirty:n,ctx:e}),j.$set(qt);const Ft={};n&2&&(Ft.$$scope={dirty:n,ctx:e}),k.$set(Ft);const At={};n&2&&(At.$$scope={dirty:n,ctx:e}),le.$set(At);const Nt={};n&2&&(Nt.$$scope={dirty:n,ctx:e}),se.$set(Nt);const Ht={};n&2&&(Ht.$$scope={dirty:n,ctx:e}),ne.$set(Ht)},i(e){Zt||(h(m.$$.fragment,e),h(_.$$.fragment,e),h(j.$$.fragment,e),h(k.$$.fragment,e),h(x.$$.fragment,e),h(ie.$$.fragment,e),h(oe.$$.fragment,e),h(le.$$.fragment,e),h(pe.$$.fragment,e),h(fe.$$.fragment,e),h(ue.$$.fragment,e),h(ce.$$.fragment,e),h(ye.$$.fragment,e),h(se.$$.fragment,e),h(Je.$$.fragment,e),h(Te.$$.fragment,e),h(Ue.$$.fragment,e),h(Ze.$$.fragment,e),h(ge.$$.fragment,e),h($e.$$.fragment,e),h(_e.$$.fragment,e),h(ve.$$.fragment,e),h(Re.$$.fragment,e),h(ne.$$.fragment,e),h(ke.$$.fragment,e),h(Ve.$$.fragment,e),Zt=!0)},o(e){T(m.$$.fragment,e),T(_.$$.fragment,e),T(j.$$.fragment,e),T(k.$$.fragment,e),T(x.$$.fragment,e),T(ie.$$.fragment,e),T(oe.$$.fragment,e),T(le.$$.fragment,e),T(pe.$$.fragment,e),T(fe.$$.fragment,e),T(ue.$$.fragment,e),T(ce.$$.fragment,e),T(ye.$$.fragment,e),T(se.$$.fragment,e),T(Je.$$.fragment,e),T(Te.$$.fragment,e),T(Ue.$$.fragment,e),T(Ze.$$.fragment,e),T(ge.$$.fragment,e),T($e.$$.fragment,e),T(_e.$$.fragment,e),T(ve.$$.fragment,e),T(Re.$$.fragment,e),T(ne.$$.fragment,e),T(ke.$$.fragment,e),T(Ve.$$.fragment,e),Zt=!1},d(e){e&&(l(M),l(i),l(c),l(w),l(Z),l(q),l(g),l(B),l(E),l(P),l(V),l(F),l($),l(A),l(K),l(I),l(N),l(O),l(v),l(Y),l(R),l(H),l(t),l(ee),l(ae),l(qe),l(Fe),l(Ae),l(Ne),l(re),l(He),l(Ye),l(me),l(xe),l(de),l(Le),l(Se),l(De),l(Me),l(Pe),l(Ke),l(Oe),l(et),l(be),l(tt),l(lt),l(he),l(st),l(nt),l(at),l(we),l(it),l(ot),l(je),l(rt),l(pt),l(Ce),l(mt),l(dt),l(ft),l(Be),l(ut),l(Mt),l(ct),l(Ge),l(yt),l(bt),l(Xe),l(Jt),l(ze),l(ht),l(Tt),l(Ee),l(Ut),l(wt),l(We)),l(a),U(m,e),U(_,e),U(j,e),U(k,e),U(x,e),U(ie,e),U(oe,e),U(le,e),U(pe,e),U(fe,e),U(ue,e),U(ce,e),U(ye,e),U(se,e),U(Je,e),U(Te,e),U(Ue,e),U(Ze,e),U(ge,e),U($e,e),U(_e,e),U(ve,e),U(Re,e),U(ne,e),U(ke,e),U(Ve,e)}}}const ul='{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit (LLM.int8() algorithm)","local":"8-bit-llmint8-algorithm","sections":[{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3}],"depth":2},{"title":"4-bit (QLoRA algorithm)","local":"4-bit-qlora-algorithm","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2},{"title":"torch.compile","local":"torchcompile","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function Ml(C){return St(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ul extends Dt{constructor(a){super(),Pt(this,a,Ml,fl,Lt,{})}}export{Ul as component};
