import{s as St,o as Dt,n as gt}from"../chunks/scheduler.53228c21.js";import{S as Pt,i as Kt,e as m,s as a,c as M,h as Ot,a as f,d as l,b as i,f as xt,g as y,j as u,k as O,l as el,m as s,n as c,t as b,o as J,p as h}from"../chunks/index.100fac89.js";import{C as tl}from"../chunks/CopyLLMTxtMenu.0ee1b69b.js";import{C as E}from"../chunks/CodeBlock.d30a6509.js";import{H as P,E as ll}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.136ef3fd.js";import{H as Lt,a as Ct}from"../chunks/HfOption.fad27e59.js";function sl(I){let o,w="Quantizing a model in 8-bit halves the memory-usage:",d,T,p=`bitsandbytes is supported in both Transformers and Diffusers, so you can quantize both the
<a href="/docs/diffusers/main/en/api/models/flux_transformer#diffusers.FluxTransformer2DModel">FluxTransformer2DModel</a> and <a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a>.`,U,Z,V="For Ada and higher-series GPUs. we recommend changing <code>torch_dtype</code> to <code>torch.bfloat16</code>.",k,v,K='<p>The <code>CLIPTextModel</code> and <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> aren’t quantized because they’re already small in size and because <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> only has a few <code>torch.nn.Linear</code> layers.</p>',z,X,q,C,ee="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter.",Q,g,L,R,Y="Let’s generate an image using our quantized models.",S,G,x=`Setting <code>device_map=&quot;auto&quot;</code> automatically fills all available space on the GPU(s) first, then the
CPU, and finally, the hard drive (the absolute slowest option) if there is still not enough memory.`,W,B,D,$,F='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/8bit.png"/>',A,j,te='When there is enough memory, you can also directly move the pipeline to the GPU with <code>.to(&quot;cuda&quot;)</code> and apply <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to optimize GPU memory usage.',N,_,le='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights. You can also save the serialized 8-bit models locally with <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',H;return X=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1RW5jb2Rlck1vZGVsJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUlMkMpJTBBJTBBdGV4dF9lbmNvZGVyXzJfOGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUlMkMpJTBBJTBBdHJhbnNmb3JtZXJfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>,)

text_encoder_2_8bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>,)

transformer_8bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),g=new E({props:{code:"dHJhbnNmb3JtZXJfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTJCJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyJTJDJTBBKQ==",highlighted:`transformer_8bit = AutoModel.from_pretrained(
    &quot;black-forest-labs/FLUX.1-dev&quot;,
    subfolder=&quot;transformer&quot;,
    quantization_config=quant_config,
<span class="hljs-addition">+   torch_dtype=torch.float32,</span>
)`,wrap:!1}}),B=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhQaXBlbGluZSUwQSUwQXBpcGUlMjAlM0QlMjBGbHV4UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXJfOGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRleHRfZW5jb2Rlcl8yJTNEdGV4dF9lbmNvZGVyXzJfOGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQXBpcGVfa3dhcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycHJvbXB0JTIyJTNBJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmhlaWdodCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJ3aWR0aCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJndWlkYW5jZV9zY2FsZSUyMiUzQSUyMDMuNSUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV9pbmZlcmVuY2Vfc3RlcHMlMjIlM0ElMjA1MCUyQyUwQSUyMCUyMCUyMCUyMCUyMm1heF9zZXF1ZW5jZV9sZW5ndGglMjIlM0ElMjA1MTIlMkMlMEElN0QlMEElMEFpbWFnZSUyMCUzRCUyMHBpcGUoKipwaXBlX2t3YXJncyUyQyUyMGdlbmVyYXRvciUzRHRvcmNoLm1hbnVhbF9zZWVkKDApJTJDKS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer_8bit,
    text_encoder_2=text_encoder_2_8bit,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)

pipe_kwargs = {
    <span class="hljs-string">&quot;prompt&quot;</span>: <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
    <span class="hljs-string">&quot;height&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;width&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;guidance_scale&quot;</span>: <span class="hljs-number">3.5</span>,
    <span class="hljs-string">&quot;num_inference_steps&quot;</span>: <span class="hljs-number">50</span>,
    <span class="hljs-string">&quot;max_sequence_length&quot;</span>: <span class="hljs-number">512</span>,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(<span class="hljs-number">0</span>),).images[<span class="hljs-number">0</span>]`,wrap:!1}}),{c(){o=m("p"),o.textContent=w,d=a(),T=m("p"),T.innerHTML=p,U=a(),Z=m("p"),Z.innerHTML=V,k=a(),v=m("blockquote"),v.innerHTML=K,z=a(),M(X.$$.fragment),q=a(),C=m("p"),C.innerHTML=ee,Q=a(),M(g.$$.fragment),L=a(),R=m("p"),R.textContent=Y,S=a(),G=m("p"),G.innerHTML=x,W=a(),M(B.$$.fragment),D=a(),$=m("div"),$.innerHTML=F,A=a(),j=m("p"),j.innerHTML=te,N=a(),_=m("p"),_.innerHTML=le,this.h()},l(t){o=f(t,"P",{"data-svelte-h":!0}),u(o)!=="svelte-4djpqq"&&(o.textContent=w),d=i(t),T=f(t,"P",{"data-svelte-h":!0}),u(T)!=="svelte-8t0085"&&(T.innerHTML=p),U=i(t),Z=f(t,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-mdqrzc"&&(Z.innerHTML=V),k=i(t),v=f(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(v)!=="svelte-1iq05of"&&(v.innerHTML=K),z=i(t),y(X.$$.fragment,t),q=i(t),C=f(t,"P",{"data-svelte-h":!0}),u(C)!=="svelte-1qozsrk"&&(C.innerHTML=ee),Q=i(t),y(g.$$.fragment,t),L=i(t),R=f(t,"P",{"data-svelte-h":!0}),u(R)!=="svelte-2bde4h"&&(R.textContent=Y),S=i(t),G=f(t,"P",{"data-svelte-h":!0}),u(G)!=="svelte-1eyrcy7"&&(G.innerHTML=x),W=i(t),y(B.$$.fragment,t),D=i(t),$=f(t,"DIV",{class:!0,"data-svelte-h":!0}),u($)!=="svelte-14afenp"&&($.innerHTML=F),A=i(t),j=f(t,"P",{"data-svelte-h":!0}),u(j)!=="svelte-uz8m1k"&&(j.innerHTML=te),N=i(t),_=f(t,"P",{"data-svelte-h":!0}),u(_)!=="svelte-13z5cfr"&&(_.innerHTML=le),this.h()},h(){O(v,"class","tip"),O($,"class","flex justify-center")},m(t,r){s(t,o,r),s(t,d,r),s(t,T,r),s(t,U,r),s(t,Z,r),s(t,k,r),s(t,v,r),s(t,z,r),c(X,t,r),s(t,q,r),s(t,C,r),s(t,Q,r),c(g,t,r),s(t,L,r),s(t,R,r),s(t,S,r),s(t,G,r),s(t,W,r),c(B,t,r),s(t,D,r),s(t,$,r),s(t,A,r),s(t,j,r),s(t,N,r),s(t,_,r),H=!0},p:gt,i(t){H||(b(X.$$.fragment,t),b(g.$$.fragment,t),b(B.$$.fragment,t),H=!0)},o(t){J(X.$$.fragment,t),J(g.$$.fragment,t),J(B.$$.fragment,t),H=!1},d(t){t&&(l(o),l(d),l(T),l(U),l(Z),l(k),l(v),l(z),l(q),l(C),l(Q),l(L),l(R),l(S),l(G),l(W),l(D),l($),l(A),l(j),l(N),l(_)),h(X,t),h(g,t),h(B,t)}}}function nl(I){let o,w="Quantizing a model in 4-bit reduces your memory-usage by 4x:",d,T,p=`bitsandbytes is supported in both Transformers and Diffusers, so you can can quantize both the
<a href="/docs/diffusers/main/en/api/models/flux_transformer#diffusers.FluxTransformer2DModel">FluxTransformer2DModel</a> and <a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a>.`,U,Z,V="For Ada and higher-series GPUs. we recommend changing <code>torch_dtype</code> to <code>torch.bfloat16</code>.",k,v,K='<p>The <code>CLIPTextModel</code> and <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> aren’t quantized because they’re already small in size and because <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> only has a few <code>torch.nn.Linear</code> layers.</p>',z,X,q,C,ee="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter.",Q,g,L,R,Y="Let’s generate an image using our quantized models.",S,G,x="Setting <code>device_map=&quot;auto&quot;</code> automatically fills all available space on the GPU(s) first, then the CPU, and finally, the hard drive (the absolute slowest option) if there is still not enough memory.",W,B,D,$,F='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/4bit.png"/>',A,j,te='When there is enough memory, you can also directly move the pipeline to the GPU with <code>.to(&quot;cuda&quot;)</code> and apply <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to optimize GPU memory usage.',N,_,le='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights. You can also save the serialized 4-bit models locally with <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',H;return X=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1RW5jb2Rlck1vZGVsJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUlMkMpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUlMkMpJTBBJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>,)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>,)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),g=new E({props:{code:"dHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTJCJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyJTJDJTBBKQ==",highlighted:`transformer_4bit = AutoModel.from_pretrained(
    &quot;black-forest-labs/FLUX.1-dev&quot;,
    subfolder=&quot;transformer&quot;,
    quantization_config=quant_config,
<span class="hljs-addition">+   torch_dtype=torch.float32,</span>
)`,wrap:!1}}),B=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhQaXBlbGluZSUwQSUwQXBpcGUlMjAlM0QlMjBGbHV4UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXJfNGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRleHRfZW5jb2Rlcl8yJTNEdGV4dF9lbmNvZGVyXzJfNGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQXBpcGVfa3dhcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycHJvbXB0JTIyJTNBJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmhlaWdodCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJ3aWR0aCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJndWlkYW5jZV9zY2FsZSUyMiUzQSUyMDMuNSUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV9pbmZlcmVuY2Vfc3RlcHMlMjIlM0ElMjA1MCUyQyUwQSUyMCUyMCUyMCUyMCUyMm1heF9zZXF1ZW5jZV9sZW5ndGglMjIlM0ElMjA1MTIlMkMlMEElN0QlMEElMEFpbWFnZSUyMCUzRCUyMHBpcGUoKipwaXBlX2t3YXJncyUyQyUyMGdlbmVyYXRvciUzRHRvcmNoLm1hbnVhbF9zZWVkKDApJTJDKS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer_4bit,
    text_encoder_2=text_encoder_2_4bit,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)

pipe_kwargs = {
    <span class="hljs-string">&quot;prompt&quot;</span>: <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
    <span class="hljs-string">&quot;height&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;width&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;guidance_scale&quot;</span>: <span class="hljs-number">3.5</span>,
    <span class="hljs-string">&quot;num_inference_steps&quot;</span>: <span class="hljs-number">50</span>,
    <span class="hljs-string">&quot;max_sequence_length&quot;</span>: <span class="hljs-number">512</span>,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(<span class="hljs-number">0</span>),).images[<span class="hljs-number">0</span>]`,wrap:!1}}),{c(){o=m("p"),o.textContent=w,d=a(),T=m("p"),T.innerHTML=p,U=a(),Z=m("p"),Z.innerHTML=V,k=a(),v=m("blockquote"),v.innerHTML=K,z=a(),M(X.$$.fragment),q=a(),C=m("p"),C.innerHTML=ee,Q=a(),M(g.$$.fragment),L=a(),R=m("p"),R.textContent=Y,S=a(),G=m("p"),G.innerHTML=x,W=a(),M(B.$$.fragment),D=a(),$=m("div"),$.innerHTML=F,A=a(),j=m("p"),j.innerHTML=te,N=a(),_=m("p"),_.innerHTML=le,this.h()},l(t){o=f(t,"P",{"data-svelte-h":!0}),u(o)!=="svelte-i07c71"&&(o.textContent=w),d=i(t),T=f(t,"P",{"data-svelte-h":!0}),u(T)!=="svelte-q3mf8r"&&(T.innerHTML=p),U=i(t),Z=f(t,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-mdqrzc"&&(Z.innerHTML=V),k=i(t),v=f(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(v)!=="svelte-1iq05of"&&(v.innerHTML=K),z=i(t),y(X.$$.fragment,t),q=i(t),C=f(t,"P",{"data-svelte-h":!0}),u(C)!=="svelte-1qozsrk"&&(C.innerHTML=ee),Q=i(t),y(g.$$.fragment,t),L=i(t),R=f(t,"P",{"data-svelte-h":!0}),u(R)!=="svelte-2bde4h"&&(R.textContent=Y),S=i(t),G=f(t,"P",{"data-svelte-h":!0}),u(G)!=="svelte-6ilppp"&&(G.innerHTML=x),W=i(t),y(B.$$.fragment,t),D=i(t),$=f(t,"DIV",{class:!0,"data-svelte-h":!0}),u($)!=="svelte-qev6y1"&&($.innerHTML=F),A=i(t),j=f(t,"P",{"data-svelte-h":!0}),u(j)!=="svelte-uz8m1k"&&(j.innerHTML=te),N=i(t),_=f(t,"P",{"data-svelte-h":!0}),u(_)!=="svelte-1wxqg1v"&&(_.innerHTML=le),this.h()},h(){O(v,"class","tip"),O($,"class","flex justify-center")},m(t,r){s(t,o,r),s(t,d,r),s(t,T,r),s(t,U,r),s(t,Z,r),s(t,k,r),s(t,v,r),s(t,z,r),c(X,t,r),s(t,q,r),s(t,C,r),s(t,Q,r),c(g,t,r),s(t,L,r),s(t,R,r),s(t,S,r),s(t,G,r),s(t,W,r),c(B,t,r),s(t,D,r),s(t,$,r),s(t,A,r),s(t,j,r),s(t,N,r),s(t,_,r),H=!0},p:gt,i(t){H||(b(X.$$.fragment,t),b(g.$$.fragment,t),b(B.$$.fragment,t),H=!0)},o(t){J(X.$$.fragment,t),J(g.$$.fragment,t),J(B.$$.fragment,t),H=!1},d(t){t&&(l(o),l(d),l(T),l(U),l(Z),l(k),l(v),l(z),l(q),l(C),l(Q),l(L),l(R),l(S),l(G),l(W),l(D),l($),l(A),l(j),l(N),l(_)),h(X,t),h(g,t),h(B,t)}}}function al(I){let o,w,d,T;return o=new Ct({props:{id:"bnb",option:"8-bit",$$slots:{default:[sl]},$$scope:{ctx:I}}}),d=new Ct({props:{id:"bnb",option:"4-bit",$$slots:{default:[nl]},$$scope:{ctx:I}}}),{c(){M(o.$$.fragment),w=a(),M(d.$$.fragment)},l(p){y(o.$$.fragment,p),w=i(p),y(d.$$.fragment,p)},m(p,U){c(o,p,U),s(p,w,U),c(d,p,U),T=!0},p(p,U){const Z={};U&2&&(Z.$$scope={dirty:U,ctx:p}),o.$set(Z);const V={};U&2&&(V.$$scope={dirty:U,ctx:p}),d.$set(V)},i(p){T||(b(o.$$.fragment,p),b(d.$$.fragment,p),T=!0)},o(p){J(o.$$.fragment,p),J(d.$$.fragment,p),T=!1},d(p){p&&l(w),h(o,p),h(d,p)}}}function il(I){let o,w;return o=new E({props:{code:"dG9yY2guX2R5bmFtby5jb25maWcuY2FwdHVyZV9keW5hbWljX291dHB1dF9zaGFwZV9vcHMlMjAlM0QlMjBUcnVlJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKSUwQXRyYW5zZm9ybWVyXzRiaXQuY29tcGlsZShmdWxsZ3JhcGglM0RUcnVlKQ==",highlighted:`torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)
transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)
transformer_4bit.<span class="hljs-built_in">compile</span>(fullgraph=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){M(o.$$.fragment)},l(d){y(o.$$.fragment,d)},m(d,T){c(o,d,T),w=!0},p:gt,i(d){w||(b(o.$$.fragment,d),w=!0)},o(d){J(o.$$.fragment,d),w=!1},d(d){h(o,d)}}}function ol(I){let o,w;return o=new E({props:{code:"cXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKSUwQXRyYW5zZm9ybWVyXzRiaXQuY29tcGlsZShmdWxsZ3JhcGglM0RUcnVlKQ==",highlighted:`quant_config = DiffusersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)
transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)
transformer_4bit.<span class="hljs-built_in">compile</span>(fullgraph=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){M(o.$$.fragment)},l(d){y(o.$$.fragment,d)},m(d,T){c(o,d,T),w=!0},p:gt,i(d){w||(b(o.$$.fragment,d),w=!0)},o(d){J(o.$$.fragment,d),w=!1},d(d){h(o,d)}}}function rl(I){let o,w,d,T;return o=new Ct({props:{id:"bnb",option:"8-bit",$$slots:{default:[il]},$$scope:{ctx:I}}}),d=new Ct({props:{id:"bnb",option:"4-bit",$$slots:{default:[ol]},$$scope:{ctx:I}}}),{c(){M(o.$$.fragment),w=a(),M(d.$$.fragment)},l(p){y(o.$$.fragment,p),w=i(p),y(d.$$.fragment,p)},m(p,U){c(o,p,U),s(p,w,U),c(d,p,U),T=!0},p(p,U){const Z={};U&2&&(Z.$$scope={dirty:U,ctx:p}),o.$set(Z);const V={};U&2&&(V.$$scope={dirty:U,ctx:p}),d.$set(V)},i(p){T||(b(o.$$.fragment,p),b(d.$$.fragment,p),T=!0)},o(p){J(o.$$.fragment,p),J(d.$$.fragment,p),T=!1},d(p){p&&l(w),h(o,p),h(d,p)}}}function pl(I){let o,w,d,T,p,U,Z,V,k,v='<a href="https://huggingface.co/docs/bitsandbytes/index" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance.',K,z,X='4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',q,C,ee=`This guide demonstrates how quantization can enable running
<a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" rel="nofollow">FLUX.1-dev</a>
on less than 16GB of VRAM and even on a free Google
Colab instance.`,Q,g,L='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/comparison.png" alt="comparison image"/>',R,Y,S="To use bitsandbytes, make sure you have the following libraries installed:",G,x,W,B,D='Now you can quantize a model by passing a <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a> to <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. This works for any model in any modality, as long as it supports loading with <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> and contains <code>torch.nn.Linear</code> layers.',$,F,A,j,te="<p>Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.</p>",N,_,le="Check your memory footprint with the <code>get_memory_footprint</code> method:",H,t,r,ie,Bt="Note that this only tells you the memory footprint of the model params and does <em>not</em> estimate the inference memory requirements.",We,oe,_t='Quantized models can be loaded from the <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a> method without needing to specify the <code>quantization_config</code> parameters:',Fe,re,Ae,pe,Ne,se,$t='<p>Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!</p>',He,de,vt="This section explores some of the specific features of 8-bit models, such as outlier thresholds and skipping module conversion.",Ye,me,xe,fe,Rt="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",Le,ue,Gt='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Se,Me,De,ye,Pe,ce,Xt='For some models, you don’t need to quantize every module to 8-bit which can actually cause instability. For example, for diffusion models like <a href="../api/pipelines/stable_diffusion/stable_diffusion_3">Stable Diffusion 3</a>, the <code>proj_out</code> module can be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ke,be,Oe,Je,et,ne,kt='<p>Learn more about its details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.</p>',tt,he,zt="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",lt,Te,st,Ue,Et='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',nt,we,at,Ze,it,je,Vt='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',ot,Ce,rt,ge,It="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",pt,Be,dt,_e,qt="Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an additional 0.4 bits/parameter.",mt,$e,ft,ve,ut,Re,Qt="Once quantized, you can dequantize a model to its original precision, but this might result in a small loss of quality. Make sure you have enough GPU RAM to fit the dequantized model.",Mt,Ge,yt,Xe,ct,ke,Wt='Speed up inference with <code>torch.compile</code>. Make sure you have the latest <code>bitsandbytes</code> installed and we also recommend installing <a href="https://pytorch.org/get-started/locally/" rel="nofollow">PyTorch nightly</a>.',bt,ae,Jt,ze,Ft="On an RTX 4090 with compilation, 4-bit Flux generation completed in 25.809 seconds versus 32.570 seconds without.",ht,Ee,At='Check out the <a href="https://gist.github.com/sayakpaul/0db9d8eeeb3d2a0e5ed7cf0d9ca19b7d" rel="nofollow">benchmarking script</a> for more details.',Tt,Ve,Ut,Ie,Nt='<li><a href="https://gist.github.com/sayakpaul/c76bd845b48759e11687ac550b99d8b4" rel="nofollow">End-to-end notebook showing Flux.1 Dev inference in a free-tier Colab</a></li> <li><a href="https://github.com/huggingface/diffusers/blob/8c661ea586bf11cb2440da740dd3c4cf84679b85/examples/dreambooth/README_hidream.md#using-quantization" rel="nofollow">Training</a></li>',wt,qe,Zt,Qe,jt;return p=new tl({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),Z=new P({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),x=new E({props:{code:"cGlwJTIwaW5zdGFsbCUyMGRpZmZ1c2VycyUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMjAtVQ==",highlighted:"pip install diffusers transformers accelerate bitsandbytes -U",wrap:!1}}),F=new Lt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[al]},$$scope:{ctx:I}}}),t=new E({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),re=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl80Yml0JTNEVHJ1ZSklMEElMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJoZi1pbnRlcm5hbC10ZXN0aW5nJTJGZmx1eC4xLWRldi1uZjQtcGtnJTIyJTJDJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;hf-internal-testing/flux.1-dev-nf4-pkg&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>
)`,wrap:!1}}),pe=new P({props:{title:"8-bit (LLM.int8() algorithm)",local:"8-bit-llmint8-algorithm",headingTag:"h2"}}),me=new P({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),Me=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3RocmVzaG9sZCUzRDEwJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),ye=new P({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),be=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3NraXBfbW9kdWxlcyUzRCU1QiUyMnByb2pfb3V0JTIyJTVEJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBTRDNUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi0zLW1lZGl1bS1kaWZmdXNlcnMlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> SD3Transformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_skip_modules=[<span class="hljs-string">&quot;proj_out&quot;</span>],
)

model_8bit = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),Je=new P({props:{title:"4-bit (QLoRA algorithm)",local:"4-bit-qlora-algorithm",headingTag:"h2"}}),Te=new P({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),we=new E({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),Ze=new P({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),Ce=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSklMEElMEF0ZXh0X2VuY29kZXJfMl80Yml0JTIwJTNEJTIwVDVFbmNvZGVyTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRleHRfZW5jb2Rlcl8yJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50X2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSklMEElMEFxdWFudF9jb25maWclMjAlM0QlMjBEaWZmdXNlcnNCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),Be=new P({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),$e=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfdXNlX2RvdWJsZV9xdWFudCUzRFRydWUlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSklMEElMEF0cmFuc2Zvcm1lcl80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),ve=new P({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),Ge=new E({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfdXNlX2RvdWJsZV9xdWFudCUzRFRydWUlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSklMEElMEF0cmFuc2Zvcm1lcl80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdC5kZXF1YW50aXplKCklMEF0cmFuc2Zvcm1lcl80Yml0LmRlcXVhbnRpemUoKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

text_encoder_2_4bit.dequantize()
transformer_4bit.dequantize()`,wrap:!1}}),Xe=new P({props:{title:"torch.compile",local:"torchcompile",headingTag:"h2"}}),ae=new Lt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[rl]},$$scope:{ctx:I}}}),Ve=new P({props:{title:"Resources",local:"resources",headingTag:"h2"}}),qe=new ll({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){o=m("meta"),w=a(),d=m("p"),T=a(),M(p.$$.fragment),U=a(),M(Z.$$.fragment),V=a(),k=m("p"),k.innerHTML=v,K=a(),z=m("p"),z.innerHTML=X,q=a(),C=m("p"),C.innerHTML=ee,Q=a(),g=m("p"),g.innerHTML=L,R=a(),Y=m("p"),Y.textContent=S,G=a(),M(x.$$.fragment),W=a(),B=m("p"),B.innerHTML=D,$=a(),M(F.$$.fragment),A=a(),j=m("blockquote"),j.innerHTML=te,N=a(),_=m("p"),_.innerHTML=le,H=a(),M(t.$$.fragment),r=a(),ie=m("p"),ie.innerHTML=Bt,We=a(),oe=m("p"),oe.innerHTML=_t,Fe=a(),M(re.$$.fragment),Ae=a(),M(pe.$$.fragment),Ne=a(),se=m("blockquote"),se.innerHTML=$t,He=a(),de=m("p"),de.textContent=vt,Ye=a(),M(me.$$.fragment),xe=a(),fe=m("p"),fe.textContent=Rt,Le=a(),ue=m("p"),ue.innerHTML=Gt,Se=a(),M(Me.$$.fragment),De=a(),M(ye.$$.fragment),Pe=a(),ce=m("p"),ce.innerHTML=Xt,Ke=a(),M(be.$$.fragment),Oe=a(),M(Je.$$.fragment),et=a(),ne=m("blockquote"),ne.innerHTML=kt,tt=a(),he=m("p"),he.textContent=zt,lt=a(),M(Te.$$.fragment),st=a(),Ue=m("p"),Ue.innerHTML=Et,nt=a(),M(we.$$.fragment),at=a(),M(Ze.$$.fragment),it=a(),je=m("p"),je.innerHTML=Vt,ot=a(),M(Ce.$$.fragment),rt=a(),ge=m("p"),ge.innerHTML=It,pt=a(),M(Be.$$.fragment),dt=a(),_e=m("p"),_e.textContent=qt,mt=a(),M($e.$$.fragment),ft=a(),M(ve.$$.fragment),ut=a(),Re=m("p"),Re.textContent=Qt,Mt=a(),M(Ge.$$.fragment),yt=a(),M(Xe.$$.fragment),ct=a(),ke=m("p"),ke.innerHTML=Wt,bt=a(),M(ae.$$.fragment),Jt=a(),ze=m("p"),ze.textContent=Ft,ht=a(),Ee=m("p"),Ee.innerHTML=At,Tt=a(),M(Ve.$$.fragment),Ut=a(),Ie=m("ul"),Ie.innerHTML=Nt,wt=a(),M(qe.$$.fragment),Zt=a(),Qe=m("p"),this.h()},l(e){const n=Ot("svelte-u9bgzb",document.head);o=f(n,"META",{name:!0,content:!0}),n.forEach(l),w=i(e),d=f(e,"P",{}),xt(d).forEach(l),T=i(e),y(p.$$.fragment,e),U=i(e),y(Z.$$.fragment,e),V=i(e),k=f(e,"P",{"data-svelte-h":!0}),u(k)!=="svelte-1m50ob7"&&(k.innerHTML=v),K=i(e),z=f(e,"P",{"data-svelte-h":!0}),u(z)!=="svelte-11pyf03"&&(z.innerHTML=X),q=i(e),C=f(e,"P",{"data-svelte-h":!0}),u(C)!=="svelte-1erhi5i"&&(C.innerHTML=ee),Q=i(e),g=f(e,"P",{"data-svelte-h":!0}),u(g)!=="svelte-ryfe7d"&&(g.innerHTML=L),R=i(e),Y=f(e,"P",{"data-svelte-h":!0}),u(Y)!=="svelte-gf36q7"&&(Y.textContent=S),G=i(e),y(x.$$.fragment,e),W=i(e),B=f(e,"P",{"data-svelte-h":!0}),u(B)!=="svelte-17fqaf"&&(B.innerHTML=D),$=i(e),y(F.$$.fragment,e),A=i(e),j=f(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(j)!=="svelte-jq136m"&&(j.innerHTML=te),N=i(e),_=f(e,"P",{"data-svelte-h":!0}),u(_)!=="svelte-139tok6"&&(_.innerHTML=le),H=i(e),y(t.$$.fragment,e),r=i(e),ie=f(e,"P",{"data-svelte-h":!0}),u(ie)!=="svelte-18sn6ad"&&(ie.innerHTML=Bt),We=i(e),oe=f(e,"P",{"data-svelte-h":!0}),u(oe)!=="svelte-10wvd51"&&(oe.innerHTML=_t),Fe=i(e),y(re.$$.fragment,e),Ae=i(e),y(pe.$$.fragment,e),Ne=i(e),se=f(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(se)!=="svelte-b7nvcq"&&(se.innerHTML=$t),He=i(e),de=f(e,"P",{"data-svelte-h":!0}),u(de)!=="svelte-1myadau"&&(de.textContent=vt),Ye=i(e),y(me.$$.fragment,e),xe=i(e),fe=f(e,"P",{"data-svelte-h":!0}),u(fe)!=="svelte-ur5rgd"&&(fe.textContent=Rt),Le=i(e),ue=f(e,"P",{"data-svelte-h":!0}),u(ue)!=="svelte-1pta66b"&&(ue.innerHTML=Gt),Se=i(e),y(Me.$$.fragment,e),De=i(e),y(ye.$$.fragment,e),Pe=i(e),ce=f(e,"P",{"data-svelte-h":!0}),u(ce)!=="svelte-1rff4ou"&&(ce.innerHTML=Xt),Ke=i(e),y(be.$$.fragment,e),Oe=i(e),y(Je.$$.fragment,e),et=i(e),ne=f(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(ne)!=="svelte-1ighp1z"&&(ne.innerHTML=kt),tt=i(e),he=f(e,"P",{"data-svelte-h":!0}),u(he)!=="svelte-7ob7j"&&(he.textContent=zt),lt=i(e),y(Te.$$.fragment,e),st=i(e),Ue=f(e,"P",{"data-svelte-h":!0}),u(Ue)!=="svelte-19knmj2"&&(Ue.innerHTML=Et),nt=i(e),y(we.$$.fragment,e),at=i(e),y(Ze.$$.fragment,e),it=i(e),je=f(e,"P",{"data-svelte-h":!0}),u(je)!=="svelte-6pxdrm"&&(je.innerHTML=Vt),ot=i(e),y(Ce.$$.fragment,e),rt=i(e),ge=f(e,"P",{"data-svelte-h":!0}),u(ge)!=="svelte-1qoc2ct"&&(ge.innerHTML=It),pt=i(e),y(Be.$$.fragment,e),dt=i(e),_e=f(e,"P",{"data-svelte-h":!0}),u(_e)!=="svelte-ep1hhf"&&(_e.textContent=qt),mt=i(e),y($e.$$.fragment,e),ft=i(e),y(ve.$$.fragment,e),ut=i(e),Re=f(e,"P",{"data-svelte-h":!0}),u(Re)!=="svelte-lubruw"&&(Re.textContent=Qt),Mt=i(e),y(Ge.$$.fragment,e),yt=i(e),y(Xe.$$.fragment,e),ct=i(e),ke=f(e,"P",{"data-svelte-h":!0}),u(ke)!=="svelte-7ubrrr"&&(ke.innerHTML=Wt),bt=i(e),y(ae.$$.fragment,e),Jt=i(e),ze=f(e,"P",{"data-svelte-h":!0}),u(ze)!=="svelte-7dnnek"&&(ze.textContent=Ft),ht=i(e),Ee=f(e,"P",{"data-svelte-h":!0}),u(Ee)!=="svelte-j15c34"&&(Ee.innerHTML=At),Tt=i(e),y(Ve.$$.fragment,e),Ut=i(e),Ie=f(e,"UL",{"data-svelte-h":!0}),u(Ie)!=="svelte-1pkgul5"&&(Ie.innerHTML=Nt),wt=i(e),y(qe.$$.fragment,e),Zt=i(e),Qe=f(e,"P",{}),xt(Qe).forEach(l),this.h()},h(){O(o,"name","hf:doc:metadata"),O(o,"content",dl),O(j,"class","warning"),O(se,"class","tip"),O(ne,"class","tip")},m(e,n){el(document.head,o),s(e,w,n),s(e,d,n),s(e,T,n),c(p,e,n),s(e,U,n),c(Z,e,n),s(e,V,n),s(e,k,n),s(e,K,n),s(e,z,n),s(e,q,n),s(e,C,n),s(e,Q,n),s(e,g,n),s(e,R,n),s(e,Y,n),s(e,G,n),c(x,e,n),s(e,W,n),s(e,B,n),s(e,$,n),c(F,e,n),s(e,A,n),s(e,j,n),s(e,N,n),s(e,_,n),s(e,H,n),c(t,e,n),s(e,r,n),s(e,ie,n),s(e,We,n),s(e,oe,n),s(e,Fe,n),c(re,e,n),s(e,Ae,n),c(pe,e,n),s(e,Ne,n),s(e,se,n),s(e,He,n),s(e,de,n),s(e,Ye,n),c(me,e,n),s(e,xe,n),s(e,fe,n),s(e,Le,n),s(e,ue,n),s(e,Se,n),c(Me,e,n),s(e,De,n),c(ye,e,n),s(e,Pe,n),s(e,ce,n),s(e,Ke,n),c(be,e,n),s(e,Oe,n),c(Je,e,n),s(e,et,n),s(e,ne,n),s(e,tt,n),s(e,he,n),s(e,lt,n),c(Te,e,n),s(e,st,n),s(e,Ue,n),s(e,nt,n),c(we,e,n),s(e,at,n),c(Ze,e,n),s(e,it,n),s(e,je,n),s(e,ot,n),c(Ce,e,n),s(e,rt,n),s(e,ge,n),s(e,pt,n),c(Be,e,n),s(e,dt,n),s(e,_e,n),s(e,mt,n),c($e,e,n),s(e,ft,n),c(ve,e,n),s(e,ut,n),s(e,Re,n),s(e,Mt,n),c(Ge,e,n),s(e,yt,n),c(Xe,e,n),s(e,ct,n),s(e,ke,n),s(e,bt,n),c(ae,e,n),s(e,Jt,n),s(e,ze,n),s(e,ht,n),s(e,Ee,n),s(e,Tt,n),c(Ve,e,n),s(e,Ut,n),s(e,Ie,n),s(e,wt,n),c(qe,e,n),s(e,Zt,n),s(e,Qe,n),jt=!0},p(e,[n]){const Ht={};n&2&&(Ht.$$scope={dirty:n,ctx:e}),F.$set(Ht);const Yt={};n&2&&(Yt.$$scope={dirty:n,ctx:e}),ae.$set(Yt)},i(e){jt||(b(p.$$.fragment,e),b(Z.$$.fragment,e),b(x.$$.fragment,e),b(F.$$.fragment,e),b(t.$$.fragment,e),b(re.$$.fragment,e),b(pe.$$.fragment,e),b(me.$$.fragment,e),b(Me.$$.fragment,e),b(ye.$$.fragment,e),b(be.$$.fragment,e),b(Je.$$.fragment,e),b(Te.$$.fragment,e),b(we.$$.fragment,e),b(Ze.$$.fragment,e),b(Ce.$$.fragment,e),b(Be.$$.fragment,e),b($e.$$.fragment,e),b(ve.$$.fragment,e),b(Ge.$$.fragment,e),b(Xe.$$.fragment,e),b(ae.$$.fragment,e),b(Ve.$$.fragment,e),b(qe.$$.fragment,e),jt=!0)},o(e){J(p.$$.fragment,e),J(Z.$$.fragment,e),J(x.$$.fragment,e),J(F.$$.fragment,e),J(t.$$.fragment,e),J(re.$$.fragment,e),J(pe.$$.fragment,e),J(me.$$.fragment,e),J(Me.$$.fragment,e),J(ye.$$.fragment,e),J(be.$$.fragment,e),J(Je.$$.fragment,e),J(Te.$$.fragment,e),J(we.$$.fragment,e),J(Ze.$$.fragment,e),J(Ce.$$.fragment,e),J(Be.$$.fragment,e),J($e.$$.fragment,e),J(ve.$$.fragment,e),J(Ge.$$.fragment,e),J(Xe.$$.fragment,e),J(ae.$$.fragment,e),J(Ve.$$.fragment,e),J(qe.$$.fragment,e),jt=!1},d(e){e&&(l(w),l(d),l(T),l(U),l(V),l(k),l(K),l(z),l(q),l(C),l(Q),l(g),l(R),l(Y),l(G),l(W),l(B),l($),l(A),l(j),l(N),l(_),l(H),l(r),l(ie),l(We),l(oe),l(Fe),l(Ae),l(Ne),l(se),l(He),l(de),l(Ye),l(xe),l(fe),l(Le),l(ue),l(Se),l(De),l(Pe),l(ce),l(Ke),l(Oe),l(et),l(ne),l(tt),l(he),l(lt),l(st),l(Ue),l(nt),l(at),l(it),l(je),l(ot),l(rt),l(ge),l(pt),l(dt),l(_e),l(mt),l(ft),l(ut),l(Re),l(Mt),l(yt),l(ct),l(ke),l(bt),l(Jt),l(ze),l(ht),l(Ee),l(Tt),l(Ut),l(Ie),l(wt),l(Zt),l(Qe)),l(o),h(p,e),h(Z,e),h(x,e),h(F,e),h(t,e),h(re,e),h(pe,e),h(me,e),h(Me,e),h(ye,e),h(be,e),h(Je,e),h(Te,e),h(we,e),h(Ze,e),h(Ce,e),h(Be,e),h($e,e),h(ve,e),h(Ge,e),h(Xe,e),h(ae,e),h(Ve,e),h(qe,e)}}}const dl='{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit (LLM.int8() algorithm)","local":"8-bit-llmint8-algorithm","sections":[{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3}],"depth":2},{"title":"4-bit (QLoRA algorithm)","local":"4-bit-qlora-algorithm","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2},{"title":"torch.compile","local":"torchcompile","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function ml(I){return Dt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Jl extends Pt{constructor(o){super(),Kt(this,o,ml,pl,St,{})}}export{Jl as component};
