import{s as Lt,o as St,n as Ie}from"../chunks/scheduler.8c3d61f6.js";import{S as Dt,i as Pt,g as m,s as i,r as y,A as Kt,h as f,f as l,c as o,j as Yt,u as b,x as u,k as We,y as Ot,a as s,v as J,d as h,t as T,w as U}from"../chunks/index.da70eac4.js";import{T as Ct}from"../chunks/Tip.6f698f24.js";import{C as W}from"../chunks/CodeBlock.a9c4becf.js";import{H as P,E as el}from"../chunks/getInferenceSnippets.cb466318.js";import{H as xt,a as jt}from"../chunks/HfOption.6c3b4e77.js";function tl($){let a,c="Quantizing a model in 8-bit halves the memory-usage:",r,M,d=`bitsandbytes is supported in both Transformers and Diffusers, so you can quantize both the
<a href="/docs/diffusers/main/en/api/models/flux_transformer#diffusers.FluxTransformer2DModel">FluxTransformer2DModel</a> and <a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a>.`,w,Z,q="For Ada and higher-series GPUs. we recommend changing <code>torch_dtype</code> to <code>torch.bfloat16</code>.",Q,j,ee='<p>The <code>CLIPTextModel</code> and <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> aren’t quantized because they’re already small in size and because <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> only has a few <code>torch.nn.Linear</code> layers.</p>',F,C,L,v,Y="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter.",S,R,E,k,K="Let’s generate an image using our quantized models.",V,G,x=`Setting <code>device_map=&quot;auto&quot;</code> automatically fills all available space on the GPU(s) first, then the
CPU, and finally, the hard drive (the absolute slowest option) if there is still not enough memory.`,D,X,z,_,A='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/8bit.png"/>',N,g,te='When there is enough memory, you can also directly move the pipeline to the GPU with <code>.to(&quot;cuda&quot;)</code> and apply <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to optimize GPU memory usage.',H,B,O='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights. You can also save the serialized 8-bit models locally with <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',I;return C=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1RW5jb2Rlck1vZGVsJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUlMkMpJTBBJTBBdGV4dF9lbmNvZGVyXzJfOGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUlMkMpJTBBJTBBdHJhbnNmb3JtZXJfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>,)

text_encoder_2_8bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>,)

transformer_8bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),R=new W({props:{code:"dHJhbnNmb3JtZXJfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTJCJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyJTJDJTBBKQ==",highlighted:`transformer_8bit = AutoModel.from_pretrained(
    &quot;black-forest-labs/FLUX.1-dev&quot;,
    subfolder=&quot;transformer&quot;,
    quantization_config=quant_config,
<span class="hljs-addition">+   torch_dtype=torch.float32,</span>
)`,wrap:!1}}),X=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhQaXBlbGluZSUwQSUwQXBpcGUlMjAlM0QlMjBGbHV4UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXJfOGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRleHRfZW5jb2Rlcl8yJTNEdGV4dF9lbmNvZGVyXzJfOGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQXBpcGVfa3dhcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycHJvbXB0JTIyJTNBJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmhlaWdodCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJ3aWR0aCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJndWlkYW5jZV9zY2FsZSUyMiUzQSUyMDMuNSUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV9pbmZlcmVuY2Vfc3RlcHMlMjIlM0ElMjA1MCUyQyUwQSUyMCUyMCUyMCUyMCUyMm1heF9zZXF1ZW5jZV9sZW5ndGglMjIlM0ElMjA1MTIlMkMlMEElN0QlMEElMEFpbWFnZSUyMCUzRCUyMHBpcGUoKipwaXBlX2t3YXJncyUyQyUyMGdlbmVyYXRvciUzRHRvcmNoLm1hbnVhbF9zZWVkKDApJTJDKS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer_8bit,
    text_encoder_2=text_encoder_2_8bit,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)

pipe_kwargs = {
    <span class="hljs-string">&quot;prompt&quot;</span>: <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
    <span class="hljs-string">&quot;height&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;width&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;guidance_scale&quot;</span>: <span class="hljs-number">3.5</span>,
    <span class="hljs-string">&quot;num_inference_steps&quot;</span>: <span class="hljs-number">50</span>,
    <span class="hljs-string">&quot;max_sequence_length&quot;</span>: <span class="hljs-number">512</span>,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(<span class="hljs-number">0</span>),).images[<span class="hljs-number">0</span>]`,wrap:!1}}),{c(){a=m("p"),a.textContent=c,r=i(),M=m("p"),M.innerHTML=d,w=i(),Z=m("p"),Z.innerHTML=q,Q=i(),j=m("blockquote"),j.innerHTML=ee,F=i(),y(C.$$.fragment),L=i(),v=m("p"),v.innerHTML=Y,S=i(),y(R.$$.fragment),E=i(),k=m("p"),k.textContent=K,V=i(),G=m("p"),G.innerHTML=x,D=i(),y(X.$$.fragment),z=i(),_=m("div"),_.innerHTML=A,N=i(),g=m("p"),g.innerHTML=te,H=i(),B=m("p"),B.innerHTML=O,this.h()},l(t){a=f(t,"P",{"data-svelte-h":!0}),u(a)!=="svelte-4djpqq"&&(a.textContent=c),r=o(t),M=f(t,"P",{"data-svelte-h":!0}),u(M)!=="svelte-8t0085"&&(M.innerHTML=d),w=o(t),Z=f(t,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-mdqrzc"&&(Z.innerHTML=q),Q=o(t),j=f(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(j)!=="svelte-1iq05of"&&(j.innerHTML=ee),F=o(t),b(C.$$.fragment,t),L=o(t),v=f(t,"P",{"data-svelte-h":!0}),u(v)!=="svelte-1qozsrk"&&(v.innerHTML=Y),S=o(t),b(R.$$.fragment,t),E=o(t),k=f(t,"P",{"data-svelte-h":!0}),u(k)!=="svelte-2bde4h"&&(k.textContent=K),V=o(t),G=f(t,"P",{"data-svelte-h":!0}),u(G)!=="svelte-1eyrcy7"&&(G.innerHTML=x),D=o(t),b(X.$$.fragment,t),z=o(t),_=f(t,"DIV",{class:!0,"data-svelte-h":!0}),u(_)!=="svelte-14afenp"&&(_.innerHTML=A),N=o(t),g=f(t,"P",{"data-svelte-h":!0}),u(g)!=="svelte-uz8m1k"&&(g.innerHTML=te),H=o(t),B=f(t,"P",{"data-svelte-h":!0}),u(B)!=="svelte-13z5cfr"&&(B.innerHTML=O),this.h()},h(){We(j,"class","tip"),We(_,"class","flex justify-center")},m(t,p){s(t,a,p),s(t,r,p),s(t,M,p),s(t,w,p),s(t,Z,p),s(t,Q,p),s(t,j,p),s(t,F,p),J(C,t,p),s(t,L,p),s(t,v,p),s(t,S,p),J(R,t,p),s(t,E,p),s(t,k,p),s(t,V,p),s(t,G,p),s(t,D,p),J(X,t,p),s(t,z,p),s(t,_,p),s(t,N,p),s(t,g,p),s(t,H,p),s(t,B,p),I=!0},p:Ie,i(t){I||(h(C.$$.fragment,t),h(R.$$.fragment,t),h(X.$$.fragment,t),I=!0)},o(t){T(C.$$.fragment,t),T(R.$$.fragment,t),T(X.$$.fragment,t),I=!1},d(t){t&&(l(a),l(r),l(M),l(w),l(Z),l(Q),l(j),l(F),l(L),l(v),l(S),l(E),l(k),l(V),l(G),l(D),l(z),l(_),l(N),l(g),l(H),l(B)),U(C,t),U(R,t),U(X,t)}}}function ll($){let a,c="Quantizing a model in 4-bit reduces your memory-usage by 4x:",r,M,d=`bitsandbytes is supported in both Transformers and Diffusers, so you can can quantize both the
<a href="/docs/diffusers/main/en/api/models/flux_transformer#diffusers.FluxTransformer2DModel">FluxTransformer2DModel</a> and <a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a>.`,w,Z,q="For Ada and higher-series GPUs. we recommend changing <code>torch_dtype</code> to <code>torch.bfloat16</code>.",Q,j,ee='<p>The <code>CLIPTextModel</code> and <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> aren’t quantized because they’re already small in size and because <a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a> only has a few <code>torch.nn.Linear</code> layers.</p>',F,C,L,v,Y="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter.",S,R,E,k,K="Let’s generate an image using our quantized models.",V,G,x="Setting <code>device_map=&quot;auto&quot;</code> automatically fills all available space on the GPU(s) first, then the CPU, and finally, the hard drive (the absolute slowest option) if there is still not enough memory.",D,X,z,_,A='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/4bit.png"/>',N,g,te='When there is enough memory, you can also directly move the pipeline to the GPU with <code>.to(&quot;cuda&quot;)</code> and apply <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to optimize GPU memory usage.',H,B,O='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights. You can also save the serialized 4-bit models locally with <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',I;return C=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1RW5jb2Rlck1vZGVsJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUlMkMpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUlMkMpJTBBJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>,)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>,)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),R=new W({props:{code:"dHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTJCJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyJTJDJTBBKQ==",highlighted:`transformer_4bit = AutoModel.from_pretrained(
    &quot;black-forest-labs/FLUX.1-dev&quot;,
    subfolder=&quot;transformer&quot;,
    quantization_config=quant_config,
<span class="hljs-addition">+   torch_dtype=torch.float32,</span>
)`,wrap:!1}}),X=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhQaXBlbGluZSUwQSUwQXBpcGUlMjAlM0QlMjBGbHV4UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXJfNGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRleHRfZW5jb2Rlcl8yJTNEdGV4dF9lbmNvZGVyXzJfNGJpdCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQXBpcGVfa3dhcmdzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycHJvbXB0JTIyJTNBJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmhlaWdodCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJ3aWR0aCUyMiUzQSUyMDEwMjQlMkMlMEElMjAlMjAlMjAlMjAlMjJndWlkYW5jZV9zY2FsZSUyMiUzQSUyMDMuNSUyQyUwQSUyMCUyMCUyMCUyMCUyMm51bV9pbmZlcmVuY2Vfc3RlcHMlMjIlM0ElMjA1MCUyQyUwQSUyMCUyMCUyMCUyMCUyMm1heF9zZXF1ZW5jZV9sZW5ndGglMjIlM0ElMjA1MTIlMkMlMEElN0QlMEElMEFpbWFnZSUyMCUzRCUyMHBpcGUoKipwaXBlX2t3YXJncyUyQyUyMGdlbmVyYXRvciUzRHRvcmNoLm1hbnVhbF9zZWVkKDApJTJDKS5pbWFnZXMlNUIwJTVE",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer_4bit,
    text_encoder_2=text_encoder_2_4bit,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)

pipe_kwargs = {
    <span class="hljs-string">&quot;prompt&quot;</span>: <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
    <span class="hljs-string">&quot;height&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;width&quot;</span>: <span class="hljs-number">1024</span>,
    <span class="hljs-string">&quot;guidance_scale&quot;</span>: <span class="hljs-number">3.5</span>,
    <span class="hljs-string">&quot;num_inference_steps&quot;</span>: <span class="hljs-number">50</span>,
    <span class="hljs-string">&quot;max_sequence_length&quot;</span>: <span class="hljs-number">512</span>,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(<span class="hljs-number">0</span>),).images[<span class="hljs-number">0</span>]`,wrap:!1}}),{c(){a=m("p"),a.textContent=c,r=i(),M=m("p"),M.innerHTML=d,w=i(),Z=m("p"),Z.innerHTML=q,Q=i(),j=m("blockquote"),j.innerHTML=ee,F=i(),y(C.$$.fragment),L=i(),v=m("p"),v.innerHTML=Y,S=i(),y(R.$$.fragment),E=i(),k=m("p"),k.textContent=K,V=i(),G=m("p"),G.innerHTML=x,D=i(),y(X.$$.fragment),z=i(),_=m("div"),_.innerHTML=A,N=i(),g=m("p"),g.innerHTML=te,H=i(),B=m("p"),B.innerHTML=O,this.h()},l(t){a=f(t,"P",{"data-svelte-h":!0}),u(a)!=="svelte-i07c71"&&(a.textContent=c),r=o(t),M=f(t,"P",{"data-svelte-h":!0}),u(M)!=="svelte-q3mf8r"&&(M.innerHTML=d),w=o(t),Z=f(t,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-mdqrzc"&&(Z.innerHTML=q),Q=o(t),j=f(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(j)!=="svelte-1iq05of"&&(j.innerHTML=ee),F=o(t),b(C.$$.fragment,t),L=o(t),v=f(t,"P",{"data-svelte-h":!0}),u(v)!=="svelte-1qozsrk"&&(v.innerHTML=Y),S=o(t),b(R.$$.fragment,t),E=o(t),k=f(t,"P",{"data-svelte-h":!0}),u(k)!=="svelte-2bde4h"&&(k.textContent=K),V=o(t),G=f(t,"P",{"data-svelte-h":!0}),u(G)!=="svelte-6ilppp"&&(G.innerHTML=x),D=o(t),b(X.$$.fragment,t),z=o(t),_=f(t,"DIV",{class:!0,"data-svelte-h":!0}),u(_)!=="svelte-qev6y1"&&(_.innerHTML=A),N=o(t),g=f(t,"P",{"data-svelte-h":!0}),u(g)!=="svelte-uz8m1k"&&(g.innerHTML=te),H=o(t),B=f(t,"P",{"data-svelte-h":!0}),u(B)!=="svelte-1wxqg1v"&&(B.innerHTML=O),this.h()},h(){We(j,"class","tip"),We(_,"class","flex justify-center")},m(t,p){s(t,a,p),s(t,r,p),s(t,M,p),s(t,w,p),s(t,Z,p),s(t,Q,p),s(t,j,p),s(t,F,p),J(C,t,p),s(t,L,p),s(t,v,p),s(t,S,p),J(R,t,p),s(t,E,p),s(t,k,p),s(t,V,p),s(t,G,p),s(t,D,p),J(X,t,p),s(t,z,p),s(t,_,p),s(t,N,p),s(t,g,p),s(t,H,p),s(t,B,p),I=!0},p:Ie,i(t){I||(h(C.$$.fragment,t),h(R.$$.fragment,t),h(X.$$.fragment,t),I=!0)},o(t){T(C.$$.fragment,t),T(R.$$.fragment,t),T(X.$$.fragment,t),I=!1},d(t){t&&(l(a),l(r),l(M),l(w),l(Z),l(Q),l(j),l(F),l(L),l(v),l(S),l(E),l(k),l(V),l(G),l(D),l(z),l(_),l(N),l(g),l(H),l(B)),U(C,t),U(R,t),U(X,t)}}}function sl($){let a,c,r,M;return a=new jt({props:{id:"bnb",option:"8-bit",$$slots:{default:[tl]},$$scope:{ctx:$}}}),r=new jt({props:{id:"bnb",option:"4-bit",$$slots:{default:[ll]},$$scope:{ctx:$}}}),{c(){y(a.$$.fragment),c=i(),y(r.$$.fragment)},l(d){b(a.$$.fragment,d),c=o(d),b(r.$$.fragment,d)},m(d,w){J(a,d,w),s(d,c,w),J(r,d,w),M=!0},p(d,w){const Z={};w&2&&(Z.$$scope={dirty:w,ctx:d}),a.$set(Z);const q={};w&2&&(q.$$scope={dirty:w,ctx:d}),r.$set(q)},i(d){M||(h(a.$$.fragment,d),h(r.$$.fragment,d),M=!0)},o(d){T(a.$$.fragment,d),T(r.$$.fragment,d),M=!1},d(d){d&&l(c),U(a,d),U(r,d)}}}function nl($){let a,c="Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.";return{c(){a=m("p"),a.innerHTML=c},l(r){a=f(r,"P",{"data-svelte-h":!0}),u(a)!=="svelte-of9sym"&&(a.innerHTML=c)},m(r,M){s(r,a,M)},p:Ie,d(r){r&&l(a)}}}function al($){let a,c='Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!';return{c(){a=m("p"),a.innerHTML=c},l(r){a=f(r,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1bb05fp"&&(a.innerHTML=c)},m(r,M){s(r,a,M)},p:Ie,d(r){r&&l(a)}}}function il($){let a,c='Learn more about its details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.';return{c(){a=m("p"),a.innerHTML=c},l(r){a=f(r,"P",{"data-svelte-h":!0}),u(a)!=="svelte-kpdzjq"&&(a.innerHTML=c)},m(r,M){s(r,a,M)},p:Ie,d(r){r&&l(a)}}}function ol($){let a,c;return a=new W({props:{code:"dG9yY2guX2R5bmFtby5jb25maWcuY2FwdHVyZV9keW5hbWljX291dHB1dF9zaGFwZV9vcHMlMjAlM0QlMjBUcnVlJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKSUwQXRyYW5zZm9ybWVyXzRiaXQuY29tcGlsZShmdWxsZ3JhcGglM0RUcnVlKQ==",highlighted:`torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)
transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)
transformer_4bit.<span class="hljs-built_in">compile</span>(fullgraph=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){y(a.$$.fragment)},l(r){b(a.$$.fragment,r)},m(r,M){J(a,r,M),c=!0},p:Ie,i(r){c||(h(a.$$.fragment,r),c=!0)},o(r){T(a.$$.fragment,r),c=!1},d(r){U(a,r)}}}function rl($){let a,c;return a=new W({props:{code:"cXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKSUwQXRyYW5zZm9ybWVyXzRiaXQuY29tcGlsZShmdWxsZ3JhcGglM0RUcnVlKQ==",highlighted:`quant_config = DiffusersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)
transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)
transformer_4bit.<span class="hljs-built_in">compile</span>(fullgraph=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){y(a.$$.fragment)},l(r){b(a.$$.fragment,r)},m(r,M){J(a,r,M),c=!0},p:Ie,i(r){c||(h(a.$$.fragment,r),c=!0)},o(r){T(a.$$.fragment,r),c=!1},d(r){U(a,r)}}}function pl($){let a,c,r,M;return a=new jt({props:{id:"bnb",option:"8-bit",$$slots:{default:[ol]},$$scope:{ctx:$}}}),r=new jt({props:{id:"bnb",option:"4-bit",$$slots:{default:[rl]},$$scope:{ctx:$}}}),{c(){y(a.$$.fragment),c=i(),y(r.$$.fragment)},l(d){b(a.$$.fragment,d),c=o(d),b(r.$$.fragment,d)},m(d,w){J(a,d,w),s(d,c,w),J(r,d,w),M=!0},p(d,w){const Z={};w&2&&(Z.$$scope={dirty:w,ctx:d}),a.$set(Z);const q={};w&2&&(q.$$scope={dirty:w,ctx:d}),r.$set(q)},i(d){M||(h(a.$$.fragment,d),h(r.$$.fragment,d),M=!0)},o(d){T(a.$$.fragment,d),T(r.$$.fragment,d),M=!1},d(d){d&&l(c),U(a,d),U(r,d)}}}function dl($){let a,c,r,M,d,w,Z,q='<a href="https://huggingface.co/docs/bitsandbytes/index" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance.',Q,j,ee='4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',F,C,L=`This guide demonstrates how quantization can enable running
<a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" rel="nofollow">FLUX.1-dev</a>
on less than 16GB of VRAM and even on a free Google
Colab instance.`,v,Y,S='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/quant-bnb/comparison.png" alt="comparison image"/>',R,E,k="To use bitsandbytes, make sure you have the following libraries installed:",K,V,G,x,D='Now you can quantize a model by passing a <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a> to <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. This works for any model in any modality, as long as it supports loading with <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> and contains <code>torch.nn.Linear</code> layers.',X,z,_,A,N,g,te="Check your memory footprint with the <code>get_memory_footprint</code> method:",H,B,O,I,t="Note that this only tells you the memory footprint of the model params and does <em>not</em> estimate the inference memory requirements.",p,ae,gt='Quantized models can be loaded from the <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a> method without needing to specify the <code>quantization_config</code> parameters:',Qe,ie,Fe,oe,Ae,le,Ne,re,Bt="This section explores some of the specific features of 8-bit models, such as outlier thresholds and skipping module conversion.",He,pe,Ye,de,_t="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",xe,me,$t='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Le,fe,Se,ue,De,Me,vt='For some models, you don’t need to quantize every module to 8-bit which can actually cause instability. For example, for diffusion models like <a href="../api/pipelines/stable_diffusion/stable_diffusion_3">Stable Diffusion 3</a>, the <code>proj_out</code> module can be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Pe,ce,Ke,ye,Oe,se,et,be,Rt="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",tt,Je,lt,he,Gt='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',st,Te,nt,Ue,at,we,Xt='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',it,Ze,ot,je,kt="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",rt,Ce,pt,ge,zt="Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an additional 0.4 bits/parameter.",dt,Be,mt,_e,ft,$e,Et="Once quantized, you can dequantize a model to its original precision, but this might result in a small loss of quality. Make sure you have enough GPU RAM to fit the dequantized model.",ut,ve,Mt,Re,ct,Ge,Vt='Speed up inference with <code>torch.compile</code>. Make sure you have the latest <code>bitsandbytes</code> installed and we also recommend installing <a href="https://pytorch.org/get-started/locally/" rel="nofollow">PyTorch nightly</a>.',yt,ne,bt,Xe,It="On an RTX 4090 with compilation, 4-bit Flux generation completed in 25.809 seconds versus 32.570 seconds without.",Jt,ke,Wt='Check out the <a href="https://gist.github.com/sayakpaul/0db9d8eeeb3d2a0e5ed7cf0d9ca19b7d" rel="nofollow">benchmarking script</a> for more details.',ht,ze,Tt,Ee,qt='<li><a href="https://gist.github.com/sayakpaul/c76bd845b48759e11687ac550b99d8b4" rel="nofollow">End-to-end notebook showing Flux.1 Dev inference in a free-tier Colab</a></li> <li><a href="https://github.com/huggingface/diffusers/blob/8c661ea586bf11cb2440da740dd3c4cf84679b85/examples/dreambooth/README_hidream.md#using-quantization" rel="nofollow">Training</a></li>',Ut,Ve,wt,qe,Zt;return d=new P({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),V=new W({props:{code:"cGlwJTIwaW5zdGFsbCUyMGRpZmZ1c2VycyUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMjAtVQ==",highlighted:"pip install diffusers transformers accelerate bitsandbytes -U",wrap:!1}}),z=new xt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[sl]},$$scope:{ctx:$}}}),A=new Ct({props:{warning:!0,$$slots:{default:[nl]},$$scope:{ctx:$}}}),B=new W({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),ie=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl80Yml0JTNEVHJ1ZSklMEElMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJoZi1pbnRlcm5hbC10ZXN0aW5nJTJGZmx1eC4xLWRldi1uZjQtcGtnJTIyJTJDJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;hf-internal-testing/flux.1-dev-nf4-pkg&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>
)`,wrap:!1}}),oe=new P({props:{title:"8-bit (LLM.int8() algorithm)",local:"8-bit-llmint8-algorithm",headingTag:"h2"}}),le=new Ct({props:{$$slots:{default:[al]},$$scope:{ctx:$}}}),pe=new P({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),fe=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3RocmVzaG9sZCUzRDEwJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),ue=new P({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),ce=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3NraXBfbW9kdWxlcyUzRCU1QiUyMnByb2pfb3V0JTIyJTVEJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBTRDNUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi0zLW1lZGl1bS1kaWZmdXNlcnMlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> SD3Transformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_skip_modules=[<span class="hljs-string">&quot;proj_out&quot;</span>],
)

model_8bit = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),ye=new P({props:{title:"4-bit (QLoRA algorithm)",local:"4-bit-qlora-algorithm",headingTag:"h2"}}),se=new Ct({props:{$$slots:{default:[il]},$$scope:{ctx:$}}}),Je=new P({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),Te=new W({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),Ue=new P({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),Ze=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSklMEElMEF0ZXh0X2VuY29kZXJfMl80Yml0JTIwJTNEJTIwVDVFbmNvZGVyTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRleHRfZW5jb2Rlcl8yJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50X2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSklMEElMEFxdWFudF9jb25maWclMjAlM0QlMjBEaWZmdXNlcnNCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBdHJhbnNmb3JtZXJfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),Ce=new P({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),Be=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfdXNlX2RvdWJsZV9xdWFudCUzRFRydWUlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSklMEElMEF0cmFuc2Zvcm1lcl80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)`,wrap:!1}}),_e=new P({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),ve=new W({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUyMGFzJTIwVHJhbnNmb3JtZXJzQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUNUVuY29kZXJNb2RlbCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFRyYW5zZm9ybWVyc0JpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfdXNlX2RvdWJsZV9xdWFudCUzRFRydWUlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdCUyMCUzRCUyMFQ1RW5jb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0ZXh0X2VuY29kZXJfMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwRGlmZnVzZXJzQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSklMEElMEF0cmFuc2Zvcm1lcl80Yml0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEEpJTBBJTBBdGV4dF9lbmNvZGVyXzJfNGJpdC5kZXF1YW50aXplKCklMEF0cmFuc2Zvcm1lcl80Yml0LmRlcXVhbnRpemUoKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5EncoderModel

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;text_encoder_2&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

transformer_4bit = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

text_encoder_2_4bit.dequantize()
transformer_4bit.dequantize()`,wrap:!1}}),Re=new P({props:{title:"torch.compile",local:"torchcompile",headingTag:"h2"}}),ne=new xt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[pl]},$$scope:{ctx:$}}}),ze=new P({props:{title:"Resources",local:"resources",headingTag:"h2"}}),Ve=new el({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){a=m("meta"),c=i(),r=m("p"),M=i(),y(d.$$.fragment),w=i(),Z=m("p"),Z.innerHTML=q,Q=i(),j=m("p"),j.innerHTML=ee,F=i(),C=m("p"),C.innerHTML=L,v=i(),Y=m("p"),Y.innerHTML=S,R=i(),E=m("p"),E.textContent=k,K=i(),y(V.$$.fragment),G=i(),x=m("p"),x.innerHTML=D,X=i(),y(z.$$.fragment),_=i(),y(A.$$.fragment),N=i(),g=m("p"),g.innerHTML=te,H=i(),y(B.$$.fragment),O=i(),I=m("p"),I.innerHTML=t,p=i(),ae=m("p"),ae.innerHTML=gt,Qe=i(),y(ie.$$.fragment),Fe=i(),y(oe.$$.fragment),Ae=i(),y(le.$$.fragment),Ne=i(),re=m("p"),re.textContent=Bt,He=i(),y(pe.$$.fragment),Ye=i(),de=m("p"),de.textContent=_t,xe=i(),me=m("p"),me.innerHTML=$t,Le=i(),y(fe.$$.fragment),Se=i(),y(ue.$$.fragment),De=i(),Me=m("p"),Me.innerHTML=vt,Pe=i(),y(ce.$$.fragment),Ke=i(),y(ye.$$.fragment),Oe=i(),y(se.$$.fragment),et=i(),be=m("p"),be.textContent=Rt,tt=i(),y(Je.$$.fragment),lt=i(),he=m("p"),he.innerHTML=Gt,st=i(),y(Te.$$.fragment),nt=i(),y(Ue.$$.fragment),at=i(),we=m("p"),we.innerHTML=Xt,it=i(),y(Ze.$$.fragment),ot=i(),je=m("p"),je.innerHTML=kt,rt=i(),y(Ce.$$.fragment),pt=i(),ge=m("p"),ge.textContent=zt,dt=i(),y(Be.$$.fragment),mt=i(),y(_e.$$.fragment),ft=i(),$e=m("p"),$e.textContent=Et,ut=i(),y(ve.$$.fragment),Mt=i(),y(Re.$$.fragment),ct=i(),Ge=m("p"),Ge.innerHTML=Vt,yt=i(),y(ne.$$.fragment),bt=i(),Xe=m("p"),Xe.textContent=It,Jt=i(),ke=m("p"),ke.innerHTML=Wt,ht=i(),y(ze.$$.fragment),Tt=i(),Ee=m("ul"),Ee.innerHTML=qt,Ut=i(),y(Ve.$$.fragment),wt=i(),qe=m("p"),this.h()},l(e){const n=Kt("svelte-u9bgzb",document.head);a=f(n,"META",{name:!0,content:!0}),n.forEach(l),c=o(e),r=f(e,"P",{}),Yt(r).forEach(l),M=o(e),b(d.$$.fragment,e),w=o(e),Z=f(e,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-1m50ob7"&&(Z.innerHTML=q),Q=o(e),j=f(e,"P",{"data-svelte-h":!0}),u(j)!=="svelte-11pyf03"&&(j.innerHTML=ee),F=o(e),C=f(e,"P",{"data-svelte-h":!0}),u(C)!=="svelte-1erhi5i"&&(C.innerHTML=L),v=o(e),Y=f(e,"P",{"data-svelte-h":!0}),u(Y)!=="svelte-ryfe7d"&&(Y.innerHTML=S),R=o(e),E=f(e,"P",{"data-svelte-h":!0}),u(E)!=="svelte-gf36q7"&&(E.textContent=k),K=o(e),b(V.$$.fragment,e),G=o(e),x=f(e,"P",{"data-svelte-h":!0}),u(x)!=="svelte-17fqaf"&&(x.innerHTML=D),X=o(e),b(z.$$.fragment,e),_=o(e),b(A.$$.fragment,e),N=o(e),g=f(e,"P",{"data-svelte-h":!0}),u(g)!=="svelte-139tok6"&&(g.innerHTML=te),H=o(e),b(B.$$.fragment,e),O=o(e),I=f(e,"P",{"data-svelte-h":!0}),u(I)!=="svelte-18sn6ad"&&(I.innerHTML=t),p=o(e),ae=f(e,"P",{"data-svelte-h":!0}),u(ae)!=="svelte-10wvd51"&&(ae.innerHTML=gt),Qe=o(e),b(ie.$$.fragment,e),Fe=o(e),b(oe.$$.fragment,e),Ae=o(e),b(le.$$.fragment,e),Ne=o(e),re=f(e,"P",{"data-svelte-h":!0}),u(re)!=="svelte-1myadau"&&(re.textContent=Bt),He=o(e),b(pe.$$.fragment,e),Ye=o(e),de=f(e,"P",{"data-svelte-h":!0}),u(de)!=="svelte-ur5rgd"&&(de.textContent=_t),xe=o(e),me=f(e,"P",{"data-svelte-h":!0}),u(me)!=="svelte-1pta66b"&&(me.innerHTML=$t),Le=o(e),b(fe.$$.fragment,e),Se=o(e),b(ue.$$.fragment,e),De=o(e),Me=f(e,"P",{"data-svelte-h":!0}),u(Me)!=="svelte-1rff4ou"&&(Me.innerHTML=vt),Pe=o(e),b(ce.$$.fragment,e),Ke=o(e),b(ye.$$.fragment,e),Oe=o(e),b(se.$$.fragment,e),et=o(e),be=f(e,"P",{"data-svelte-h":!0}),u(be)!=="svelte-7ob7j"&&(be.textContent=Rt),tt=o(e),b(Je.$$.fragment,e),lt=o(e),he=f(e,"P",{"data-svelte-h":!0}),u(he)!=="svelte-19knmj2"&&(he.innerHTML=Gt),st=o(e),b(Te.$$.fragment,e),nt=o(e),b(Ue.$$.fragment,e),at=o(e),we=f(e,"P",{"data-svelte-h":!0}),u(we)!=="svelte-6pxdrm"&&(we.innerHTML=Xt),it=o(e),b(Ze.$$.fragment,e),ot=o(e),je=f(e,"P",{"data-svelte-h":!0}),u(je)!=="svelte-1qoc2ct"&&(je.innerHTML=kt),rt=o(e),b(Ce.$$.fragment,e),pt=o(e),ge=f(e,"P",{"data-svelte-h":!0}),u(ge)!=="svelte-ep1hhf"&&(ge.textContent=zt),dt=o(e),b(Be.$$.fragment,e),mt=o(e),b(_e.$$.fragment,e),ft=o(e),$e=f(e,"P",{"data-svelte-h":!0}),u($e)!=="svelte-lubruw"&&($e.textContent=Et),ut=o(e),b(ve.$$.fragment,e),Mt=o(e),b(Re.$$.fragment,e),ct=o(e),Ge=f(e,"P",{"data-svelte-h":!0}),u(Ge)!=="svelte-7ubrrr"&&(Ge.innerHTML=Vt),yt=o(e),b(ne.$$.fragment,e),bt=o(e),Xe=f(e,"P",{"data-svelte-h":!0}),u(Xe)!=="svelte-7dnnek"&&(Xe.textContent=It),Jt=o(e),ke=f(e,"P",{"data-svelte-h":!0}),u(ke)!=="svelte-j15c34"&&(ke.innerHTML=Wt),ht=o(e),b(ze.$$.fragment,e),Tt=o(e),Ee=f(e,"UL",{"data-svelte-h":!0}),u(Ee)!=="svelte-1pkgul5"&&(Ee.innerHTML=qt),Ut=o(e),b(Ve.$$.fragment,e),wt=o(e),qe=f(e,"P",{}),Yt(qe).forEach(l),this.h()},h(){We(a,"name","hf:doc:metadata"),We(a,"content",ml)},m(e,n){Ot(document.head,a),s(e,c,n),s(e,r,n),s(e,M,n),J(d,e,n),s(e,w,n),s(e,Z,n),s(e,Q,n),s(e,j,n),s(e,F,n),s(e,C,n),s(e,v,n),s(e,Y,n),s(e,R,n),s(e,E,n),s(e,K,n),J(V,e,n),s(e,G,n),s(e,x,n),s(e,X,n),J(z,e,n),s(e,_,n),J(A,e,n),s(e,N,n),s(e,g,n),s(e,H,n),J(B,e,n),s(e,O,n),s(e,I,n),s(e,p,n),s(e,ae,n),s(e,Qe,n),J(ie,e,n),s(e,Fe,n),J(oe,e,n),s(e,Ae,n),J(le,e,n),s(e,Ne,n),s(e,re,n),s(e,He,n),J(pe,e,n),s(e,Ye,n),s(e,de,n),s(e,xe,n),s(e,me,n),s(e,Le,n),J(fe,e,n),s(e,Se,n),J(ue,e,n),s(e,De,n),s(e,Me,n),s(e,Pe,n),J(ce,e,n),s(e,Ke,n),J(ye,e,n),s(e,Oe,n),J(se,e,n),s(e,et,n),s(e,be,n),s(e,tt,n),J(Je,e,n),s(e,lt,n),s(e,he,n),s(e,st,n),J(Te,e,n),s(e,nt,n),J(Ue,e,n),s(e,at,n),s(e,we,n),s(e,it,n),J(Ze,e,n),s(e,ot,n),s(e,je,n),s(e,rt,n),J(Ce,e,n),s(e,pt,n),s(e,ge,n),s(e,dt,n),J(Be,e,n),s(e,mt,n),J(_e,e,n),s(e,ft,n),s(e,$e,n),s(e,ut,n),J(ve,e,n),s(e,Mt,n),J(Re,e,n),s(e,ct,n),s(e,Ge,n),s(e,yt,n),J(ne,e,n),s(e,bt,n),s(e,Xe,n),s(e,Jt,n),s(e,ke,n),s(e,ht,n),J(ze,e,n),s(e,Tt,n),s(e,Ee,n),s(e,Ut,n),J(Ve,e,n),s(e,wt,n),s(e,qe,n),Zt=!0},p(e,[n]){const Qt={};n&2&&(Qt.$$scope={dirty:n,ctx:e}),z.$set(Qt);const Ft={};n&2&&(Ft.$$scope={dirty:n,ctx:e}),A.$set(Ft);const At={};n&2&&(At.$$scope={dirty:n,ctx:e}),le.$set(At);const Nt={};n&2&&(Nt.$$scope={dirty:n,ctx:e}),se.$set(Nt);const Ht={};n&2&&(Ht.$$scope={dirty:n,ctx:e}),ne.$set(Ht)},i(e){Zt||(h(d.$$.fragment,e),h(V.$$.fragment,e),h(z.$$.fragment,e),h(A.$$.fragment,e),h(B.$$.fragment,e),h(ie.$$.fragment,e),h(oe.$$.fragment,e),h(le.$$.fragment,e),h(pe.$$.fragment,e),h(fe.$$.fragment,e),h(ue.$$.fragment,e),h(ce.$$.fragment,e),h(ye.$$.fragment,e),h(se.$$.fragment,e),h(Je.$$.fragment,e),h(Te.$$.fragment,e),h(Ue.$$.fragment,e),h(Ze.$$.fragment,e),h(Ce.$$.fragment,e),h(Be.$$.fragment,e),h(_e.$$.fragment,e),h(ve.$$.fragment,e),h(Re.$$.fragment,e),h(ne.$$.fragment,e),h(ze.$$.fragment,e),h(Ve.$$.fragment,e),Zt=!0)},o(e){T(d.$$.fragment,e),T(V.$$.fragment,e),T(z.$$.fragment,e),T(A.$$.fragment,e),T(B.$$.fragment,e),T(ie.$$.fragment,e),T(oe.$$.fragment,e),T(le.$$.fragment,e),T(pe.$$.fragment,e),T(fe.$$.fragment,e),T(ue.$$.fragment,e),T(ce.$$.fragment,e),T(ye.$$.fragment,e),T(se.$$.fragment,e),T(Je.$$.fragment,e),T(Te.$$.fragment,e),T(Ue.$$.fragment,e),T(Ze.$$.fragment,e),T(Ce.$$.fragment,e),T(Be.$$.fragment,e),T(_e.$$.fragment,e),T(ve.$$.fragment,e),T(Re.$$.fragment,e),T(ne.$$.fragment,e),T(ze.$$.fragment,e),T(Ve.$$.fragment,e),Zt=!1},d(e){e&&(l(c),l(r),l(M),l(w),l(Z),l(Q),l(j),l(F),l(C),l(v),l(Y),l(R),l(E),l(K),l(G),l(x),l(X),l(_),l(N),l(g),l(H),l(O),l(I),l(p),l(ae),l(Qe),l(Fe),l(Ae),l(Ne),l(re),l(He),l(Ye),l(de),l(xe),l(me),l(Le),l(Se),l(De),l(Me),l(Pe),l(Ke),l(Oe),l(et),l(be),l(tt),l(lt),l(he),l(st),l(nt),l(at),l(we),l(it),l(ot),l(je),l(rt),l(pt),l(ge),l(dt),l(mt),l(ft),l($e),l(ut),l(Mt),l(ct),l(Ge),l(yt),l(bt),l(Xe),l(Jt),l(ke),l(ht),l(Tt),l(Ee),l(Ut),l(wt),l(qe)),l(a),U(d,e),U(V,e),U(z,e),U(A,e),U(B,e),U(ie,e),U(oe,e),U(le,e),U(pe,e),U(fe,e),U(ue,e),U(ce,e),U(ye,e),U(se,e),U(Je,e),U(Te,e),U(Ue,e),U(Ze,e),U(Ce,e),U(Be,e),U(_e,e),U(ve,e),U(Re,e),U(ne,e),U(ze,e),U(Ve,e)}}}const ml='{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit (LLM.int8() algorithm)","local":"8-bit-llmint8-algorithm","sections":[{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3}],"depth":2},{"title":"4-bit (QLoRA algorithm)","local":"4-bit-qlora-algorithm","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2},{"title":"torch.compile","local":"torchcompile","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function fl($){return St(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class hl extends Dt{constructor(a){super(),Pt(this,a,fl,dl,Lt,{})}}export{hl as component};
