import{s as At,o as Kt,n as Ot}from"../chunks/scheduler.8a2cc2fa.js";import{S as te,i as ee,e as o,s as a,c as p,h as ne,a as l,d as e,b as s,f as N,g as b,j as x,k as F,l as i,m as r,n as c,t as f,o as _,p as h}from"../chunks/index.7079e750.js";import{C as ae,H as lt,E as se}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.b2e4ed49.js";import{D as J}from"../chunks/Docstring.7dc3c45d.js";import{C as ie}from"../chunks/CodeBlock.af9da676.js";import{E as re}from"../chunks/ExampleCodeBlock.d2544fd1.js";function oe(mt){let d,z="Example:",M,$,v;return $=new ie({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5ubiUyMGFzJTIwbm4lMEElMEFpbXBvcnQlMjBiaXRzYW5kYnl0ZXMlMjBhcyUyMGJuYiUwQWZyb20lMjBibmIubm4lMjBpbXBvcnQlMjBMaW5lYXI0Yml0JTBBJTBBZnAxNl9tb2RlbCUyMCUzRCUyMG5uLlNlcXVlbnRpYWwoJTBBJTIwJTIwJTIwJTIwbm4uTGluZWFyKDY0JTJDJTIwNjQpJTJDJTBBJTIwJTIwJTIwJTIwbm4uTGluZWFyKDY0JTJDJTIwNjQpJTBBKSUwQSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMG5uLlNlcXVlbnRpYWwoJTBBJTIwJTIwJTIwJTIwTGluZWFyNGJpdCg2NCUyQyUyMDY0KSUyQyUwQSUyMCUyMCUyMCUyMExpbmVhcjRiaXQoNjQlMkMlMjA2NCklMEEpJTBBJTBBcXVhbnRpemVkX21vZGVsLmxvYWRfc3RhdGVfZGljdChmcDE2X21vZGVsLnN0YXRlX2RpY3QoKSklMEFxdWFudGl6ZWRfbW9kZWwlMjAlM0QlMjBxdWFudGl6ZWRfbW9kZWwudG8oMCklMjAlMjMlMjBRdWFudGl6YXRpb24lMjBoYXBwZW5zJTIwaGVyZQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn

<span class="hljs-keyword">import</span> bitsandbytes <span class="hljs-keyword">as</span> bnb
<span class="hljs-keyword">from</span> bnb.nn <span class="hljs-keyword">import</span> Linear4bit

fp16_model = nn.Sequential(
    nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>),
    nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)
)

quantized_model = nn.Sequential(
    Linear4bit(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>),
    Linear4bit(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)
)

quantized_model.load_state_dict(fp16_model.state_dict())
quantized_model = quantized_model.to(<span class="hljs-number">0</span>) <span class="hljs-comment"># Quantization happens here</span>`,wrap:!1}}),{c(){d=o("p"),d.textContent=z,M=a(),p($.$$.fragment)},l(m){d=l(m,"P",{"data-svelte-h":!0}),x(d)!=="svelte-11lpom8"&&(d.textContent=z),M=s(m),b($.$$.fragment,m)},m(m,L){r(m,d,L),r(m,M,L),c($,m,L),v=!0},p:Ot,i(m){v||(f($.$$.fragment,m),v=!0)},o(m){_($.$$.fragment,m),v=!1},d(m){m&&(e(d),e(M)),h($,m)}}}function le(mt){let d,z,M,$,v,m,L,dt,I,Rt='<a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> is a finetuning method that quantizes a model to 4-bits and adds a set of low-rank adaptation (LoRA) weights to the model and tuning them through the quantized weights. This method also introduces a new data type, 4-bit NormalFloat (<code>LinearNF4</code>) in addition to the standard Float4 data type (<code>LinearFP4</code>). <code>LinearNF4</code> is a quantization data type for normally distributed data and can improve performance.',ut,B,pt,y,k,xt,S,Wt=`This class is the base module for the 4-bit quantization algorithm presented in <a href="https://arxiv.org/abs/2305.14314" rel="nofollow">QLoRA</a>.
QLoRA 4-bit linear layers uses blockwise k-bit quantization under the hood, with the possibility of selecting various
compute datatypes such as FP4 and NF4.`,Mt,H,Qt=`In order to quantize a linear layer one should first load the original fp16 / bf16 weights into
the Linear4bit module, then call <code>quantized_module.to(&quot;cuda&quot;)</code> to quantize the fp16 / bf16 weights.`,Nt,q,Ft,j,E,Ct,A,Dt="Initialize Linear4bit class.",bt,R,ct,T,W,qt,K,Ut="Implements the FP4 data type.",jt,O,Q,ft,D,_t,u,U,Pt,tt,Gt="Implements the NF4 data type.",Jt,et,Xt=`Constructs a quantization data type where each bin has equal area under a standard normal distribution N(0, 1) that
is normalized into the range [-1, 1].`,zt,nt,Vt='For more information read the paper: QLoRA: Efficient Finetuning of Quantized LLMs (<a href="https://arxiv.org/abs/2305.14314" rel="nofollow">https://arxiv.org/abs/2305.14314</a>)',It,at,Yt=`Implementation of the NF4 data type in bitsandbytes can be found in the <code>create_normal_map</code> function in
the <code>functional.py</code> file: <a href="https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236" rel="nofollow">https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236</a>.`,Bt,st,G,ht,X,yt,C,V,kt,P,Y,Et,it,Zt="Initialize self.  See help(type(self)) for accurate signature.",gt,Z,$t,ot,vt;return v=new ae({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),L=new lt({props:{title:"4-bit quantization",local:"4-bit-quantization",headingTag:"h1"}}),B=new lt({props:{title:"Linear4bit",local:"bitsandbytes.nn.Linear4bit",headingTag:"h2"}}),k=new J({props:{name:"class bitsandbytes.nn.Linear4bit",anchor:"bitsandbytes.nn.Linear4bit",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_type",val:" = 'fp4'"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/bitsandbytes/nn/modules.py#L422"}}),q=new re({props:{anchor:"bitsandbytes.nn.Linear4bit.example",$$slots:{default:[oe]},$$scope:{ctx:mt}}}),E=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.Linear4bit.__init__",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_type",val:" = 'fp4'"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.Linear4bit.__init__.input_features",description:`<strong>input_features</strong> (<code>str</code>) &#x2014;
Number of input features of the linear layer.`,name:"input_features"},{anchor:"bitsandbytes.nn.Linear4bit.__init__.output_features",description:`<strong>output_features</strong> (<code>str</code>) &#x2014;
Number of output features of the linear layer.`,name:"output_features"},{anchor:"bitsandbytes.nn.Linear4bit.__init__.bias",description:`<strong>bias</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the linear class uses the bias term as well.`,name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/bitsandbytes/nn/modules.py#L455"}}),R=new lt({props:{title:"LinearFP4",local:"bitsandbytes.nn.LinearFP4",headingTag:"h2"}}),W=new J({props:{name:"class bitsandbytes.nn.LinearFP4",anchor:"bitsandbytes.nn.LinearFP4",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/bitsandbytes/nn/modules.py#L558"}}),Q=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.LinearFP4.__init__",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.LinearFP4.__init__.input_features",description:`<strong>input_features</strong> (<code>str</code>) &#x2014;
Number of input features of the linear layer.`,name:"input_features"},{anchor:"bitsandbytes.nn.LinearFP4.__init__.output_features",description:`<strong>output_features</strong> (<code>str</code>) &#x2014;
Number of output features of the linear layer.`,name:"output_features"},{anchor:"bitsandbytes.nn.LinearFP4.__init__.bias",description:`<strong>bias</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the linear class uses the bias term as well.`,name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/bitsandbytes/nn/modules.py#L563"}}),D=new lt({props:{title:"LinearNF4",local:"bitsandbytes.nn.LinearNF4",headingTag:"h2"}}),U=new J({props:{name:"class bitsandbytes.nn.LinearNF4",anchor:"bitsandbytes.nn.LinearNF4",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/bitsandbytes/nn/modules.py#L594"}}),G=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.LinearNF4.__init__",parameters:[{name:"input_features",val:""},{name:"output_features",val:""},{name:"bias",val:" = True"},{name:"compute_dtype",val:" = None"},{name:"compress_statistics",val:" = True"},{name:"quant_storage",val:" = torch.uint8"},{name:"device",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.LinearNF4.__init__.input_features",description:`<strong>input_features</strong> (<code>str</code>) &#x2014;
Number of input features of the linear layer.`,name:"input_features"},{anchor:"bitsandbytes.nn.LinearNF4.__init__.output_features",description:`<strong>output_features</strong> (<code>str</code>) &#x2014;
Number of output features of the linear layer.`,name:"output_features"},{anchor:"bitsandbytes.nn.LinearNF4.__init__.bias",description:`<strong>bias</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014;
Whether the linear class uses the bias term as well.`,name:"bias"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/bitsandbytes/nn/modules.py#L606"}}),X=new lt({props:{title:"Params4bit",local:"bitsandbytes.nn.Params4bit",headingTag:"h2"}}),V=new J({props:{name:"class bitsandbytes.nn.Params4bit",anchor:"bitsandbytes.nn.Params4bit",parameters:[{name:"data",val:": typing.Optional[torch.Tensor] = None"},{name:"requires_grad",val:" = False"},{name:"quant_state",val:": typing.Optional[bitsandbytes.functional.QuantState] = None"},{name:"blocksize",val:": typing.Optional[int] = None"},{name:"compress_statistics",val:": bool = True"},{name:"quant_type",val:": str = 'fp4'"},{name:"quant_storage",val:": dtype = torch.uint8"},{name:"module",val:": typing.Optional[ForwardRef('Linear4bit')] = None"},{name:"bnb_quantized",val:": bool = False"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/bitsandbytes/nn/modules.py#L212"}}),Y=new J({props:{name:"__init__",anchor:"bitsandbytes.nn.Params4bit.__init__",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}]}}),Z=new se({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/nn/linear4bit.mdx"}}),{c(){d=o("meta"),z=a(),M=o("p"),$=a(),p(v.$$.fragment),m=a(),p(L.$$.fragment),dt=a(),I=o("p"),I.innerHTML=Rt,ut=a(),p(B.$$.fragment),pt=a(),y=o("div"),p(k.$$.fragment),xt=a(),S=o("p"),S.innerHTML=Wt,Mt=a(),H=o("p"),H.innerHTML=Qt,Nt=a(),p(q.$$.fragment),Ft=a(),j=o("div"),p(E.$$.fragment),Ct=a(),A=o("p"),A.textContent=Dt,bt=a(),p(R.$$.fragment),ct=a(),T=o("div"),p(W.$$.fragment),qt=a(),K=o("p"),K.textContent=Ut,jt=a(),O=o("div"),p(Q.$$.fragment),ft=a(),p(D.$$.fragment),_t=a(),u=o("div"),p(U.$$.fragment),Pt=a(),tt=o("p"),tt.textContent=Gt,Jt=a(),et=o("p"),et.textContent=Xt,zt=a(),nt=o("p"),nt.innerHTML=Vt,It=a(),at=o("p"),at.innerHTML=Yt,Bt=a(),st=o("div"),p(G.$$.fragment),ht=a(),p(X.$$.fragment),yt=a(),C=o("div"),p(V.$$.fragment),kt=a(),P=o("div"),p(Y.$$.fragment),Et=a(),it=o("p"),it.textContent=Zt,gt=a(),p(Z.$$.fragment),$t=a(),ot=o("p"),this.h()},l(t){const n=ne("svelte-u9bgzb",document.head);d=l(n,"META",{name:!0,content:!0}),n.forEach(e),z=s(t),M=l(t,"P",{}),N(M).forEach(e),$=s(t),b(v.$$.fragment,t),m=s(t),b(L.$$.fragment,t),dt=s(t),I=l(t,"P",{"data-svelte-h":!0}),x(I)!=="svelte-j7lmsi"&&(I.innerHTML=Rt),ut=s(t),b(B.$$.fragment,t),pt=s(t),y=l(t,"DIV",{class:!0});var g=N(y);b(k.$$.fragment,g),xt=s(g),S=l(g,"P",{"data-svelte-h":!0}),x(S)!=="svelte-b56pxf"&&(S.innerHTML=Wt),Mt=s(g),H=l(g,"P",{"data-svelte-h":!0}),x(H)!=="svelte-yrth4q"&&(H.innerHTML=Qt),Nt=s(g),b(q.$$.fragment,g),Ft=s(g),j=l(g,"DIV",{class:!0});var wt=N(j);b(E.$$.fragment,wt),Ct=s(wt),A=l(wt,"P",{"data-svelte-h":!0}),x(A)!=="svelte-9bim4q"&&(A.textContent=Dt),wt.forEach(e),g.forEach(e),bt=s(t),b(R.$$.fragment,t),ct=s(t),T=l(t,"DIV",{class:!0});var rt=N(T);b(W.$$.fragment,rt),qt=s(rt),K=l(rt,"P",{"data-svelte-h":!0}),x(K)!=="svelte-1wlk05v"&&(K.textContent=Ut),jt=s(rt),O=l(rt,"DIV",{class:!0});var St=N(O);b(Q.$$.fragment,St),St.forEach(e),rt.forEach(e),ft=s(t),b(D.$$.fragment,t),_t=s(t),u=l(t,"DIV",{class:!0});var w=N(u);b(U.$$.fragment,w),Pt=s(w),tt=l(w,"P",{"data-svelte-h":!0}),x(tt)!=="svelte-5d8bnh"&&(tt.textContent=Gt),Jt=s(w),et=l(w,"P",{"data-svelte-h":!0}),x(et)!=="svelte-4zan40"&&(et.textContent=Xt),zt=s(w),nt=l(w,"P",{"data-svelte-h":!0}),x(nt)!=="svelte-1ditbm8"&&(nt.innerHTML=Vt),It=s(w),at=l(w,"P",{"data-svelte-h":!0}),x(at)!=="svelte-weczdq"&&(at.innerHTML=Yt),Bt=s(w),st=l(w,"DIV",{class:!0});var Ht=N(st);b(G.$$.fragment,Ht),Ht.forEach(e),w.forEach(e),ht=s(t),b(X.$$.fragment,t),yt=s(t),C=l(t,"DIV",{class:!0});var Lt=N(C);b(V.$$.fragment,Lt),kt=s(Lt),P=l(Lt,"DIV",{class:!0});var Tt=N(P);b(Y.$$.fragment,Tt),Et=s(Tt),it=l(Tt,"P",{"data-svelte-h":!0}),x(it)!=="svelte-gef1cn"&&(it.textContent=Zt),Tt.forEach(e),Lt.forEach(e),gt=s(t),b(Z.$$.fragment,t),$t=s(t),ot=l(t,"P",{}),N(ot).forEach(e),this.h()},h(){F(d,"name","hf:doc:metadata"),F(d,"content",me),F(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(st,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),F(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,n){i(document.head,d),r(t,z,n),r(t,M,n),r(t,$,n),c(v,t,n),r(t,m,n),c(L,t,n),r(t,dt,n),r(t,I,n),r(t,ut,n),c(B,t,n),r(t,pt,n),r(t,y,n),c(k,y,null),i(y,xt),i(y,S),i(y,Mt),i(y,H),i(y,Nt),c(q,y,null),i(y,Ft),i(y,j),c(E,j,null),i(j,Ct),i(j,A),r(t,bt,n),c(R,t,n),r(t,ct,n),r(t,T,n),c(W,T,null),i(T,qt),i(T,K),i(T,jt),i(T,O),c(Q,O,null),r(t,ft,n),c(D,t,n),r(t,_t,n),r(t,u,n),c(U,u,null),i(u,Pt),i(u,tt),i(u,Jt),i(u,et),i(u,zt),i(u,nt),i(u,It),i(u,at),i(u,Bt),i(u,st),c(G,st,null),r(t,ht,n),c(X,t,n),r(t,yt,n),r(t,C,n),c(V,C,null),i(C,kt),i(C,P),c(Y,P,null),i(P,Et),i(P,it),r(t,gt,n),c(Z,t,n),r(t,$t,n),r(t,ot,n),vt=!0},p(t,[n]){const g={};n&2&&(g.$$scope={dirty:n,ctx:t}),q.$set(g)},i(t){vt||(f(v.$$.fragment,t),f(L.$$.fragment,t),f(B.$$.fragment,t),f(k.$$.fragment,t),f(q.$$.fragment,t),f(E.$$.fragment,t),f(R.$$.fragment,t),f(W.$$.fragment,t),f(Q.$$.fragment,t),f(D.$$.fragment,t),f(U.$$.fragment,t),f(G.$$.fragment,t),f(X.$$.fragment,t),f(V.$$.fragment,t),f(Y.$$.fragment,t),f(Z.$$.fragment,t),vt=!0)},o(t){_(v.$$.fragment,t),_(L.$$.fragment,t),_(B.$$.fragment,t),_(k.$$.fragment,t),_(q.$$.fragment,t),_(E.$$.fragment,t),_(R.$$.fragment,t),_(W.$$.fragment,t),_(Q.$$.fragment,t),_(D.$$.fragment,t),_(U.$$.fragment,t),_(G.$$.fragment,t),_(X.$$.fragment,t),_(V.$$.fragment,t),_(Y.$$.fragment,t),_(Z.$$.fragment,t),vt=!1},d(t){t&&(e(z),e(M),e($),e(m),e(dt),e(I),e(ut),e(pt),e(y),e(bt),e(ct),e(T),e(ft),e(_t),e(u),e(ht),e(yt),e(C),e(gt),e($t),e(ot)),e(d),h(v,t),h(L,t),h(B,t),h(k),h(q),h(E),h(R,t),h(W),h(Q),h(D,t),h(U),h(G),h(X,t),h(V),h(Y),h(Z,t)}}}const me='{"title":"4-bit quantization","local":"4-bit-quantization","sections":[{"title":"Linear4bit","local":"bitsandbytes.nn.Linear4bit","sections":[],"depth":2},{"title":"LinearFP4","local":"bitsandbytes.nn.LinearFP4","sections":[],"depth":2},{"title":"LinearNF4","local":"bitsandbytes.nn.LinearNF4","sections":[],"depth":2},{"title":"Params4bit","local":"bitsandbytes.nn.Params4bit","sections":[],"depth":2}],"depth":1}';function de(mt){return Kt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class he extends te{constructor(d){super(),ee(this,d,de,le,At,{})}}export{he as component};
