import{s as xe,o as Le,n as He}from"../chunks/scheduler.d6170356.js";import{S as Ye,i as _e,g as s,s as n,r as m,A as ke,h as o,f as l,c as i,j as Ie,u as r,x as M,k as je,y as Be,a,v as c,d as u,t as f,w}from"../chunks/index.fcd4cc08.js";import{T as Qe}from"../chunks/Tip.b09c67cf.js";import{C as A}from"../chunks/CodeBlock.db16bf50.js";import{H as Me,E as Se}from"../chunks/getInferenceSnippets.0f6acb26.js";function We(R){let p,h='You can now deploy any llama.cpp compatible GGUF on Hugging Face Endpoints, read more about it <a href="https://huggingface.co/docs/inference-endpoints/en/others/llamacpp_container" rel="nofollow">here</a>';return{c(){p=s("p"),p.innerHTML=h},l(y){p=o(y,"P",{"data-svelte-h":!0}),M(p)!=="svelte-t4s0t"&&(p.innerHTML=h)},m(y,E){a(y,p,E)},p:He,d(y){y&&l(p)}}}function qe(R){let p,h,y,E,U,N,d,V,$,ye='Llama.cpp allows you to download and run inference on a GGUF simply by providing a path to the Hugging Face repo path and the file name. llama.cpp downloads the model checkpoint and automatically caches it. The location of the cache is defined by <code>LLAMA_CACHE</code> environment variable; read more about it <a href="https://github.com/ggerganov/llama.cpp/pull/7826" rel="nofollow">here</a>.',P,T,de='You can install llama.cpp through brew (works on Mac and Linux), or you can build it from source. There are also pre-built binaries and Docker images that you can <a href="https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#usage" rel="nofollow">check in the official documentation</a>.',Z,b,z,g,X,J,he="or, on windows via winget",D,C,O,v,K,G,Ue="Step 1: Clone llama.cpp from GitHub.",ee,I,te,j,$e="Step 2: Move into the llama.cpp folder and build it. You can also add hardware-specific flags (for ex: <code>-DGGML_CUDA=1</code> for Nvidia GPUs).",le,x,ae,L,Te='Note: for other hardware support (for ex: AMD ROCm, Intel SYCL), please refer to <a href="https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md" rel="nofollow">llama.cpp’s build guide</a>',ne,H,be="Once installed, you can use the <code>llama-cli</code> or <code>llama-server</code> as follows:",ie,Y,pe,_,ge="Note: You can explicitly add <code>-no-cnv</code> to run the CLI in raw completion mode (non-chat mode).",se,k,Je="Additionally, you can invoke an OpenAI spec chat completions endpoint directly using the llama.cpp server:",oe,B,me,Q,Ce="After running the server you can simply utilise the endpoint as below:",re,S,ce,W,ve="Replace <code>-hf</code> with any valid Hugging Face hub repo name - off you go! 🦙",ue,q,fe,F,we;return U=new Me({props:{title:"GGUF usage with llama.cpp",local:"gguf-usage-with-llamacpp",headingTag:"h1"}}),d=new Qe({props:{warning:!1,$$slots:{default:[We]},$$scope:{ctx:R}}}),b=new Me({props:{title:"Option 1: Install with brew/ winget",local:"option-1-install-with-brew-winget",headingTag:"h3"}}),g=new A({props:{code:"YnJldyUyMGluc3RhbGwlMjBsbGFtYS5jcHA=",highlighted:"brew install llama.cpp",wrap:!1}}),C=new A({props:{code:"d2luZ2V0JTIwaW5zdGFsbCUyMGxsYW1hLmNwcA==",highlighted:"winget install llama.cpp",wrap:!1}}),v=new Me({props:{title:"Option 2: build from source",local:"option-2-build-from-source",headingTag:"h3"}}),I=new A({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZnZ2VyZ2Fub3YlMkZsbGFtYS5jcHA=",highlighted:'git clone https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/ggerganov/</span>llama.cpp',wrap:!1}}),x=new A({props:{code:"Y2QlMjBsbGFtYS5jcHAlMEFjbWFrZSUyMC1CJTIwYnVpbGQlMjAlMjAlMjAlMjMlMjBvcHRpb25hbGx5JTJDJTIwYWRkJTIwLURHR01MX0NVREElM0RPTiUyMHRvJTIwYWN0aXZhdGUlMjBDVURBJTBBY21ha2UlMjAtLWJ1aWxkJTIwYnVpbGQlMjAtLWNvbmZpZyUyMFJlbGVhc2U=",highlighted:`cd <span class="hljs-keyword">llama.cpp
</span>cmake -<span class="hljs-keyword">B </span><span class="hljs-keyword">build </span>  <span class="hljs-comment"># optionally, add -DGGML_CUDA=ON to activate CUDA</span>
cmake --<span class="hljs-keyword">build </span><span class="hljs-keyword">build </span>--<span class="hljs-built_in">config</span> Release`,wrap:!1}}),Y=new A({props:{code:"bGxhbWEtY2xpJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),B=new A({props:{code:"bGxhbWEtc2VydmVyJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),S=new A({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEEtSCUyMCUyMkNvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbiUyMiUyMCU1QyUwQS1IJTIwJTIyQXV0aG9yaXphdGlvbiUzQSUyMEJlYXJlciUyMG5vLWtleSUyMiUyMCU1QyUwQS1kJTIwJyU3QiUwQSUyMm1lc3NhZ2VzJTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhbiUyMEFJJTIwYXNzaXN0YW50LiUyMFlvdXIlMjB0b3AlMjBwcmlvcml0eSUyMGlzJTIwYWNoaWV2aW5nJTIwdXNlciUyMGZ1bGZpbG1lbnQlMjB2aWElMjBoZWxwaW5nJTIwdGhlbSUyMHdpdGglMjB0aGVpciUyMHJlcXVlc3RzLiUyMiUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCUyMldyaXRlJTIwYSUyMGxpbWVyaWNrJTIwYWJvdXQlMjBQeXRob24lMjBleGNlcHRpb25zJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTVEJTBBJTdEJw==",highlighted:`curl http://localhost:8080/v1/chat/completions \\
-H <span class="hljs-string">&quot;Content-Type: application/json&quot;</span> \\
-H <span class="hljs-string">&quot;Authorization: Bearer no-key&quot;</span> \\
-d <span class="hljs-string">&#x27;{
&quot;messages&quot;: [
    {
        &quot;role&quot;: &quot;system&quot;,
        &quot;content&quot;: &quot;You are an AI assistant. Your top priority is achieving user fulfilment via helping them with their requests.&quot;
    },
    {
        &quot;role&quot;: &quot;user&quot;,
        &quot;content&quot;: &quot;Write a limerick about Python exceptions&quot;
    }
  ]
}&#x27;</span>`,wrap:!1}}),q=new Se({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/gguf-llamacpp.md"}}),{c(){p=s("meta"),h=n(),y=s("p"),E=n(),m(U.$$.fragment),N=n(),m(d.$$.fragment),V=n(),$=s("p"),$.innerHTML=ye,P=n(),T=s("p"),T.innerHTML=de,Z=n(),m(b.$$.fragment),z=n(),m(g.$$.fragment),X=n(),J=s("p"),J.textContent=he,D=n(),m(C.$$.fragment),O=n(),m(v.$$.fragment),K=n(),G=s("p"),G.textContent=Ue,ee=n(),m(I.$$.fragment),te=n(),j=s("p"),j.innerHTML=$e,le=n(),m(x.$$.fragment),ae=n(),L=s("p"),L.innerHTML=Te,ne=n(),H=s("p"),H.innerHTML=be,ie=n(),m(Y.$$.fragment),pe=n(),_=s("p"),_.innerHTML=ge,se=n(),k=s("p"),k.textContent=Je,oe=n(),m(B.$$.fragment),me=n(),Q=s("p"),Q.textContent=Ce,re=n(),m(S.$$.fragment),ce=n(),W=s("p"),W.innerHTML=ve,ue=n(),m(q.$$.fragment),fe=n(),F=s("p"),this.h()},l(e){const t=ke("svelte-u9bgzb",document.head);p=o(t,"META",{name:!0,content:!0}),t.forEach(l),h=i(e),y=o(e,"P",{}),Ie(y).forEach(l),E=i(e),r(U.$$.fragment,e),N=i(e),r(d.$$.fragment,e),V=i(e),$=o(e,"P",{"data-svelte-h":!0}),M($)!=="svelte-llq74"&&($.innerHTML=ye),P=i(e),T=o(e,"P",{"data-svelte-h":!0}),M(T)!=="svelte-kldsj5"&&(T.innerHTML=de),Z=i(e),r(b.$$.fragment,e),z=i(e),r(g.$$.fragment,e),X=i(e),J=o(e,"P",{"data-svelte-h":!0}),M(J)!=="svelte-1but92h"&&(J.textContent=he),D=i(e),r(C.$$.fragment,e),O=i(e),r(v.$$.fragment,e),K=i(e),G=o(e,"P",{"data-svelte-h":!0}),M(G)!=="svelte-1xohd7n"&&(G.textContent=Ue),ee=i(e),r(I.$$.fragment,e),te=i(e),j=o(e,"P",{"data-svelte-h":!0}),M(j)!=="svelte-1r0jgu0"&&(j.innerHTML=$e),le=i(e),r(x.$$.fragment,e),ae=i(e),L=o(e,"P",{"data-svelte-h":!0}),M(L)!=="svelte-3pqgmd"&&(L.innerHTML=Te),ne=i(e),H=o(e,"P",{"data-svelte-h":!0}),M(H)!=="svelte-56179e"&&(H.innerHTML=be),ie=i(e),r(Y.$$.fragment,e),pe=i(e),_=o(e,"P",{"data-svelte-h":!0}),M(_)!=="svelte-1pxj7s7"&&(_.innerHTML=ge),se=i(e),k=o(e,"P",{"data-svelte-h":!0}),M(k)!=="svelte-1aqbixr"&&(k.textContent=Je),oe=i(e),r(B.$$.fragment,e),me=i(e),Q=o(e,"P",{"data-svelte-h":!0}),M(Q)!=="svelte-1qbn3ou"&&(Q.textContent=Ce),re=i(e),r(S.$$.fragment,e),ce=i(e),W=o(e,"P",{"data-svelte-h":!0}),M(W)!=="svelte-rc3gcp"&&(W.innerHTML=ve),ue=i(e),r(q.$$.fragment,e),fe=i(e),F=o(e,"P",{}),Ie(F).forEach(l),this.h()},h(){je(p,"name","hf:doc:metadata"),je(p,"content",Ae)},m(e,t){Be(document.head,p),a(e,h,t),a(e,y,t),a(e,E,t),c(U,e,t),a(e,N,t),c(d,e,t),a(e,V,t),a(e,$,t),a(e,P,t),a(e,T,t),a(e,Z,t),c(b,e,t),a(e,z,t),c(g,e,t),a(e,X,t),a(e,J,t),a(e,D,t),c(C,e,t),a(e,O,t),c(v,e,t),a(e,K,t),a(e,G,t),a(e,ee,t),c(I,e,t),a(e,te,t),a(e,j,t),a(e,le,t),c(x,e,t),a(e,ae,t),a(e,L,t),a(e,ne,t),a(e,H,t),a(e,ie,t),c(Y,e,t),a(e,pe,t),a(e,_,t),a(e,se,t),a(e,k,t),a(e,oe,t),c(B,e,t),a(e,me,t),a(e,Q,t),a(e,re,t),c(S,e,t),a(e,ce,t),a(e,W,t),a(e,ue,t),c(q,e,t),a(e,fe,t),a(e,F,t),we=!0},p(e,[t]){const Ge={};t&2&&(Ge.$$scope={dirty:t,ctx:e}),d.$set(Ge)},i(e){we||(u(U.$$.fragment,e),u(d.$$.fragment,e),u(b.$$.fragment,e),u(g.$$.fragment,e),u(C.$$.fragment,e),u(v.$$.fragment,e),u(I.$$.fragment,e),u(x.$$.fragment,e),u(Y.$$.fragment,e),u(B.$$.fragment,e),u(S.$$.fragment,e),u(q.$$.fragment,e),we=!0)},o(e){f(U.$$.fragment,e),f(d.$$.fragment,e),f(b.$$.fragment,e),f(g.$$.fragment,e),f(C.$$.fragment,e),f(v.$$.fragment,e),f(I.$$.fragment,e),f(x.$$.fragment,e),f(Y.$$.fragment,e),f(B.$$.fragment,e),f(S.$$.fragment,e),f(q.$$.fragment,e),we=!1},d(e){e&&(l(h),l(y),l(E),l(N),l(V),l($),l(P),l(T),l(Z),l(z),l(X),l(J),l(D),l(O),l(K),l(G),l(ee),l(te),l(j),l(le),l(ae),l(L),l(ne),l(H),l(ie),l(pe),l(_),l(se),l(k),l(oe),l(me),l(Q),l(re),l(ce),l(W),l(ue),l(fe),l(F)),l(p),w(U,e),w(d,e),w(b,e),w(g,e),w(C,e),w(v,e),w(I,e),w(x,e),w(Y,e),w(B,e),w(S,e),w(q,e)}}}const Ae='{"title":"GGUF usage with llama.cpp","local":"gguf-usage-with-llamacpp","sections":[{"title":"Option 1: Install with brew/ winget","local":"option-1-install-with-brew-winget","sections":[],"depth":3},{"title":"Option 2: build from source","local":"option-2-build-from-source","sections":[],"depth":3}],"depth":1}';function Ee(R){return Le(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ze extends Ye{constructor(p){super(),_e(this,p,Ee,qe,xe,{})}}export{Ze as component};
