import{s as Ll,n as Dl,o as Kl}from"../chunks/scheduler.e0212c12.js";import{S as Pl,i as Ol,e as w,s as n,c as J,h as ls,a as y,d as t,b as a,f as zl,g as M,j as b,k as Hl,l as ss,m as e,n as T,t as i,o as p,p as m}from"../chunks/index.a585a37c.js";import{C as ts,H as U,E as es}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.b7b693fd.js";import{D as ns,C as j}from"../chunks/DocNotebookDropdown.1738c255.js";function as($l){let d,P,D,O,c,ll,I,sl,u,tl,o,El='<em>작성자: <a href="https://github.com/MartinEls" rel="nofollow">Martin Elstner</a> , 번역: <a href="https://github.com/whybe-choi" rel="nofollow">최용빈</a></em>',el,r,Rl='허깅페이스의 데이터셋은 파켓(parquet) 파일에 의존합니다. 우리는 빠른 인메모리 데이터베이스 시스템인 <a href="https://huggingface.co/docs/hub/en/datasets-duckdb" rel="nofollow">DuckDB를 사용하여 이 파일들과 상호작용</a>할 수 있습니다. DuckDB의 기능 중 하나는 <a href="https://duckdb.org/docs/extensions/vss.html" rel="nofollow">벡터 유사도 검색</a>으로, 인덱스 유무에 관계없이 사용할 수 있습니다.',nl,f,al,h,Jl,C,Ml,Z,Al="먼저, 검색할 데이터셋에 대한 임베딩을 생성해야 합니다. 우리는 <code>sentence-transformers</code> 라이브러리를 사용하여 데이터셋에 대한 임베딩을 생성할 것입니다.",Tl,W,il,_,Gl='이제 허브에서 <a href="https://huggingface.co/datasets/ai-blueprint/fineweb-bbc-news" rel="nofollow">ai-blueprint/fineweb-bbc-news</a> 데이터셋을 로드해 보겠습니다.',pl,B,ml,k,Yl="이제 데이터셋에 대한 임베딩을 생성할 수 있습니다. 일반적으로 정밀도(precision)를 잃지 않기 위해 데이터를 더 작은 배치로 나누고 싶을 수 있지만, 이 예제에서는 데이터셋의 전체 텍스트에 대한 임베딩만 생성하겠습니다.",wl,V,yl,g,Nl="이제 임베딩이 포함된 데이터셋을 허브에 다시 업로드할 수 있습니다.",bl,$,dl,E,Ul,R,Xl="이제 <code>duckdb</code>를 사용하여 데이터셋에서 벡터 검색을 수행할 수 있습니다. 이때 인덱스를 사용하거나 사용하지 않을 수 있습니다. 인덱스를 <strong>활용하지 않고</strong> 검색하는 것은 더 느리지만 더 정확하고, 인덱스를 <strong>활용하여</strong> 검색하는 것은 더 빠르지만 덜 정확합니다.",jl,A,cl,G,ql="인덱스를 활용하지 않고 검색하려면 <code>duckdb</code> 라이브러리를 사용하여 데이터셋에 연결하고 벡터 검색을 수행할 수 있습니다. 이는 느린 작업지만 일반적으로 약 10만 행까지의 작은 데이터셋에서는 충분히 빠르게 동작합니다. 즉, 우리가 사용하는 데이터셋에 대해서 쿼리를 날리는 것은 다소 느릴 것입니다.",Il,Y,ul,N,ol,X,Fl="이 접근법은 데이터셋의 로컬 복사본을 생성하고 이를 사용하여 인덱스를 생성합니다. 약간의 경미한 오버헤드가 있지만 한번 생성한 후에는 검색 속도가 상당히 향상될 것입니다.",rl,q,fl,F,Ql="이제 인덱스를 사용하여 벡터 검색을 수행할 수 있으며, 결과는 즉시 반환됩니다.",hl,Q,Cl,S,Sl="이 쿼리는 응답시간을 30초에서 1초 미만으로 줄이며, 무거운 벡터 검색 엔진을 배포할 필요가 없고 저장소는 허브에서 처리됩니다.",Zl,x,Wl,v,xl="우리는 <code>duckbd</code>를 사용하여 허브에서 벡터 검색을 수행하는 방법을 보았습니다. 10만 행 미만의 작은 데이터셋의 경우, 허브를 벡터 검색 벡엔드로 사용하여 인덱스 없이 벡터 검색을 수행할 수 있지만, 더 큰 데이터셋의 경우 <code>vss</code> 확장 프로그램으로 인덱스를 생성하면서 로컬 검색을 수행하고 허브를 스토리지 백엔드로 사용해야 합니다.",_l,z,Bl,H,vl='<li><a href="https://huggingface.co/docs/hub/en/datasets-duckdb" rel="nofollow">허깅페이스에서의 벡터 검색</a></li> <li><a href="https://duckdb.org/docs/extensions/vss.html" rel="nofollow">DuckDB를 사용한 벡터 검색 인덱싱</a></li>',kl,L,Vl,K,gl;return c=new ns({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/main/notebooks/ko/vector_search_with_hub_as_backend.ipynb"}]}}),I=new ts({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),u=new U({props:{title:"허깅페이스 허브를 백엔드로 사용한 벡터 검색",local:"허깅페이스-허브를-백엔드로-사용한-벡터-검색",headingTag:"h1"}}),f=new U({props:{title:"의존성 설치",local:"의존성-설치",headingTag:"h2"}}),h=new j({props:{code:"IXBpcCUyMGluc3RhbGwlMjBkYXRhc2V0cyUyMGR1Y2tkYiUyMHNlbnRlbmNlLXRyYW5zZm9ybWVycyUyMG1vZGVsMnZlYyUyMC1x",highlighted:"!pip install datasets duckdb sentence-transformers model2vec -q",wrap:!1}}),C=new U({props:{title:"데이터셋에 대한 임베딩 생성",local:"데이터셋에-대한-임베딩-생성",headingTag:"h2"}}),W=new j({props:{code:"ZnJvbSUyMHNlbnRlbmNlX3RyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNlbnRlbmNlVHJhbnNmb3JtZXIlMEFmcm9tJTIwc2VudGVuY2VfdHJhbnNmb3JtZXJzLm1vZGVscyUyMGltcG9ydCUyMFN0YXRpY0VtYmVkZGluZyUwQSUwQXN0YXRpY19lbWJlZGRpbmclMjAlM0QlMjBTdGF0aWNFbWJlZGRpbmcuZnJvbV9tb2RlbDJ2ZWMoJTIybWluaXNobGFiJTJGcG90aW9uLWJhc2UtOE0lMjIpJTBBbW9kZWwlMjAlM0QlMjBTZW50ZW5jZVRyYW5zZm9ybWVyKG1vZHVsZXMlM0QlNUJzdGF0aWNfZW1iZWRkaW5nJTVEKQ==",highlighted:`<span class="hljs-keyword">from</span> sentence_transformers <span class="hljs-keyword">import</span> SentenceTransformer
<span class="hljs-keyword">from</span> sentence_transformers.models <span class="hljs-keyword">import</span> StaticEmbedding

static_embedding = StaticEmbedding.from_model2vec(<span class="hljs-string">&quot;minishlab/potion-base-8M&quot;</span>)
model = SentenceTransformer(modules=[static_embedding])`,wrap:!1}}),B=new j({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cyUyMik=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

ds = load_dataset(<span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news&quot;</span>)`,wrap:!1}}),V=new j({props:{code:"ZGVmJTIwY3JlYXRlX2VtYmVkZGluZ3MoYmF0Y2gpJTNBJTBBJTIwJTIwJTIwJTIwZW1iZWRkaW5ncyUyMCUzRCUyMG1vZGVsLmVuY29kZShiYXRjaCU1QiUyMnRleHQlMjIlNUQlMkMlMjBjb252ZXJ0X3RvX251bXB5JTNEVHJ1ZSklMEElMjAlMjAlMjAlMjBiYXRjaCU1QiUyMmVtYmVkZGluZ3MlMjIlNUQlMjAlM0QlMjBlbWJlZGRpbmdzLnRvbGlzdCgpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwYmF0Y2glMEElMEFkcyUyMCUzRCUyMGRzLm1hcChjcmVhdGVfZW1iZWRkaW5ncyUyQyUyMGJhdGNoZWQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">create_embeddings</span>(<span class="hljs-params">batch</span>):
    embeddings = model.encode(batch[<span class="hljs-string">&quot;text&quot;</span>], convert_to_numpy=<span class="hljs-literal">True</span>)
    batch[<span class="hljs-string">&quot;embeddings&quot;</span>] = embeddings.tolist()
    <span class="hljs-keyword">return</span> batch

ds = ds.<span class="hljs-built_in">map</span>(create_embeddings, batched=<span class="hljs-literal">True</span>)`,wrap:!1}}),$=new j({props:{code:"ZHMucHVzaF90b19odWIoJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cy1lbWJlZGRpbmdzJTIyKQ==",highlighted:'ds.push_to_hub(<span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news-embeddings&quot;</span>)',wrap:!1}}),E=new U({props:{title:"허깅페이스 허브에서 벡터 검색",local:"허깅페이스-허브에서-벡터-검색",headingTag:"h2"}}),A=new U({props:{title:"인덱스를 활용하지 않고 검색하기",local:"인덱스를-활용하지-않고-검색하기",headingTag:"h3"}}),Y=new j({props:{code:"aW1wb3J0JTIwZHVja2RiJTBBZnJvbSUyMHR5cGluZyUyMGltcG9ydCUyMExpc3QlMEElMEFkZWYlMjBzaW1pbGFyaXR5X3NlYXJjaF93aXRob3V0X2R1Y2tkYl9pbmRleCglMEElMjAlMjAlMjAlMjBxdWVyeSUzQSUyMHN0ciUyQyUwQSUyMCUyMCUyMCUyMGslM0ElMjBpbnQlMjAlM0QlMjA1JTJDJTBBJTIwJTIwJTIwJTIwZGF0YXNldF9uYW1lJTNBJTIwc3RyJTIwJTNEJTIwJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cy1lbWJlZGRpbmdzJTIyJTJDJTBBJTIwJTIwJTIwJTIwZW1iZWRkaW5nX2NvbHVtbiUzQSUyMHN0ciUyMCUzRCUyMCUyMmVtYmVkZGluZ3MlMjIlMkMlMEEpJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwJUVDJTlEJUI4JUVCJThEJUIxJUVDJThBJUE0JUVCJUE1JUJDJTIwJUVDJTlDJTg0JUVEJTk1JUI0JTIwJUVDJTgyJUFDJUVDJTlBJUE5JUVEJTk1JTlDJTIwJUVCJUFBJUE4JUVCJThEJUI4JUVBJUIzJUJDJTIwJUVCJThGJTk5JUVDJTlEJUJDJUVEJTk1JTlDJTIwJUVCJUFBJUE4JUVCJThEJUI4JUVDJTlEJTg0JTIwJUVDJTgyJUFDJUVDJTlBJUE5JTBBJTIwJTIwJTIwJTIwcXVlcnlfdmVjdG9yJTIwJTNEJTIwbW9kZWwuZW5jb2RlKHF1ZXJ5KSUwQSUyMCUyMCUyMCUyMGVtYmVkZGluZ19kaW0lMjAlM0QlMjBtb2RlbC5nZXRfc2VudGVuY2VfZW1iZWRkaW5nX2RpbWVuc2lvbigpJTBBJTBBJTIwJTIwJTIwJTIwc3FsJTIwJTNEJTIwZiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFNFTEVDVCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMColMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhcnJheV9jb3NpbmVfZGlzdGFuY2UoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCZW1iZWRkaW5nX2NvbHVtbiU3RCUzQSUzQWZsb2F0JTVCJTdCZW1iZWRkaW5nX2RpbSU3RCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QnF1ZXJ5X3ZlY3Rvci50b2xpc3QoKSU3RCUzQSUzQWZsb2F0JTVCJTdCZW1iZWRkaW5nX2RpbSU3RCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMjBhcyUyMGRpc3RhbmNlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRlJPTSUyMCdoZiUzQSUyRiUyRmRhdGFzZXRzJTJGJTdCZGF0YXNldF9uYW1lJTdEJTJGKiolMkYqLnBhcnF1ZXQnJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwT1JERVIlMjBCWSUyMGRpc3RhbmNlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTElNSVQlMjAlN0JrJTdEJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZHVja2RiLnNxbChzcWwpLnRvX2RmKCklMEElMEFzaW1pbGFyaXR5X3NlYXJjaF93aXRob3V0X2R1Y2tkYl9pbmRleCglMjJXaGF0JTIwaXMlMjB0aGUlMjBmdXR1cmUlMjBvZiUyMEFJJTNGJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> duckdb
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">List</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">similarity_search_without_duckdb_index</span>(<span class="hljs-params">
    query: <span class="hljs-built_in">str</span>,
    k: <span class="hljs-built_in">int</span> = <span class="hljs-number">5</span>,
    dataset_name: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news-embeddings&quot;</span>,
    embedding_column: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;embeddings&quot;</span>,
</span>):
    <span class="hljs-comment"># 인덱스를 위해 사용한 모델과 동일한 모델을 사용</span>
    query_vector = model.encode(query)
    embedding_dim = model.get_sentence_embedding_dimension()

    sql = <span class="hljs-string">f&quot;&quot;&quot;
        SELECT
            *,
            array_cosine_distance(
                <span class="hljs-subst">{embedding_column}</span>::float[<span class="hljs-subst">{embedding_dim}</span>],
                <span class="hljs-subst">{query_vector.tolist()}</span>::float[<span class="hljs-subst">{embedding_dim}</span>]
            ) as distance
        FROM &#x27;hf://datasets/<span class="hljs-subst">{dataset_name}</span>/**/*.parquet&#x27;
        ORDER BY distance
        LIMIT <span class="hljs-subst">{k}</span>
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> duckdb.sql(sql).to_df()

similarity_search_without_duckdb_index(<span class="hljs-string">&quot;What is the future of AI?&quot;</span>)`,wrap:!1}}),N=new U({props:{title:"인덱스를 활용하여 검색하기",local:"인덱스를-활용하여-검색하기",headingTag:"h3"}}),q=new j({props:{code:"aW1wb3J0JTIwZHVja2RiJTBBJTBBZGVmJTIwX3NldHVwX3ZzcygpJTNBJTBBJTIwJTIwJTIwJTIwZHVja2RiLnNxbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVyeSUzRCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMElOU1RBTEwlMjB2c3MlM0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBMT0FEJTIwdnNzJTNCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwKSUwQWRlZiUyMF9kcm9wX3RhYmxlKHRhYmxlX25hbWUpJTNBJTBBJTIwJTIwJTIwJTIwZHVja2RiLnNxbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVyeSUzRGYlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBEUk9QJTIwVEFCTEUlMjBJRiUyMEVYSVNUUyUyMCU3QnRhYmxlX25hbWUlN0QlM0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjApJTBBJTBBZGVmJTIwX2NyZWF0ZV90YWJsZShkYXRhc2V0X25hbWUlMkMlMjB0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklM0ElMEElMjAlMjAlMjAlMjBkdWNrZGIuc3FsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXJ5JTNEZiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMENSRUFURSUyMFRBQkxFJTIwJTdCdGFibGVfbmFtZSU3RCUyMEFTJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU0VMRUNUJTIwKiUyQyUyMCU3QmVtYmVkZGluZ19jb2x1bW4lN0QlM0ElM0FmbG9hdCU1QiU3Qm1vZGVsLmdldF9zZW50ZW5jZV9lbWJlZGRpbmdfZGltZW5zaW9uKCklN0QlNUQlMjBhcyUyMCU3QmVtYmVkZGluZ19jb2x1bW4lN0RfZmxvYXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBGUk9NJTIwJ2hmJTNBJTJGJTJGZGF0YXNldHMlMkYlN0JkYXRhc2V0X25hbWUlN0QlMkYqKiUyRioucGFycXVldCclM0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjApJTBBJTBBZGVmJTIwX2NyZWF0ZV9pbmRleCh0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklM0ElMEElMjAlMjAlMjAlMjBkdWNrZGIuc3FsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXJ5JTNEZiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMENSRUFURSUyMElOREVYJTIwbXlfaG5zd19pbmRleCUyME9OJTIwJTdCdGFibGVfbmFtZSU3RCUyMFVTSU5HJTIwSE5TVyUyMCglN0JlbWJlZGRpbmdfY29sdW1uJTdEX2Zsb2F0KSUyMFdJVEglMjAobWV0cmljJTIwJTNEJTIwJ2Nvc2luZScpJTNCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwKSUwQSUwQWRlZiUyMGNyZWF0ZV9pbmRleChkYXRhc2V0X25hbWUlMkMlMjB0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklM0ElMEElMjAlMjAlMjAlMjBfc2V0dXBfdnNzKCklMEElMjAlMjAlMjAlMjBfZHJvcF90YWJsZSh0YWJsZV9uYW1lKSUwQSUyMCUyMCUyMCUyMF9jcmVhdGVfdGFibGUoZGF0YXNldF9uYW1lJTJDJTIwdGFibGVfbmFtZSUyQyUyMGVtYmVkZGluZ19jb2x1bW4pJTBBJTIwJTIwJTIwJTIwX2NyZWF0ZV9pbmRleCh0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklMEElMEFjcmVhdGVfaW5kZXgoJTBBJTIwJTIwJTIwJTIwZGF0YXNldF9uYW1lJTNEJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cy1lbWJlZGRpbmdzJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFibGVfbmFtZSUzRCUyMmZpbmV3ZWJfYmJjX25ld3NfZW1iZWRkaW5ncyUyMiUyQyUwQSUyMCUyMCUyMCUyMGVtYmVkZGluZ19jb2x1bW4lM0QlMjJlbWJlZGRpbmdzJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> duckdb

<span class="hljs-keyword">def</span> <span class="hljs-title function_">_setup_vss</span>():
    duckdb.sql(
        query=<span class="hljs-string">&quot;&quot;&quot;
        INSTALL vss;
        LOAD vss;
        &quot;&quot;&quot;</span>
    )
<span class="hljs-keyword">def</span> <span class="hljs-title function_">_drop_table</span>(<span class="hljs-params">table_name</span>):
    duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        DROP TABLE IF EXISTS <span class="hljs-subst">{table_name}</span>;
        &quot;&quot;&quot;</span>
    )

<span class="hljs-keyword">def</span> <span class="hljs-title function_">_create_table</span>(<span class="hljs-params">dataset_name, table_name, embedding_column</span>):
    duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        CREATE TABLE <span class="hljs-subst">{table_name}</span> AS
        SELECT *, <span class="hljs-subst">{embedding_column}</span>::float[<span class="hljs-subst">{model.get_sentence_embedding_dimension()}</span>] as <span class="hljs-subst">{embedding_column}</span>_float
        FROM &#x27;hf://datasets/<span class="hljs-subst">{dataset_name}</span>/**/*.parquet&#x27;;
        &quot;&quot;&quot;</span>
    )

<span class="hljs-keyword">def</span> <span class="hljs-title function_">_create_index</span>(<span class="hljs-params">table_name, embedding_column</span>):
    duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        CREATE INDEX my_hnsw_index ON <span class="hljs-subst">{table_name}</span> USING HNSW (<span class="hljs-subst">{embedding_column}</span>_float) WITH (metric = &#x27;cosine&#x27;);
        &quot;&quot;&quot;</span>
    )

<span class="hljs-keyword">def</span> <span class="hljs-title function_">create_index</span>(<span class="hljs-params">dataset_name, table_name, embedding_column</span>):
    _setup_vss()
    _drop_table(table_name)
    _create_table(dataset_name, table_name, embedding_column)
    _create_index(table_name, embedding_column)

create_index(
    dataset_name=<span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news-embeddings&quot;</span>,
    table_name=<span class="hljs-string">&quot;fineweb_bbc_news_embeddings&quot;</span>,
    embedding_column=<span class="hljs-string">&quot;embeddings&quot;</span>
)`,wrap:!1}}),Q=new j({props:{code:"ZGVmJTIwc2ltaWxhcml0eV9zZWFyY2hfd2l0aF9kdWNrZGJfaW5kZXgoJTBBJTIwJTIwJTIwJTIwcXVlcnklM0ElMjBzdHIlMkMlMEElMjAlMjAlMjAlMjBrJTNBJTIwaW50JTIwJTNEJTIwNSUyQyUwQSUyMCUyMCUyMCUyMHRhYmxlX25hbWUlM0ElMjBzdHIlMjAlM0QlMjAlMjJmaW5ld2ViX2JiY19uZXdzX2VtYmVkZGluZ3MlMjIlMkMlMEElMjAlMjAlMjAlMjBlbWJlZGRpbmdfY29sdW1uJTNBJTIwc3RyJTIwJTNEJTIwJTIyZW1iZWRkaW5ncyUyMiUwQSklM0ElMEElMjAlMjAlMjAlMjBlbWJlZGRpbmclMjAlM0QlMjBtb2RlbC5lbmNvZGUocXVlcnkpLnRvbGlzdCgpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZHVja2RiLnNxbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVyeSUzRGYlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBTRUxFQ1QlMjAqJTJDJTIwYXJyYXlfY29zaW5lX2Rpc3RhbmNlKCU3QmVtYmVkZGluZ19jb2x1bW4lN0RfZmxvYXQlMkMlMjAlN0JlbWJlZGRpbmclN0QlM0ElM0FGTE9BVCU1QiU3Qm1vZGVsLmdldF9zZW50ZW5jZV9lbWJlZGRpbmdfZGltZW5zaW9uKCklN0QlNUQpJTIwYXMlMjBkaXN0YW5jZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEZST00lMjAlN0J0YWJsZV9uYW1lJTdEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwT1JERVIlMjBCWSUyMGRpc3RhbmNlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTElNSVQlMjAlN0JrJTdEJTNCJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwKS50b19kZigpJTBBJTBBc2ltaWxhcml0eV9zZWFyY2hfd2l0aF9kdWNrZGJfaW5kZXgoJTIyV2hhdCUyMGlzJTIwdGhlJTIwZnV0dXJlJTIwb2YlMjBBSSUzRiUyMik=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">similarity_search_with_duckdb_index</span>(<span class="hljs-params">
    query: <span class="hljs-built_in">str</span>,
    k: <span class="hljs-built_in">int</span> = <span class="hljs-number">5</span>,
    table_name: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;fineweb_bbc_news_embeddings&quot;</span>,
    embedding_column: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;embeddings&quot;</span>
</span>):
    embedding = model.encode(query).tolist()
    <span class="hljs-keyword">return</span> duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        SELECT *, array_cosine_distance(<span class="hljs-subst">{embedding_column}</span>_float, <span class="hljs-subst">{embedding}</span>::FLOAT[<span class="hljs-subst">{model.get_sentence_embedding_dimension()}</span>]) as distance
        FROM <span class="hljs-subst">{table_name}</span>
        ORDER BY distance
        LIMIT <span class="hljs-subst">{k}</span>;
    &quot;&quot;&quot;</span>
    ).to_df()

similarity_search_with_duckdb_index(<span class="hljs-string">&quot;What is the future of AI?&quot;</span>)`,wrap:!1}}),x=new U({props:{title:"결론",local:"결론",headingTag:"h2"}}),z=new U({props:{title:"더 알아보기",local:"더-알아보기",headingTag:"h2"}}),L=new es({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/ko/vector_search_with_hub_as_backend.md"}}),{c(){d=w("meta"),P=n(),D=w("p"),O=n(),J(c.$$.fragment),ll=n(),J(I.$$.fragment),sl=n(),J(u.$$.fragment),tl=n(),o=w("p"),o.innerHTML=El,el=n(),r=w("p"),r.innerHTML=Rl,nl=n(),J(f.$$.fragment),al=n(),J(h.$$.fragment),Jl=n(),J(C.$$.fragment),Ml=n(),Z=w("p"),Z.innerHTML=Al,Tl=n(),J(W.$$.fragment),il=n(),_=w("p"),_.innerHTML=Gl,pl=n(),J(B.$$.fragment),ml=n(),k=w("p"),k.textContent=Yl,wl=n(),J(V.$$.fragment),yl=n(),g=w("p"),g.textContent=Nl,bl=n(),J($.$$.fragment),dl=n(),J(E.$$.fragment),Ul=n(),R=w("p"),R.innerHTML=Xl,jl=n(),J(A.$$.fragment),cl=n(),G=w("p"),G.innerHTML=ql,Il=n(),J(Y.$$.fragment),ul=n(),J(N.$$.fragment),ol=n(),X=w("p"),X.textContent=Fl,rl=n(),J(q.$$.fragment),fl=n(),F=w("p"),F.textContent=Ql,hl=n(),J(Q.$$.fragment),Cl=n(),S=w("p"),S.textContent=Sl,Zl=n(),J(x.$$.fragment),Wl=n(),v=w("p"),v.innerHTML=xl,_l=n(),J(z.$$.fragment),Bl=n(),H=w("ul"),H.innerHTML=vl,kl=n(),J(L.$$.fragment),Vl=n(),K=w("p"),this.h()},l(l){const s=ls("svelte-u9bgzb",document.head);d=y(s,"META",{name:!0,content:!0}),s.forEach(t),P=a(l),D=y(l,"P",{}),zl(D).forEach(t),O=a(l),M(c.$$.fragment,l),ll=a(l),M(I.$$.fragment,l),sl=a(l),M(u.$$.fragment,l),tl=a(l),o=y(l,"P",{"data-svelte-h":!0}),b(o)!=="svelte-732dc3"&&(o.innerHTML=El),el=a(l),r=y(l,"P",{"data-svelte-h":!0}),b(r)!=="svelte-z8wvfb"&&(r.innerHTML=Rl),nl=a(l),M(f.$$.fragment,l),al=a(l),M(h.$$.fragment,l),Jl=a(l),M(C.$$.fragment,l),Ml=a(l),Z=y(l,"P",{"data-svelte-h":!0}),b(Z)!=="svelte-1s2w2c"&&(Z.innerHTML=Al),Tl=a(l),M(W.$$.fragment,l),il=a(l),_=y(l,"P",{"data-svelte-h":!0}),b(_)!=="svelte-1ci2cq3"&&(_.innerHTML=Gl),pl=a(l),M(B.$$.fragment,l),ml=a(l),k=y(l,"P",{"data-svelte-h":!0}),b(k)!=="svelte-1qo93gw"&&(k.textContent=Yl),wl=a(l),M(V.$$.fragment,l),yl=a(l),g=y(l,"P",{"data-svelte-h":!0}),b(g)!=="svelte-1g329lk"&&(g.textContent=Nl),bl=a(l),M($.$$.fragment,l),dl=a(l),M(E.$$.fragment,l),Ul=a(l),R=y(l,"P",{"data-svelte-h":!0}),b(R)!=="svelte-80cs59"&&(R.innerHTML=Xl),jl=a(l),M(A.$$.fragment,l),cl=a(l),G=y(l,"P",{"data-svelte-h":!0}),b(G)!=="svelte-96zzjt"&&(G.innerHTML=ql),Il=a(l),M(Y.$$.fragment,l),ul=a(l),M(N.$$.fragment,l),ol=a(l),X=y(l,"P",{"data-svelte-h":!0}),b(X)!=="svelte-kxctf7"&&(X.textContent=Fl),rl=a(l),M(q.$$.fragment,l),fl=a(l),F=y(l,"P",{"data-svelte-h":!0}),b(F)!=="svelte-80uvij"&&(F.textContent=Ql),hl=a(l),M(Q.$$.fragment,l),Cl=a(l),S=y(l,"P",{"data-svelte-h":!0}),b(S)!=="svelte-53ypoa"&&(S.textContent=Sl),Zl=a(l),M(x.$$.fragment,l),Wl=a(l),v=y(l,"P",{"data-svelte-h":!0}),b(v)!=="svelte-12niir1"&&(v.innerHTML=xl),_l=a(l),M(z.$$.fragment,l),Bl=a(l),H=y(l,"UL",{"data-svelte-h":!0}),b(H)!=="svelte-vdnjht"&&(H.innerHTML=vl),kl=a(l),M(L.$$.fragment,l),Vl=a(l),K=y(l,"P",{}),zl(K).forEach(t),this.h()},h(){Hl(d,"name","hf:doc:metadata"),Hl(d,"content",Js)},m(l,s){ss(document.head,d),e(l,P,s),e(l,D,s),e(l,O,s),T(c,l,s),e(l,ll,s),T(I,l,s),e(l,sl,s),T(u,l,s),e(l,tl,s),e(l,o,s),e(l,el,s),e(l,r,s),e(l,nl,s),T(f,l,s),e(l,al,s),T(h,l,s),e(l,Jl,s),T(C,l,s),e(l,Ml,s),e(l,Z,s),e(l,Tl,s),T(W,l,s),e(l,il,s),e(l,_,s),e(l,pl,s),T(B,l,s),e(l,ml,s),e(l,k,s),e(l,wl,s),T(V,l,s),e(l,yl,s),e(l,g,s),e(l,bl,s),T($,l,s),e(l,dl,s),T(E,l,s),e(l,Ul,s),e(l,R,s),e(l,jl,s),T(A,l,s),e(l,cl,s),e(l,G,s),e(l,Il,s),T(Y,l,s),e(l,ul,s),T(N,l,s),e(l,ol,s),e(l,X,s),e(l,rl,s),T(q,l,s),e(l,fl,s),e(l,F,s),e(l,hl,s),T(Q,l,s),e(l,Cl,s),e(l,S,s),e(l,Zl,s),T(x,l,s),e(l,Wl,s),e(l,v,s),e(l,_l,s),T(z,l,s),e(l,Bl,s),e(l,H,s),e(l,kl,s),T(L,l,s),e(l,Vl,s),e(l,K,s),gl=!0},p:Dl,i(l){gl||(i(c.$$.fragment,l),i(I.$$.fragment,l),i(u.$$.fragment,l),i(f.$$.fragment,l),i(h.$$.fragment,l),i(C.$$.fragment,l),i(W.$$.fragment,l),i(B.$$.fragment,l),i(V.$$.fragment,l),i($.$$.fragment,l),i(E.$$.fragment,l),i(A.$$.fragment,l),i(Y.$$.fragment,l),i(N.$$.fragment,l),i(q.$$.fragment,l),i(Q.$$.fragment,l),i(x.$$.fragment,l),i(z.$$.fragment,l),i(L.$$.fragment,l),gl=!0)},o(l){p(c.$$.fragment,l),p(I.$$.fragment,l),p(u.$$.fragment,l),p(f.$$.fragment,l),p(h.$$.fragment,l),p(C.$$.fragment,l),p(W.$$.fragment,l),p(B.$$.fragment,l),p(V.$$.fragment,l),p($.$$.fragment,l),p(E.$$.fragment,l),p(A.$$.fragment,l),p(Y.$$.fragment,l),p(N.$$.fragment,l),p(q.$$.fragment,l),p(Q.$$.fragment,l),p(x.$$.fragment,l),p(z.$$.fragment,l),p(L.$$.fragment,l),gl=!1},d(l){l&&(t(P),t(D),t(O),t(ll),t(sl),t(tl),t(o),t(el),t(r),t(nl),t(al),t(Jl),t(Ml),t(Z),t(Tl),t(il),t(_),t(pl),t(ml),t(k),t(wl),t(yl),t(g),t(bl),t(dl),t(Ul),t(R),t(jl),t(cl),t(G),t(Il),t(ul),t(ol),t(X),t(rl),t(fl),t(F),t(hl),t(Cl),t(S),t(Zl),t(Wl),t(v),t(_l),t(Bl),t(H),t(kl),t(Vl),t(K)),t(d),m(c,l),m(I,l),m(u,l),m(f,l),m(h,l),m(C,l),m(W,l),m(B,l),m(V,l),m($,l),m(E,l),m(A,l),m(Y,l),m(N,l),m(q,l),m(Q,l),m(x,l),m(z,l),m(L,l)}}}const Js='{"title":"허깅페이스 허브를 백엔드로 사용한 벡터 검색","local":"허깅페이스-허브를-백엔드로-사용한-벡터-검색","sections":[{"title":"의존성 설치","local":"의존성-설치","sections":[],"depth":2},{"title":"데이터셋에 대한 임베딩 생성","local":"데이터셋에-대한-임베딩-생성","sections":[],"depth":2},{"title":"허깅페이스 허브에서 벡터 검색","local":"허깅페이스-허브에서-벡터-검색","sections":[{"title":"인덱스를 활용하지 않고 검색하기","local":"인덱스를-활용하지-않고-검색하기","sections":[],"depth":3},{"title":"인덱스를 활용하여 검색하기","local":"인덱스를-활용하여-검색하기","sections":[],"depth":3}],"depth":2},{"title":"결론","local":"결론","sections":[],"depth":2},{"title":"더 알아보기","local":"더-알아보기","sections":[],"depth":2}],"depth":1}';function Ms($l){return Kl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ws extends Pl{constructor(d){super(),Ol(this,d,Ms,as,Ll,{})}}export{ws as component};
