import{S as ga,i as va,s as ya,e as t,k as p,w as L,t as h,M as ba,c as r,d as a,m,a as o,x as W,h as c,b as i,G as e,g as n,y as R,L as _a,q as D,o as C,B as F,v as ka}from"../chunks/vendor-hf-doc-builder.js";import{I as bs}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as wa}from"../chunks/CodeBlock-hf-doc-builder.js";function $a(Os){let d,es,w,b,H,j,_s,U,ks,ts,g,_,J,A,$s,O,js,rs,E,P,As,Es,os,f,q,Ps,M,qs,Ms,Is,Z,Ts,xs,K,zs,Ss,Q,Ns,Bs,V,Gs,ls,v,k,X,I,Ls,Y,Ws,is,S,Rs,ns,N,Ds,ps,T,ms,B,Cs,hs,y,$,ss,x,Fs,as,Hs,cs,G,Us,fs,z,us;return j=new bs({}),A=new bs({}),I=new bs({}),T=new wa({props:{code:`import os; import psutil; import timeit
from datasets import load_dataset

mem_before = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024)
wiki = load_dataset("wikipedia", "20220301.en", split="train")
mem_after = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024)

print(f"RAM memory used: {(mem_after - mem_before)} MB")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> os; <span class="hljs-keyword">import</span> psutil; <span class="hljs-keyword">import</span> timeit
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># Process.memory_info is expressed in bytes, so convert to megabytes </span>
<span class="hljs-meta">&gt;&gt;&gt; </span>mem_before = psutil.Process(os.getpid()).memory_info().rss / (<span class="hljs-number">1024</span> * <span class="hljs-number">1024</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>wiki = load_dataset(<span class="hljs-string">&quot;wikipedia&quot;</span>, <span class="hljs-string">&quot;20220301.en&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mem_after = psutil.Process(os.getpid()).memory_info().rss / (<span class="hljs-number">1024</span> * <span class="hljs-number">1024</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;RAM memory used: <span class="hljs-subst">{(mem_after - mem_before)}</span> MB&quot;</span>)
RAM memory used: <span class="hljs-number">50</span> MB`}}),x=new bs({}),z=new wa({props:{code:`s = """batch_size = 1000
for batch in wiki.iter(batch_size):
    ...
"""

time = timeit.timeit(stmt=s, number=1, globals=globals())
print(f"Time to iterate over the {wiki.dataset_size >> 30} GB dataset: {time:.1f} sec, "
      f"ie. {float(wiki.dataset_size >> 27)/time:.1f} Gb/s")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>s = <span class="hljs-string">&quot;&quot;&quot;batch_size = 1000
<span class="hljs-meta">... </span>for batch in wiki.iter(batch_size):
<span class="hljs-meta">... </span>    ...
<span class="hljs-meta">... </span>&quot;&quot;&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>time = timeit.timeit(stmt=s, number=<span class="hljs-number">1</span>, <span class="hljs-built_in">globals</span>=<span class="hljs-built_in">globals</span>())
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time to iterate over the <span class="hljs-subst">{wiki.dataset_size &gt;&gt; <span class="hljs-number">30</span>}</span> GB dataset: <span class="hljs-subst">{time:<span class="hljs-number">.1</span>f}</span> sec, &quot;</span>
<span class="hljs-meta">... </span>      <span class="hljs-string">f&quot;ie. <span class="hljs-subst">{<span class="hljs-built_in">float</span>(wiki.dataset_size &gt;&gt; <span class="hljs-number">27</span>)/time:<span class="hljs-number">.1</span>f}</span> Gb/s&quot;</span>)
Time to iterate over the <span class="hljs-number">18</span> GB dataset: <span class="hljs-number">31.8</span> sec, ie. <span class="hljs-number">4.8</span> Gb/s`}}),{c(){d=t("meta"),es=p(),w=t("h1"),b=t("a"),H=t("span"),L(j.$$.fragment),_s=p(),U=t("span"),ks=h("Datasets \u{1F91D} Arrow"),ts=p(),g=t("h2"),_=t("a"),J=t("span"),L(A.$$.fragment),$s=p(),O=t("span"),js=h("What is Arrow?"),rs=p(),E=t("p"),P=t("a"),As=h("Arrow"),Es=h(" enables large amounts of data to be processed and moved quickly. It is a specific data format that stores data in a columnar memory layout. This provides several significant advantages:"),os=p(),f=t("ul"),q=t("li"),Ps=h("Arrow\u2019s standard format allows "),M=t("a"),qs=h("zero-copy reads"),Ms=h(" which removes virtually all serialization overhead."),Is=p(),Z=t("li"),Ts=h("Arrow is language-agnostic so it supports different programming languages."),xs=p(),K=t("li"),zs=h("Arrow is column-oriented so it is faster at querying and processing slices or columns of data."),Ss=p(),Q=t("li"),Ns=h("Arrow allows for copy-free hand-offs to standard machine learning tools such as NumPy, Pandas, PyTorch, and TensorFlow."),Bs=p(),V=t("li"),Gs=h("Arrow supports many, possibly nested, column types."),ls=p(),v=t("h2"),k=t("a"),X=t("span"),L(I.$$.fragment),Ls=p(),Y=t("span"),Ws=h("Memory-mapping"),is=p(),S=t("p"),Rs=h(`\u{1F917} Datasets uses Arrow for its local caching system. It allows datasets to be backed by an on-disk cache, which is memory-mapped for fast lookup.
This architecture allows for large datasets to be used on machines with relatively small device memory.`),ns=p(),N=t("p"),Ds=h("For example, loading the full English Wikipedia dataset only takes a few MB of RAM:"),ps=p(),L(T.$$.fragment),ms=p(),B=t("p"),Cs=h(`This is possible because the Arrow data is actually memory-mapped from disk, and not loaded in memory.
Memory-mapping allows access to data on disk, and leverages virtual memory capabilities for fast lookups.`),hs=p(),y=t("h2"),$=t("a"),ss=t("span"),L(x.$$.fragment),Fs=p(),as=t("span"),Hs=h("Performance"),cs=p(),G=t("p"),Us=h("Iterating over a memory-mapped dataset using Arrow is fast. Iterating over Wikipedia on a laptop gives you speeds of 1-3 Gbit/s:"),fs=p(),L(z.$$.fragment),this.h()},l(s){const l=ba('[data-svelte="svelte-1phssyn"]',document.head);d=r(l,"META",{name:!0,content:!0}),l.forEach(a),es=m(s),w=r(s,"H1",{class:!0});var ds=o(w);b=r(ds,"A",{id:!0,class:!0,href:!0});var Zs=o(b);H=r(Zs,"SPAN",{});var Ks=o(H);W(j.$$.fragment,Ks),Ks.forEach(a),Zs.forEach(a),_s=m(ds),U=r(ds,"SPAN",{});var Qs=o(U);ks=c(Qs,"Datasets \u{1F91D} Arrow"),Qs.forEach(a),ds.forEach(a),ts=m(s),g=r(s,"H2",{class:!0});var ws=o(g);_=r(ws,"A",{id:!0,class:!0,href:!0});var Vs=o(_);J=r(Vs,"SPAN",{});var Xs=o(J);W(A.$$.fragment,Xs),Xs.forEach(a),Vs.forEach(a),$s=m(ws),O=r(ws,"SPAN",{});var Ys=o(O);js=c(Ys,"What is Arrow?"),Ys.forEach(a),ws.forEach(a),rs=m(s),E=r(s,"P",{});var Js=o(E);P=r(Js,"A",{href:!0,rel:!0});var sa=o(P);As=c(sa,"Arrow"),sa.forEach(a),Es=c(Js," enables large amounts of data to be processed and moved quickly. It is a specific data format that stores data in a columnar memory layout. This provides several significant advantages:"),Js.forEach(a),os=m(s),f=r(s,"UL",{});var u=o(f);q=r(u,"LI",{});var gs=o(q);Ps=c(gs,"Arrow\u2019s standard format allows "),M=r(gs,"A",{href:!0,rel:!0});var aa=o(M);qs=c(aa,"zero-copy reads"),aa.forEach(a),Ms=c(gs," which removes virtually all serialization overhead."),gs.forEach(a),Is=m(u),Z=r(u,"LI",{});var ea=o(Z);Ts=c(ea,"Arrow is language-agnostic so it supports different programming languages."),ea.forEach(a),xs=m(u),K=r(u,"LI",{});var ta=o(K);zs=c(ta,"Arrow is column-oriented so it is faster at querying and processing slices or columns of data."),ta.forEach(a),Ss=m(u),Q=r(u,"LI",{});var ra=o(Q);Ns=c(ra,"Arrow allows for copy-free hand-offs to standard machine learning tools such as NumPy, Pandas, PyTorch, and TensorFlow."),ra.forEach(a),Bs=m(u),V=r(u,"LI",{});var oa=o(V);Gs=c(oa,"Arrow supports many, possibly nested, column types."),oa.forEach(a),u.forEach(a),ls=m(s),v=r(s,"H2",{class:!0});var vs=o(v);k=r(vs,"A",{id:!0,class:!0,href:!0});var la=o(k);X=r(la,"SPAN",{});var ia=o(X);W(I.$$.fragment,ia),ia.forEach(a),la.forEach(a),Ls=m(vs),Y=r(vs,"SPAN",{});var na=o(Y);Ws=c(na,"Memory-mapping"),na.forEach(a),vs.forEach(a),is=m(s),S=r(s,"P",{});var pa=o(S);Rs=c(pa,`\u{1F917} Datasets uses Arrow for its local caching system. It allows datasets to be backed by an on-disk cache, which is memory-mapped for fast lookup.
This architecture allows for large datasets to be used on machines with relatively small device memory.`),pa.forEach(a),ns=m(s),N=r(s,"P",{});var ma=o(N);Ds=c(ma,"For example, loading the full English Wikipedia dataset only takes a few MB of RAM:"),ma.forEach(a),ps=m(s),W(T.$$.fragment,s),ms=m(s),B=r(s,"P",{});var ha=o(B);Cs=c(ha,`This is possible because the Arrow data is actually memory-mapped from disk, and not loaded in memory.
Memory-mapping allows access to data on disk, and leverages virtual memory capabilities for fast lookups.`),ha.forEach(a),hs=m(s),y=r(s,"H2",{class:!0});var ys=o(y);$=r(ys,"A",{id:!0,class:!0,href:!0});var ca=o($);ss=r(ca,"SPAN",{});var fa=o(ss);W(x.$$.fragment,fa),fa.forEach(a),ca.forEach(a),Fs=m(ys),as=r(ys,"SPAN",{});var ua=o(as);Hs=c(ua,"Performance"),ua.forEach(a),ys.forEach(a),cs=m(s),G=r(s,"P",{});var da=o(G);Us=c(da,"Iterating over a memory-mapped dataset using Arrow is fast. Iterating over Wikipedia on a laptop gives you speeds of 1-3 Gbit/s:"),da.forEach(a),fs=m(s),W(z.$$.fragment,s),this.h()},h(){i(d,"name","hf:doc:metadata"),i(d,"content",JSON.stringify(ja)),i(b,"id","datasets-arrow"),i(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(b,"href","#datasets-arrow"),i(w,"class","relative group"),i(_,"id","what-is-arrow"),i(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(_,"href","#what-is-arrow"),i(g,"class","relative group"),i(P,"href","https://arrow.apache.org/"),i(P,"rel","nofollow"),i(M,"href","https://en.wikipedia.org/wiki/Zero-copy"),i(M,"rel","nofollow"),i(k,"id","memorymapping"),i(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(k,"href","#memorymapping"),i(v,"class","relative group"),i($,"id","performance"),i($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i($,"href","#performance"),i(y,"class","relative group")},m(s,l){e(document.head,d),n(s,es,l),n(s,w,l),e(w,b),e(b,H),R(j,H,null),e(w,_s),e(w,U),e(U,ks),n(s,ts,l),n(s,g,l),e(g,_),e(_,J),R(A,J,null),e(g,$s),e(g,O),e(O,js),n(s,rs,l),n(s,E,l),e(E,P),e(P,As),e(E,Es),n(s,os,l),n(s,f,l),e(f,q),e(q,Ps),e(q,M),e(M,qs),e(q,Ms),e(f,Is),e(f,Z),e(Z,Ts),e(f,xs),e(f,K),e(K,zs),e(f,Ss),e(f,Q),e(Q,Ns),e(f,Bs),e(f,V),e(V,Gs),n(s,ls,l),n(s,v,l),e(v,k),e(k,X),R(I,X,null),e(v,Ls),e(v,Y),e(Y,Ws),n(s,is,l),n(s,S,l),e(S,Rs),n(s,ns,l),n(s,N,l),e(N,Ds),n(s,ps,l),R(T,s,l),n(s,ms,l),n(s,B,l),e(B,Cs),n(s,hs,l),n(s,y,l),e(y,$),e($,ss),R(x,ss,null),e(y,Fs),e(y,as),e(as,Hs),n(s,cs,l),n(s,G,l),e(G,Us),n(s,fs,l),R(z,s,l),us=!0},p:_a,i(s){us||(D(j.$$.fragment,s),D(A.$$.fragment,s),D(I.$$.fragment,s),D(T.$$.fragment,s),D(x.$$.fragment,s),D(z.$$.fragment,s),us=!0)},o(s){C(j.$$.fragment,s),C(A.$$.fragment,s),C(I.$$.fragment,s),C(T.$$.fragment,s),C(x.$$.fragment,s),C(z.$$.fragment,s),us=!1},d(s){a(d),s&&a(es),s&&a(w),F(j),s&&a(ts),s&&a(g),F(A),s&&a(rs),s&&a(E),s&&a(os),s&&a(f),s&&a(ls),s&&a(v),F(I),s&&a(is),s&&a(S),s&&a(ns),s&&a(N),s&&a(ps),F(T,s),s&&a(ms),s&&a(B),s&&a(hs),s&&a(y),F(x),s&&a(cs),s&&a(G),s&&a(fs),F(z,s)}}}const ja={local:"datasets-arrow",sections:[{local:"what-is-arrow",title:"What is Arrow?"},{local:"memorymapping",title:"Memory-mapping"},{local:"performance",title:"Performance"}],title:"Datasets \u{1F91D} Arrow"};function Aa(Os){return ka(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ma extends ga{constructor(d){super();va(this,d,Aa,$a,ya,{})}}export{Ma as default,ja as metadata};
