import{s as Wt,n as Ht,o as Lt}from"../chunks/scheduler.d6170356.js";import{S as Rt,i as Xt,g as i,s as l,r as h,A as At,h as d,f as a,c as n,j as Gt,u,x as m,k as z,y as Dt,a as s,v as c,d as g,t as b,w}from"../chunks/index.fcd4cc08.js";import{C as ct}from"../chunks/CodeBlock.7b16bdef.js";import{H as ut,E as Qt}from"../chunks/EditOnGithub.da2b595c.js";function zt(gt){let f,I,D,V,M,Y,y,bt=`<a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> is a library for writing I/O pipelines for large datasets.
Its sequential I/O and sharding features make it especially useful for streaming large-scale datasets to a DataLoader.`,F,$,N,C,wt=`A WebDataset file is a TAR archive containing a series of data files.
All successive data files with the same prefix are considered to be part of the same example (e.g., an image/audio file and its label or metadata):`,S,p,Mt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format-dark.png"/>',P,v,yt="Labels and metadata can be in a <code>.json</code> file, in a <code>.txt</code> (for a caption, a description), or in a <code>.cls</code> (for a class index).",E,k,$t=`A large scale WebDataset is made of many files called shards, where each shard is a TAR archive.
Each shard is often ~1GB but the full dataset can be multiple terabytes!`,B,x,q,j,Ct=`Streaming TAR archives is fast because it reads contiguous chunks of data.
It can be orders of magnitude faster than reading separate data files one by one.`,K,U,vt="WebDataset streaming offers high-speed performance both when reading from disk and from cloud storage, which makes it an ideal format to feed to a DataLoader:",O,r,kt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream-dark.png"/>',tt,T,xt='For example here is how to stream the <a href="https://huggingface.co/datasets/timm/imagenet-12k-wds" rel="nofollow">timm/imagenet-12k-wds</a> dataset directly from Hugging Face:',et,J,jt='First you need to <a href="/docs/huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',at,_,st,Z,Ut="And then you can stream the dataset with WebDataset:",lt,G,nt,W,it,H,Tt=`Generally, datasets in WebDataset formats are already shuffled and ready to feed to a DataLoader.
But you can still reshuffle the data with WebDataset’s approximate shuffling.`,dt,L,Jt="In addition to shuffling the list of shards, WebDataset uses a buffer to shuffle a dataset without any cost to speed:",mt,o,_t='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle-dark.png"/>',ft,R,Zt="To shuffle a list of sharded files and randomly sample from the shuffle buffer:",pt,X,rt,A,ot,Q,ht;return M=new ut({props:{title:"WebDataset",local:"webdataset",headingTag:"h1"}}),$=new ut({props:{title:"The WebDataset format",local:"the-webdataset-format",headingTag:"h2"}}),x=new ut({props:{title:"Streaming",local:"streaming",headingTag:"h2"}}),_=new ct({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),G=new ct({props:{code:"aW1wb3J0JTIwd2ViZGF0YXNldCUyMGFzJTIwd2RzJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF90b2tlbiUwQWZyb20lMjB0b3JjaC51dGlscy5kYXRhJTIwaW1wb3J0JTIwRGF0YUxvYWRlciUwQSUwQWhmX3Rva2VuJTIwJTNEJTIwZ2V0X3Rva2VuKCklMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZ0aW1tJTJGaW1hZ2VuZXQtMTJrLXdkcyUyRnJlc29sdmUlMkZtYWluJTJGaW1hZ2VuZXQxMmstdHJhaW4tJTdCJTdCMDAwMC4uMTAyMyU3RCU3RC50YXIlMjIlMEF1cmwlMjAlM0QlMjBmJTIycGlwZSUzQWN1cmwlMjAtcyUyMC1MJTIwJTdCdXJsJTdEJTIwLUglMjAnQXV0aG9yaXphdGlvbiUzQUJlYXJlciUyMCU3QmhmX3Rva2VuJTdEJyUyMiUwQWRhdGFzZXQlMjAlM0QlMjB3ZHMuV2ViRGF0YXNldCh1cmwpLmRlY29kZSgpJTBBZGF0YWxvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIoZGF0YXNldCUyQyUyMGJhdGNoX3NpemUlM0Q2NCUyQyUyMG51bV93b3JrZXJzJTNENCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> webdataset <span class="hljs-keyword">as</span> wds
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_token
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>hf_token = get_token()
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://huggingface.co/datasets/timm/imagenet-12k-wds/resolve/main/imagenet12k-train-{{0000..1023}}.tar&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">f&quot;pipe:curl -s -L <span class="hljs-subst">{url}</span> -H &#x27;Authorization:Bearer <span class="hljs-subst">{hf_token}</span>&#x27;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = wds.WebDataset(url).decode()
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(dataset, batch_size=<span class="hljs-number">64</span>, num_workers=<span class="hljs-number">4</span>)`,wrap:!1}}),W=new ut({props:{title:"Shuffle",local:"shuffle",headingTag:"h2"}}),X=new ct({props:{code:"YnVmZmVyX3NpemUlMjAlM0QlMjAxMDAwJTBBZGF0YXNldCUyMCUzRCUyMCglMEElMjAlMjAlMjAlMjB3ZHMuV2ViRGF0YXNldCh1cmwlMkMlMjBzaGFyZHNodWZmbGUlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMC5zaHVmZmxlKGJ1ZmZlcl9zaXplKSUwQSUyMCUyMCUyMCUyMC5kZWNvZGUoKSUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>buffer_size = <span class="hljs-number">1000</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = (
<span class="hljs-meta">... </span>    wds.WebDataset(url, shardshuffle=<span class="hljs-literal">True</span>)
<span class="hljs-meta">... </span>    .shuffle(buffer_size)
<span class="hljs-meta">... </span>    .decode()
<span class="hljs-meta">... </span>)`,wrap:!1}}),A=new Qt({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-webdataset.md"}}),{c(){f=i("meta"),I=l(),D=i("p"),V=l(),h(M.$$.fragment),Y=l(),y=i("p"),y.innerHTML=bt,F=l(),h($.$$.fragment),N=l(),C=i("p"),C.textContent=wt,S=l(),p=i("div"),p.innerHTML=Mt,P=l(),v=i("p"),v.innerHTML=yt,E=l(),k=i("p"),k.textContent=$t,B=l(),h(x.$$.fragment),q=l(),j=i("p"),j.textContent=Ct,K=l(),U=i("p"),U.textContent=vt,O=l(),r=i("div"),r.innerHTML=kt,tt=l(),T=i("p"),T.innerHTML=xt,et=l(),J=i("p"),J.innerHTML=jt,at=l(),h(_.$$.fragment),st=l(),Z=i("p"),Z.textContent=Ut,lt=l(),h(G.$$.fragment),nt=l(),h(W.$$.fragment),it=l(),H=i("p"),H.textContent=Tt,dt=l(),L=i("p"),L.textContent=Jt,mt=l(),o=i("div"),o.innerHTML=_t,ft=l(),R=i("p"),R.textContent=Zt,pt=l(),h(X.$$.fragment),rt=l(),h(A.$$.fragment),ot=l(),Q=i("p"),this.h()},l(t){const e=At("svelte-u9bgzb",document.head);f=d(e,"META",{name:!0,content:!0}),e.forEach(a),I=n(t),D=d(t,"P",{}),Gt(D).forEach(a),V=n(t),u(M.$$.fragment,t),Y=n(t),y=d(t,"P",{"data-svelte-h":!0}),m(y)!=="svelte-1x0ilw7"&&(y.innerHTML=bt),F=n(t),u($.$$.fragment,t),N=n(t),C=d(t,"P",{"data-svelte-h":!0}),m(C)!=="svelte-s28fzx"&&(C.textContent=wt),S=n(t),p=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(p)!=="svelte-1bdrcp9"&&(p.innerHTML=Mt),P=n(t),v=d(t,"P",{"data-svelte-h":!0}),m(v)!=="svelte-7sb7k1"&&(v.innerHTML=yt),E=n(t),k=d(t,"P",{"data-svelte-h":!0}),m(k)!=="svelte-1pmozez"&&(k.textContent=$t),B=n(t),u(x.$$.fragment,t),q=n(t),j=d(t,"P",{"data-svelte-h":!0}),m(j)!=="svelte-d0lfwq"&&(j.textContent=Ct),K=n(t),U=d(t,"P",{"data-svelte-h":!0}),m(U)!=="svelte-1ski7pk"&&(U.textContent=vt),O=n(t),r=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(r)!=="svelte-16x5sv3"&&(r.innerHTML=kt),tt=n(t),T=d(t,"P",{"data-svelte-h":!0}),m(T)!=="svelte-kd5fni"&&(T.innerHTML=xt),et=n(t),J=d(t,"P",{"data-svelte-h":!0}),m(J)!=="svelte-1jdzygp"&&(J.innerHTML=jt),at=n(t),u(_.$$.fragment,t),st=n(t),Z=d(t,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-3qvfc"&&(Z.textContent=Ut),lt=n(t),u(G.$$.fragment,t),nt=n(t),u(W.$$.fragment,t),it=n(t),H=d(t,"P",{"data-svelte-h":!0}),m(H)!=="svelte-1qsq6cj"&&(H.textContent=Tt),dt=n(t),L=d(t,"P",{"data-svelte-h":!0}),m(L)!=="svelte-e5b3kw"&&(L.textContent=Jt),mt=n(t),o=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(o)!=="svelte-15c8cpf"&&(o.innerHTML=_t),ft=n(t),R=d(t,"P",{"data-svelte-h":!0}),m(R)!=="svelte-1qykgdn"&&(R.textContent=Zt),pt=n(t),u(X.$$.fragment,t),rt=n(t),u(A.$$.fragment,t),ot=n(t),Q=d(t,"P",{}),Gt(Q).forEach(a),this.h()},h(){z(f,"name","hf:doc:metadata"),z(f,"content",It),z(p,"class","flex justify-center"),z(r,"class","flex justify-center"),z(o,"class","flex justify-center")},m(t,e){Dt(document.head,f),s(t,I,e),s(t,D,e),s(t,V,e),c(M,t,e),s(t,Y,e),s(t,y,e),s(t,F,e),c($,t,e),s(t,N,e),s(t,C,e),s(t,S,e),s(t,p,e),s(t,P,e),s(t,v,e),s(t,E,e),s(t,k,e),s(t,B,e),c(x,t,e),s(t,q,e),s(t,j,e),s(t,K,e),s(t,U,e),s(t,O,e),s(t,r,e),s(t,tt,e),s(t,T,e),s(t,et,e),s(t,J,e),s(t,at,e),c(_,t,e),s(t,st,e),s(t,Z,e),s(t,lt,e),c(G,t,e),s(t,nt,e),c(W,t,e),s(t,it,e),s(t,H,e),s(t,dt,e),s(t,L,e),s(t,mt,e),s(t,o,e),s(t,ft,e),s(t,R,e),s(t,pt,e),c(X,t,e),s(t,rt,e),c(A,t,e),s(t,ot,e),s(t,Q,e),ht=!0},p:Ht,i(t){ht||(g(M.$$.fragment,t),g($.$$.fragment,t),g(x.$$.fragment,t),g(_.$$.fragment,t),g(G.$$.fragment,t),g(W.$$.fragment,t),g(X.$$.fragment,t),g(A.$$.fragment,t),ht=!0)},o(t){b(M.$$.fragment,t),b($.$$.fragment,t),b(x.$$.fragment,t),b(_.$$.fragment,t),b(G.$$.fragment,t),b(W.$$.fragment,t),b(X.$$.fragment,t),b(A.$$.fragment,t),ht=!1},d(t){t&&(a(I),a(D),a(V),a(Y),a(y),a(F),a(N),a(C),a(S),a(p),a(P),a(v),a(E),a(k),a(B),a(q),a(j),a(K),a(U),a(O),a(r),a(tt),a(T),a(et),a(J),a(at),a(st),a(Z),a(lt),a(nt),a(it),a(H),a(dt),a(L),a(mt),a(o),a(ft),a(R),a(pt),a(rt),a(ot),a(Q)),a(f),w(M,t),w($,t),w(x,t),w(_,t),w(G,t),w(W,t),w(X,t),w(A,t)}}}const It='{"title":"WebDataset","local":"webdataset","sections":[{"title":"The WebDataset format","local":"the-webdataset-format","sections":[],"depth":2},{"title":"Streaming","local":"streaming","sections":[],"depth":2},{"title":"Shuffle","local":"shuffle","sections":[],"depth":2}],"depth":1}';function Vt(gt){return Lt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends Rt{constructor(f){super(),Xt(this,f,Vt,zt,Wt,{})}}export{Pt as component};
