import{S as Cl,i as Ol,s as zl,e as r,k as i,w as m,t as n,M as Fl,c as p,d as a,m as c,a as o,x as u,h as l,b as d,G as t,g as h,y as f,q as g,o as j,B as b,v as Nl}from"../chunks/vendor-hf-doc-builder.js";import{T as Il}from"../chunks/Tip-hf-doc-builder.js";import{I as M}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as w}from"../chunks/CodeBlock-hf-doc-builder.js";function Ul(Oa){let y,Y,_,k,O;return{c(){y=r("p"),Y=n("A "),_=r("a"),k=n("Dataset"),O=n(" object is a wrapper of an Arrow table, which allows fast zero-copy reads from arrays in the dataset to PyTorch tensors."),this.h()},l($){y=p($,"P",{});var x=o(y);Y=l(x,"A "),_=p(x,"A",{href:!0});var G=o(_);k=l(G,"Dataset"),G.forEach(a),O=l(x," object is a wrapper of an Arrow table, which allows fast zero-copy reads from arrays in the dataset to PyTorch tensors."),x.forEach(a),this.h()},h(){d(_,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset")},m($,x){h($,y,x),t(y,Y),t(y,_),t(_,k),t(y,O)},d($){$&&a(y)}}}function Bl(Oa){let y,Y,_,k,O,$,x,G,Nt,za,v,It,Ks,Ut,Bt,Qs,Ht,Rt,Vs,Mt,Yt,Xs,Gt,Wt,Fa,z,W,Zs,is,Jt,sa,Kt,Na,Cs,Qt,Ia,A,Vt,aa,Xt,Zt,Os,se,ae,Ua,cs,Ba,J,Ha,K,te,ta,ee,ne,Ra,ds,Ma,F,Q,ea,ms,le,na,re,Ya,zs,pe,Ga,us,Wa,V,oe,la,he,ie,Ja,fs,Ka,N,X,ra,gs,ce,pa,de,Qa,js,Fs,me,ue,Va,bs,Xa,Ns,fe,Za,T,ge,Is,je,be,Us,ye,_e,st,ys,at,_s,tt,I,Z,oa,vs,ve,ha,we,et,D,$e,ia,ke,De,Bs,Ee,qe,ca,Pe,xe,nt,ws,lt,U,ss,da,$s,Ae,ma,Te,rt,Hs,Le,pt,B,as,ua,ks,Se,fa,Ce,ot,L,Oe,ga,ze,Fe,ja,Ne,Ie,ht,E,Ue,ba,Be,He,ya,Re,Me,_a,Ye,Ge,it,Ds,ct,H,ts,va,Es,We,wa,Je,dt,es,Ke,$a,Qe,Ve,mt,qs,ut,Rs,Xe,ft,Ps,gt,S,Ze,ka,sn,an,Da,tn,en,jt,xs,bt,q,nn,Ea,ln,rn,qa,pn,on,Pa,hn,cn,yt,R,ns,xa,As,dn,Aa,mn,_t,P,un,Ta,fn,gn,La,jn,bn,Sa,yn,_n,vt,Ts,wt,ls,vn,Ca,wn,$n,$t,Ls,kt,Ms,kn,Dt;return $=new M({}),is=new M({}),cs=new w({props:{code:`from datasets import Dataset
data = [[1, 2],[3, 4]]
ds = Dataset.from_dict({"data": data})
ds = ds.with_format("torch")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>])}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
         [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}`}}),J=new Il({props:{$$slots:{default:[Ul]},$$scope:{ctx:Oa}}}),ds=new w({props:{code:`import torch
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
ds = ds.with_format("torch", device=device)
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span>device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>, device=device)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)}`}}),ms=new M({}),us=new w({props:{code:`from datasets import Dataset
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
ds = Dataset.from_dict({"data": data})
ds = ds.with_format("torch")
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: [tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>]), tensor([<span class="hljs-number">3</span>, <span class="hljs-number">4</span>])]}`}}),fs=new w({props:{code:`from datasets import Dataset, Features, Array2D
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
features = Features({"data": Array2D(shape=(2, 2), dtype='int32')})
ds = Dataset.from_dict({"data": data}, features=features)
ds = ds.with_format("torch")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Array2D
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;data&quot;</span>: Array2D(shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=<span class="hljs-string">&#x27;int32&#x27;</span>)})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}, features=features)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
         [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
          [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],
 
         [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],
          [<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]])}`}}),gs=new M({}),bs=new w({props:{code:`from datasets import Dataset, Features, ClassLabel
labels = [0, 0, 1]
features = Features({"label": ClassLabel(names=["negative", "positive"])})
ds = Dataset.from_dict({"label": labels}, features=features) 
ds = ds.with_format("torch")  
ds[:3]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, ClassLabel
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;label&quot;</span>: ClassLabel(names=[<span class="hljs-string">&quot;negative&quot;</span>, <span class="hljs-string">&quot;positive&quot;</span>])})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;label&quot;</span>: labels}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">3</span>]
{<span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>])}`}}),ys=new w({props:{code:`from datasets import Dataset, Features, Audio, Image
images = ["path/to/image.png"] * 10
features = Features({"image": Image()})
ds = Dataset.from_dict({"image": images}, features=features) 
ds = ds.with_format("torch")
ds[0]["image"].shape
ds[0]
ds[:2]["image"].shape
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>images = [<span class="hljs-string">&quot;path/to/image.png&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;image&quot;</span>: Image()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;image&quot;</span>: images}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>].shape
torch.Size([<span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: tensor([[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         ...,
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]], dtype=torch.uint8)}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>][<span class="hljs-string">&quot;image&quot;</span>].shape
torch.Size([<span class="hljs-number">2</span>, <span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: tensor([[[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          ...,
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]]], dtype=torch.uint8)}`}}),_s=new w({props:{code:`from datasets import Dataset, Features, Audio, Image
audio = ["path/to/audio.wav"] * 10
features = Features({"audio": Audio()})
ds = Dataset.from_dict({"audio": audio}, features=features) 
ds = ds.with_format("torch")  
ds[0]["audio"]["array"]
ds[0]["audio"]["sampling_rate"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = [<span class="hljs-string">&quot;path/to/audio.wav&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;audio&quot;</span>: Audio()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;audio&quot;</span>: audio}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]
tensor([ <span class="hljs-number">6.1035e-05</span>,  <span class="hljs-number">1.5259e-05</span>,  <span class="hljs-number">1.6785e-04</span>,  ..., -<span class="hljs-number">1.5259e-05</span>,
        -<span class="hljs-number">1.5259e-05</span>,  <span class="hljs-number">1.5259e-05</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;sampling_rate&quot;</span>]
tensor(<span class="hljs-number">44100</span>)`}}),vs=new M({}),ws=new w({props:{code:`import numpy as np
from datasets import Dataset 
from torch.utils.data import DataLoader
data = np.random.rand(16)
label = np.random.randint(0, 2, size=16)
ds = Dataset.from_dict({"data": data, "label": label}).with_format("torch")
dataloader = DataLoader(ds, batch_size=4)
for batch in dataloader:
    print(batch)                                                                                            `,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset 
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-meta">&gt;&gt;&gt; </span>data = np.random.rand(<span class="hljs-number">16</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>label = np.random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>, size=<span class="hljs-number">16</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data, <span class="hljs-string">&quot;label&quot;</span>: label}).with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, batch_size=<span class="hljs-number">4</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> dataloader:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(batch)                                                                                            
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.0047</span>, <span class="hljs-number">0.4979</span>, <span class="hljs-number">0.6726</span>, <span class="hljs-number">0.8105</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>])}
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.4832</span>, <span class="hljs-number">0.2723</span>, <span class="hljs-number">0.4259</span>, <span class="hljs-number">0.2224</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>])}
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.5837</span>, <span class="hljs-number">0.3444</span>, <span class="hljs-number">0.4658</span>, <span class="hljs-number">0.6417</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>])}
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.7022</span>, <span class="hljs-number">0.1225</span>, <span class="hljs-number">0.7228</span>, <span class="hljs-number">0.8259</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>])}`}}),$s=new M({}),ks=new M({}),Ds=new w({props:{code:`import numpy as np
from datasets import Dataset, load_from_disk
from torch.utils.data import DataLoader
data = np.random.rand(10_000)
Dataset.from_dict({"data": data}).save_to_disk("my_dataset")
ds = load_from_disk("my_dataset").with_format("torch")
dataloader = DataLoader(ds, batch_size=32, num_workers=4)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, load_from_disk
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-meta">&gt;&gt;&gt; </span>data = np.random.rand(<span class="hljs-number">10_000</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}).save_to_disk(<span class="hljs-string">&quot;my_dataset&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_from_disk(<span class="hljs-string">&quot;my_dataset&quot;</span>).with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, batch_size=<span class="hljs-number">32</span>, num_workers=<span class="hljs-number">4</span>)`}}),Es=new M({}),qs=new w({props:{code:"batch = [dataset[idx] for idx in range(start, end)]",highlighted:'batch = [dataset[idx] <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(start, end)]'}}),Ps=new w({props:{code:`batch = dataset[start:end]
# or
batch = dataset[list_of_indices]`,highlighted:`batch = dataset[start:end]
<span class="hljs-comment"># or</span>
batch = dataset[list_of_indices]`}}),xs=new w({props:{code:`from torch.utils.data.sampler import BatchSampler, RandomSampler
sampler = BatchSampler(RandomSampler(ds), batch_size=32, drop_last=False)
dataloader = DataLoader(ds, sampler=sampler)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data.sampler <span class="hljs-keyword">import</span> BatchSampler, RandomSampler
<span class="hljs-meta">&gt;&gt;&gt; </span>sampler = BatchSampler(RandomSampler(ds), batch_size=<span class="hljs-number">32</span>, drop_last=<span class="hljs-literal">False</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, sampler=sampler)`}}),As=new M({}),Ts=new w({props:{code:`import numpy as np
from datasets import Dataset, load_dataset
from torch.utils.data import DataLoader
data = np.random.rand(10_000)
Dataset.from_dict({"data": data}).push_to_hub("<username>/my_dataset")  # Upload to the Hugging Face Hub
ds = load_dataset("<username>/my_dataset", streaming=True, split="train")
dataloader = DataLoader(ds, batch_size=32)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-meta">&gt;&gt;&gt; </span>data = np.random.rand(<span class="hljs-number">10_000</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}).push_to_hub(<span class="hljs-string">&quot;&lt;username&gt;/my_dataset&quot;</span>)  <span class="hljs-comment"># Upload to the Hugging Face Hub</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&quot;&lt;username&gt;/my_dataset&quot;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, batch_size=<span class="hljs-number">32</span>)`}}),Ls=new w({props:{code:`ds = load_dataset("c4", "en", streaming=True, split="train")
ds.n_shards
dataloader = DataLoader(ds, batch_size=32, num_workers=4)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&quot;c4&quot;</span>, <span class="hljs-string">&quot;en&quot;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds.n_shards
<span class="hljs-number">1024</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, batch_size=<span class="hljs-number">32</span>, num_workers=<span class="hljs-number">4</span>)`}}),{c(){y=r("meta"),Y=i(),_=r("h1"),k=r("a"),O=r("span"),m($.$$.fragment),x=i(),G=r("span"),Nt=n("Use with PyTorch"),za=i(),v=r("p"),It=n("This document is a quick introduction to using "),Ks=r("code"),Ut=n("datasets"),Bt=n(` with PyTorch, with a particular focus on how to get
`),Qs=r("code"),Ht=n("torch.Tensor"),Rt=n(" objects out of our datasets, and how to use a PyTorch "),Vs=r("code"),Mt=n("DataLoader"),Yt=n(" and a Hugging Face "),Xs=r("code"),Gt=n("Dataset"),Wt=n(`
with the best performance.`),Fa=i(),z=r("h2"),W=r("a"),Zs=r("span"),m(is.$$.fragment),Jt=i(),sa=r("span"),Kt=n("Dataset format"),Na=i(),Cs=r("p"),Qt=n("By default, datasets return regular python objects: integers, floats, strings, lists, etc."),Ia=i(),A=r("p"),Vt=n("To get PyTorch tensors instead, you can set the format of the dataset to "),aa=r("code"),Xt=n("pytorch"),Zt=n(" using "),Os=r("a"),se=n("Dataset.with_format()"),ae=n(":"),Ua=i(),m(cs.$$.fragment),Ba=i(),m(J.$$.fragment),Ha=i(),K=r("p"),te=n("To load the data as tensors on a GPU, specify the "),ta=r("code"),ee=n("device"),ne=n(" argument:"),Ra=i(),m(ds.$$.fragment),Ma=i(),F=r("h2"),Q=r("a"),ea=r("span"),m(ms.$$.fragment),le=i(),na=r("span"),re=n("N-dimensional arrays"),Ya=i(),zs=r("p"),pe=n(`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a PyTorch formatted dataset outputs nested lists instead of a single tensor:`),Ga=i(),m(us.$$.fragment),Wa=i(),V=r("p"),oe=n("To get a single tensor, you must explicitly use the "),la=r("code"),he=n("Array"),ie=n(" feature type and specify the shape of your tensors:"),Ja=i(),m(fs.$$.fragment),Ka=i(),N=r("h2"),X=r("a"),ra=r("span"),m(gs.$$.fragment),ce=i(),pa=r("span"),de=n("Other feature types"),Qa=i(),js=r("p"),Fs=r("a"),me=n("ClassLabel"),ue=n(" data are properly converted to tensors:"),Va=i(),m(bs.$$.fragment),Xa=i(),Ns=r("p"),fe=n("String and binary objects are unchanged, since PyTorch only supports numbers."),Za=i(),T=r("p"),ge=n("The "),Is=r("a"),je=n("Image"),be=n(" and "),Us=r("a"),ye=n("Audio"),_e=n(" feature types are also supported:"),st=i(),m(ys.$$.fragment),at=i(),m(_s.$$.fragment),tt=i(),I=r("h2"),Z=r("a"),oa=r("span"),m(vs.$$.fragment),ve=i(),ha=r("span"),we=n("Data loading"),et=i(),D=r("p"),$e=n("Like "),ia=r("code"),ke=n("torch.utils.data.Dataset"),De=n(" objects, a "),Bs=r("a"),Ee=n("Dataset"),qe=n(" can be passed directly to a PyTorch "),ca=r("code"),Pe=n("DataLoader"),xe=n(":"),nt=i(),m(ws.$$.fragment),lt=i(),U=r("h3"),ss=r("a"),da=r("span"),m($s.$$.fragment),Ae=i(),ma=r("span"),Te=n("Optimize data loading"),rt=i(),Hs=r("p"),Le=n(`There are several ways you can increase the speed your data is loaded which can save you time, especially if you are working with large datasets.
PyTorch offers parallelized data loading, retrieving batches of indices instead of individually, and streaming to iterate over the dataset without downloading it on disk.`),pt=i(),B=r("h4"),as=r("a"),ua=r("span"),m(ks.$$.fragment),Se=i(),fa=r("span"),Ce=n("Use multiple Workers"),ot=i(),L=r("p"),Oe=n("You can parallelize data loading with the "),ga=r("code"),ze=n("num_workers"),Fe=n(" argument of a PyTorch "),ja=r("code"),Ne=n("DataLoader"),Ie=n(" and get a higher throughput."),ht=i(),E=r("p"),Ue=n("Under the hood, the "),ba=r("code"),Be=n("DataLoader"),He=n(" starts "),ya=r("code"),Re=n("num_workers"),Me=n(` processes.
Each process reloads the dataset passed to the `),_a=r("code"),Ye=n("DataLoader"),Ge=n(` and is used to query examples.
Reloading the dataset inside a worker doesn\u2019t fill up your RAM, since it simply memory-maps the dataset again from your disk.`),it=i(),m(Ds.$$.fragment),ct=i(),H=r("h4"),ts=r("a"),va=r("span"),m(Es.$$.fragment),We=i(),wa=r("span"),Je=n("Use a BatchSampler"),dt=i(),es=r("p"),Ke=n("By default, the PyTorch "),$a=r("code"),Qe=n("DataLoader"),Ve=n(" load batches of data from a dataset one by one like this:"),mt=i(),m(qs.$$.fragment),ut=i(),Rs=r("p"),Xe=n(`Unfortunately, this does numerous read operations on the dataset.
It is more efficient to query batches of examples using a list:`),ft=i(),m(Ps.$$.fragment),gt=i(),S=r("p"),Ze=n("For the PyTorch "),ka=r("code"),sn=n("DataLoader"),an=n(" to query batches using a list, you can use a "),Da=r("code"),tn=n("BatchSampler"),en=n(":"),jt=i(),m(xs.$$.fragment),bt=i(),q=r("p"),nn=n("Moreover, this is particularly useful if you used "),Ea=r("code"),ln=n("set_transform"),rn=n(` to apply a transform on-the-fly when examples are accessed.
You must use a `),qa=r("code"),pn=n("BatchSampler"),on=n(" if you want the transform to be given full batches instead of receiving "),Pa=r("code"),hn=n("batch_size"),cn=n(" times one single element."),yt=i(),R=r("h3"),ns=r("a"),xa=r("span"),m(As.$$.fragment),dn=i(),Aa=r("span"),mn=n("Stream data"),_t=i(),P=r("p"),un=n(`Loading a dataset in streaming mode allows one to iterate over the dataset without downloading it on disk.
An iterable dataset from `),Ta=r("code"),fn=n("datasets"),gn=n(" inherits from "),La=r("code"),jn=n("torch.utils.data.IterableDataset"),bn=n(" so you can pass it to a "),Sa=r("code"),yn=n("DataLoader"),_n=n(":"),vt=i(),m(Ts.$$.fragment),wt=i(),ls=r("p"),vn=n("If the dataset is split in several shards (i.e. if the dataset consists of multiple data files), then you can stream in parallel using "),Ca=r("code"),wn=n("num_workers"),$n=n(":"),$t=i(),m(Ls.$$.fragment),kt=i(),Ms=r("p"),kn=n("In this case each worker will be given a subset of the list of shards to stream from."),this.h()},l(s){const e=Fl('[data-svelte="svelte-1phssyn"]',document.head);y=p(e,"META",{name:!0,content:!0}),e.forEach(a),Y=c(s),_=p(s,"H1",{class:!0});var Ss=o(_);k=p(Ss,"A",{id:!0,class:!0,href:!0});var En=o(k);O=p(En,"SPAN",{});var qn=o(O);u($.$$.fragment,qn),qn.forEach(a),En.forEach(a),x=c(Ss),G=p(Ss,"SPAN",{});var Pn=o(G);Nt=l(Pn,"Use with PyTorch"),Pn.forEach(a),Ss.forEach(a),za=c(s),v=p(s,"P",{});var C=o(v);It=l(C,"This document is a quick introduction to using "),Ks=p(C,"CODE",{});var xn=o(Ks);Ut=l(xn,"datasets"),xn.forEach(a),Bt=l(C,` with PyTorch, with a particular focus on how to get
`),Qs=p(C,"CODE",{});var An=o(Qs);Ht=l(An,"torch.Tensor"),An.forEach(a),Rt=l(C," objects out of our datasets, and how to use a PyTorch "),Vs=p(C,"CODE",{});var Tn=o(Vs);Mt=l(Tn,"DataLoader"),Tn.forEach(a),Yt=l(C," and a Hugging Face "),Xs=p(C,"CODE",{});var Ln=o(Xs);Gt=l(Ln,"Dataset"),Ln.forEach(a),Wt=l(C,`
with the best performance.`),C.forEach(a),Fa=c(s),z=p(s,"H2",{class:!0});var Et=o(z);W=p(Et,"A",{id:!0,class:!0,href:!0});var Sn=o(W);Zs=p(Sn,"SPAN",{});var Cn=o(Zs);u(is.$$.fragment,Cn),Cn.forEach(a),Sn.forEach(a),Jt=c(Et),sa=p(Et,"SPAN",{});var On=o(sa);Kt=l(On,"Dataset format"),On.forEach(a),Et.forEach(a),Na=c(s),Cs=p(s,"P",{});var zn=o(Cs);Qt=l(zn,"By default, datasets return regular python objects: integers, floats, strings, lists, etc."),zn.forEach(a),Ia=c(s),A=p(s,"P",{});var Ys=o(A);Vt=l(Ys,"To get PyTorch tensors instead, you can set the format of the dataset to "),aa=p(Ys,"CODE",{});var Fn=o(aa);Xt=l(Fn,"pytorch"),Fn.forEach(a),Zt=l(Ys," using "),Os=p(Ys,"A",{href:!0});var Nn=o(Os);se=l(Nn,"Dataset.with_format()"),Nn.forEach(a),ae=l(Ys,":"),Ys.forEach(a),Ua=c(s),u(cs.$$.fragment,s),Ba=c(s),u(J.$$.fragment,s),Ha=c(s),K=p(s,"P",{});var qt=o(K);te=l(qt,"To load the data as tensors on a GPU, specify the "),ta=p(qt,"CODE",{});var In=o(ta);ee=l(In,"device"),In.forEach(a),ne=l(qt," argument:"),qt.forEach(a),Ra=c(s),u(ds.$$.fragment,s),Ma=c(s),F=p(s,"H2",{class:!0});var Pt=o(F);Q=p(Pt,"A",{id:!0,class:!0,href:!0});var Un=o(Q);ea=p(Un,"SPAN",{});var Bn=o(ea);u(ms.$$.fragment,Bn),Bn.forEach(a),Un.forEach(a),le=c(Pt),na=p(Pt,"SPAN",{});var Hn=o(na);re=l(Hn,"N-dimensional arrays"),Hn.forEach(a),Pt.forEach(a),Ya=c(s),zs=p(s,"P",{});var Rn=o(zs);pe=l(Rn,`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a PyTorch formatted dataset outputs nested lists instead of a single tensor:`),Rn.forEach(a),Ga=c(s),u(us.$$.fragment,s),Wa=c(s),V=p(s,"P",{});var xt=o(V);oe=l(xt,"To get a single tensor, you must explicitly use the "),la=p(xt,"CODE",{});var Mn=o(la);he=l(Mn,"Array"),Mn.forEach(a),ie=l(xt," feature type and specify the shape of your tensors:"),xt.forEach(a),Ja=c(s),u(fs.$$.fragment,s),Ka=c(s),N=p(s,"H2",{class:!0});var At=o(N);X=p(At,"A",{id:!0,class:!0,href:!0});var Yn=o(X);ra=p(Yn,"SPAN",{});var Gn=o(ra);u(gs.$$.fragment,Gn),Gn.forEach(a),Yn.forEach(a),ce=c(At),pa=p(At,"SPAN",{});var Wn=o(pa);de=l(Wn,"Other feature types"),Wn.forEach(a),At.forEach(a),Qa=c(s),js=p(s,"P",{});var Dn=o(js);Fs=p(Dn,"A",{href:!0});var Jn=o(Fs);me=l(Jn,"ClassLabel"),Jn.forEach(a),ue=l(Dn," data are properly converted to tensors:"),Dn.forEach(a),Va=c(s),u(bs.$$.fragment,s),Xa=c(s),Ns=p(s,"P",{});var Kn=o(Ns);fe=l(Kn,"String and binary objects are unchanged, since PyTorch only supports numbers."),Kn.forEach(a),Za=c(s),T=p(s,"P",{});var Gs=o(T);ge=l(Gs,"The "),Is=p(Gs,"A",{href:!0});var Qn=o(Is);je=l(Qn,"Image"),Qn.forEach(a),be=l(Gs," and "),Us=p(Gs,"A",{href:!0});var Vn=o(Us);ye=l(Vn,"Audio"),Vn.forEach(a),_e=l(Gs," feature types are also supported:"),Gs.forEach(a),st=c(s),u(ys.$$.fragment,s),at=c(s),u(_s.$$.fragment,s),tt=c(s),I=p(s,"H2",{class:!0});var Tt=o(I);Z=p(Tt,"A",{id:!0,class:!0,href:!0});var Xn=o(Z);oa=p(Xn,"SPAN",{});var Zn=o(oa);u(vs.$$.fragment,Zn),Zn.forEach(a),Xn.forEach(a),ve=c(Tt),ha=p(Tt,"SPAN",{});var sl=o(ha);we=l(sl,"Data loading"),sl.forEach(a),Tt.forEach(a),et=c(s),D=p(s,"P",{});var rs=o(D);$e=l(rs,"Like "),ia=p(rs,"CODE",{});var al=o(ia);ke=l(al,"torch.utils.data.Dataset"),al.forEach(a),De=l(rs," objects, a "),Bs=p(rs,"A",{href:!0});var tl=o(Bs);Ee=l(tl,"Dataset"),tl.forEach(a),qe=l(rs," can be passed directly to a PyTorch "),ca=p(rs,"CODE",{});var el=o(ca);Pe=l(el,"DataLoader"),el.forEach(a),xe=l(rs,":"),rs.forEach(a),nt=c(s),u(ws.$$.fragment,s),lt=c(s),U=p(s,"H3",{class:!0});var Lt=o(U);ss=p(Lt,"A",{id:!0,class:!0,href:!0});var nl=o(ss);da=p(nl,"SPAN",{});var ll=o(da);u($s.$$.fragment,ll),ll.forEach(a),nl.forEach(a),Ae=c(Lt),ma=p(Lt,"SPAN",{});var rl=o(ma);Te=l(rl,"Optimize data loading"),rl.forEach(a),Lt.forEach(a),rt=c(s),Hs=p(s,"P",{});var pl=o(Hs);Le=l(pl,`There are several ways you can increase the speed your data is loaded which can save you time, especially if you are working with large datasets.
PyTorch offers parallelized data loading, retrieving batches of indices instead of individually, and streaming to iterate over the dataset without downloading it on disk.`),pl.forEach(a),pt=c(s),B=p(s,"H4",{class:!0});var St=o(B);as=p(St,"A",{id:!0,class:!0,href:!0});var ol=o(as);ua=p(ol,"SPAN",{});var hl=o(ua);u(ks.$$.fragment,hl),hl.forEach(a),ol.forEach(a),Se=c(St),fa=p(St,"SPAN",{});var il=o(fa);Ce=l(il,"Use multiple Workers"),il.forEach(a),St.forEach(a),ot=c(s),L=p(s,"P",{});var Ws=o(L);Oe=l(Ws,"You can parallelize data loading with the "),ga=p(Ws,"CODE",{});var cl=o(ga);ze=l(cl,"num_workers"),cl.forEach(a),Fe=l(Ws," argument of a PyTorch "),ja=p(Ws,"CODE",{});var dl=o(ja);Ne=l(dl,"DataLoader"),dl.forEach(a),Ie=l(Ws," and get a higher throughput."),Ws.forEach(a),ht=c(s),E=p(s,"P",{});var ps=o(E);Ue=l(ps,"Under the hood, the "),ba=p(ps,"CODE",{});var ml=o(ba);Be=l(ml,"DataLoader"),ml.forEach(a),He=l(ps," starts "),ya=p(ps,"CODE",{});var ul=o(ya);Re=l(ul,"num_workers"),ul.forEach(a),Me=l(ps,` processes.
Each process reloads the dataset passed to the `),_a=p(ps,"CODE",{});var fl=o(_a);Ye=l(fl,"DataLoader"),fl.forEach(a),Ge=l(ps,` and is used to query examples.
Reloading the dataset inside a worker doesn\u2019t fill up your RAM, since it simply memory-maps the dataset again from your disk.`),ps.forEach(a),it=c(s),u(Ds.$$.fragment,s),ct=c(s),H=p(s,"H4",{class:!0});var Ct=o(H);ts=p(Ct,"A",{id:!0,class:!0,href:!0});var gl=o(ts);va=p(gl,"SPAN",{});var jl=o(va);u(Es.$$.fragment,jl),jl.forEach(a),gl.forEach(a),We=c(Ct),wa=p(Ct,"SPAN",{});var bl=o(wa);Je=l(bl,"Use a BatchSampler"),bl.forEach(a),Ct.forEach(a),dt=c(s),es=p(s,"P",{});var Ot=o(es);Ke=l(Ot,"By default, the PyTorch "),$a=p(Ot,"CODE",{});var yl=o($a);Qe=l(yl,"DataLoader"),yl.forEach(a),Ve=l(Ot," load batches of data from a dataset one by one like this:"),Ot.forEach(a),mt=c(s),u(qs.$$.fragment,s),ut=c(s),Rs=p(s,"P",{});var _l=o(Rs);Xe=l(_l,`Unfortunately, this does numerous read operations on the dataset.
It is more efficient to query batches of examples using a list:`),_l.forEach(a),ft=c(s),u(Ps.$$.fragment,s),gt=c(s),S=p(s,"P",{});var Js=o(S);Ze=l(Js,"For the PyTorch "),ka=p(Js,"CODE",{});var vl=o(ka);sn=l(vl,"DataLoader"),vl.forEach(a),an=l(Js," to query batches using a list, you can use a "),Da=p(Js,"CODE",{});var wl=o(Da);tn=l(wl,"BatchSampler"),wl.forEach(a),en=l(Js,":"),Js.forEach(a),jt=c(s),u(xs.$$.fragment,s),bt=c(s),q=p(s,"P",{});var os=o(q);nn=l(os,"Moreover, this is particularly useful if you used "),Ea=p(os,"CODE",{});var $l=o(Ea);ln=l($l,"set_transform"),$l.forEach(a),rn=l(os,` to apply a transform on-the-fly when examples are accessed.
You must use a `),qa=p(os,"CODE",{});var kl=o(qa);pn=l(kl,"BatchSampler"),kl.forEach(a),on=l(os," if you want the transform to be given full batches instead of receiving "),Pa=p(os,"CODE",{});var Dl=o(Pa);hn=l(Dl,"batch_size"),Dl.forEach(a),cn=l(os," times one single element."),os.forEach(a),yt=c(s),R=p(s,"H3",{class:!0});var zt=o(R);ns=p(zt,"A",{id:!0,class:!0,href:!0});var El=o(ns);xa=p(El,"SPAN",{});var ql=o(xa);u(As.$$.fragment,ql),ql.forEach(a),El.forEach(a),dn=c(zt),Aa=p(zt,"SPAN",{});var Pl=o(Aa);mn=l(Pl,"Stream data"),Pl.forEach(a),zt.forEach(a),_t=c(s),P=p(s,"P",{});var hs=o(P);un=l(hs,`Loading a dataset in streaming mode allows one to iterate over the dataset without downloading it on disk.
An iterable dataset from `),Ta=p(hs,"CODE",{});var xl=o(Ta);fn=l(xl,"datasets"),xl.forEach(a),gn=l(hs," inherits from "),La=p(hs,"CODE",{});var Al=o(La);jn=l(Al,"torch.utils.data.IterableDataset"),Al.forEach(a),bn=l(hs," so you can pass it to a "),Sa=p(hs,"CODE",{});var Tl=o(Sa);yn=l(Tl,"DataLoader"),Tl.forEach(a),_n=l(hs,":"),hs.forEach(a),vt=c(s),u(Ts.$$.fragment,s),wt=c(s),ls=p(s,"P",{});var Ft=o(ls);vn=l(Ft,"If the dataset is split in several shards (i.e. if the dataset consists of multiple data files), then you can stream in parallel using "),Ca=p(Ft,"CODE",{});var Ll=o(Ca);wn=l(Ll,"num_workers"),Ll.forEach(a),$n=l(Ft,":"),Ft.forEach(a),$t=c(s),u(Ls.$$.fragment,s),kt=c(s),Ms=p(s,"P",{});var Sl=o(Ms);kn=l(Sl,"In this case each worker will be given a subset of the list of shards to stream from."),Sl.forEach(a),this.h()},h(){d(y,"name","hf:doc:metadata"),d(y,"content",JSON.stringify(Hl)),d(k,"id","use-with-pytorch"),d(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(k,"href","#use-with-pytorch"),d(_,"class","relative group"),d(W,"id","dataset-format"),d(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(W,"href","#dataset-format"),d(z,"class","relative group"),d(Os,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset.with_format"),d(Q,"id","ndimensional-arrays"),d(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Q,"href","#ndimensional-arrays"),d(F,"class","relative group"),d(X,"id","other-feature-types"),d(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(X,"href","#other-feature-types"),d(N,"class","relative group"),d(Fs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.ClassLabel"),d(Is,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Image"),d(Us,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Audio"),d(Z,"id","data-loading"),d(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Z,"href","#data-loading"),d(I,"class","relative group"),d(Bs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset"),d(ss,"id","optimize-data-loading"),d(ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ss,"href","#optimize-data-loading"),d(U,"class","relative group"),d(as,"id","use-multiple-workers"),d(as,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(as,"href","#use-multiple-workers"),d(B,"class","relative group"),d(ts,"id","use-a-batchsampler"),d(ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ts,"href","#use-a-batchsampler"),d(H,"class","relative group"),d(ns,"id","stream-data"),d(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ns,"href","#stream-data"),d(R,"class","relative group")},m(s,e){t(document.head,y),h(s,Y,e),h(s,_,e),t(_,k),t(k,O),f($,O,null),t(_,x),t(_,G),t(G,Nt),h(s,za,e),h(s,v,e),t(v,It),t(v,Ks),t(Ks,Ut),t(v,Bt),t(v,Qs),t(Qs,Ht),t(v,Rt),t(v,Vs),t(Vs,Mt),t(v,Yt),t(v,Xs),t(Xs,Gt),t(v,Wt),h(s,Fa,e),h(s,z,e),t(z,W),t(W,Zs),f(is,Zs,null),t(z,Jt),t(z,sa),t(sa,Kt),h(s,Na,e),h(s,Cs,e),t(Cs,Qt),h(s,Ia,e),h(s,A,e),t(A,Vt),t(A,aa),t(aa,Xt),t(A,Zt),t(A,Os),t(Os,se),t(A,ae),h(s,Ua,e),f(cs,s,e),h(s,Ba,e),f(J,s,e),h(s,Ha,e),h(s,K,e),t(K,te),t(K,ta),t(ta,ee),t(K,ne),h(s,Ra,e),f(ds,s,e),h(s,Ma,e),h(s,F,e),t(F,Q),t(Q,ea),f(ms,ea,null),t(F,le),t(F,na),t(na,re),h(s,Ya,e),h(s,zs,e),t(zs,pe),h(s,Ga,e),f(us,s,e),h(s,Wa,e),h(s,V,e),t(V,oe),t(V,la),t(la,he),t(V,ie),h(s,Ja,e),f(fs,s,e),h(s,Ka,e),h(s,N,e),t(N,X),t(X,ra),f(gs,ra,null),t(N,ce),t(N,pa),t(pa,de),h(s,Qa,e),h(s,js,e),t(js,Fs),t(Fs,me),t(js,ue),h(s,Va,e),f(bs,s,e),h(s,Xa,e),h(s,Ns,e),t(Ns,fe),h(s,Za,e),h(s,T,e),t(T,ge),t(T,Is),t(Is,je),t(T,be),t(T,Us),t(Us,ye),t(T,_e),h(s,st,e),f(ys,s,e),h(s,at,e),f(_s,s,e),h(s,tt,e),h(s,I,e),t(I,Z),t(Z,oa),f(vs,oa,null),t(I,ve),t(I,ha),t(ha,we),h(s,et,e),h(s,D,e),t(D,$e),t(D,ia),t(ia,ke),t(D,De),t(D,Bs),t(Bs,Ee),t(D,qe),t(D,ca),t(ca,Pe),t(D,xe),h(s,nt,e),f(ws,s,e),h(s,lt,e),h(s,U,e),t(U,ss),t(ss,da),f($s,da,null),t(U,Ae),t(U,ma),t(ma,Te),h(s,rt,e),h(s,Hs,e),t(Hs,Le),h(s,pt,e),h(s,B,e),t(B,as),t(as,ua),f(ks,ua,null),t(B,Se),t(B,fa),t(fa,Ce),h(s,ot,e),h(s,L,e),t(L,Oe),t(L,ga),t(ga,ze),t(L,Fe),t(L,ja),t(ja,Ne),t(L,Ie),h(s,ht,e),h(s,E,e),t(E,Ue),t(E,ba),t(ba,Be),t(E,He),t(E,ya),t(ya,Re),t(E,Me),t(E,_a),t(_a,Ye),t(E,Ge),h(s,it,e),f(Ds,s,e),h(s,ct,e),h(s,H,e),t(H,ts),t(ts,va),f(Es,va,null),t(H,We),t(H,wa),t(wa,Je),h(s,dt,e),h(s,es,e),t(es,Ke),t(es,$a),t($a,Qe),t(es,Ve),h(s,mt,e),f(qs,s,e),h(s,ut,e),h(s,Rs,e),t(Rs,Xe),h(s,ft,e),f(Ps,s,e),h(s,gt,e),h(s,S,e),t(S,Ze),t(S,ka),t(ka,sn),t(S,an),t(S,Da),t(Da,tn),t(S,en),h(s,jt,e),f(xs,s,e),h(s,bt,e),h(s,q,e),t(q,nn),t(q,Ea),t(Ea,ln),t(q,rn),t(q,qa),t(qa,pn),t(q,on),t(q,Pa),t(Pa,hn),t(q,cn),h(s,yt,e),h(s,R,e),t(R,ns),t(ns,xa),f(As,xa,null),t(R,dn),t(R,Aa),t(Aa,mn),h(s,_t,e),h(s,P,e),t(P,un),t(P,Ta),t(Ta,fn),t(P,gn),t(P,La),t(La,jn),t(P,bn),t(P,Sa),t(Sa,yn),t(P,_n),h(s,vt,e),f(Ts,s,e),h(s,wt,e),h(s,ls,e),t(ls,vn),t(ls,Ca),t(Ca,wn),t(ls,$n),h(s,$t,e),f(Ls,s,e),h(s,kt,e),h(s,Ms,e),t(Ms,kn),Dt=!0},p(s,[e]){const Ss={};e&2&&(Ss.$$scope={dirty:e,ctx:s}),J.$set(Ss)},i(s){Dt||(g($.$$.fragment,s),g(is.$$.fragment,s),g(cs.$$.fragment,s),g(J.$$.fragment,s),g(ds.$$.fragment,s),g(ms.$$.fragment,s),g(us.$$.fragment,s),g(fs.$$.fragment,s),g(gs.$$.fragment,s),g(bs.$$.fragment,s),g(ys.$$.fragment,s),g(_s.$$.fragment,s),g(vs.$$.fragment,s),g(ws.$$.fragment,s),g($s.$$.fragment,s),g(ks.$$.fragment,s),g(Ds.$$.fragment,s),g(Es.$$.fragment,s),g(qs.$$.fragment,s),g(Ps.$$.fragment,s),g(xs.$$.fragment,s),g(As.$$.fragment,s),g(Ts.$$.fragment,s),g(Ls.$$.fragment,s),Dt=!0)},o(s){j($.$$.fragment,s),j(is.$$.fragment,s),j(cs.$$.fragment,s),j(J.$$.fragment,s),j(ds.$$.fragment,s),j(ms.$$.fragment,s),j(us.$$.fragment,s),j(fs.$$.fragment,s),j(gs.$$.fragment,s),j(bs.$$.fragment,s),j(ys.$$.fragment,s),j(_s.$$.fragment,s),j(vs.$$.fragment,s),j(ws.$$.fragment,s),j($s.$$.fragment,s),j(ks.$$.fragment,s),j(Ds.$$.fragment,s),j(Es.$$.fragment,s),j(qs.$$.fragment,s),j(Ps.$$.fragment,s),j(xs.$$.fragment,s),j(As.$$.fragment,s),j(Ts.$$.fragment,s),j(Ls.$$.fragment,s),Dt=!1},d(s){a(y),s&&a(Y),s&&a(_),b($),s&&a(za),s&&a(v),s&&a(Fa),s&&a(z),b(is),s&&a(Na),s&&a(Cs),s&&a(Ia),s&&a(A),s&&a(Ua),b(cs,s),s&&a(Ba),b(J,s),s&&a(Ha),s&&a(K),s&&a(Ra),b(ds,s),s&&a(Ma),s&&a(F),b(ms),s&&a(Ya),s&&a(zs),s&&a(Ga),b(us,s),s&&a(Wa),s&&a(V),s&&a(Ja),b(fs,s),s&&a(Ka),s&&a(N),b(gs),s&&a(Qa),s&&a(js),s&&a(Va),b(bs,s),s&&a(Xa),s&&a(Ns),s&&a(Za),s&&a(T),s&&a(st),b(ys,s),s&&a(at),b(_s,s),s&&a(tt),s&&a(I),b(vs),s&&a(et),s&&a(D),s&&a(nt),b(ws,s),s&&a(lt),s&&a(U),b($s),s&&a(rt),s&&a(Hs),s&&a(pt),s&&a(B),b(ks),s&&a(ot),s&&a(L),s&&a(ht),s&&a(E),s&&a(it),b(Ds,s),s&&a(ct),s&&a(H),b(Es),s&&a(dt),s&&a(es),s&&a(mt),b(qs,s),s&&a(ut),s&&a(Rs),s&&a(ft),b(Ps,s),s&&a(gt),s&&a(S),s&&a(jt),b(xs,s),s&&a(bt),s&&a(q),s&&a(yt),s&&a(R),b(As),s&&a(_t),s&&a(P),s&&a(vt),b(Ts,s),s&&a(wt),s&&a(ls),s&&a($t),b(Ls,s),s&&a(kt),s&&a(Ms)}}}const Hl={local:"use-with-pytorch",sections:[{local:"dataset-format",title:"Dataset format"},{local:"ndimensional-arrays",title:"N-dimensional arrays"},{local:"other-feature-types",title:"Other feature types"},{local:"data-loading",sections:[{local:"optimize-data-loading",sections:[{local:"use-multiple-workers",title:"Use multiple Workers"},{local:"use-a-batchsampler",title:"Use a BatchSampler"}],title:"Optimize data loading"},{local:"stream-data",title:"Stream data"}],title:"Data loading"}],title:"Use with PyTorch"};function Rl(Oa){return Nl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Jl extends Cl{constructor(y){super();Ol(this,y,Rl,Bl,zl,{})}}export{Jl as default,Hl as metadata};
