import{S as Fr,i as Nr,s as Ir,e as l,k as h,w as f,t as e,M as Sr,c as o,d as t,m as d,a as r,x as m,h as n,b as c,G as a,g as i,y as g,q as b,o as y,B as j,v as Hr}from"../chunks/vendor-hf-doc-builder.js";import{T as Lr}from"../chunks/Tip-hf-doc-builder.js";import{I as ms}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as S}from"../chunks/CodeBlock-hf-doc-builder.js";function Ur(it){let w,Z,E,P,z;return{c(){w=l("p"),Z=e("A "),E=l("a"),P=e("Dataset"),z=e(" object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to TensorFlow tensors."),this.h()},l(C){w=o(C,"P",{});var H=r(w);Z=n(H,"A "),E=o(H,"A",{href:!0});var ss=r(E);P=n(ss,"Dataset"),ss.forEach(t),z=n(H," object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to TensorFlow tensors."),H.forEach(t),this.h()},h(){c(E,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset")},m(C,H){i(C,w,H),a(w,Z),a(w,E),a(E,P),a(w,z)},d(C){C&&t(w)}}}function Yr(it){let w,Z,E,P,z,C,H,ss,ie,ht,T,he,ta,de,ce,ea,ue,fe,na,me,ge,la,be,ye,dt,B,as,oa,gs,je,ra,we,ct,Rs,ve,ut,ts,_e,pa,$e,Ee,ft,bs,mt,es,gt,L,De,ia,ke,Te,ha,qe,xe,bt,ys,yt,K,ns,da,js,Ae,ca,Ce,jt,ls,Oe,ua,Pe,Fe,wt,ws,vt,Ws,Ne,_t,vs,$t,G,os,fa,_s,Ie,ma,Se,Et,$s,Ms,He,Le,Dt,Es,kt,zs,Ue,Tt,Ds,qt,Bs,Ye,xt,ks,At,Ks,Re,Ct,U,We,Gs,Me,ze,Js,Be,Ke,Ot,Ts,Pt,qs,Ft,J,rs,ga,xs,Ge,ba,Je,Nt,D,Qe,ya,Ve,Xe,ja,Ze,sn,wa,an,tn,va,en,nn,_a,ln,on,It,v,rn,$a,pn,hn,Ea,dn,cn,Da,un,fn,ka,mn,gn,Ta,bn,yn,qa,jn,wn,St,_,vn,xa,_n,$n,Aa,En,Dn,Ca,kn,Tn,As,qn,xn,Cs,An,Cn,Oa,On,Pn,Ht,Q,ps,Pa,Os,Fn,Qs,Nn,Fa,In,Lt,is,Sn,Na,Hn,Ln,Ut,Ps,Yt,F,Un,Ia,Yn,Rn,Sa,Wn,Mn,Ha,zn,Bn,Rt,Fs,Wt,$,Kn,Vs,Gn,Jn,La,Qn,Vn,Ns,Xn,Zn,Ua,sl,al,Is,tl,el,Ss,nl,ll,Mt,V,hs,Ya,Hs,ol,Ra,rl,zt,u,pl,Wa,il,hl,Ma,dl,cl,za,ul,fl,Ba,ml,gl,Ka,bl,yl,Ga,jl,wl,Ja,vl,_l,Bt,Y,$l,Qa,El,Dl,Va,kl,Tl,Kt,R,Ls,ql,Xa,xl,Al,Cl,I,Ol,Za,Pl,Fl,st,Nl,Il,at,Sl,Hl,Ll,O,Ul,tt,Yl,Rl,et,Wl,Ml,nt,zl,Bl,lt,Kl,Gl,Gt,X,ds,ot,Us,Jl,rt,Ql,Jt,cs,Vl,pt,Xl,Zl,Qt;return C=new ms({}),gs=new ms({}),bs=new S({props:{code:`from datasets import Dataset
data = [[1, 2],[3, 4]]
ds = Dataset.from_dict({"data": [[1, 2],[3, 4]]})
ds = ds.with_format("tf")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=int64, numpy=array([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>])&gt;}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
       [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])&gt;}`}}),es=new Lr({props:{$$slots:{default:[Ur]},$$scope:{ctx:it}}}),ys=new S({props:{code:"ds[:]",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
       [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])&gt;}`}}),js=new ms({}),ws=new S({props:{code:`from datasets import Dataset
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
ds = Dataset.from_dict({"data": data})
ds = ds.with_format("tf")
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.RaggedTensor [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]&gt;}`}}),vs=new S({props:{code:`from datasets import Dataset, Features, Array2D
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
features = Features({"data": Array2D(shape=(2, 2), dtype='int32')})
ds = Dataset.from_dict({"data": data}, features=features)
ds = ds.with_format("tf")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Array2D
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;data&quot;</span>: Array2D(shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=<span class="hljs-string">&#x27;int32&#x27;</span>)})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}, features=features)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
 array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
        [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])&gt;}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
 array([[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
         [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],
 
        [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],
         [<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]])&gt;}`}}),_s=new ms({}),Es=new S({props:{code:`from datasets import Dataset, Features, ClassLabel
labels = [0, 0, 1]
features = Features({"label": ClassLabel(names=["negative", "positive"])})
ds = Dataset.from_dict({"label": labels}, features=features) 
ds = ds.with_format("tf")  
ds[:3]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, ClassLabel
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;label&quot;</span>: ClassLabel(names=[<span class="hljs-string">&quot;negative&quot;</span>, <span class="hljs-string">&quot;positive&quot;</span>])})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;label&quot;</span>: labels}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">3</span>]
{<span class="hljs-string">&#x27;label&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">3</span>,), dtype=int64, numpy=array([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>])&gt;}`}}),Ds=new S({props:{code:`from datasets import Dataset, Features 
text = ["foo", "bar"]
data = [0, 1] 
ds = Dataset.from_dict({"text": text, "data": data})  
ds = ds.with_format("tf") 
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features 
<span class="hljs-meta">&gt;&gt;&gt; </span>text = [<span class="hljs-string">&quot;foo&quot;</span>, <span class="hljs-string">&quot;bar&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>] 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;text&quot;</span>: text, <span class="hljs-string">&quot;data&quot;</span>: data})  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;text&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=string, numpy=array([<span class="hljs-string">b&#x27;foo&#x27;</span>, <span class="hljs-string">b&#x27;bar&#x27;</span>], dtype=<span class="hljs-built_in">object</span>)&gt;,
 <span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=int64, numpy=array([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>])&gt;}`}}),ks=new S({props:{code:`ds = ds.with_format("tf", columns=["data"], output_all_columns=True)
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>, columns=[<span class="hljs-string">&quot;data&quot;</span>], output_all_columns=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=int64, numpy=array([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>])&gt;,
 <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&#x27;foo&#x27;</span>, <span class="hljs-string">&#x27;bar&#x27;</span>]}`}}),Ts=new S({props:{code:`from datasets import Dataset, Features, Audio, Image
images = ["path/to/image.png"] * 10
features = Features({"image": Image()})
ds = Dataset.from_dict({"image": images}, features=features) 
ds = ds.with_format("tf")  
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>images = [<span class="hljs-string">&quot;path/to/image.png&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;image&quot;</span>: Image()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;image&quot;</span>: images}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>), dtype=uint8, numpy=
 array([[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         ...,
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]], dtype=uint8)&gt;}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>), dtype=uint8, numpy=
 array([[[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          ...,
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]]], dtype=uint8)&gt;}`}}),qs=new S({props:{code:`from datasets import Dataset, Features, Audio, Image
audio = ["path/to/audio.wav"] * 10
features = Features({"audio": Audio()})
ds = Dataset.from_dict({"audio": audio}, features=features) 
ds = ds.with_format("tf")  
ds[0]["audio"]["array"]
ds[0]["audio"]["sampling_rate"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = [<span class="hljs-string">&quot;path/to/audio.wav&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;audio&quot;</span>: Audio()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;audio&quot;</span>: audio}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]
&lt;tf.Tensor: shape=(<span class="hljs-number">202311</span>,), dtype=float32, numpy=
array([ <span class="hljs-number">6.1035156e-05</span>,  <span class="hljs-number">1.5258789e-05</span>,  <span class="hljs-number">1.6784668e-04</span>, ...,
       -<span class="hljs-number">1.5258789e-05</span>, -<span class="hljs-number">1.5258789e-05</span>,  <span class="hljs-number">1.5258789e-05</span>], dtype=float32)&gt;
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;sampling_rate&quot;</span>]
&lt;tf.Tensor: shape=(), dtype=int32, numpy=<span class="hljs-number">44100</span>&gt;`}}),xs=new ms({}),Os=new ms({}),Ps=new S({props:{code:`from datasets import Dataset
data = {"inputs": [[1, 2],[3, 4]], "labels": [0, 1]}
ds = Dataset.from_dict(data)
tf_ds = ds.to_tf_dataset(`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = {<span class="hljs-string">&quot;inputs&quot;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], <span class="hljs-string">&quot;labels&quot;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict(data)
<span class="hljs-meta">&gt;&gt;&gt; </span>tf_ds = ds.to_tf_dataset(
            columns=[<span class="hljs-string">&quot;inputs&quot;</span>],
            label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
            batch_size=<span class="hljs-number">2</span>,
            shuffle=<span class="hljs-literal">True</span>
            )`}}),Fs=new S({props:{code:"model.fit(tf_ds, epochs=2)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(tf_ds, epochs=<span class="hljs-number">2</span>)'}}),Hs=new ms({}),Us=new ms({}),{c(){w=l("meta"),Z=h(),E=l("h1"),P=l("a"),z=l("span"),f(C.$$.fragment),H=h(),ss=l("span"),ie=e("Using Datasets with TensorFlow"),ht=h(),T=l("p"),he=e("This document is a quick introduction to using "),ta=l("code"),de=e("datasets"),ce=e(` with TensorFlow, with a particular focus on how to get
`),ea=l("code"),ue=e("tf.Tensor"),fe=e(" objects out of our datasets, and how to stream data from Hugging Face "),na=l("code"),me=e("Dataset"),ge=e(` objects to Keras methods
like `),la=l("code"),be=e("model.fit()"),ye=e("."),dt=h(),B=l("h2"),as=l("a"),oa=l("span"),f(gs.$$.fragment),je=h(),ra=l("span"),we=e("Dataset format"),ct=h(),Rs=l("p"),ve=e("By default, datasets return regular Python objects: integers, floats, strings, lists, etc."),ut=h(),ts=l("p"),_e=e("To get TensorFlow tensors instead, you can set the format of the dataset to "),pa=l("code"),$e=e("tf"),Ee=e(":"),ft=h(),f(bs.$$.fragment),mt=h(),f(es.$$.fragment),gt=h(),L=l("p"),De=e("This can be useful for converting your dataset to a dict of "),ia=l("code"),ke=e("Tensor"),Te=e(` objects, or for writing a generator to load TF
samples from it. If you wish to convert the entire dataset to `),ha=l("code"),qe=e("Tensor"),xe=e(", simply query the full dataset:"),bt=h(),f(ys.$$.fragment),yt=h(),K=l("h2"),ns=l("a"),da=l("span"),f(js.$$.fragment),Ae=h(),ca=l("span"),Ce=e("N-dimensional arrays"),jt=h(),ls=l("p"),Oe=e(`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a TensorFlow formatted dataset outputs a `),ua=l("code"),Pe=e("RaggedTensor"),Fe=e(" instead of a single tensor:"),wt=h(),f(ws.$$.fragment),vt=h(),Ws=l("p"),Ne=e("To get a single tensor, you must explicitly use the Array feature type and specify the shape of your tensors:"),_t=h(),f(vs.$$.fragment),$t=h(),G=l("h2"),os=l("a"),fa=l("span"),f(_s.$$.fragment),Ie=h(),ma=l("span"),Se=e("Other feature types"),Et=h(),$s=l("p"),Ms=l("a"),He=e("ClassLabel"),Le=e(" data are properly converted to tensors:"),Dt=h(),f(Es.$$.fragment),kt=h(),zs=l("p"),Ue=e("Strings and binary objects are also supported:"),Tt=h(),f(Ds.$$.fragment),qt=h(),Bs=l("p"),Ye=e("You can also explicitly format certain columns and leave the other columns unformatted:"),xt=h(),f(ks.$$.fragment),At=h(),Ks=l("p"),Re=e("String and binary objects are unchanged, since PyTorch only supports numbers."),Ct=h(),U=l("p"),We=e("The "),Gs=l("a"),Me=e("Image"),ze=e(" and "),Js=l("a"),Be=e("Audio"),Ke=e(" feature types are also supported:"),Ot=h(),f(Ts.$$.fragment),Pt=h(),f(qs.$$.fragment),Ft=h(),J=l("h2"),rs=l("a"),ga=l("span"),f(xs.$$.fragment),Ge=h(),ba=l("span"),Je=e("Data loading"),Nt=h(),D=l("p"),Qe=e(`Although you can load individual samples and batches just by indexing into your dataset, this won\u2019t work if you want
to use Keras methods like `),ya=l("code"),Ve=e("fit()"),Xe=e(" and "),ja=l("code"),Ze=e("predict()"),sn=e(`. You could write a generator function that shuffles and loads batches
from your dataset and `),wa=l("code"),an=e("fit()"),tn=e(` on that, but that sounds like a lot of unnecessary work. Instead, if you want to stream
data from your dataset on-the-fly, we recommend converting your dataset to a `),va=l("code"),en=e("tf.data.Dataset"),nn=e(` using the
`),_a=l("code"),ln=e("to_tf_dataset()"),on=e(" method."),It=h(),v=l("p"),rn=e("The "),$a=l("code"),pn=e("tf.data.Dataset"),hn=e(` class covers a wide range of use-cases - it is often created from Tensors in memory, or using a load function to read files on disc
or external storage. The dataset can be transformed arbitrarily with the `),Ea=l("code"),dn=e("map()"),cn=e(" method, or methods like "),Da=l("code"),un=e("batch()"),fn=e(`
and `),ka=l("code"),mn=e("shuffle()"),gn=e(` can be used to create a dataset that\u2019s ready for training. These methods do not modify the stored data
in any way - instead, the methods build a data pipeline graph that will be executed when the dataset is iterated over,
usually during model training or inference. This is different from the `),Ta=l("code"),bn=e("map()"),yn=e(" method of Hugging Face "),qa=l("code"),jn=e("Dataset"),wn=e(` objects,
which runs the map function immediately and saves the new or changed columns.`),St=h(),_=l("p"),vn=e("Since the entire data preprocessing pipeline can be compiled in a "),xa=l("code"),_n=e("tf.data.Dataset"),$n=e(`, this approach allows for massively
parallel, asynchronous data loading and training. However, the requirement for graph compilation can be a limitation,
particularly for Hugging Face tokenizers, which are usually not (yet!) compilable as part of a TF graph. As a result,
we usually advise pre-processing the dataset as a Hugging Face dataset, where arbitrary Python functions can be
used, and then converting to `),Aa=l("code"),En=e("tf.data.Dataset"),Dn=e(" afterwards using "),Ca=l("code"),kn=e("to_tf_dataset()"),Tn=e(` to get a batched dataset ready for
training. To see examples of this approach, please see the `),As=l("a"),qn=e("examples"),xn=e(" or "),Cs=l("a"),An=e("notebooks"),Cn=e(" for "),Oa=l("code"),On=e("transformers"),Pn=e("."),Ht=h(),Q=l("h3"),ps=l("a"),Pa=l("span"),f(Os.$$.fragment),Fn=h(),Qs=l("span"),Nn=e("Using "),Fa=l("code"),In=e("to_tf_dataset()"),Lt=h(),is=l("p"),Sn=e("Using "),Na=l("code"),Hn=e("to_tf_dataset()"),Ln=e(" is straightforward. Once your dataset is preprocessed and ready, simply call it like so:"),Ut=h(),f(Ps.$$.fragment),Yt=h(),F=l("p"),Un=e("The returned "),Ia=l("code"),Yn=e("tf_ds"),Rn=e(" object here is now fully ready to train on, and can be passed directly to "),Sa=l("code"),Wn=e("model.fit()"),Mn=e(`! Note
that you set the batch size when creating the dataset, and so you don\u2019t need to specify it when calling `),Ha=l("code"),zn=e("fit()"),Bn=e(":"),Rt=h(),f(Fs.$$.fragment),Wt=h(),$=l("p"),Kn=e("For a full description of the arguments, please see the "),Vs=l("a"),Gn=e("to_tf_dataset()"),Jn=e(` documentation. In many cases,
you will also need to add a `),La=l("code"),Qn=e("collate_fn"),Vn=e(` to your call. This is a function that takes multiple elements of the dataset
and combines them into a single batch. When all elements have the same length, the built-in default collator will
suffice, but for more complex tasks a custom collator may be necessary. In particular, many tasks have samples
with varying sequence lengths which will require a `),Ns=l("a"),Xn=e("data collator"),Zn=e(` that can pad batches correctly. You can see examples
of this in the `),Ua=l("code"),sl=e("transformers"),al=e(" NLP "),Is=l("a"),tl=e("examples"),el=e(` and
`),Ss=l("a"),nl=e("notebooks"),ll=e(", where variable sequence lengths are very common."),Mt=h(),V=l("h3"),hs=l("a"),Ya=l("span"),f(Hs.$$.fragment),ol=h(),Ra=l("span"),rl=e("When to use to_tf_dataset"),zt=h(),u=l("p"),pl=e(`The astute reader may have noticed at this point that we have offered two approaches to achieve the same goal - if you
want to pass your dataset to a TensorFlow model, you can either convert the dataset to a `),Wa=l("code"),il=e("Tensor"),hl=e(" or "),Ma=l("code"),dl=e("dict"),cl=e(" of "),za=l("code"),ul=e("Tensors"),fl=e(`
using `),Ba=l("code"),ml=e(".with_format('tf')"),gl=e(", or you can convert the dataset to a "),Ka=l("code"),bl=e("tf.data.Dataset"),yl=e(" with "),Ga=l("code"),jl=e("to_tf_dataset()"),wl=e(`. Either of these
can be passed to `),Ja=l("code"),vl=e("model.fit()"),_l=e(", so which should you choose?"),Bt=h(),Y=l("p"),$l=e("The key thing to recognize is that when you convert the whole dataset to "),Qa=l("code"),El=e("Tensor"),Dl=e(`s, it is static and fully loaded into
RAM. This is simple and convenient, but if any of the following apply, you should probably use `),Va=l("code"),kl=e("to_tf_dataset()"),Tl=e(`
instead:`),Kt=h(),R=l("ul"),Ls=l("li"),ql=e("Your dataset is too large to fit in RAM. "),Xa=l("code"),xl=e("to_tf_dataset()"),Al=e(` streams only one batch at a time, so even very large
datasets can be handled with this method.`),Cl=h(),I=l("li"),Ol=e("You want to apply random transformations using "),Za=l("code"),Pl=e("dataset.with_transform()"),Fl=e(" or the "),st=l("code"),Nl=e("collate_fn"),Il=e(`. This is
common in several modalities, such as image augmentations when training vision models, or random masking when training
masked language models. Using `),at=l("code"),Sl=e("to_tf_dataset()"),Hl=e(` will apply those transformations
at the moment when a batch is loaded, which means the same samples will get different augmentations each time
they are loaded. This is usually what you want.`),Ll=h(),O=l("li"),Ul=e(`Your data has a variable dimension, such as input texts in NLP that consist of varying
numbers of tokens. When you create a batch with samples with a variable dimension, the standard solution is to
pad the shorter samples to the length of the longest one. When you stream samples from a dataset with `),tt=l("code"),Yl=e("to_tf_dataset"),Rl=e(`,
you can apply this padding to each batch via your `),et=l("code"),Wl=e("collate_fn"),Ml=e(`. However, if you want to convert
such a dataset to dense `),nt=l("code"),zl=e("Tensor"),Bl=e("s, then you will have to pad samples to the length of the longest sample in "),lt=l("em"),Kl=e(`the
entire dataset!`),Gl=e(" This can result in huge amounts of padding, which wastes memory and reduces your model\u2019s speed."),Gt=h(),X=l("h3"),ds=l("a"),ot=l("span"),f(Us.$$.fragment),Jl=h(),rt=l("span"),Ql=e("Caveats and limitations"),Jt=h(),cs=l("p"),Vl=e("Right now, "),pt=l("code"),Xl=e("to_tf_dataset()"),Zl=e(" always return a batched dataset - we will add support for unbatched datasets soon!"),this.h()},l(s){const p=Sr('[data-svelte="svelte-1phssyn"]',document.head);w=o(p,"META",{name:!0,content:!0}),p.forEach(t),Z=d(s),E=o(s,"H1",{class:!0});var Ys=r(E);P=o(Ys,"A",{id:!0,class:!0,href:!0});var to=r(P);z=o(to,"SPAN",{});var eo=r(z);m(C.$$.fragment,eo),eo.forEach(t),to.forEach(t),H=d(Ys),ss=o(Ys,"SPAN",{});var no=r(ss);ie=n(no,"Using Datasets with TensorFlow"),no.forEach(t),Ys.forEach(t),ht=d(s),T=o(s,"P",{});var W=r(T);he=n(W,"This document is a quick introduction to using "),ta=o(W,"CODE",{});var lo=r(ta);de=n(lo,"datasets"),lo.forEach(t),ce=n(W,` with TensorFlow, with a particular focus on how to get
`),ea=o(W,"CODE",{});var oo=r(ea);ue=n(oo,"tf.Tensor"),oo.forEach(t),fe=n(W," objects out of our datasets, and how to stream data from Hugging Face "),na=o(W,"CODE",{});var ro=r(na);me=n(ro,"Dataset"),ro.forEach(t),ge=n(W,` objects to Keras methods
like `),la=o(W,"CODE",{});var po=r(la);be=n(po,"model.fit()"),po.forEach(t),ye=n(W,"."),W.forEach(t),dt=d(s),B=o(s,"H2",{class:!0});var Vt=r(B);as=o(Vt,"A",{id:!0,class:!0,href:!0});var io=r(as);oa=o(io,"SPAN",{});var ho=r(oa);m(gs.$$.fragment,ho),ho.forEach(t),io.forEach(t),je=d(Vt),ra=o(Vt,"SPAN",{});var co=r(ra);we=n(co,"Dataset format"),co.forEach(t),Vt.forEach(t),ct=d(s),Rs=o(s,"P",{});var uo=r(Rs);ve=n(uo,"By default, datasets return regular Python objects: integers, floats, strings, lists, etc."),uo.forEach(t),ut=d(s),ts=o(s,"P",{});var Xt=r(ts);_e=n(Xt,"To get TensorFlow tensors instead, you can set the format of the dataset to "),pa=o(Xt,"CODE",{});var fo=r(pa);$e=n(fo,"tf"),fo.forEach(t),Ee=n(Xt,":"),Xt.forEach(t),ft=d(s),m(bs.$$.fragment,s),mt=d(s),m(es.$$.fragment,s),gt=d(s),L=o(s,"P",{});var Xs=r(L);De=n(Xs,"This can be useful for converting your dataset to a dict of "),ia=o(Xs,"CODE",{});var mo=r(ia);ke=n(mo,"Tensor"),mo.forEach(t),Te=n(Xs,` objects, or for writing a generator to load TF
samples from it. If you wish to convert the entire dataset to `),ha=o(Xs,"CODE",{});var go=r(ha);qe=n(go,"Tensor"),go.forEach(t),xe=n(Xs,", simply query the full dataset:"),Xs.forEach(t),bt=d(s),m(ys.$$.fragment,s),yt=d(s),K=o(s,"H2",{class:!0});var Zt=r(K);ns=o(Zt,"A",{id:!0,class:!0,href:!0});var bo=r(ns);da=o(bo,"SPAN",{});var yo=r(da);m(js.$$.fragment,yo),yo.forEach(t),bo.forEach(t),Ae=d(Zt),ca=o(Zt,"SPAN",{});var jo=r(ca);Ce=n(jo,"N-dimensional arrays"),jo.forEach(t),Zt.forEach(t),jt=d(s),ls=o(s,"P",{});var se=r(ls);Oe=n(se,`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a TensorFlow formatted dataset outputs a `),ua=o(se,"CODE",{});var wo=r(ua);Pe=n(wo,"RaggedTensor"),wo.forEach(t),Fe=n(se," instead of a single tensor:"),se.forEach(t),wt=d(s),m(ws.$$.fragment,s),vt=d(s),Ws=o(s,"P",{});var vo=r(Ws);Ne=n(vo,"To get a single tensor, you must explicitly use the Array feature type and specify the shape of your tensors:"),vo.forEach(t),_t=d(s),m(vs.$$.fragment,s),$t=d(s),G=o(s,"H2",{class:!0});var ae=r(G);os=o(ae,"A",{id:!0,class:!0,href:!0});var _o=r(os);fa=o(_o,"SPAN",{});var $o=r(fa);m(_s.$$.fragment,$o),$o.forEach(t),_o.forEach(t),Ie=d(ae),ma=o(ae,"SPAN",{});var Eo=r(ma);Se=n(Eo,"Other feature types"),Eo.forEach(t),ae.forEach(t),Et=d(s),$s=o(s,"P",{});var so=r($s);Ms=o(so,"A",{href:!0});var Do=r(Ms);He=n(Do,"ClassLabel"),Do.forEach(t),Le=n(so," data are properly converted to tensors:"),so.forEach(t),Dt=d(s),m(Es.$$.fragment,s),kt=d(s),zs=o(s,"P",{});var ko=r(zs);Ue=n(ko,"Strings and binary objects are also supported:"),ko.forEach(t),Tt=d(s),m(Ds.$$.fragment,s),qt=d(s),Bs=o(s,"P",{});var To=r(Bs);Ye=n(To,"You can also explicitly format certain columns and leave the other columns unformatted:"),To.forEach(t),xt=d(s),m(ks.$$.fragment,s),At=d(s),Ks=o(s,"P",{});var qo=r(Ks);Re=n(qo,"String and binary objects are unchanged, since PyTorch only supports numbers."),qo.forEach(t),Ct=d(s),U=o(s,"P",{});var Zs=r(U);We=n(Zs,"The "),Gs=o(Zs,"A",{href:!0});var xo=r(Gs);Me=n(xo,"Image"),xo.forEach(t),ze=n(Zs," and "),Js=o(Zs,"A",{href:!0});var Ao=r(Js);Be=n(Ao,"Audio"),Ao.forEach(t),Ke=n(Zs," feature types are also supported:"),Zs.forEach(t),Ot=d(s),m(Ts.$$.fragment,s),Pt=d(s),m(qs.$$.fragment,s),Ft=d(s),J=o(s,"H2",{class:!0});var te=r(J);rs=o(te,"A",{id:!0,class:!0,href:!0});var Co=r(rs);ga=o(Co,"SPAN",{});var Oo=r(ga);m(xs.$$.fragment,Oo),Oo.forEach(t),Co.forEach(t),Ge=d(te),ba=o(te,"SPAN",{});var Po=r(ba);Je=n(Po,"Data loading"),Po.forEach(t),te.forEach(t),Nt=d(s),D=o(s,"P",{});var N=r(D);Qe=n(N,`Although you can load individual samples and batches just by indexing into your dataset, this won\u2019t work if you want
to use Keras methods like `),ya=o(N,"CODE",{});var Fo=r(ya);Ve=n(Fo,"fit()"),Fo.forEach(t),Xe=n(N," and "),ja=o(N,"CODE",{});var No=r(ja);Ze=n(No,"predict()"),No.forEach(t),sn=n(N,`. You could write a generator function that shuffles and loads batches
from your dataset and `),wa=o(N,"CODE",{});var Io=r(wa);an=n(Io,"fit()"),Io.forEach(t),tn=n(N,` on that, but that sounds like a lot of unnecessary work. Instead, if you want to stream
data from your dataset on-the-fly, we recommend converting your dataset to a `),va=o(N,"CODE",{});var So=r(va);en=n(So,"tf.data.Dataset"),So.forEach(t),nn=n(N,` using the
`),_a=o(N,"CODE",{});var Ho=r(_a);ln=n(Ho,"to_tf_dataset()"),Ho.forEach(t),on=n(N," method."),N.forEach(t),It=d(s),v=o(s,"P",{});var q=r(v);rn=n(q,"The "),$a=o(q,"CODE",{});var Lo=r($a);pn=n(Lo,"tf.data.Dataset"),Lo.forEach(t),hn=n(q,` class covers a wide range of use-cases - it is often created from Tensors in memory, or using a load function to read files on disc
or external storage. The dataset can be transformed arbitrarily with the `),Ea=o(q,"CODE",{});var Uo=r(Ea);dn=n(Uo,"map()"),Uo.forEach(t),cn=n(q," method, or methods like "),Da=o(q,"CODE",{});var Yo=r(Da);un=n(Yo,"batch()"),Yo.forEach(t),fn=n(q,`
and `),ka=o(q,"CODE",{});var Ro=r(ka);mn=n(Ro,"shuffle()"),Ro.forEach(t),gn=n(q,` can be used to create a dataset that\u2019s ready for training. These methods do not modify the stored data
in any way - instead, the methods build a data pipeline graph that will be executed when the dataset is iterated over,
usually during model training or inference. This is different from the `),Ta=o(q,"CODE",{});var Wo=r(Ta);bn=n(Wo,"map()"),Wo.forEach(t),yn=n(q," method of Hugging Face "),qa=o(q,"CODE",{});var Mo=r(qa);jn=n(Mo,"Dataset"),Mo.forEach(t),wn=n(q,` objects,
which runs the map function immediately and saves the new or changed columns.`),q.forEach(t),St=d(s),_=o(s,"P",{});var x=r(_);vn=n(x,"Since the entire data preprocessing pipeline can be compiled in a "),xa=o(x,"CODE",{});var zo=r(xa);_n=n(zo,"tf.data.Dataset"),zo.forEach(t),$n=n(x,`, this approach allows for massively
parallel, asynchronous data loading and training. However, the requirement for graph compilation can be a limitation,
particularly for Hugging Face tokenizers, which are usually not (yet!) compilable as part of a TF graph. As a result,
we usually advise pre-processing the dataset as a Hugging Face dataset, where arbitrary Python functions can be
used, and then converting to `),Aa=o(x,"CODE",{});var Bo=r(Aa);En=n(Bo,"tf.data.Dataset"),Bo.forEach(t),Dn=n(x," afterwards using "),Ca=o(x,"CODE",{});var Ko=r(Ca);kn=n(Ko,"to_tf_dataset()"),Ko.forEach(t),Tn=n(x,` to get a batched dataset ready for
training. To see examples of this approach, please see the `),As=o(x,"A",{href:!0,rel:!0});var Go=r(As);qn=n(Go,"examples"),Go.forEach(t),xn=n(x," or "),Cs=o(x,"A",{href:!0,rel:!0});var Jo=r(Cs);An=n(Jo,"notebooks"),Jo.forEach(t),Cn=n(x," for "),Oa=o(x,"CODE",{});var Qo=r(Oa);On=n(Qo,"transformers"),Qo.forEach(t),Pn=n(x,"."),x.forEach(t),Ht=d(s),Q=o(s,"H3",{class:!0});var ee=r(Q);ps=o(ee,"A",{id:!0,class:!0,href:!0});var Vo=r(ps);Pa=o(Vo,"SPAN",{});var Xo=r(Pa);m(Os.$$.fragment,Xo),Xo.forEach(t),Vo.forEach(t),Fn=d(ee),Qs=o(ee,"SPAN",{});var ao=r(Qs);Nn=n(ao,"Using "),Fa=o(ao,"CODE",{});var Zo=r(Fa);In=n(Zo,"to_tf_dataset()"),Zo.forEach(t),ao.forEach(t),ee.forEach(t),Lt=d(s),is=o(s,"P",{});var ne=r(is);Sn=n(ne,"Using "),Na=o(ne,"CODE",{});var sr=r(Na);Hn=n(sr,"to_tf_dataset()"),sr.forEach(t),Ln=n(ne," is straightforward. Once your dataset is preprocessed and ready, simply call it like so:"),ne.forEach(t),Ut=d(s),m(Ps.$$.fragment,s),Yt=d(s),F=o(s,"P",{});var us=r(F);Un=n(us,"The returned "),Ia=o(us,"CODE",{});var ar=r(Ia);Yn=n(ar,"tf_ds"),ar.forEach(t),Rn=n(us," object here is now fully ready to train on, and can be passed directly to "),Sa=o(us,"CODE",{});var tr=r(Sa);Wn=n(tr,"model.fit()"),tr.forEach(t),Mn=n(us,`! Note
that you set the batch size when creating the dataset, and so you don\u2019t need to specify it when calling `),Ha=o(us,"CODE",{});var er=r(Ha);zn=n(er,"fit()"),er.forEach(t),Bn=n(us,":"),us.forEach(t),Rt=d(s),m(Fs.$$.fragment,s),Wt=d(s),$=o(s,"P",{});var A=r($);Kn=n(A,"For a full description of the arguments, please see the "),Vs=o(A,"A",{href:!0});var nr=r(Vs);Gn=n(nr,"to_tf_dataset()"),nr.forEach(t),Jn=n(A,` documentation. In many cases,
you will also need to add a `),La=o(A,"CODE",{});var lr=r(La);Qn=n(lr,"collate_fn"),lr.forEach(t),Vn=n(A,` to your call. This is a function that takes multiple elements of the dataset
and combines them into a single batch. When all elements have the same length, the built-in default collator will
suffice, but for more complex tasks a custom collator may be necessary. In particular, many tasks have samples
with varying sequence lengths which will require a `),Ns=o(A,"A",{href:!0,rel:!0});var or=r(Ns);Xn=n(or,"data collator"),or.forEach(t),Zn=n(A,` that can pad batches correctly. You can see examples
of this in the `),Ua=o(A,"CODE",{});var rr=r(Ua);sl=n(rr,"transformers"),rr.forEach(t),al=n(A," NLP "),Is=o(A,"A",{href:!0,rel:!0});var pr=r(Is);tl=n(pr,"examples"),pr.forEach(t),el=n(A,` and
`),Ss=o(A,"A",{href:!0,rel:!0});var ir=r(Ss);nl=n(ir,"notebooks"),ir.forEach(t),ll=n(A,", where variable sequence lengths are very common."),A.forEach(t),Mt=d(s),V=o(s,"H3",{class:!0});var le=r(V);hs=o(le,"A",{id:!0,class:!0,href:!0});var hr=r(hs);Ya=o(hr,"SPAN",{});var dr=r(Ya);m(Hs.$$.fragment,dr),dr.forEach(t),hr.forEach(t),ol=d(le),Ra=o(le,"SPAN",{});var cr=r(Ra);rl=n(cr,"When to use to_tf_dataset"),cr.forEach(t),le.forEach(t),zt=d(s),u=o(s,"P",{});var k=r(u);pl=n(k,`The astute reader may have noticed at this point that we have offered two approaches to achieve the same goal - if you
want to pass your dataset to a TensorFlow model, you can either convert the dataset to a `),Wa=o(k,"CODE",{});var ur=r(Wa);il=n(ur,"Tensor"),ur.forEach(t),hl=n(k," or "),Ma=o(k,"CODE",{});var fr=r(Ma);dl=n(fr,"dict"),fr.forEach(t),cl=n(k," of "),za=o(k,"CODE",{});var mr=r(za);ul=n(mr,"Tensors"),mr.forEach(t),fl=n(k,`
using `),Ba=o(k,"CODE",{});var gr=r(Ba);ml=n(gr,".with_format('tf')"),gr.forEach(t),gl=n(k,", or you can convert the dataset to a "),Ka=o(k,"CODE",{});var br=r(Ka);bl=n(br,"tf.data.Dataset"),br.forEach(t),yl=n(k," with "),Ga=o(k,"CODE",{});var yr=r(Ga);jl=n(yr,"to_tf_dataset()"),yr.forEach(t),wl=n(k,`. Either of these
can be passed to `),Ja=o(k,"CODE",{});var jr=r(Ja);vl=n(jr,"model.fit()"),jr.forEach(t),_l=n(k,", so which should you choose?"),k.forEach(t),Bt=d(s),Y=o(s,"P",{});var sa=r(Y);$l=n(sa,"The key thing to recognize is that when you convert the whole dataset to "),Qa=o(sa,"CODE",{});var wr=r(Qa);El=n(wr,"Tensor"),wr.forEach(t),Dl=n(sa,`s, it is static and fully loaded into
RAM. This is simple and convenient, but if any of the following apply, you should probably use `),Va=o(sa,"CODE",{});var vr=r(Va);kl=n(vr,"to_tf_dataset()"),vr.forEach(t),Tl=n(sa,`
instead:`),sa.forEach(t),Kt=d(s),R=o(s,"UL",{});var aa=r(R);Ls=o(aa,"LI",{});var oe=r(Ls);ql=n(oe,"Your dataset is too large to fit in RAM. "),Xa=o(oe,"CODE",{});var _r=r(Xa);xl=n(_r,"to_tf_dataset()"),_r.forEach(t),Al=n(oe,` streams only one batch at a time, so even very large
datasets can be handled with this method.`),oe.forEach(t),Cl=d(aa),I=o(aa,"LI",{});var fs=r(I);Ol=n(fs,"You want to apply random transformations using "),Za=o(fs,"CODE",{});var $r=r(Za);Pl=n($r,"dataset.with_transform()"),$r.forEach(t),Fl=n(fs," or the "),st=o(fs,"CODE",{});var Er=r(st);Nl=n(Er,"collate_fn"),Er.forEach(t),Il=n(fs,`. This is
common in several modalities, such as image augmentations when training vision models, or random masking when training
masked language models. Using `),at=o(fs,"CODE",{});var Dr=r(at);Sl=n(Dr,"to_tf_dataset()"),Dr.forEach(t),Hl=n(fs,` will apply those transformations
at the moment when a batch is loaded, which means the same samples will get different augmentations each time
they are loaded. This is usually what you want.`),fs.forEach(t),Ll=d(aa),O=o(aa,"LI",{});var M=r(O);Ul=n(M,`Your data has a variable dimension, such as input texts in NLP that consist of varying
numbers of tokens. When you create a batch with samples with a variable dimension, the standard solution is to
pad the shorter samples to the length of the longest one. When you stream samples from a dataset with `),tt=o(M,"CODE",{});var kr=r(tt);Yl=n(kr,"to_tf_dataset"),kr.forEach(t),Rl=n(M,`,
you can apply this padding to each batch via your `),et=o(M,"CODE",{});var Tr=r(et);Wl=n(Tr,"collate_fn"),Tr.forEach(t),Ml=n(M,`. However, if you want to convert
such a dataset to dense `),nt=o(M,"CODE",{});var qr=r(nt);zl=n(qr,"Tensor"),qr.forEach(t),Bl=n(M,"s, then you will have to pad samples to the length of the longest sample in "),lt=o(M,"EM",{});var xr=r(lt);Kl=n(xr,`the
entire dataset!`),xr.forEach(t),Gl=n(M," This can result in huge amounts of padding, which wastes memory and reduces your model\u2019s speed."),M.forEach(t),aa.forEach(t),Gt=d(s),X=o(s,"H3",{class:!0});var re=r(X);ds=o(re,"A",{id:!0,class:!0,href:!0});var Ar=r(ds);ot=o(Ar,"SPAN",{});var Cr=r(ot);m(Us.$$.fragment,Cr),Cr.forEach(t),Ar.forEach(t),Jl=d(re),rt=o(re,"SPAN",{});var Or=r(rt);Ql=n(Or,"Caveats and limitations"),Or.forEach(t),re.forEach(t),Jt=d(s),cs=o(s,"P",{});var pe=r(cs);Vl=n(pe,"Right now, "),pt=o(pe,"CODE",{});var Pr=r(pt);Xl=n(Pr,"to_tf_dataset()"),Pr.forEach(t),Zl=n(pe," always return a batched dataset - we will add support for unbatched datasets soon!"),pe.forEach(t),this.h()},h(){c(w,"name","hf:doc:metadata"),c(w,"content",JSON.stringify(Rr)),c(P,"id","using-datasets-with-tensorflow"),c(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(P,"href","#using-datasets-with-tensorflow"),c(E,"class","relative group"),c(as,"id","dataset-format"),c(as,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(as,"href","#dataset-format"),c(B,"class","relative group"),c(ns,"id","ndimensional-arrays"),c(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ns,"href","#ndimensional-arrays"),c(K,"class","relative group"),c(os,"id","other-feature-types"),c(os,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(os,"href","#other-feature-types"),c(G,"class","relative group"),c(Ms,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.ClassLabel"),c(Gs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Image"),c(Js,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Audio"),c(rs,"id","data-loading"),c(rs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(rs,"href","#data-loading"),c(J,"class","relative group"),c(As,"href","https://github.com/huggingface/transformers/tree/main/examples"),c(As,"rel","nofollow"),c(Cs,"href","https://huggingface.co/docs/transformers/notebooks"),c(Cs,"rel","nofollow"),c(ps,"id","using-totfdataset"),c(ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ps,"href","#using-totfdataset"),c(Q,"class","relative group"),c(Vs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset"),c(Ns,"href","https://huggingface.co/docs/transformers/main/en/main_classes/data_collator"),c(Ns,"rel","nofollow"),c(Is,"href","https://github.com/huggingface/transformers/tree/main/examples"),c(Is,"rel","nofollow"),c(Ss,"href","https://huggingface.co/docs/transformers/notebooks"),c(Ss,"rel","nofollow"),c(hs,"id","when-to-use-totfdataset"),c(hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(hs,"href","#when-to-use-totfdataset"),c(V,"class","relative group"),c(ds,"id","caveats-and-limitations"),c(ds,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ds,"href","#caveats-and-limitations"),c(X,"class","relative group")},m(s,p){a(document.head,w),i(s,Z,p),i(s,E,p),a(E,P),a(P,z),g(C,z,null),a(E,H),a(E,ss),a(ss,ie),i(s,ht,p),i(s,T,p),a(T,he),a(T,ta),a(ta,de),a(T,ce),a(T,ea),a(ea,ue),a(T,fe),a(T,na),a(na,me),a(T,ge),a(T,la),a(la,be),a(T,ye),i(s,dt,p),i(s,B,p),a(B,as),a(as,oa),g(gs,oa,null),a(B,je),a(B,ra),a(ra,we),i(s,ct,p),i(s,Rs,p),a(Rs,ve),i(s,ut,p),i(s,ts,p),a(ts,_e),a(ts,pa),a(pa,$e),a(ts,Ee),i(s,ft,p),g(bs,s,p),i(s,mt,p),g(es,s,p),i(s,gt,p),i(s,L,p),a(L,De),a(L,ia),a(ia,ke),a(L,Te),a(L,ha),a(ha,qe),a(L,xe),i(s,bt,p),g(ys,s,p),i(s,yt,p),i(s,K,p),a(K,ns),a(ns,da),g(js,da,null),a(K,Ae),a(K,ca),a(ca,Ce),i(s,jt,p),i(s,ls,p),a(ls,Oe),a(ls,ua),a(ua,Pe),a(ls,Fe),i(s,wt,p),g(ws,s,p),i(s,vt,p),i(s,Ws,p),a(Ws,Ne),i(s,_t,p),g(vs,s,p),i(s,$t,p),i(s,G,p),a(G,os),a(os,fa),g(_s,fa,null),a(G,Ie),a(G,ma),a(ma,Se),i(s,Et,p),i(s,$s,p),a($s,Ms),a(Ms,He),a($s,Le),i(s,Dt,p),g(Es,s,p),i(s,kt,p),i(s,zs,p),a(zs,Ue),i(s,Tt,p),g(Ds,s,p),i(s,qt,p),i(s,Bs,p),a(Bs,Ye),i(s,xt,p),g(ks,s,p),i(s,At,p),i(s,Ks,p),a(Ks,Re),i(s,Ct,p),i(s,U,p),a(U,We),a(U,Gs),a(Gs,Me),a(U,ze),a(U,Js),a(Js,Be),a(U,Ke),i(s,Ot,p),g(Ts,s,p),i(s,Pt,p),g(qs,s,p),i(s,Ft,p),i(s,J,p),a(J,rs),a(rs,ga),g(xs,ga,null),a(J,Ge),a(J,ba),a(ba,Je),i(s,Nt,p),i(s,D,p),a(D,Qe),a(D,ya),a(ya,Ve),a(D,Xe),a(D,ja),a(ja,Ze),a(D,sn),a(D,wa),a(wa,an),a(D,tn),a(D,va),a(va,en),a(D,nn),a(D,_a),a(_a,ln),a(D,on),i(s,It,p),i(s,v,p),a(v,rn),a(v,$a),a($a,pn),a(v,hn),a(v,Ea),a(Ea,dn),a(v,cn),a(v,Da),a(Da,un),a(v,fn),a(v,ka),a(ka,mn),a(v,gn),a(v,Ta),a(Ta,bn),a(v,yn),a(v,qa),a(qa,jn),a(v,wn),i(s,St,p),i(s,_,p),a(_,vn),a(_,xa),a(xa,_n),a(_,$n),a(_,Aa),a(Aa,En),a(_,Dn),a(_,Ca),a(Ca,kn),a(_,Tn),a(_,As),a(As,qn),a(_,xn),a(_,Cs),a(Cs,An),a(_,Cn),a(_,Oa),a(Oa,On),a(_,Pn),i(s,Ht,p),i(s,Q,p),a(Q,ps),a(ps,Pa),g(Os,Pa,null),a(Q,Fn),a(Q,Qs),a(Qs,Nn),a(Qs,Fa),a(Fa,In),i(s,Lt,p),i(s,is,p),a(is,Sn),a(is,Na),a(Na,Hn),a(is,Ln),i(s,Ut,p),g(Ps,s,p),i(s,Yt,p),i(s,F,p),a(F,Un),a(F,Ia),a(Ia,Yn),a(F,Rn),a(F,Sa),a(Sa,Wn),a(F,Mn),a(F,Ha),a(Ha,zn),a(F,Bn),i(s,Rt,p),g(Fs,s,p),i(s,Wt,p),i(s,$,p),a($,Kn),a($,Vs),a(Vs,Gn),a($,Jn),a($,La),a(La,Qn),a($,Vn),a($,Ns),a(Ns,Xn),a($,Zn),a($,Ua),a(Ua,sl),a($,al),a($,Is),a(Is,tl),a($,el),a($,Ss),a(Ss,nl),a($,ll),i(s,Mt,p),i(s,V,p),a(V,hs),a(hs,Ya),g(Hs,Ya,null),a(V,ol),a(V,Ra),a(Ra,rl),i(s,zt,p),i(s,u,p),a(u,pl),a(u,Wa),a(Wa,il),a(u,hl),a(u,Ma),a(Ma,dl),a(u,cl),a(u,za),a(za,ul),a(u,fl),a(u,Ba),a(Ba,ml),a(u,gl),a(u,Ka),a(Ka,bl),a(u,yl),a(u,Ga),a(Ga,jl),a(u,wl),a(u,Ja),a(Ja,vl),a(u,_l),i(s,Bt,p),i(s,Y,p),a(Y,$l),a(Y,Qa),a(Qa,El),a(Y,Dl),a(Y,Va),a(Va,kl),a(Y,Tl),i(s,Kt,p),i(s,R,p),a(R,Ls),a(Ls,ql),a(Ls,Xa),a(Xa,xl),a(Ls,Al),a(R,Cl),a(R,I),a(I,Ol),a(I,Za),a(Za,Pl),a(I,Fl),a(I,st),a(st,Nl),a(I,Il),a(I,at),a(at,Sl),a(I,Hl),a(R,Ll),a(R,O),a(O,Ul),a(O,tt),a(tt,Yl),a(O,Rl),a(O,et),a(et,Wl),a(O,Ml),a(O,nt),a(nt,zl),a(O,Bl),a(O,lt),a(lt,Kl),a(O,Gl),i(s,Gt,p),i(s,X,p),a(X,ds),a(ds,ot),g(Us,ot,null),a(X,Jl),a(X,rt),a(rt,Ql),i(s,Jt,p),i(s,cs,p),a(cs,Vl),a(cs,pt),a(pt,Xl),a(cs,Zl),Qt=!0},p(s,[p]){const Ys={};p&2&&(Ys.$$scope={dirty:p,ctx:s}),es.$set(Ys)},i(s){Qt||(b(C.$$.fragment,s),b(gs.$$.fragment,s),b(bs.$$.fragment,s),b(es.$$.fragment,s),b(ys.$$.fragment,s),b(js.$$.fragment,s),b(ws.$$.fragment,s),b(vs.$$.fragment,s),b(_s.$$.fragment,s),b(Es.$$.fragment,s),b(Ds.$$.fragment,s),b(ks.$$.fragment,s),b(Ts.$$.fragment,s),b(qs.$$.fragment,s),b(xs.$$.fragment,s),b(Os.$$.fragment,s),b(Ps.$$.fragment,s),b(Fs.$$.fragment,s),b(Hs.$$.fragment,s),b(Us.$$.fragment,s),Qt=!0)},o(s){y(C.$$.fragment,s),y(gs.$$.fragment,s),y(bs.$$.fragment,s),y(es.$$.fragment,s),y(ys.$$.fragment,s),y(js.$$.fragment,s),y(ws.$$.fragment,s),y(vs.$$.fragment,s),y(_s.$$.fragment,s),y(Es.$$.fragment,s),y(Ds.$$.fragment,s),y(ks.$$.fragment,s),y(Ts.$$.fragment,s),y(qs.$$.fragment,s),y(xs.$$.fragment,s),y(Os.$$.fragment,s),y(Ps.$$.fragment,s),y(Fs.$$.fragment,s),y(Hs.$$.fragment,s),y(Us.$$.fragment,s),Qt=!1},d(s){t(w),s&&t(Z),s&&t(E),j(C),s&&t(ht),s&&t(T),s&&t(dt),s&&t(B),j(gs),s&&t(ct),s&&t(Rs),s&&t(ut),s&&t(ts),s&&t(ft),j(bs,s),s&&t(mt),j(es,s),s&&t(gt),s&&t(L),s&&t(bt),j(ys,s),s&&t(yt),s&&t(K),j(js),s&&t(jt),s&&t(ls),s&&t(wt),j(ws,s),s&&t(vt),s&&t(Ws),s&&t(_t),j(vs,s),s&&t($t),s&&t(G),j(_s),s&&t(Et),s&&t($s),s&&t(Dt),j(Es,s),s&&t(kt),s&&t(zs),s&&t(Tt),j(Ds,s),s&&t(qt),s&&t(Bs),s&&t(xt),j(ks,s),s&&t(At),s&&t(Ks),s&&t(Ct),s&&t(U),s&&t(Ot),j(Ts,s),s&&t(Pt),j(qs,s),s&&t(Ft),s&&t(J),j(xs),s&&t(Nt),s&&t(D),s&&t(It),s&&t(v),s&&t(St),s&&t(_),s&&t(Ht),s&&t(Q),j(Os),s&&t(Lt),s&&t(is),s&&t(Ut),j(Ps,s),s&&t(Yt),s&&t(F),s&&t(Rt),j(Fs,s),s&&t(Wt),s&&t($),s&&t(Mt),s&&t(V),j(Hs),s&&t(zt),s&&t(u),s&&t(Bt),s&&t(Y),s&&t(Kt),s&&t(R),s&&t(Gt),s&&t(X),j(Us),s&&t(Jt),s&&t(cs)}}}const Rr={local:"using-datasets-with-tensorflow",sections:[{local:"dataset-format",title:"Dataset format"},{local:"ndimensional-arrays",title:"N-dimensional arrays"},{local:"other-feature-types",title:"Other feature types"},{local:"data-loading",sections:[{local:"using-totfdataset",title:"Using `to_tf_dataset()`"},{local:"when-to-use-totfdataset",title:"When to use to_tf_dataset"},{local:"caveats-and-limitations",title:"Caveats and limitations"}],title:"Data loading"}],title:"Using Datasets with TensorFlow"};function Wr(it){return Hr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Gr extends Fr{constructor(w){super();Nr(this,w,Wr,Yr,Ir,{})}}export{Gr as default,Rr as metadata};
