import{S as Qi,i as Xi,s as Zi,e as n,k as c,w as _,t,M as eh,c as r,d as a,m as f,a as o,x as v,h as l,b as h,N as Ki,G as s,g as i,y as j,q as b,o as $,B as x,v as sh,L as ah}from"../chunks/vendor-hf-doc-builder.js";import{T as xt}from"../chunks/Tip-hf-doc-builder.js";import{I as N}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as I}from"../chunks/CodeBlock-hf-doc-builder.js";import{F as th,M as lh}from"../chunks/Markdown-hf-doc-builder.js";import"../chunks/IconTensorflow-hf-doc-builder.js";function nh(q){let d,w,m,y,k,g,E,u;return{c(){d=n("p"),w=t("An "),m=n("a"),y=t("IterableDataset"),k=t(" is useful for iterative jobs like training a model. You shouldn\u2019t use a "),g=n("a"),E=t("IterableDataset"),u=t(" for jobs that require random access to examples because you have to iterate all over it using a for loop. Getting the last example in an iterable dataset would require you to iterate over all the previous examples."),this.h()},l(D){d=r(D,"P",{});var A=o(d);w=l(A,"An "),m=r(A,"A",{href:!0});var T=o(m);y=l(T,"IterableDataset"),T.forEach(a),k=l(A," is useful for iterative jobs like training a model. You shouldn\u2019t use a "),g=r(A,"A",{href:!0});var M=o(g);E=l(M,"IterableDataset"),M.forEach(a),u=l(A," for jobs that require random access to examples because you have to iterate all over it using a for loop. Getting the last example in an iterable dataset would require you to iterate over all the previous examples."),A.forEach(a),this.h()},h(){h(m,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset"),h(g,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset")},m(D,A){i(D,d,A),s(d,w),s(d,m),s(m,y),s(d,k),s(d,g),s(g,E),s(d,u)},d(D){D&&a(d)}}}function rh(q){let d,w,m,y;return{c(){d=n("p"),w=n("a"),m=t("IterableDataset.shuffle()"),y=t(" will also shuffle the order of the shards if the dataset is sharded into multiple files."),this.h()},l(k){d=r(k,"P",{});var g=o(d);w=r(g,"A",{href:!0});var E=o(w);m=l(E,"IterableDataset.shuffle()"),E.forEach(a),y=l(g," will also shuffle the order of the shards if the dataset is sharded into multiple files."),g.forEach(a),this.h()},h(){h(w,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.shuffle")},m(k,g){i(k,d,g),s(d,w),s(w,m),s(d,y)},d(k){k&&a(d)}}}function oh(q){let d,w,m,y,k,g,E,u,D,A,T,M,de;return{c(){d=n("p"),w=n("code"),m=t("take"),y=t(" and "),k=n("code"),g=t("skip"),E=t(" prevent future calls to "),u=n("code"),D=t("shuffle"),A=t(" because they lock in the order of the shards. You should "),T=n("code"),M=t("shuffle"),de=t(" your dataset before splitting it.")},l(S){d=r(S,"P",{});var P=o(d);w=r(P,"CODE",{});var Is=o(w);m=l(Is,"take"),Is.forEach(a),y=l(P," and "),k=r(P,"CODE",{});var Ts=o(k);g=l(Ts,"skip"),Ts.forEach(a),E=l(P," prevent future calls to "),u=r(P,"CODE",{});var me=o(u);D=l(me,"shuffle"),me.forEach(a),A=l(P," because they lock in the order of the shards. You should "),T=r(P,"CODE",{});var Ps=o(T);M=l(Ps,"shuffle"),Ps.forEach(a),de=l(P," your dataset before splitting it."),P.forEach(a)},m(S,P){i(S,d,P),s(d,w),s(w,m),s(d,y),s(d,k),s(k,g),s(d,E),s(d,u),s(u,D),s(d,A),s(d,T),s(T,M),s(d,de)},d(S){S&&a(d)}}}function ph(q){let d,w,m,y,k,g,E,u;return{c(){d=n("p"),w=t("Casting only works if the original feature type and new feature type are compatible. For example, you can cast a column with the feature type "),m=n("code"),y=t("Value('int32')"),k=t(" to "),g=n("code"),E=t("Value('bool')"),u=t(" if the original column only contains ones and zeros.")},l(D){d=r(D,"P",{});var A=o(d);w=l(A,"Casting only works if the original feature type and new feature type are compatible. For example, you can cast a column with the feature type "),m=r(A,"CODE",{});var T=o(m);y=l(T,"Value('int32')"),T.forEach(a),k=l(A," to "),g=r(A,"CODE",{});var M=o(g);E=l(M,"Value('bool')"),M.forEach(a),u=l(A," if the original column only contains ones and zeros."),A.forEach(a)},m(D,A){i(D,d,A),s(d,w),s(d,m),s(m,y),s(d,k),s(d,g),s(g,E),s(d,u)},d(D){D&&a(d)}}}function ih(q){let d,w,m,y,k;return{c(){d=n("p"),w=t("See other examples of batch processing in the "),m=n("a"),y=t("batched map processing"),k=t(" documentation. They work the same for iterable datasets."),this.h()},l(g){d=r(g,"P",{});var E=o(d);w=l(E,"See other examples of batch processing in the "),m=r(E,"A",{href:!0});var u=o(m);y=l(u,"batched map processing"),u.forEach(a),k=l(E," documentation. They work the same for iterable datasets."),E.forEach(a),this.h()},h(){h(m,"href","./process#batch-processing")},m(g,E){i(g,d,E),s(d,w),s(d,m),s(m,y),s(d,k)},d(g){g&&a(d)}}}function hh(q){let d,w,m,y,k,g,E;return d=new I({props:{code:`seed, buffer_size = 42, 10_000
dataset = dataset.shuffle(seed, buffer_size=buffer_size)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>seed, buffer_size = <span class="hljs-number">42</span>, <span class="hljs-number">10_000</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.shuffle(seed, buffer_size=buffer_size)`}}),g=new I({props:{code:`import torch
from torch.utils.data import DataLoader
from transformers import AutoModelForMaskedLM, DataCollatorForLanguageModeling
from tqdm import tqdm
dataset = dataset.with_format("torch")
dataloader = DataLoader(dataset, collate_fn=DataCollatorForLanguageModeling(tokenizer))
device = 'cuda' if torch.cuda.is_available() else 'cpu' 
model = AutoModelForMaskedLM.from_pretrained("distilbert-base-uncased")
model.train().to(device)
optimizer = torch.optim.AdamW(params=model.parameters(), lr=1e-5)
for epoch in range(3):
    dataset.set_epoch(epoch)
    for i, batch in enumerate(tqdm(dataloader, total=5)):
        if i == 5:
            break
        batch = {k: v.to(device) for k, v in batch.items()}
        outputs = model(**batch)
        loss = outputs[0]
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        if i % 10 == 0:
            print(f"loss: {loss}")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM, DataCollatorForLanguageModeling
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(dataset, collate_fn=DataCollatorForLanguageModeling(tokenizer))
<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&#x27;cuda&#x27;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&#x27;cpu&#x27;</span> 
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.train().to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = torch.optim.AdamW(params=model.parameters(), lr=<span class="hljs-number">1e-5</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>):
<span class="hljs-meta">... </span>    dataset.set_epoch(epoch)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(tqdm(dataloader, total=<span class="hljs-number">5</span>)):
<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> i == <span class="hljs-number">5</span>:
<span class="hljs-meta">... </span>            <span class="hljs-keyword">break</span>
<span class="hljs-meta">... </span>        batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
<span class="hljs-meta">... </span>        outputs = model(**batch)
<span class="hljs-meta">... </span>        loss = outputs[<span class="hljs-number">0</span>]
<span class="hljs-meta">... </span>        loss.backward()
<span class="hljs-meta">... </span>        optimizer.step()
<span class="hljs-meta">... </span>        optimizer.zero_grad()
<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> i % <span class="hljs-number">10</span> == <span class="hljs-number">0</span>:
<span class="hljs-meta">... </span>            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;loss: <span class="hljs-subst">{loss}</span>&quot;</span>)`}}),{c(){_(d.$$.fragment),w=c(),m=n("p"),y=t("Lastly, create a simple training loop and start training:"),k=c(),_(g.$$.fragment)},l(u){v(d.$$.fragment,u),w=f(u),m=r(u,"P",{});var D=o(m);y=l(D,"Lastly, create a simple training loop and start training:"),D.forEach(a),k=f(u),v(g.$$.fragment,u)},m(u,D){j(d,u,D),i(u,w,D),i(u,m,D),s(m,y),i(u,k,D),j(g,u,D),E=!0},p:ah,i(u){E||(b(d.$$.fragment,u),b(g.$$.fragment,u),E=!0)},o(u){$(d.$$.fragment,u),$(g.$$.fragment,u),E=!1},d(u){x(d,u),u&&a(w),u&&a(m),u&&a(k),x(g,u)}}}function ch(q){let d,w;return d=new lh({props:{$$slots:{default:[hh]},$$scope:{ctx:q}}}),{c(){_(d.$$.fragment)},l(m){v(d.$$.fragment,m)},m(m,y){j(d,m,y),w=!0},p(m,y){const k={};y&2&&(k.$$scope={dirty:y,ctx:m}),d.$set(k)},i(m){w||(b(d.$$.fragment,m),w=!0)},o(m){$(d.$$.fragment,m),w=!1},d(m){x(d,m)}}}function fh(q){let d,w,m,y,k,g,E,u,D,A,T,M,de,S,P,Is,Ts,me,Ps,on,ja,pn,wt,G,qs,tp,hn,Ss,lp,yt,O,cn,Ge,fn,dn,ba,mn,un,Cs,gn,_n,kt,Ke,Et,H,vn,zs,jn,bn,Ns,$n,xn,Dt,ue,At,K,ge,$a,Qe,wn,xa,yn,It,V,kn,Ms,En,Dn,Ls,An,In,Os,Tn,Pn,Tt,F,qn,wa,Sn,Cn,ya,zn,Nn,Vs,Mn,Ln,Pt,Xe,qt,_e,St,Q,ve,ka,Ze,On,Ea,Vn,Ct,je,Fn,Da,Rn,Yn,zt,be,Hn,Aa,Un,Bn,Nt,es,Mt,X,$e,Ia,ss,Jn,Ta,Wn,Lt,Fs,Gn,Ot,Rs,xe,Ys,Kn,Qn,Pa,Xn,Zn,Vt,as,Ft,Hs,we,Us,er,sr,qa,ar,tr,Rt,ts,Yt,ye,Ht,Bs,Ut,Z,ke,Sa,ls,lr,Ca,nr,Bt,ee,Js,rr,or,Ws,pr,ir,Jt,ns,Wt,Ee,hr,za,cr,fr,Gt,rs,Kt,U,dr,Na,mr,ur,Ma,gr,_r,Qt,z,vr,La,jr,br,Oa,$r,xr,Va,wr,yr,Fa,kr,Er,Xt,se,De,Ra,os,Dr,Ya,Ar,Zt,Gs,Ir,el,ae,Ae,Ha,ps,Tr,Ua,Pr,sl,Ie,qr,Ks,Sr,Cr,al,Te,zr,Qs,Nr,Mr,tl,is,ll,te,Pe,Ba,hs,Lr,Ja,Or,nl,qe,Vr,Xs,Fr,Rr,rl,cs,ol,le,Se,Wa,fs,Yr,Ga,Hr,pl,L,Zs,Ur,Br,Ka,Jr,Wr,Qa,Gr,Kr,Xa,Qr,Xr,il,ds,hl,Ce,cl,ze,Zr,ea,eo,so,fl,ms,dl,ne,Ne,Za,us,ao,et,to,ml,C,lo,sa,no,ro,aa,oo,po,ta,io,ho,la,co,fo,na,mo,uo,ul,ra,go,gl,B,_o,oa,vo,jo,st,bo,$o,_l,gs,vl,Me,xo,pa,wo,yo,jl,_s,bl,Le,ko,ia,Eo,Do,$l,J,Ao,at,Io,To,ha,Po,qo,xl,vs,wl,re,Oe,tt,js,So,lt,Co,yl,Y,ca,zo,No,nt,Mo,Lo,rt,Oo,Vo,kl,oe,Ve,ot,bs,Fo,pt,Ro,El,$s,Dl,Fe,Al,pe,Re,it,xs,Yo,ht,Ho,Il,Ye,Uo,fa,Bo,Jo,Tl,ws,Pl,ie,da,Wo,Go,ct,Ko,Qo,ql,ys,Sl,he,He,ft,ks,Xo,dt,Zo,Cl,Es,ma,ep,sp,zl,Ue,Nl;return g=new N({}),Ke=new I({props:{code:`from datasets import load_dataset
dataset = load_dataset('oscar', "unshuffled_deduplicated_en", split='train', streaming=True)
print(next(iter(dataset)))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;oscar&#x27;</span>, <span class="hljs-string">&quot;unshuffled_deduplicated_en&quot;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(dataset)))
{<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Mtendere Village was inspired by the vision of Chief Napoleon Dzombe, which he shared with John Blanchard during his first visit to Malawi. Chief Napoleon conveyed the desperate need for a program to intervene and care for the orphans and vulnerable children (OVC) in Malawi, and John committed to help...</span>`}}),ue=new xt({props:{$$slots:{default:[nh]},$$scope:{ctx:q}}}),Qe=new N({}),Xe=new I({props:{code:`from datasets import load_dataset
dataset = load_dataset('oscar', "unshuffled_deduplicated_en", split='train', streaming=True)
shuffled_dataset = dataset.shuffle(seed=42, buffer_size=10_000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;oscar&#x27;</span>, <span class="hljs-string">&quot;unshuffled_deduplicated_en&quot;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>shuffled_dataset = dataset.shuffle(seed=<span class="hljs-number">42</span>, buffer_size=<span class="hljs-number">10_000</span>)`}}),_e=new xt({props:{$$slots:{default:[rh]},$$scope:{ctx:q}}}),Ze=new N({}),es=new I({props:{code:`for epoch in range(epochs):
    shuffled_dataset.set_epoch(epoch)
    for example in shuffled_dataset:
        ...`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(epochs):
<span class="hljs-meta">... </span>    shuffled_dataset.set_epoch(epoch)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> shuffled_dataset:
<span class="hljs-meta">... </span>        ...`}}),ss=new N({}),as=new I({props:{code:`dataset = load_dataset('oscar', "unshuffled_deduplicated_en", split='train', streaming=True)
dataset_head = dataset.take(2)
list(dataset_head)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;oscar&#x27;</span>, <span class="hljs-string">&quot;unshuffled_deduplicated_en&quot;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset_head = dataset.take(<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">list</span>(dataset_head)
[{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Mtendere Village was...&#x27;</span>}, {<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Lily James cannot fight the music...&#x27;</span>}]`}}),ts=new I({props:{code:"train_dataset = shuffled_dataset.skip(1000)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>train_dataset = shuffled_dataset.skip(<span class="hljs-number">1000</span>)'}}),ye=new xt({props:{warning:!0,$$slots:{default:[oh]},$$scope:{ctx:q}}}),ls=new N({}),ns=new I({props:{code:`from datasets import interleave_datasets
en_dataset = load_dataset('oscar', "unshuffled_deduplicated_en", split='train', streaming=True)
fr_dataset = load_dataset('oscar', "unshuffled_deduplicated_fr", split='train', streaming=True)

multilingual_dataset = interleave_datasets([en_dataset, fr_dataset])
list(multilingual_dataset.take(2))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> interleave_datasets
<span class="hljs-meta">&gt;&gt;&gt; </span>en_dataset = load_dataset(<span class="hljs-string">&#x27;oscar&#x27;</span>, <span class="hljs-string">&quot;unshuffled_deduplicated_en&quot;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>fr_dataset = load_dataset(<span class="hljs-string">&#x27;oscar&#x27;</span>, <span class="hljs-string">&quot;unshuffled_deduplicated_fr&quot;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>, streaming=<span class="hljs-literal">True</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>multilingual_dataset = interleave_datasets([en_dataset, fr_dataset])
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">list</span>(multilingual_dataset.take(<span class="hljs-number">2</span>))
[{<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Mtendere Village was inspired by the vision...&#x27;</span>}, {<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&quot;M\xE9dia de d\xE9bat d&#x27;id\xE9es, de culture et de litt\xE9rature...&quot;</span>}]`}}),rs=new I({props:{code:`multilingual_dataset_with_oversampling = interleave_datasets([en_dataset, fr_dataset], probabilities=[0.8, 0.2], seed=42)
list(multilingual_dataset_with_oversampling.take(2))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>multilingual_dataset_with_oversampling = interleave_datasets([en_dataset, fr_dataset], probabilities=[<span class="hljs-number">0.8</span>, <span class="hljs-number">0.2</span>], seed=<span class="hljs-number">42</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">list</span>(multilingual_dataset_with_oversampling.take(<span class="hljs-number">2</span>))
[{<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Mtendere Village was inspired by the vision...&#x27;</span>}, {<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Lily James cannot fight the music...&#x27;</span>}]`}}),os=new N({}),ps=new N({}),is=new I({props:{code:`from datasets import load_dataset
dataset = load_dataset('mc4', 'en', streaming=True, split='train')
dataset = dataset.rename_column("text", "content")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;mc4&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.rename_column(<span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>)`}}),hs=new N({}),cs=new I({props:{code:`from datasets import load_dataset
dataset = load_dataset('mc4', 'en', streaming=True, split='train')
dataset = dataset.remove_columns('timestamp')`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;mc4&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.remove_columns(<span class="hljs-string">&#x27;timestamp&#x27;</span>)`}}),fs=new N({}),ds=new I({props:{code:`from datasets import load_dataset
dataset = load_dataset('glue', 'mrpc', split='train')
dataset.features

from datasets import ClassLabel, Value
new_features = dataset.features.copy()
new_features["label"] = ClassLabel(names=['negative', 'positive'])
new_features["idx"] = Value('int64')
dataset = dataset.cast(new_features)
dataset.features`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;glue&#x27;</span>, <span class="hljs-string">&#x27;mrpc&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.features
{<span class="hljs-string">&#x27;sentence1&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
<span class="hljs-string">&#x27;sentence2&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
<span class="hljs-string">&#x27;label&#x27;</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">&#x27;not_equivalent&#x27;</span>, <span class="hljs-string">&#x27;equivalent&#x27;</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
<span class="hljs-string">&#x27;idx&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> ClassLabel, Value
<span class="hljs-meta">&gt;&gt;&gt; </span>new_features = dataset.features.copy()
<span class="hljs-meta">&gt;&gt;&gt; </span>new_features[<span class="hljs-string">&quot;label&quot;</span>] = ClassLabel(names=[<span class="hljs-string">&#x27;negative&#x27;</span>, <span class="hljs-string">&#x27;positive&#x27;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>new_features[<span class="hljs-string">&quot;idx&quot;</span>] = Value(<span class="hljs-string">&#x27;int64&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast(new_features)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.features
{<span class="hljs-string">&#x27;sentence1&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
<span class="hljs-string">&#x27;sentence2&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
<span class="hljs-string">&#x27;label&#x27;</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">&#x27;negative&#x27;</span>, <span class="hljs-string">&#x27;positive&#x27;</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
<span class="hljs-string">&#x27;idx&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int64&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`}}),Ce=new xt({props:{$$slots:{default:[ph]},$$scope:{ctx:q}}}),ms=new I({props:{code:`dataset.features

dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
dataset.features`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.features
{<span class="hljs-string">&#x27;audio&#x27;</span>: Audio(sampling_rate=<span class="hljs-number">44100</span>, mono=<span class="hljs-literal">True</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.features
{<span class="hljs-string">&#x27;audio&#x27;</span>: Audio(sampling_rate=<span class="hljs-number">16000</span>, mono=<span class="hljs-literal">True</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`}}),us=new N({}),gs=new I({props:{code:`def add_prefix(example):
    example['text'] = 'My text: ' + example['text']
    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">add_prefix</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    example[<span class="hljs-string">&#x27;text&#x27;</span>] = <span class="hljs-string">&#x27;My text: &#x27;</span> + example[<span class="hljs-string">&#x27;text&#x27;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),_s=new I({props:{code:`from datasets import load_dataset
dataset = load_dataset('oscar', 'unshuffled_deduplicated_en', streaming=True, split='train')
updated_dataset = dataset.map(add_prefix)
list(updated_dataset.take(3))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;oscar&#x27;</span>, <span class="hljs-string">&#x27;unshuffled_deduplicated_en&#x27;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset = dataset.<span class="hljs-built_in">map</span>(add_prefix)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">list</span>(updated_dataset.take(<span class="hljs-number">3</span>))
[{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My text: Mtendere Village was inspired by...&#x27;</span>},
 {<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My text: Lily James cannot fight the music...&#x27;</span>},
 {<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My text: &quot;I\\&#x27;d love to help kickstart...&#x27;</span>}]`}}),vs=new I({props:{code:`updated_dataset = dataset.map(add_prefix, remove_columns=["id"])
list(updated_dataset.take(3))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset = dataset.<span class="hljs-built_in">map</span>(add_prefix, remove_columns=[<span class="hljs-string">&quot;id&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">list</span>(updated_dataset.take(<span class="hljs-number">3</span>))
[{<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My text: Mtendere Village was inspired by...&#x27;</span>},
 {<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My text: Lily James cannot fight the music...&#x27;</span>},
 {<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My text: &quot;I\\&#x27;d love to help kickstart...&#x27;</span>}]`}}),js=new N({}),bs=new N({}),$s=new I({props:{code:`from datasets import load_dataset
from transformers import AutoTokenizer
dataset = load_dataset("mc4", "en", streaming=True, split="train")
tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
def encode(examples):
    return tokenizer(examples['text'], truncation=True, padding='max_length')
dataset = dataset.map(encode, batched=True, remove_columns=["text", "timestamp", "url"])
next(iter(dataset))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;mc4&quot;</span>, <span class="hljs-string">&quot;en&quot;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&#x27;distilbert-base-uncased&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">encode</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&#x27;text&#x27;</span>], truncation=<span class="hljs-literal">True</span>, padding=<span class="hljs-string">&#x27;max_length&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(encode, batched=<span class="hljs-literal">True</span>, remove_columns=[<span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;timestamp&quot;</span>, <span class="hljs-string">&quot;url&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(dataset))
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: <span class="hljs-number">101</span>, <span class="hljs-number">8466</span>, <span class="hljs-number">1018</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">4029</span>, <span class="hljs-number">2475</span>, <span class="hljs-number">2062</span>, <span class="hljs-number">18558</span>, <span class="hljs-number">3100</span>, <span class="hljs-number">2061</span>, ...,<span class="hljs-number">1106</span>, <span class="hljs-number">3739</span>, <span class="hljs-number">102</span>],
<span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Fe=new xt({props:{$$slots:{default:[ih]},$$scope:{ctx:q}}}),xs=new N({}),ws=new I({props:{code:`from datasets import load_dataset
dataset = load_dataset('oscar', 'unshuffled_deduplicated_en', streaming=True, split='train')
start_with_ar = dataset.filter(lambda example: example['text'].startswith('Ar'))
next(iter(start_with_ar))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;oscar&#x27;</span>, <span class="hljs-string">&#x27;unshuffled_deduplicated_en&#x27;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>start_with_ar = dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> example: example[<span class="hljs-string">&#x27;text&#x27;</span>].startswith(<span class="hljs-string">&#x27;Ar&#x27;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(start_with_ar))
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Are you looking for Number the Stars (Essential Modern Classics)?...&#x27;</span>}`}}),ys=new I({props:{code:`even_dataset = dataset.filter(lambda example, idx: idx % 2 == 0, with_indices=True)
list(even_dataset.take(3))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>even_dataset = dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> example, idx: idx % <span class="hljs-number">2</span> == <span class="hljs-number">0</span>, with_indices=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">list</span>(even_dataset.take(<span class="hljs-number">3</span>))
[{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Mtendere Village was inspired by the vision of Chief Napoleon Dzombe, ...&#x27;</span>},
 {<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;&quot;I\\&#x27;d love to help kickstart continued development! And 0 EUR/month...&#x27;</span>},
 {<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Are you looking for Number the Stars (Essential Modern Classics)? Normally, ...&#x27;</span>}]`}}),ks=new N({}),Ue=new th({props:{pytorch:!0,tensorflow:!1,jax:!1,$$slots:{pytorch:[ch]},$$scope:{ctx:q}}}),{c(){d=n("meta"),w=c(),m=n("h1"),y=n("a"),k=n("span"),_(g.$$.fragment),E=c(),u=n("span"),D=t("Stream"),A=c(),T=n("p"),M=t("Dataset streaming lets you work with a dataset without downloading it. The data is streamed as you iterate over the dataset. This is especially helpful when:"),de=c(),S=n("ul"),P=n("li"),Is=t("You don\u2019t want to wait for an extremely large dataset to download."),Ts=c(),me=n("li"),Ps=t("The dataset size exceeds the amount of disk space on your computer."),on=c(),ja=n("li"),pn=t("You want to quickly explore just a few samples of a dataset."),wt=c(),G=n("div"),qs=n("img"),hn=c(),Ss=n("img"),yt=c(),O=n("p"),cn=t("For example, the English split of the "),Ge=n("a"),fn=t("OSCAR"),dn=t(" dataset is 1.2 terabytes, but you can use it instantly with streaming. Stream a dataset by setting "),ba=n("code"),mn=t("streaming=True"),un=t(" in "),Cs=n("a"),gn=t("load_dataset()"),_n=t(" as shown below:"),kt=c(),_(Ke.$$.fragment),Et=c(),H=n("p"),vn=t("Loading a dataset in streaming mode creates a new dataset type instance (instead of the classic "),zs=n("a"),jn=t("Dataset"),bn=t(" object), known as an "),Ns=n("a"),$n=t("IterableDataset"),xn=t(". This special type of dataset has its own set of processing methods shown below."),Dt=c(),_(ue.$$.fragment),At=c(),K=n("h2"),ge=n("a"),$a=n("span"),_(Qe.$$.fragment),wn=c(),xa=n("span"),yn=t("Shuffle"),It=c(),V=n("p"),kn=t("Like a regular "),Ms=n("a"),En=t("Dataset"),Dn=t(" object, you can also shuffle a "),Ls=n("a"),An=t("IterableDataset"),In=t(" with "),Os=n("a"),Tn=t("IterableDataset.shuffle()"),Pn=t("."),Tt=c(),F=n("p"),qn=t("The "),wa=n("code"),Sn=t("buffer_size"),Cn=t(" argument controls the size of the buffer to randomly sample examples from. Let\u2019s say your dataset has one million examples, and you set the "),ya=n("code"),zn=t("buffer_size"),Nn=t(" to ten thousand. "),Vs=n("a"),Mn=t("IterableDataset.shuffle()"),Ln=t(" will randomly select examples from the first ten thousand examples in the buffer. Selected examples in the buffer are replaced with new examples. By default, the buffer size is 1,000."),Pt=c(),_(Xe.$$.fragment),qt=c(),_(_e.$$.fragment),St=c(),Q=n("h2"),ve=n("a"),ka=n("span"),_(Ze.$$.fragment),On=c(),Ea=n("span"),Vn=t("Reshuffle"),Ct=c(),je=n("p"),Fn=t("Sometimes you may want to reshuffle the dataset after each epoch. This will require you to set a different seed for each epoch. Use "),Da=n("code"),Rn=t("IterableDataset.set_epoch()"),Yn=t(" in between epochs to tell the dataset what epoch you\u2019re on."),zt=c(),be=n("p"),Hn=t("Your seed effectively becomes: "),Aa=n("code"),Un=t("initial seed + current epoch"),Bn=t("."),Nt=c(),_(es.$$.fragment),Mt=c(),X=n("h2"),$e=n("a"),Ia=n("span"),_(ss.$$.fragment),Jn=c(),Ta=n("span"),Wn=t("Split dataset"),Lt=c(),Fs=n("p"),Gn=t("You can split your dataset one of two ways:"),Ot=c(),Rs=n("ul"),xe=n("li"),Ys=n("a"),Kn=t("IterableDataset.take()"),Qn=t(" returns the first "),Pa=n("code"),Xn=t("n"),Zn=t(" examples in a dataset:"),Vt=c(),_(as.$$.fragment),Ft=c(),Hs=n("ul"),we=n("li"),Us=n("a"),er=t("IterableDataset.skip()"),sr=t(" omits the first "),qa=n("code"),ar=t("n"),tr=t(" examples in a dataset and returns the remaining examples:"),Rt=c(),_(ts.$$.fragment),Yt=c(),_(ye.$$.fragment),Ht=c(),Bs=n("a"),Ut=c(),Z=n("h2"),ke=n("a"),Sa=n("span"),_(ls.$$.fragment),lr=c(),Ca=n("span"),nr=t("Interleave"),Bt=c(),ee=n("p"),Js=n("a"),rr=t("interleave_datasets()"),or=t(" can combine an "),Ws=n("a"),pr=t("IterableDataset"),ir=t(" with other datasets. The combined dataset returns alternating examples from each of the original datasets."),Jt=c(),_(ns.$$.fragment),Wt=c(),Ee=n("p"),hr=t("Define sampling probabilities from each of the original datasets for more control over how each of them are sampled and combined. Set the "),za=n("code"),cr=t("probabilities"),fr=t(" argument with your desired sampling probabilities:"),Gt=c(),_(rs.$$.fragment),Kt=c(),U=n("p"),dr=t("Around 80% of the final dataset is made of the "),Na=n("code"),mr=t("en_dataset"),ur=t(", and 20% of the "),Ma=n("code"),gr=t("fr_dataset"),_r=t("."),Qt=c(),z=n("p"),vr=t("You can also specify the "),La=n("code"),jr=t("stopping_strategy"),br=t(". The default strategy, "),Oa=n("code"),$r=t("first_exhausted"),xr=t(`, is a subsampling strategy, i.e the dataset construction is stopped as soon one of the dataset runs out of samples.
You can specify `),Va=n("code"),wr=t("stopping_strategy=all_exhausted"),yr=t(` to execute an oversampling strategy. In this case, the dataset construction is stopped as soon as every samples in every dataset has been added at least once. In practice, it means that if a dataset is exhausted, it will return to the beginning of this dataset until the stop criterion has been reached.
Note that if no sampling probabilities are specified, the new dataset will have `),Fa=n("code"),kr=t("max_length_datasets*nb_dataset samples"),Er=t("."),Xt=c(),se=n("h2"),De=n("a"),Ra=n("span"),_(os.$$.fragment),Dr=c(),Ya=n("span"),Ar=t("Rename, remove, and cast"),Zt=c(),Gs=n("p"),Ir=t("The following methods allow you to modify the columns of a dataset. These methods are useful for renaming or removing columns and changing columns to a new set of features."),el=c(),ae=n("h3"),Ae=n("a"),Ha=n("span"),_(ps.$$.fragment),Tr=c(),Ua=n("span"),Pr=t("Rename"),sl=c(),Ie=n("p"),qr=t("Use "),Ks=n("a"),Sr=t("IterableDataset.rename_column()"),Cr=t(" when you need to rename a column in your dataset. Features associated with the original column are actually moved under the new column name, instead of just replacing the original column in-place."),al=c(),Te=n("p"),zr=t("Provide "),Qs=n("a"),Nr=t("IterableDataset.rename_column()"),Mr=t(" with the name of the original column, and the new column name:"),tl=c(),_(is.$$.fragment),ll=c(),te=n("h3"),Pe=n("a"),Ba=n("span"),_(hs.$$.fragment),Lr=c(),Ja=n("span"),Or=t("Remove"),nl=c(),qe=n("p"),Vr=t("When you need to remove one or more columns, give "),Xs=n("a"),Fr=t("IterableDataset.remove_columns()"),Rr=t(" the name of the column to remove. Remove more than one column by providing a list of column names:"),rl=c(),_(cs.$$.fragment),ol=c(),le=n("h3"),Se=n("a"),Wa=n("span"),_(fs.$$.fragment),Yr=c(),Ga=n("span"),Hr=t("Cast"),pl=c(),L=n("p"),Zs=n("a"),Ur=t("IterableDataset.cast()"),Br=t(" changes the feature type of one or more columns. This method takes your new "),Ka=n("code"),Jr=t("Features"),Wr=t(" as its argument. The following sample code shows how to change the feature types of "),Qa=n("code"),Gr=t("ClassLabel"),Kr=t(" and "),Xa=n("code"),Qr=t("Value"),Xr=t(":"),il=c(),_(ds.$$.fragment),hl=c(),_(Ce.$$.fragment),cl=c(),ze=n("p"),Zr=t("Use "),ea=n("a"),eo=t("IterableDataset.cast_column()"),so=t(" to change the feature type of just one column. Pass the column name and its new feature type as arguments:"),fl=c(),_(ms.$$.fragment),dl=c(),ne=n("h2"),Ne=n("a"),Za=n("span"),_(us.$$.fragment),ao=c(),et=n("span"),to=t("Map"),ml=c(),C=n("p"),lo=t("Similar to the "),sa=n("a"),no=t("Dataset.map()"),ro=t(" function for a regular "),aa=n("a"),oo=t("Dataset"),po=t(", \u{1F917}  Datasets features "),ta=n("a"),io=t("IterableDataset.map()"),ho=t(" for processing an "),la=n("a"),co=t("IterableDataset"),fo=t(`.
`),na=n("a"),mo=t("IterableDataset.map()"),uo=t(" applies processing on-the-fly when examples are streamed."),ul=c(),ra=n("p"),go=t("It allows you to apply a processing function to each example in a dataset, independently or in batches. This function can even create new rows and columns."),gl=c(),B=n("p"),_o=t("The following example demonstrates how to tokenize a "),oa=n("a"),vo=t("IterableDataset"),jo=t(". The function needs to accept and output a "),st=n("code"),bo=t("dict"),$o=t(":"),_l=c(),_(gs.$$.fragment),vl=c(),Me=n("p"),xo=t("Next, apply this function to the dataset with "),pa=n("a"),wo=t("IterableDataset.map()"),yo=t(":"),jl=c(),_(_s.$$.fragment),bl=c(),Le=n("p"),ko=t("Let\u2019s take a look at another example, except this time, you will remove a column with "),ia=n("a"),Eo=t("IterableDataset.map()"),Do=t(". When you remove a column, it is only removed after the example has been provided to the mapped function. This allows the mapped function to use the content of the columns before they are removed."),$l=c(),J=n("p"),Ao=t("Specify the column to remove with the "),at=n("code"),Io=t("remove_columns"),To=t(" argument in "),ha=n("a"),Po=t("IterableDataset.map()"),qo=t(":"),xl=c(),_(vs.$$.fragment),wl=c(),re=n("h3"),Oe=n("a"),tt=n("span"),_(js.$$.fragment),So=c(),lt=n("span"),Co=t("Batch processing"),yl=c(),Y=n("p"),ca=n("a"),zo=t("IterableDataset.map()"),No=t(" also supports working with batches of examples. Operate on batches by setting "),nt=n("code"),Mo=t("batched=True"),Lo=t(". The default batch size is 1000, but you can adjust it with the "),rt=n("code"),Oo=t("batch_size"),Vo=t(" argument. This opens the door to many interesting applications such as tokenization, splitting long sentences into shorter chunks, and data augmentation."),kl=c(),oe=n("h4"),Ve=n("a"),ot=n("span"),_(bs.$$.fragment),Fo=c(),pt=n("span"),Ro=t("Tokenization"),El=c(),_($s.$$.fragment),Dl=c(),_(Fe.$$.fragment),Al=c(),pe=n("h3"),Re=n("a"),it=n("span"),_(xs.$$.fragment),Yo=c(),ht=n("span"),Ho=t("Filter"),Il=c(),Ye=n("p"),Uo=t("You can filter rows in the dataset based on a predicate function using "),fa=n("a"),Bo=t("Dataset.filter()"),Jo=t(". It returns rows that match a specified condition:"),Tl=c(),_(ws.$$.fragment),Pl=c(),ie=n("p"),da=n("a"),Wo=t("Dataset.filter()"),Go=t(" can also filter by indices if you set "),ct=n("code"),Ko=t("with_indices=True"),Qo=t(":"),ql=c(),_(ys.$$.fragment),Sl=c(),he=n("h2"),He=n("a"),ft=n("span"),_(ks.$$.fragment),Xo=c(),dt=n("span"),Zo=t("Stream in a training loop"),Cl=c(),Es=n("p"),ma=n("a"),ep=t("IterableDataset"),sp=t(" can be integrated into a training loop. First, shuffle the dataset:"),zl=c(),_(Ue.$$.fragment),this.h()},l(e){const p=eh('[data-svelte="svelte-1phssyn"]',document.head);d=r(p,"META",{name:!0,content:!0}),p.forEach(a),w=f(e),m=r(e,"H1",{class:!0});var Ds=o(m);y=r(Ds,"A",{id:!0,class:!0,href:!0});var mt=o(y);k=r(mt,"SPAN",{});var ut=o(k);v(g.$$.fragment,ut),ut.forEach(a),mt.forEach(a),E=f(Ds),u=r(Ds,"SPAN",{});var gt=o(u);D=l(gt,"Stream"),gt.forEach(a),Ds.forEach(a),A=f(e),T=r(e,"P",{});var _t=o(T);M=l(_t,"Dataset streaming lets you work with a dataset without downloading it. The data is streamed as you iterate over the dataset. This is especially helpful when:"),_t.forEach(a),de=f(e),S=r(e,"UL",{});var ce=o(S);P=r(ce,"LI",{});var np=o(P);Is=l(np,"You don\u2019t want to wait for an extremely large dataset to download."),np.forEach(a),Ts=f(ce),me=r(ce,"LI",{});var rp=o(me);Ps=l(rp,"The dataset size exceeds the amount of disk space on your computer."),rp.forEach(a),on=f(ce),ja=r(ce,"LI",{});var op=o(ja);pn=l(op,"You want to quickly explore just a few samples of a dataset."),op.forEach(a),ce.forEach(a),wt=f(e),G=r(e,"DIV",{class:!0});var Ml=o(G);qs=r(Ml,"IMG",{class:!0,src:!0}),hn=f(Ml),Ss=r(Ml,"IMG",{class:!0,src:!0}),Ml.forEach(a),yt=f(e),O=r(e,"P",{});var Be=o(O);cn=l(Be,"For example, the English split of the "),Ge=r(Be,"A",{href:!0,rel:!0});var pp=o(Ge);fn=l(pp,"OSCAR"),pp.forEach(a),dn=l(Be," dataset is 1.2 terabytes, but you can use it instantly with streaming. Stream a dataset by setting "),ba=r(Be,"CODE",{});var ip=o(ba);mn=l(ip,"streaming=True"),ip.forEach(a),un=l(Be," in "),Cs=r(Be,"A",{href:!0});var hp=o(Cs);gn=l(hp,"load_dataset()"),hp.forEach(a),_n=l(Be," as shown below:"),Be.forEach(a),kt=f(e),v(Ke.$$.fragment,e),Et=f(e),H=r(e,"P",{});var ua=o(H);vn=l(ua,"Loading a dataset in streaming mode creates a new dataset type instance (instead of the classic "),zs=r(ua,"A",{href:!0});var cp=o(zs);jn=l(cp,"Dataset"),cp.forEach(a),bn=l(ua," object), known as an "),Ns=r(ua,"A",{href:!0});var fp=o(Ns);$n=l(fp,"IterableDataset"),fp.forEach(a),xn=l(ua,". This special type of dataset has its own set of processing methods shown below."),ua.forEach(a),Dt=f(e),v(ue.$$.fragment,e),At=f(e),K=r(e,"H2",{class:!0});var Ll=o(K);ge=r(Ll,"A",{id:!0,class:!0,href:!0});var dp=o(ge);$a=r(dp,"SPAN",{});var mp=o($a);v(Qe.$$.fragment,mp),mp.forEach(a),dp.forEach(a),wn=f(Ll),xa=r(Ll,"SPAN",{});var up=o(xa);yn=l(up,"Shuffle"),up.forEach(a),Ll.forEach(a),It=f(e),V=r(e,"P",{});var Je=o(V);kn=l(Je,"Like a regular "),Ms=r(Je,"A",{href:!0});var gp=o(Ms);En=l(gp,"Dataset"),gp.forEach(a),Dn=l(Je," object, you can also shuffle a "),Ls=r(Je,"A",{href:!0});var _p=o(Ls);An=l(_p,"IterableDataset"),_p.forEach(a),In=l(Je," with "),Os=r(Je,"A",{href:!0});var vp=o(Os);Tn=l(vp,"IterableDataset.shuffle()"),vp.forEach(a),Pn=l(Je,"."),Je.forEach(a),Tt=f(e),F=r(e,"P",{});var We=o(F);qn=l(We,"The "),wa=r(We,"CODE",{});var jp=o(wa);Sn=l(jp,"buffer_size"),jp.forEach(a),Cn=l(We," argument controls the size of the buffer to randomly sample examples from. Let\u2019s say your dataset has one million examples, and you set the "),ya=r(We,"CODE",{});var bp=o(ya);zn=l(bp,"buffer_size"),bp.forEach(a),Nn=l(We," to ten thousand. "),Vs=r(We,"A",{href:!0});var $p=o(Vs);Mn=l($p,"IterableDataset.shuffle()"),$p.forEach(a),Ln=l(We," will randomly select examples from the first ten thousand examples in the buffer. Selected examples in the buffer are replaced with new examples. By default, the buffer size is 1,000."),We.forEach(a),Pt=f(e),v(Xe.$$.fragment,e),qt=f(e),v(_e.$$.fragment,e),St=f(e),Q=r(e,"H2",{class:!0});var Ol=o(Q);ve=r(Ol,"A",{id:!0,class:!0,href:!0});var xp=o(ve);ka=r(xp,"SPAN",{});var wp=o(ka);v(Ze.$$.fragment,wp),wp.forEach(a),xp.forEach(a),On=f(Ol),Ea=r(Ol,"SPAN",{});var yp=o(Ea);Vn=l(yp,"Reshuffle"),yp.forEach(a),Ol.forEach(a),Ct=f(e),je=r(e,"P",{});var Vl=o(je);Fn=l(Vl,"Sometimes you may want to reshuffle the dataset after each epoch. This will require you to set a different seed for each epoch. Use "),Da=r(Vl,"CODE",{});var kp=o(Da);Rn=l(kp,"IterableDataset.set_epoch()"),kp.forEach(a),Yn=l(Vl," in between epochs to tell the dataset what epoch you\u2019re on."),Vl.forEach(a),zt=f(e),be=r(e,"P",{});var Fl=o(be);Hn=l(Fl,"Your seed effectively becomes: "),Aa=r(Fl,"CODE",{});var Ep=o(Aa);Un=l(Ep,"initial seed + current epoch"),Ep.forEach(a),Bn=l(Fl,"."),Fl.forEach(a),Nt=f(e),v(es.$$.fragment,e),Mt=f(e),X=r(e,"H2",{class:!0});var Rl=o(X);$e=r(Rl,"A",{id:!0,class:!0,href:!0});var Dp=o($e);Ia=r(Dp,"SPAN",{});var Ap=o(Ia);v(ss.$$.fragment,Ap),Ap.forEach(a),Dp.forEach(a),Jn=f(Rl),Ta=r(Rl,"SPAN",{});var Ip=o(Ta);Wn=l(Ip,"Split dataset"),Ip.forEach(a),Rl.forEach(a),Lt=f(e),Fs=r(e,"P",{});var Tp=o(Fs);Gn=l(Tp,"You can split your dataset one of two ways:"),Tp.forEach(a),Ot=f(e),Rs=r(e,"UL",{});var Pp=o(Rs);xe=r(Pp,"LI",{});var vt=o(xe);Ys=r(vt,"A",{href:!0});var qp=o(Ys);Kn=l(qp,"IterableDataset.take()"),qp.forEach(a),Qn=l(vt," returns the first "),Pa=r(vt,"CODE",{});var Sp=o(Pa);Xn=l(Sp,"n"),Sp.forEach(a),Zn=l(vt," examples in a dataset:"),vt.forEach(a),Pp.forEach(a),Vt=f(e),v(as.$$.fragment,e),Ft=f(e),Hs=r(e,"UL",{});var Cp=o(Hs);we=r(Cp,"LI",{});var jt=o(we);Us=r(jt,"A",{href:!0});var zp=o(Us);er=l(zp,"IterableDataset.skip()"),zp.forEach(a),sr=l(jt," omits the first "),qa=r(jt,"CODE",{});var Np=o(qa);ar=l(Np,"n"),Np.forEach(a),tr=l(jt," examples in a dataset and returns the remaining examples:"),jt.forEach(a),Cp.forEach(a),Rt=f(e),v(ts.$$.fragment,e),Yt=f(e),v(ye.$$.fragment,e),Ht=f(e),Bs=r(e,"A",{id:!0}),o(Bs).forEach(a),Ut=f(e),Z=r(e,"H2",{class:!0});var Yl=o(Z);ke=r(Yl,"A",{id:!0,class:!0,href:!0});var Mp=o(ke);Sa=r(Mp,"SPAN",{});var Lp=o(Sa);v(ls.$$.fragment,Lp),Lp.forEach(a),Mp.forEach(a),lr=f(Yl),Ca=r(Yl,"SPAN",{});var Op=o(Ca);nr=l(Op,"Interleave"),Op.forEach(a),Yl.forEach(a),Bt=f(e),ee=r(e,"P",{});var bt=o(ee);Js=r(bt,"A",{href:!0});var Vp=o(Js);rr=l(Vp,"interleave_datasets()"),Vp.forEach(a),or=l(bt," can combine an "),Ws=r(bt,"A",{href:!0});var Fp=o(Ws);pr=l(Fp,"IterableDataset"),Fp.forEach(a),ir=l(bt," with other datasets. The combined dataset returns alternating examples from each of the original datasets."),bt.forEach(a),Jt=f(e),v(ns.$$.fragment,e),Wt=f(e),Ee=r(e,"P",{});var Hl=o(Ee);hr=l(Hl,"Define sampling probabilities from each of the original datasets for more control over how each of them are sampled and combined. Set the "),za=r(Hl,"CODE",{});var Rp=o(za);cr=l(Rp,"probabilities"),Rp.forEach(a),fr=l(Hl," argument with your desired sampling probabilities:"),Hl.forEach(a),Gt=f(e),v(rs.$$.fragment,e),Kt=f(e),U=r(e,"P",{});var ga=o(U);dr=l(ga,"Around 80% of the final dataset is made of the "),Na=r(ga,"CODE",{});var Yp=o(Na);mr=l(Yp,"en_dataset"),Yp.forEach(a),ur=l(ga,", and 20% of the "),Ma=r(ga,"CODE",{});var Hp=o(Ma);gr=l(Hp,"fr_dataset"),Hp.forEach(a),_r=l(ga,"."),ga.forEach(a),Qt=f(e),z=r(e,"P",{});var W=o(z);vr=l(W,"You can also specify the "),La=r(W,"CODE",{});var Up=o(La);jr=l(Up,"stopping_strategy"),Up.forEach(a),br=l(W,". The default strategy, "),Oa=r(W,"CODE",{});var Bp=o(Oa);$r=l(Bp,"first_exhausted"),Bp.forEach(a),xr=l(W,`, is a subsampling strategy, i.e the dataset construction is stopped as soon one of the dataset runs out of samples.
You can specify `),Va=r(W,"CODE",{});var Jp=o(Va);wr=l(Jp,"stopping_strategy=all_exhausted"),Jp.forEach(a),yr=l(W,` to execute an oversampling strategy. In this case, the dataset construction is stopped as soon as every samples in every dataset has been added at least once. In practice, it means that if a dataset is exhausted, it will return to the beginning of this dataset until the stop criterion has been reached.
Note that if no sampling probabilities are specified, the new dataset will have `),Fa=r(W,"CODE",{});var Wp=o(Fa);kr=l(Wp,"max_length_datasets*nb_dataset samples"),Wp.forEach(a),Er=l(W,"."),W.forEach(a),Xt=f(e),se=r(e,"H2",{class:!0});var Ul=o(se);De=r(Ul,"A",{id:!0,class:!0,href:!0});var Gp=o(De);Ra=r(Gp,"SPAN",{});var Kp=o(Ra);v(os.$$.fragment,Kp),Kp.forEach(a),Gp.forEach(a),Dr=f(Ul),Ya=r(Ul,"SPAN",{});var Qp=o(Ya);Ar=l(Qp,"Rename, remove, and cast"),Qp.forEach(a),Ul.forEach(a),Zt=f(e),Gs=r(e,"P",{});var Xp=o(Gs);Ir=l(Xp,"The following methods allow you to modify the columns of a dataset. These methods are useful for renaming or removing columns and changing columns to a new set of features."),Xp.forEach(a),el=f(e),ae=r(e,"H3",{class:!0});var Bl=o(ae);Ae=r(Bl,"A",{id:!0,class:!0,href:!0});var Zp=o(Ae);Ha=r(Zp,"SPAN",{});var ei=o(Ha);v(ps.$$.fragment,ei),ei.forEach(a),Zp.forEach(a),Tr=f(Bl),Ua=r(Bl,"SPAN",{});var si=o(Ua);Pr=l(si,"Rename"),si.forEach(a),Bl.forEach(a),sl=f(e),Ie=r(e,"P",{});var Jl=o(Ie);qr=l(Jl,"Use "),Ks=r(Jl,"A",{href:!0});var ai=o(Ks);Sr=l(ai,"IterableDataset.rename_column()"),ai.forEach(a),Cr=l(Jl," when you need to rename a column in your dataset. Features associated with the original column are actually moved under the new column name, instead of just replacing the original column in-place."),Jl.forEach(a),al=f(e),Te=r(e,"P",{});var Wl=o(Te);zr=l(Wl,"Provide "),Qs=r(Wl,"A",{href:!0});var ti=o(Qs);Nr=l(ti,"IterableDataset.rename_column()"),ti.forEach(a),Mr=l(Wl," with the name of the original column, and the new column name:"),Wl.forEach(a),tl=f(e),v(is.$$.fragment,e),ll=f(e),te=r(e,"H3",{class:!0});var Gl=o(te);Pe=r(Gl,"A",{id:!0,class:!0,href:!0});var li=o(Pe);Ba=r(li,"SPAN",{});var ni=o(Ba);v(hs.$$.fragment,ni),ni.forEach(a),li.forEach(a),Lr=f(Gl),Ja=r(Gl,"SPAN",{});var ri=o(Ja);Or=l(ri,"Remove"),ri.forEach(a),Gl.forEach(a),nl=f(e),qe=r(e,"P",{});var Kl=o(qe);Vr=l(Kl,"When you need to remove one or more columns, give "),Xs=r(Kl,"A",{href:!0});var oi=o(Xs);Fr=l(oi,"IterableDataset.remove_columns()"),oi.forEach(a),Rr=l(Kl," the name of the column to remove. Remove more than one column by providing a list of column names:"),Kl.forEach(a),rl=f(e),v(cs.$$.fragment,e),ol=f(e),le=r(e,"H3",{class:!0});var Ql=o(le);Se=r(Ql,"A",{id:!0,class:!0,href:!0});var pi=o(Se);Wa=r(pi,"SPAN",{});var ii=o(Wa);v(fs.$$.fragment,ii),ii.forEach(a),pi.forEach(a),Yr=f(Ql),Ga=r(Ql,"SPAN",{});var hi=o(Ga);Hr=l(hi,"Cast"),hi.forEach(a),Ql.forEach(a),pl=f(e),L=r(e,"P",{});var fe=o(L);Zs=r(fe,"A",{href:!0});var ci=o(Zs);Ur=l(ci,"IterableDataset.cast()"),ci.forEach(a),Br=l(fe," changes the feature type of one or more columns. This method takes your new "),Ka=r(fe,"CODE",{});var fi=o(Ka);Jr=l(fi,"Features"),fi.forEach(a),Wr=l(fe," as its argument. The following sample code shows how to change the feature types of "),Qa=r(fe,"CODE",{});var di=o(Qa);Gr=l(di,"ClassLabel"),di.forEach(a),Kr=l(fe," and "),Xa=r(fe,"CODE",{});var mi=o(Xa);Qr=l(mi,"Value"),mi.forEach(a),Xr=l(fe,":"),fe.forEach(a),il=f(e),v(ds.$$.fragment,e),hl=f(e),v(Ce.$$.fragment,e),cl=f(e),ze=r(e,"P",{});var Xl=o(ze);Zr=l(Xl,"Use "),ea=r(Xl,"A",{href:!0});var ui=o(ea);eo=l(ui,"IterableDataset.cast_column()"),ui.forEach(a),so=l(Xl," to change the feature type of just one column. Pass the column name and its new feature type as arguments:"),Xl.forEach(a),fl=f(e),v(ms.$$.fragment,e),dl=f(e),ne=r(e,"H2",{class:!0});var Zl=o(ne);Ne=r(Zl,"A",{id:!0,class:!0,href:!0});var gi=o(Ne);Za=r(gi,"SPAN",{});var _i=o(Za);v(us.$$.fragment,_i),_i.forEach(a),gi.forEach(a),ao=f(Zl),et=r(Zl,"SPAN",{});var vi=o(et);to=l(vi,"Map"),vi.forEach(a),Zl.forEach(a),ml=f(e),C=r(e,"P",{});var R=o(C);lo=l(R,"Similar to the "),sa=r(R,"A",{href:!0});var ji=o(sa);no=l(ji,"Dataset.map()"),ji.forEach(a),ro=l(R," function for a regular "),aa=r(R,"A",{href:!0});var bi=o(aa);oo=l(bi,"Dataset"),bi.forEach(a),po=l(R,", \u{1F917}  Datasets features "),ta=r(R,"A",{href:!0});var $i=o(ta);io=l($i,"IterableDataset.map()"),$i.forEach(a),ho=l(R," for processing an "),la=r(R,"A",{href:!0});var xi=o(la);co=l(xi,"IterableDataset"),xi.forEach(a),fo=l(R,`.
`),na=r(R,"A",{href:!0});var wi=o(na);mo=l(wi,"IterableDataset.map()"),wi.forEach(a),uo=l(R," applies processing on-the-fly when examples are streamed."),R.forEach(a),ul=f(e),ra=r(e,"P",{});var yi=o(ra);go=l(yi,"It allows you to apply a processing function to each example in a dataset, independently or in batches. This function can even create new rows and columns."),yi.forEach(a),gl=f(e),B=r(e,"P",{});var _a=o(B);_o=l(_a,"The following example demonstrates how to tokenize a "),oa=r(_a,"A",{href:!0});var ki=o(oa);vo=l(ki,"IterableDataset"),ki.forEach(a),jo=l(_a,". The function needs to accept and output a "),st=r(_a,"CODE",{});var Ei=o(st);bo=l(Ei,"dict"),Ei.forEach(a),$o=l(_a,":"),_a.forEach(a),_l=f(e),v(gs.$$.fragment,e),vl=f(e),Me=r(e,"P",{});var en=o(Me);xo=l(en,"Next, apply this function to the dataset with "),pa=r(en,"A",{href:!0});var Di=o(pa);wo=l(Di,"IterableDataset.map()"),Di.forEach(a),yo=l(en,":"),en.forEach(a),jl=f(e),v(_s.$$.fragment,e),bl=f(e),Le=r(e,"P",{});var sn=o(Le);ko=l(sn,"Let\u2019s take a look at another example, except this time, you will remove a column with "),ia=r(sn,"A",{href:!0});var Ai=o(ia);Eo=l(Ai,"IterableDataset.map()"),Ai.forEach(a),Do=l(sn,". When you remove a column, it is only removed after the example has been provided to the mapped function. This allows the mapped function to use the content of the columns before they are removed."),sn.forEach(a),$l=f(e),J=r(e,"P",{});var va=o(J);Ao=l(va,"Specify the column to remove with the "),at=r(va,"CODE",{});var Ii=o(at);Io=l(Ii,"remove_columns"),Ii.forEach(a),To=l(va," argument in "),ha=r(va,"A",{href:!0});var Ti=o(ha);Po=l(Ti,"IterableDataset.map()"),Ti.forEach(a),qo=l(va,":"),va.forEach(a),xl=f(e),v(vs.$$.fragment,e),wl=f(e),re=r(e,"H3",{class:!0});var an=o(re);Oe=r(an,"A",{id:!0,class:!0,href:!0});var Pi=o(Oe);tt=r(Pi,"SPAN",{});var qi=o(tt);v(js.$$.fragment,qi),qi.forEach(a),Pi.forEach(a),So=f(an),lt=r(an,"SPAN",{});var Si=o(lt);Co=l(Si,"Batch processing"),Si.forEach(a),an.forEach(a),yl=f(e),Y=r(e,"P",{});var As=o(Y);ca=r(As,"A",{href:!0});var Ci=o(ca);zo=l(Ci,"IterableDataset.map()"),Ci.forEach(a),No=l(As," also supports working with batches of examples. Operate on batches by setting "),nt=r(As,"CODE",{});var zi=o(nt);Mo=l(zi,"batched=True"),zi.forEach(a),Lo=l(As,". The default batch size is 1000, but you can adjust it with the "),rt=r(As,"CODE",{});var Ni=o(rt);Oo=l(Ni,"batch_size"),Ni.forEach(a),Vo=l(As," argument. This opens the door to many interesting applications such as tokenization, splitting long sentences into shorter chunks, and data augmentation."),As.forEach(a),kl=f(e),oe=r(e,"H4",{class:!0});var tn=o(oe);Ve=r(tn,"A",{id:!0,class:!0,href:!0});var Mi=o(Ve);ot=r(Mi,"SPAN",{});var Li=o(ot);v(bs.$$.fragment,Li),Li.forEach(a),Mi.forEach(a),Fo=f(tn),pt=r(tn,"SPAN",{});var Oi=o(pt);Ro=l(Oi,"Tokenization"),Oi.forEach(a),tn.forEach(a),El=f(e),v($s.$$.fragment,e),Dl=f(e),v(Fe.$$.fragment,e),Al=f(e),pe=r(e,"H3",{class:!0});var ln=o(pe);Re=r(ln,"A",{id:!0,class:!0,href:!0});var Vi=o(Re);it=r(Vi,"SPAN",{});var Fi=o(it);v(xs.$$.fragment,Fi),Fi.forEach(a),Vi.forEach(a),Yo=f(ln),ht=r(ln,"SPAN",{});var Ri=o(ht);Ho=l(Ri,"Filter"),Ri.forEach(a),ln.forEach(a),Il=f(e),Ye=r(e,"P",{});var nn=o(Ye);Uo=l(nn,"You can filter rows in the dataset based on a predicate function using "),fa=r(nn,"A",{href:!0});var Yi=o(fa);Bo=l(Yi,"Dataset.filter()"),Yi.forEach(a),Jo=l(nn,". It returns rows that match a specified condition:"),nn.forEach(a),Tl=f(e),v(ws.$$.fragment,e),Pl=f(e),ie=r(e,"P",{});var $t=o(ie);da=r($t,"A",{href:!0});var Hi=o(da);Wo=l(Hi,"Dataset.filter()"),Hi.forEach(a),Go=l($t," can also filter by indices if you set "),ct=r($t,"CODE",{});var Ui=o(ct);Ko=l(Ui,"with_indices=True"),Ui.forEach(a),Qo=l($t,":"),$t.forEach(a),ql=f(e),v(ys.$$.fragment,e),Sl=f(e),he=r(e,"H2",{class:!0});var rn=o(he);He=r(rn,"A",{id:!0,class:!0,href:!0});var Bi=o(He);ft=r(Bi,"SPAN",{});var Ji=o(ft);v(ks.$$.fragment,Ji),Ji.forEach(a),Bi.forEach(a),Xo=f(rn),dt=r(rn,"SPAN",{});var Wi=o(dt);Zo=l(Wi,"Stream in a training loop"),Wi.forEach(a),rn.forEach(a),Cl=f(e),Es=r(e,"P",{});var ap=o(Es);ma=r(ap,"A",{href:!0});var Gi=o(ma);ep=l(Gi,"IterableDataset"),Gi.forEach(a),sp=l(ap," can be integrated into a training loop. First, shuffle the dataset:"),ap.forEach(a),zl=f(e),v(Ue.$$.fragment,e),this.h()},h(){h(d,"name","hf:doc:metadata"),h(d,"content",JSON.stringify(dh)),h(y,"id","stream"),h(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(y,"href","#stream"),h(m,"class","relative group"),h(qs,"class","block dark:hidden"),Ki(qs.src,tp="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/streaming.gif")||h(qs,"src",tp),h(Ss,"class","hidden dark:block"),Ki(Ss.src,lp="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/streaming-dark.gif")||h(Ss,"src",lp),h(G,"class","flex justify-center"),h(Ge,"href","https://huggingface.co/datasets/oscar"),h(Ge,"rel","nofollow"),h(Cs,"href","/docs/datasets/v2.8.0/en/package_reference/loading_methods#datasets.load_dataset"),h(zs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset"),h(Ns,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset"),h(ge,"id","shuffle"),h(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ge,"href","#shuffle"),h(K,"class","relative group"),h(Ms,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset"),h(Ls,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset"),h(Os,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.shuffle"),h(Vs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.shuffle"),h(ve,"id","reshuffle"),h(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ve,"href","#reshuffle"),h(Q,"class","relative group"),h($e,"id","split-dataset"),h($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h($e,"href","#split-dataset"),h(X,"class","relative group"),h(Ys,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.take"),h(Us,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.skip"),h(Bs,"id","interleave_datasets"),h(ke,"id","interleave"),h(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ke,"href","#interleave"),h(Z,"class","relative group"),h(Js,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.interleave_datasets"),h(Ws,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset"),h(De,"id","rename-remove-and-cast"),h(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(De,"href","#rename-remove-and-cast"),h(se,"class","relative group"),h(Ae,"id","rename"),h(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ae,"href","#rename"),h(ae,"class","relative group"),h(Ks,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.rename_column"),h(Qs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.rename_column"),h(Pe,"id","remove"),h(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Pe,"href","#remove"),h(te,"class","relative group"),h(Xs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.remove_columns"),h(Se,"id","cast"),h(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Se,"href","#cast"),h(le,"class","relative group"),h(Zs,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.cast"),h(ea,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.cast_column"),h(Ne,"id","map"),h(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ne,"href","#map"),h(ne,"class","relative group"),h(sa,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset.map"),h(aa,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset"),h(ta,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.map"),h(la,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset"),h(na,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.map"),h(oa,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset"),h(pa,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.map"),h(ia,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.map"),h(ha,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.map"),h(Oe,"id","batch-processing"),h(Oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Oe,"href","#batch-processing"),h(re,"class","relative group"),h(ca,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset.map"),h(Ve,"id","tokenization"),h(Ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ve,"href","#tokenization"),h(oe,"class","relative group"),h(Re,"id","filter"),h(Re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Re,"href","#filter"),h(pe,"class","relative group"),h(fa,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset.filter"),h(da,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset.filter"),h(He,"id","stream-in-a-training-loop"),h(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(He,"href","#stream-in-a-training-loop"),h(he,"class","relative group"),h(ma,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.IterableDataset")},m(e,p){s(document.head,d),i(e,w,p),i(e,m,p),s(m,y),s(y,k),j(g,k,null),s(m,E),s(m,u),s(u,D),i(e,A,p),i(e,T,p),s(T,M),i(e,de,p),i(e,S,p),s(S,P),s(P,Is),s(S,Ts),s(S,me),s(me,Ps),s(S,on),s(S,ja),s(ja,pn),i(e,wt,p),i(e,G,p),s(G,qs),s(G,hn),s(G,Ss),i(e,yt,p),i(e,O,p),s(O,cn),s(O,Ge),s(Ge,fn),s(O,dn),s(O,ba),s(ba,mn),s(O,un),s(O,Cs),s(Cs,gn),s(O,_n),i(e,kt,p),j(Ke,e,p),i(e,Et,p),i(e,H,p),s(H,vn),s(H,zs),s(zs,jn),s(H,bn),s(H,Ns),s(Ns,$n),s(H,xn),i(e,Dt,p),j(ue,e,p),i(e,At,p),i(e,K,p),s(K,ge),s(ge,$a),j(Qe,$a,null),s(K,wn),s(K,xa),s(xa,yn),i(e,It,p),i(e,V,p),s(V,kn),s(V,Ms),s(Ms,En),s(V,Dn),s(V,Ls),s(Ls,An),s(V,In),s(V,Os),s(Os,Tn),s(V,Pn),i(e,Tt,p),i(e,F,p),s(F,qn),s(F,wa),s(wa,Sn),s(F,Cn),s(F,ya),s(ya,zn),s(F,Nn),s(F,Vs),s(Vs,Mn),s(F,Ln),i(e,Pt,p),j(Xe,e,p),i(e,qt,p),j(_e,e,p),i(e,St,p),i(e,Q,p),s(Q,ve),s(ve,ka),j(Ze,ka,null),s(Q,On),s(Q,Ea),s(Ea,Vn),i(e,Ct,p),i(e,je,p),s(je,Fn),s(je,Da),s(Da,Rn),s(je,Yn),i(e,zt,p),i(e,be,p),s(be,Hn),s(be,Aa),s(Aa,Un),s(be,Bn),i(e,Nt,p),j(es,e,p),i(e,Mt,p),i(e,X,p),s(X,$e),s($e,Ia),j(ss,Ia,null),s(X,Jn),s(X,Ta),s(Ta,Wn),i(e,Lt,p),i(e,Fs,p),s(Fs,Gn),i(e,Ot,p),i(e,Rs,p),s(Rs,xe),s(xe,Ys),s(Ys,Kn),s(xe,Qn),s(xe,Pa),s(Pa,Xn),s(xe,Zn),i(e,Vt,p),j(as,e,p),i(e,Ft,p),i(e,Hs,p),s(Hs,we),s(we,Us),s(Us,er),s(we,sr),s(we,qa),s(qa,ar),s(we,tr),i(e,Rt,p),j(ts,e,p),i(e,Yt,p),j(ye,e,p),i(e,Ht,p),i(e,Bs,p),i(e,Ut,p),i(e,Z,p),s(Z,ke),s(ke,Sa),j(ls,Sa,null),s(Z,lr),s(Z,Ca),s(Ca,nr),i(e,Bt,p),i(e,ee,p),s(ee,Js),s(Js,rr),s(ee,or),s(ee,Ws),s(Ws,pr),s(ee,ir),i(e,Jt,p),j(ns,e,p),i(e,Wt,p),i(e,Ee,p),s(Ee,hr),s(Ee,za),s(za,cr),s(Ee,fr),i(e,Gt,p),j(rs,e,p),i(e,Kt,p),i(e,U,p),s(U,dr),s(U,Na),s(Na,mr),s(U,ur),s(U,Ma),s(Ma,gr),s(U,_r),i(e,Qt,p),i(e,z,p),s(z,vr),s(z,La),s(La,jr),s(z,br),s(z,Oa),s(Oa,$r),s(z,xr),s(z,Va),s(Va,wr),s(z,yr),s(z,Fa),s(Fa,kr),s(z,Er),i(e,Xt,p),i(e,se,p),s(se,De),s(De,Ra),j(os,Ra,null),s(se,Dr),s(se,Ya),s(Ya,Ar),i(e,Zt,p),i(e,Gs,p),s(Gs,Ir),i(e,el,p),i(e,ae,p),s(ae,Ae),s(Ae,Ha),j(ps,Ha,null),s(ae,Tr),s(ae,Ua),s(Ua,Pr),i(e,sl,p),i(e,Ie,p),s(Ie,qr),s(Ie,Ks),s(Ks,Sr),s(Ie,Cr),i(e,al,p),i(e,Te,p),s(Te,zr),s(Te,Qs),s(Qs,Nr),s(Te,Mr),i(e,tl,p),j(is,e,p),i(e,ll,p),i(e,te,p),s(te,Pe),s(Pe,Ba),j(hs,Ba,null),s(te,Lr),s(te,Ja),s(Ja,Or),i(e,nl,p),i(e,qe,p),s(qe,Vr),s(qe,Xs),s(Xs,Fr),s(qe,Rr),i(e,rl,p),j(cs,e,p),i(e,ol,p),i(e,le,p),s(le,Se),s(Se,Wa),j(fs,Wa,null),s(le,Yr),s(le,Ga),s(Ga,Hr),i(e,pl,p),i(e,L,p),s(L,Zs),s(Zs,Ur),s(L,Br),s(L,Ka),s(Ka,Jr),s(L,Wr),s(L,Qa),s(Qa,Gr),s(L,Kr),s(L,Xa),s(Xa,Qr),s(L,Xr),i(e,il,p),j(ds,e,p),i(e,hl,p),j(Ce,e,p),i(e,cl,p),i(e,ze,p),s(ze,Zr),s(ze,ea),s(ea,eo),s(ze,so),i(e,fl,p),j(ms,e,p),i(e,dl,p),i(e,ne,p),s(ne,Ne),s(Ne,Za),j(us,Za,null),s(ne,ao),s(ne,et),s(et,to),i(e,ml,p),i(e,C,p),s(C,lo),s(C,sa),s(sa,no),s(C,ro),s(C,aa),s(aa,oo),s(C,po),s(C,ta),s(ta,io),s(C,ho),s(C,la),s(la,co),s(C,fo),s(C,na),s(na,mo),s(C,uo),i(e,ul,p),i(e,ra,p),s(ra,go),i(e,gl,p),i(e,B,p),s(B,_o),s(B,oa),s(oa,vo),s(B,jo),s(B,st),s(st,bo),s(B,$o),i(e,_l,p),j(gs,e,p),i(e,vl,p),i(e,Me,p),s(Me,xo),s(Me,pa),s(pa,wo),s(Me,yo),i(e,jl,p),j(_s,e,p),i(e,bl,p),i(e,Le,p),s(Le,ko),s(Le,ia),s(ia,Eo),s(Le,Do),i(e,$l,p),i(e,J,p),s(J,Ao),s(J,at),s(at,Io),s(J,To),s(J,ha),s(ha,Po),s(J,qo),i(e,xl,p),j(vs,e,p),i(e,wl,p),i(e,re,p),s(re,Oe),s(Oe,tt),j(js,tt,null),s(re,So),s(re,lt),s(lt,Co),i(e,yl,p),i(e,Y,p),s(Y,ca),s(ca,zo),s(Y,No),s(Y,nt),s(nt,Mo),s(Y,Lo),s(Y,rt),s(rt,Oo),s(Y,Vo),i(e,kl,p),i(e,oe,p),s(oe,Ve),s(Ve,ot),j(bs,ot,null),s(oe,Fo),s(oe,pt),s(pt,Ro),i(e,El,p),j($s,e,p),i(e,Dl,p),j(Fe,e,p),i(e,Al,p),i(e,pe,p),s(pe,Re),s(Re,it),j(xs,it,null),s(pe,Yo),s(pe,ht),s(ht,Ho),i(e,Il,p),i(e,Ye,p),s(Ye,Uo),s(Ye,fa),s(fa,Bo),s(Ye,Jo),i(e,Tl,p),j(ws,e,p),i(e,Pl,p),i(e,ie,p),s(ie,da),s(da,Wo),s(ie,Go),s(ie,ct),s(ct,Ko),s(ie,Qo),i(e,ql,p),j(ys,e,p),i(e,Sl,p),i(e,he,p),s(he,He),s(He,ft),j(ks,ft,null),s(he,Xo),s(he,dt),s(dt,Zo),i(e,Cl,p),i(e,Es,p),s(Es,ma),s(ma,ep),s(Es,sp),i(e,zl,p),j(Ue,e,p),Nl=!0},p(e,[p]){const Ds={};p&2&&(Ds.$$scope={dirty:p,ctx:e}),ue.$set(Ds);const mt={};p&2&&(mt.$$scope={dirty:p,ctx:e}),_e.$set(mt);const ut={};p&2&&(ut.$$scope={dirty:p,ctx:e}),ye.$set(ut);const gt={};p&2&&(gt.$$scope={dirty:p,ctx:e}),Ce.$set(gt);const _t={};p&2&&(_t.$$scope={dirty:p,ctx:e}),Fe.$set(_t);const ce={};p&2&&(ce.$$scope={dirty:p,ctx:e}),Ue.$set(ce)},i(e){Nl||(b(g.$$.fragment,e),b(Ke.$$.fragment,e),b(ue.$$.fragment,e),b(Qe.$$.fragment,e),b(Xe.$$.fragment,e),b(_e.$$.fragment,e),b(Ze.$$.fragment,e),b(es.$$.fragment,e),b(ss.$$.fragment,e),b(as.$$.fragment,e),b(ts.$$.fragment,e),b(ye.$$.fragment,e),b(ls.$$.fragment,e),b(ns.$$.fragment,e),b(rs.$$.fragment,e),b(os.$$.fragment,e),b(ps.$$.fragment,e),b(is.$$.fragment,e),b(hs.$$.fragment,e),b(cs.$$.fragment,e),b(fs.$$.fragment,e),b(ds.$$.fragment,e),b(Ce.$$.fragment,e),b(ms.$$.fragment,e),b(us.$$.fragment,e),b(gs.$$.fragment,e),b(_s.$$.fragment,e),b(vs.$$.fragment,e),b(js.$$.fragment,e),b(bs.$$.fragment,e),b($s.$$.fragment,e),b(Fe.$$.fragment,e),b(xs.$$.fragment,e),b(ws.$$.fragment,e),b(ys.$$.fragment,e),b(ks.$$.fragment,e),b(Ue.$$.fragment,e),Nl=!0)},o(e){$(g.$$.fragment,e),$(Ke.$$.fragment,e),$(ue.$$.fragment,e),$(Qe.$$.fragment,e),$(Xe.$$.fragment,e),$(_e.$$.fragment,e),$(Ze.$$.fragment,e),$(es.$$.fragment,e),$(ss.$$.fragment,e),$(as.$$.fragment,e),$(ts.$$.fragment,e),$(ye.$$.fragment,e),$(ls.$$.fragment,e),$(ns.$$.fragment,e),$(rs.$$.fragment,e),$(os.$$.fragment,e),$(ps.$$.fragment,e),$(is.$$.fragment,e),$(hs.$$.fragment,e),$(cs.$$.fragment,e),$(fs.$$.fragment,e),$(ds.$$.fragment,e),$(Ce.$$.fragment,e),$(ms.$$.fragment,e),$(us.$$.fragment,e),$(gs.$$.fragment,e),$(_s.$$.fragment,e),$(vs.$$.fragment,e),$(js.$$.fragment,e),$(bs.$$.fragment,e),$($s.$$.fragment,e),$(Fe.$$.fragment,e),$(xs.$$.fragment,e),$(ws.$$.fragment,e),$(ys.$$.fragment,e),$(ks.$$.fragment,e),$(Ue.$$.fragment,e),Nl=!1},d(e){a(d),e&&a(w),e&&a(m),x(g),e&&a(A),e&&a(T),e&&a(de),e&&a(S),e&&a(wt),e&&a(G),e&&a(yt),e&&a(O),e&&a(kt),x(Ke,e),e&&a(Et),e&&a(H),e&&a(Dt),x(ue,e),e&&a(At),e&&a(K),x(Qe),e&&a(It),e&&a(V),e&&a(Tt),e&&a(F),e&&a(Pt),x(Xe,e),e&&a(qt),x(_e,e),e&&a(St),e&&a(Q),x(Ze),e&&a(Ct),e&&a(je),e&&a(zt),e&&a(be),e&&a(Nt),x(es,e),e&&a(Mt),e&&a(X),x(ss),e&&a(Lt),e&&a(Fs),e&&a(Ot),e&&a(Rs),e&&a(Vt),x(as,e),e&&a(Ft),e&&a(Hs),e&&a(Rt),x(ts,e),e&&a(Yt),x(ye,e),e&&a(Ht),e&&a(Bs),e&&a(Ut),e&&a(Z),x(ls),e&&a(Bt),e&&a(ee),e&&a(Jt),x(ns,e),e&&a(Wt),e&&a(Ee),e&&a(Gt),x(rs,e),e&&a(Kt),e&&a(U),e&&a(Qt),e&&a(z),e&&a(Xt),e&&a(se),x(os),e&&a(Zt),e&&a(Gs),e&&a(el),e&&a(ae),x(ps),e&&a(sl),e&&a(Ie),e&&a(al),e&&a(Te),e&&a(tl),x(is,e),e&&a(ll),e&&a(te),x(hs),e&&a(nl),e&&a(qe),e&&a(rl),x(cs,e),e&&a(ol),e&&a(le),x(fs),e&&a(pl),e&&a(L),e&&a(il),x(ds,e),e&&a(hl),x(Ce,e),e&&a(cl),e&&a(ze),e&&a(fl),x(ms,e),e&&a(dl),e&&a(ne),x(us),e&&a(ml),e&&a(C),e&&a(ul),e&&a(ra),e&&a(gl),e&&a(B),e&&a(_l),x(gs,e),e&&a(vl),e&&a(Me),e&&a(jl),x(_s,e),e&&a(bl),e&&a(Le),e&&a($l),e&&a(J),e&&a(xl),x(vs,e),e&&a(wl),e&&a(re),x(js),e&&a(yl),e&&a(Y),e&&a(kl),e&&a(oe),x(bs),e&&a(El),x($s,e),e&&a(Dl),x(Fe,e),e&&a(Al),e&&a(pe),x(xs),e&&a(Il),e&&a(Ye),e&&a(Tl),x(ws,e),e&&a(Pl),e&&a(ie),e&&a(ql),x(ys,e),e&&a(Sl),e&&a(he),x(ks),e&&a(Cl),e&&a(Es),e&&a(zl),x(Ue,e)}}}const dh={local:"stream",sections:[{local:"shuffle",title:"Shuffle"},{local:"reshuffle",title:"Reshuffle"},{local:"split-dataset",title:"Split dataset"},{local:"interleave",title:"Interleave"},{local:"rename-remove-and-cast",sections:[{local:"rename",title:"Rename"},{local:"remove",title:"Remove"},{local:"cast",title:"Cast"}],title:"Rename, remove, and cast"},{local:"map",sections:[{local:"batch-processing",sections:[{local:"tokenization",title:"Tokenization"}],title:"Batch processing"},{local:"filter",title:"Filter"}],title:"Map"},{local:"stream-in-a-training-loop",title:"Stream in a training loop"}],title:"Stream"};function mh(q){return sh(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $h extends Qi{constructor(d){super();Xi(this,d,mh,fh,Zi,{})}}export{$h as default,dh as metadata};
