import{S as Mn,i as Un,s as Kn,e as a,k as d,w as c,t as i,M as Wn,c as o,d as e,m as f,a as r,x as h,h as p,b as u,G as s,g as n,y as m,q as g,o as _,B as v,v as Jn}from"../chunks/vendor-hf-doc-builder.js";import{T as Qn}from"../chunks/Tip-hf-doc-builder.js";import{I as D}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as $}from"../chunks/CodeBlock-hf-doc-builder.js";function Vn(Ss){let y,M,w,j,C,b,_t,z;return{c(){y=a("p"),M=i("Remember to define your credentials in your "),w=a("a"),j=i("FileSystem instance"),C=d(),b=a("code"),_t=i("fs"),z=i(" whenever you are interacting with a private cloud storage."),this.h()},l(L){y=o(L,"P",{});var E=r(y);M=p(E,"Remember to define your credentials in your "),w=o(E,"A",{href:!0});var A=r(w);j=p(A,"FileSystem instance"),A.forEach(e),C=f(E),b=o(E,"CODE",{});var de=r(b);_t=p(de,"fs"),de.forEach(e),z=p(E," whenever you are interacting with a private cloud storage."),E.forEach(e),this.h()},h(){u(w,"href","#set-up-your-cloud-storage-filesystem")},m(L,E){n(L,y,E),s(y,M),s(y,w),s(w,j),s(y,C),s(y,b),s(b,_t),s(y,z)},d(L){L&&e(y)}}}function Xn(Ss){let y,M,w,j,C,b,_t,z,L,E,A,de,ke,ao,oo,As,U,je,vt,qe,ro,lo,Se,no,io,k,yt,Ae,po,fo,De,$t,uo,co,wt,Te,ho,mo,Pe,bt,go,_o,Et,Ce,vo,yo,ze,kt,$o,wo,jt,Le,bo,Eo,Ne,qt,ko,jo,St,Oe,qo,So,Fe,At,Ao,Ds,fe,Do,Ts,N,K,Ie,Dt,To,xe,Po,Ps,O,W,He,Tt,Co,Be,zo,Cs,ue,Ge,Lo,zs,Pt,Ls,Ct,Ye,No,Ns,q,Oo,Re,Fo,Io,Me,xo,Ho,Ue,Bo,Go,Os,zt,Fs,Lt,Ke,Yo,Is,Nt,xs,F,J,We,Ot,Ro,Je,Mo,Hs,ce,Qe,Uo,Bs,Ft,Gs,It,Ve,Ko,Ys,xt,Rs,Ht,Xe,Wo,Ms,Bt,Us,I,Q,Ze,Gt,Jo,ts,Qo,Ks,he,es,Vo,Ws,Yt,Js,Rt,ss,Xo,Qs,Mt,Vs,Ut,as,Zo,Xs,Kt,Zs,x,V,os,Wt,tr,rs,er,ta,H,X,ls,Jt,sr,ns,ar,ea,S,or,is,rr,lr,ps,nr,ir,ds,pr,dr,sa,Z,fr,fs,ur,cr,aa,tt,et,hr,us,mr,gr,me,_r,vr,cs,yr,oa,st,$r,ge,wr,br,ra,Qt,la,at,Er,_e,kr,jr,na,Vt,ia,ot,qr,ve,Sr,Ar,pa,Xt,da,rt,Dr,hs,Tr,Pr,fa,lt,Cr,ms,zr,Lr,ua,Zt,ca,B,nt,gs,te,Nr,_s,Or,ha,ye,Fr,ma,$e,Ir,ga,ee,_a,it,xr,se,Hr,Br,va,G,pt,vs,ae,Gr,ys,Yr,ya,dt,Rr,we,Mr,Ur,$a,oe,wa,ft,ba,Y,ut,$s,re,Kr,ws,Wr,Ea,T,Jr,bs,Qr,Vr,Es,Xr,Zr,ka,le,ja,R,ct,ks,ne,tl,js,el,qa,ht,sl,be,al,ol,Sa,ie,Aa;return b=new D({}),Dt=new D({}),Tt=new D({}),Pt=new $({props:{code:"pip install s3fs",highlighted:'<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">pip install s3fs</span>'}}),zt=new $({props:{code:`storage_options = {"anon": True}  # for anonymous connection
storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key}  # for private buckets
import botocore
s3_session = botocore.session.Session(profile="my_profile_name")
storage_options = {"session": s3_session}`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;anon&quot;</span>: <span class="hljs-literal">True</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;key&quot;</span>: aws_access_key_id, <span class="hljs-string">&quot;secret&quot;</span>: aws_secret_access_key}  <span class="hljs-comment"># for private buckets</span>
<span class="hljs-comment"># or use a botocore session</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> botocore
<span class="hljs-meta">&gt;&gt;&gt; </span>s3_session = botocore.session.Session(profile=<span class="hljs-string">&quot;my_profile_name&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;session&quot;</span>: s3_session}`}}),Nt=new $({props:{code:`import s3fs
fs = s3fs.S3FileSystem(**storage_options)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> s3fs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = s3fs.S3FileSystem(**storage_options)`}}),Ot=new D({}),Ft=new $({props:{code:`conda install -c conda-forge gcsfs
pip install gcsfs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">conda install -c conda-forge gcsfs</span>
# or install with pip
<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">pip install gcsfs</span>`}}),xt=new $({props:{code:`storage_options={"token": "anon"}  # for anonymous connection
storage_options={"project": "my-google-project"}
storage_options={"project": "my-google-project", "token": TOKEN}`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;token&quot;</span>: <span class="hljs-string">&quot;anon&quot;</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials of your default gcloud credentials or from the google metadata service</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;project&quot;</span>: <span class="hljs-string">&quot;my-google-project&quot;</span>}
<span class="hljs-comment"># or use your credentials from elsewhere, see the documentation at https://gcsfs.readthedocs.io/</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;project&quot;</span>: <span class="hljs-string">&quot;my-google-project&quot;</span>, <span class="hljs-string">&quot;token&quot;</span>: TOKEN}`}}),Bt=new $({props:{code:`import gcsfs
fs = gcsfs.GCSFileSystem(**storage_options)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> gcsfs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = gcsfs.GCSFileSystem(**storage_options)`}}),Gt=new D({}),Yt=new $({props:{code:`conda install -c conda-forge adlfs
pip install adlfs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">conda install -c conda-forge adlfs</span>
# or install with pip
<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">pip install adlfs</span>`}}),Mt=new $({props:{code:`storage_options = {"anon": True}  # for anonymous connection
storage_options = {"account_name": ACCOUNT_NAME, "account_key": ACCOUNT_KEY}  # gen 2 filesystem
storage_options={"tenant_id": TENANT_ID, "client_id": CLIENT_ID, "client_secret": CLIENT_SECRET}`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;anon&quot;</span>: <span class="hljs-literal">True</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;account_name&quot;</span>: ACCOUNT_NAME, <span class="hljs-string">&quot;account_key&quot;</span>: ACCOUNT_KEY}  <span class="hljs-comment"># gen 2 filesystem</span>
<span class="hljs-comment"># or use your credentials with the gen 1 filesystem</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;tenant_id&quot;</span>: TENANT_ID, <span class="hljs-string">&quot;client_id&quot;</span>: CLIENT_ID, <span class="hljs-string">&quot;client_secret&quot;</span>: CLIENT_SECRET}`}}),Kt=new $({props:{code:`import adlfs
fs = adlfs.AzureBlobFileSystem(**storage_options)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> adlfs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = adlfs.AzureBlobFileSystem(**storage_options)`}}),Wt=new D({}),Jt=new D({}),Qt=new $({props:{code:`output_dir = "s3://my-bucket/imdb"
builder = load_dataset_builder("imdb")
builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;imdb&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`}}),Vt=new $({props:{code:`output_dir = "s3://my-bucket/imdb"
builder = load_dataset_builder("path/to/local/loading_script/loading_script.py")
builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;path/to/local/loading_script/loading_script.py&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`}}),Xt=new $({props:{code:`data_files = {"train": ["path/to/train.csv"]}
output_dir = "s3://my-bucket/imdb"
builder = load_dataset_builder("csv", data_files=data_files)
builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/train.csv&quot;</span>]}
<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`}}),Zt=new $({props:{code:'builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet", max_shard_size="1GB")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>, max_shard_size=<span class="hljs-string">&quot;1GB&quot;</span>)'}}),te=new D({}),ee=new $({props:{code:`import dask.dataframe as dd

df = dd.read_parquet(output_dir, storage_options=storage_options)

# or if your dataset is split into train/valid/test
df_train = dd.read_parquet(output_dir + f"/{builder.name}-train-*.parquet", storage_options=storage_options)
df_valid = dd.read_parquet(output_dir + f"/{builder.name}-validation-*.parquet", storage_options=storage_options)
df_test = dd.read_parquet(output_dir + f"/{builder.name}-test-*.parquet", storage_options=storage_options)`,highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df = dd.read_parquet(output_dir, storage_options=storage_options)

<span class="hljs-comment"># or if your dataset is split into train/valid/test</span>
df_train = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-train-*.parquet&quot;</span>, storage_options=storage_options)
df_valid = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-validation-*.parquet&quot;</span>, storage_options=storage_options)
df_test = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-test-*.parquet&quot;</span>, storage_options=storage_options)`}}),ae=new D({}),oe=new $({props:{code:`encoded_dataset.save_to_disk("s3://my-private-datasets/imdb/train", storage_options=storage_options)
encoded_dataset.save_to_disk("gcs://my-private-datasets/imdb/train", storage_options=storage_options)
encoded_dataset.save_to_disk("adl://my-private-datasets/imdb/train", storage_options=storage_options)`,highlighted:`<span class="hljs-comment"># saves encoded_dataset to amazon s3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;s3://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)
<span class="hljs-comment"># saves encoded_dataset to google cloud storage</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;gcs://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)
<span class="hljs-comment"># saves encoded_dataset to microsoft azure blob/datalake</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;adl://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)`}}),ft=new Qn({props:{$$slots:{default:[Vn]},$$scope:{ctx:Ss}}}),re=new D({}),le=new $({props:{code:'fs.ls("my-private-datasets/imdb/train")',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>fs.ls(<span class="hljs-string">&quot;my-private-datasets/imdb/train&quot;</span>)
[<span class="hljs-string">&quot;dataset_info.json.json&quot;</span>,<span class="hljs-string">&quot;dataset.arrow&quot;</span>,<span class="hljs-string">&quot;state.json&quot;</span>]`}}),ne=new D({}),ie=new $({props:{code:`from datasets import load_from_disk
dataset = load_from_disk("s3://a-public-datasets/imdb/train", storage_options=storage_options)  
print(len(dataset))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_from_disk
<span class="hljs-comment"># load encoded_dataset from cloud storage</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_from_disk(<span class="hljs-string">&quot;s3://a-public-datasets/imdb/train&quot;</span>, storage_options=storage_options)  
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(dataset))
<span class="hljs-number">25000</span>`}}),{c(){y=a("meta"),M=d(),w=a("h1"),j=a("a"),C=a("span"),c(b.$$.fragment),_t=d(),z=a("span"),L=i("Cloud storage"),E=d(),A=a("p"),de=i("\u{1F917} Datasets supports access to cloud storage providers through a "),ke=a("code"),ao=i("fsspec"),oo=i(` FileSystem implementations.
You can save and load datasets from any cloud storage in a Pythonic way.
Take a look at the following table for some example of supported cloud storage providers:`),As=d(),U=a("table"),je=a("thead"),vt=a("tr"),qe=a("th"),ro=i("Storage provider"),lo=d(),Se=a("th"),no=i("Filesystem implementation"),io=d(),k=a("tbody"),yt=a("tr"),Ae=a("td"),po=i("Amazon S3"),fo=d(),De=a("td"),$t=a("a"),uo=i("s3fs"),co=d(),wt=a("tr"),Te=a("td"),ho=i("Google Cloud Storage"),mo=d(),Pe=a("td"),bt=a("a"),go=i("gcsfs"),_o=d(),Et=a("tr"),Ce=a("td"),vo=i("Azure Blob/DataLake"),yo=d(),ze=a("td"),kt=a("a"),$o=i("adlfs"),wo=d(),jt=a("tr"),Le=a("td"),bo=i("Dropbox"),Eo=d(),Ne=a("td"),qt=a("a"),ko=i("dropboxdrivefs"),jo=d(),St=a("tr"),Oe=a("td"),qo=i("Google Drive"),So=d(),Fe=a("td"),At=a("a"),Ao=i("gdrivefs"),Ds=d(),fe=a("p"),Do=i(`This guide will show you how to save and load datasets with any cloud storage.
Here are examples for S3, Google Cloud Storage and Azure Blob Storage.`),Ts=d(),N=a("h2"),K=a("a"),Ie=a("span"),c(Dt.$$.fragment),To=d(),xe=a("span"),Po=i("Set up your cloud storage FileSystem"),Ps=d(),O=a("h3"),W=a("a"),He=a("span"),c(Tt.$$.fragment),Co=d(),Be=a("span"),zo=i("Amazon S3"),Cs=d(),ue=a("ol"),Ge=a("li"),Lo=i("Install the S3 FileSystem implementation:"),zs=d(),c(Pt.$$.fragment),Ls=d(),Ct=a("ol"),Ye=a("li"),No=i("Define your credentials"),Ns=d(),q=a("p"),Oo=i("To use an anonymous connection, use "),Re=a("code"),Fo=i("anon=True"),Io=i(`.
Otherwise, include your `),Me=a("code"),xo=i("aws_access_key_id"),Ho=i(" and "),Ue=a("code"),Bo=i("aws_secret_access_key"),Go=i(" whenever you are interacting with a private S3 bucket."),Os=d(),c(zt.$$.fragment),Fs=d(),Lt=a("ol"),Ke=a("li"),Yo=i("Create your FileSystem instance"),Is=d(),c(Nt.$$.fragment),xs=d(),F=a("h3"),J=a("a"),We=a("span"),c(Ot.$$.fragment),Ro=d(),Je=a("span"),Mo=i("Google Cloud Storage"),Hs=d(),ce=a("ol"),Qe=a("li"),Uo=i("Install the Google Cloud Storage implementation:"),Bs=d(),c(Ft.$$.fragment),Gs=d(),It=a("ol"),Ve=a("li"),Ko=i("Define your credentials"),Ys=d(),c(xt.$$.fragment),Rs=d(),Ht=a("ol"),Xe=a("li"),Wo=i("Create your FileSystem instance"),Ms=d(),c(Bt.$$.fragment),Us=d(),I=a("h3"),Q=a("a"),Ze=a("span"),c(Gt.$$.fragment),Jo=d(),ts=a("span"),Qo=i("Azure Blob Storage"),Ks=d(),he=a("ol"),es=a("li"),Vo=i("Install the Azure Blob Storage implementation:"),Ws=d(),c(Yt.$$.fragment),Js=d(),Rt=a("ol"),ss=a("li"),Xo=i("Define your credentials"),Qs=d(),c(Mt.$$.fragment),Vs=d(),Ut=a("ol"),as=a("li"),Zo=i("Create your FileSystem instance"),Xs=d(),c(Kt.$$.fragment),Zs=d(),x=a("h2"),V=a("a"),os=a("span"),c(Wt.$$.fragment),tr=d(),rs=a("span"),er=i("Load and Save your datasets using your cloud storage FileSystem"),ta=d(),H=a("h3"),X=a("a"),ls=a("span"),c(Jt.$$.fragment),sr=d(),ns=a("span"),ar=i("Download and prepare a dataset into a cloud storage"),ea=d(),S=a("p"),or=i("You can download and prepare a dataset into your cloud storage by specifying a remote "),is=a("code"),rr=i("output_dir"),lr=i(" in "),ps=a("code"),nr=i("download_and_prepare"),ir=i(`.
Don\u2019t forget to use the previously defined `),ds=a("code"),pr=i("storage_options"),dr=i(" containing your credentials to write into a private cloud storage."),sa=d(),Z=a("p"),fr=i("The "),fs=a("code"),ur=i("download_and_prepare"),cr=i(" method works in two steps:"),aa=d(),tt=a("ol"),et=a("li"),hr=i("it first downloads the raw data files (if any) in your local cache. You can set your cache directory by passing "),us=a("code"),mr=i("cache_dir"),gr=i(" to "),me=a("a"),_r=i("load_dataset_builder()"),vr=d(),cs=a("li"),yr=i("then it generates the dataset in Arrow or Parquet format in your cloud storage by iterating over the raw data files."),oa=d(),st=a("p"),$r=i("Load a dataset builder from the Hugging Face Hub (see "),ge=a("a"),wr=i("how to load from the Hugging Face Hub"),br=i("):"),ra=d(),c(Qt.$$.fragment),la=d(),at=a("p"),Er=i("Load a dataset builder using a loading script (see "),_e=a("a"),kr=i("how to load a local loading script"),jr=i("):"),na=d(),c(Vt.$$.fragment),ia=d(),ot=a("p"),qr=i("Use your own data files (see "),ve=a("a"),Sr=i("how to load local and remote files"),Ar=i("):"),pa=d(),c(Xt.$$.fragment),da=d(),rt=a("p"),Dr=i("It is highly recommended to save the files as compressed Parquet files to optimize I/O by specifying "),hs=a("code"),Tr=i('file_format="parquet"'),Pr=i(`.
Otherwise the dataset is saved as an uncompressed Arrow file.`),fa=d(),lt=a("p"),Cr=i("You can also specify the size of the shards using "),ms=a("code"),zr=i("max_shard_size"),Lr=i(" (default is 500MB):"),ua=d(),c(Zt.$$.fragment),ca=d(),B=a("h4"),nt=a("a"),gs=a("span"),c(te.$$.fragment),Nr=d(),_s=a("span"),Or=i("Dask"),ha=d(),ye=a("p"),Fr=i(`Dask is a parallel computing library and it has a pandas-like API for working with larger than memory Parquet datasets in parallel.
Dask can use multiple threads or processes on a single machine, or a cluster of machines to process data in parallel.
Dask supports local data but also data from a cloud storage.`),ma=d(),$e=a("p"),Ir=i("Therefore you can load a dataset saved as sharded Parquet files in Dask with"),ga=d(),c(ee.$$.fragment),_a=d(),it=a("p"),xr=i("You can find more about dask dataframes in their "),se=a("a"),Hr=i("documentation"),Br=i("."),va=d(),G=a("h2"),pt=a("a"),vs=a("span"),c(ae.$$.fragment),Gr=d(),ys=a("span"),Yr=i("Saving serialized datasets"),ya=d(),dt=a("p"),Rr=i("After you have processed your dataset, you can save it to your cloud storage with "),we=a("a"),Mr=i("Dataset.save_to_disk()"),Ur=i(":"),$a=d(),c(oe.$$.fragment),wa=d(),c(ft.$$.fragment),ba=d(),Y=a("h2"),ut=a("a"),$s=a("span"),c(re.$$.fragment),Kr=d(),ws=a("span"),Wr=i("Listing serialized datasets"),Ea=d(),T=a("p"),Jr=i("List files from a cloud storage with your FileSystem instance "),bs=a("code"),Qr=i("fs"),Vr=i(", using "),Es=a("code"),Xr=i("fs.ls"),Zr=i(":"),ka=d(),c(le.$$.fragment),ja=d(),R=a("h3"),ct=a("a"),ks=a("span"),c(ne.$$.fragment),tl=d(),js=a("span"),el=i("Load serialized datasets"),qa=d(),ht=a("p"),sl=i("When you are ready to use your dataset again, reload it with "),be=a("a"),al=i("Dataset.load_from_disk()"),ol=i(":"),Sa=d(),c(ie.$$.fragment),this.h()},l(t){const l=Wn('[data-svelte="svelte-1phssyn"]',document.head);y=o(l,"META",{name:!0,content:!0}),l.forEach(e),M=f(t),w=o(t,"H1",{class:!0});var pe=r(w);j=o(pe,"A",{id:!0,class:!0,href:!0});var rl=r(j);C=o(rl,"SPAN",{});var ll=r(C);h(b.$$.fragment,ll),ll.forEach(e),rl.forEach(e),_t=f(pe),z=o(pe,"SPAN",{});var nl=r(z);L=p(nl,"Cloud storage"),nl.forEach(e),pe.forEach(e),E=f(t),A=o(t,"P",{});var Da=r(A);de=p(Da,"\u{1F917} Datasets supports access to cloud storage providers through a "),ke=o(Da,"CODE",{});var il=r(ke);ao=p(il,"fsspec"),il.forEach(e),oo=p(Da,` FileSystem implementations.
You can save and load datasets from any cloud storage in a Pythonic way.
Take a look at the following table for some example of supported cloud storage providers:`),Da.forEach(e),As=f(t),U=o(t,"TABLE",{});var Ta=r(U);je=o(Ta,"THEAD",{});var pl=r(je);vt=o(pl,"TR",{});var Pa=r(vt);qe=o(Pa,"TH",{});var dl=r(qe);ro=p(dl,"Storage provider"),dl.forEach(e),lo=f(Pa),Se=o(Pa,"TH",{});var fl=r(Se);no=p(fl,"Filesystem implementation"),fl.forEach(e),Pa.forEach(e),pl.forEach(e),io=f(Ta),k=o(Ta,"TBODY",{});var P=r(k);yt=o(P,"TR",{});var Ca=r(yt);Ae=o(Ca,"TD",{});var ul=r(Ae);po=p(ul,"Amazon S3"),ul.forEach(e),fo=f(Ca),De=o(Ca,"TD",{});var cl=r(De);$t=o(cl,"A",{href:!0,rel:!0});var hl=r($t);uo=p(hl,"s3fs"),hl.forEach(e),cl.forEach(e),Ca.forEach(e),co=f(P),wt=o(P,"TR",{});var za=r(wt);Te=o(za,"TD",{});var ml=r(Te);ho=p(ml,"Google Cloud Storage"),ml.forEach(e),mo=f(za),Pe=o(za,"TD",{});var gl=r(Pe);bt=o(gl,"A",{href:!0,rel:!0});var _l=r(bt);go=p(_l,"gcsfs"),_l.forEach(e),gl.forEach(e),za.forEach(e),_o=f(P),Et=o(P,"TR",{});var La=r(Et);Ce=o(La,"TD",{});var vl=r(Ce);vo=p(vl,"Azure Blob/DataLake"),vl.forEach(e),yo=f(La),ze=o(La,"TD",{});var yl=r(ze);kt=o(yl,"A",{href:!0,rel:!0});var $l=r(kt);$o=p($l,"adlfs"),$l.forEach(e),yl.forEach(e),La.forEach(e),wo=f(P),jt=o(P,"TR",{});var Na=r(jt);Le=o(Na,"TD",{});var wl=r(Le);bo=p(wl,"Dropbox"),wl.forEach(e),Eo=f(Na),Ne=o(Na,"TD",{});var bl=r(Ne);qt=o(bl,"A",{href:!0,rel:!0});var El=r(qt);ko=p(El,"dropboxdrivefs"),El.forEach(e),bl.forEach(e),Na.forEach(e),jo=f(P),St=o(P,"TR",{});var Oa=r(St);Oe=o(Oa,"TD",{});var kl=r(Oe);qo=p(kl,"Google Drive"),kl.forEach(e),So=f(Oa),Fe=o(Oa,"TD",{});var jl=r(Fe);At=o(jl,"A",{href:!0,rel:!0});var ql=r(At);Ao=p(ql,"gdrivefs"),ql.forEach(e),jl.forEach(e),Oa.forEach(e),P.forEach(e),Ta.forEach(e),Ds=f(t),fe=o(t,"P",{});var Sl=r(fe);Do=p(Sl,`This guide will show you how to save and load datasets with any cloud storage.
Here are examples for S3, Google Cloud Storage and Azure Blob Storage.`),Sl.forEach(e),Ts=f(t),N=o(t,"H2",{class:!0});var Fa=r(N);K=o(Fa,"A",{id:!0,class:!0,href:!0});var Al=r(K);Ie=o(Al,"SPAN",{});var Dl=r(Ie);h(Dt.$$.fragment,Dl),Dl.forEach(e),Al.forEach(e),To=f(Fa),xe=o(Fa,"SPAN",{});var Tl=r(xe);Po=p(Tl,"Set up your cloud storage FileSystem"),Tl.forEach(e),Fa.forEach(e),Ps=f(t),O=o(t,"H3",{class:!0});var Ia=r(O);W=o(Ia,"A",{id:!0,class:!0,href:!0});var Pl=r(W);He=o(Pl,"SPAN",{});var Cl=r(He);h(Tt.$$.fragment,Cl),Cl.forEach(e),Pl.forEach(e),Co=f(Ia),Be=o(Ia,"SPAN",{});var zl=r(Be);zo=p(zl,"Amazon S3"),zl.forEach(e),Ia.forEach(e),Cs=f(t),ue=o(t,"OL",{});var Ll=r(ue);Ge=o(Ll,"LI",{});var Nl=r(Ge);Lo=p(Nl,"Install the S3 FileSystem implementation:"),Nl.forEach(e),Ll.forEach(e),zs=f(t),h(Pt.$$.fragment,t),Ls=f(t),Ct=o(t,"OL",{start:!0});var Ol=r(Ct);Ye=o(Ol,"LI",{});var Fl=r(Ye);No=p(Fl,"Define your credentials"),Fl.forEach(e),Ol.forEach(e),Ns=f(t),q=o(t,"P",{});var mt=r(q);Oo=p(mt,"To use an anonymous connection, use "),Re=o(mt,"CODE",{});var Il=r(Re);Fo=p(Il,"anon=True"),Il.forEach(e),Io=p(mt,`.
Otherwise, include your `),Me=o(mt,"CODE",{});var xl=r(Me);xo=p(xl,"aws_access_key_id"),xl.forEach(e),Ho=p(mt," and "),Ue=o(mt,"CODE",{});var Hl=r(Ue);Bo=p(Hl,"aws_secret_access_key"),Hl.forEach(e),Go=p(mt," whenever you are interacting with a private S3 bucket."),mt.forEach(e),Os=f(t),h(zt.$$.fragment,t),Fs=f(t),Lt=o(t,"OL",{start:!0});var Bl=r(Lt);Ke=o(Bl,"LI",{});var Gl=r(Ke);Yo=p(Gl,"Create your FileSystem instance"),Gl.forEach(e),Bl.forEach(e),Is=f(t),h(Nt.$$.fragment,t),xs=f(t),F=o(t,"H3",{class:!0});var xa=r(F);J=o(xa,"A",{id:!0,class:!0,href:!0});var Yl=r(J);We=o(Yl,"SPAN",{});var Rl=r(We);h(Ot.$$.fragment,Rl),Rl.forEach(e),Yl.forEach(e),Ro=f(xa),Je=o(xa,"SPAN",{});var Ml=r(Je);Mo=p(Ml,"Google Cloud Storage"),Ml.forEach(e),xa.forEach(e),Hs=f(t),ce=o(t,"OL",{});var Ul=r(ce);Qe=o(Ul,"LI",{});var Kl=r(Qe);Uo=p(Kl,"Install the Google Cloud Storage implementation:"),Kl.forEach(e),Ul.forEach(e),Bs=f(t),h(Ft.$$.fragment,t),Gs=f(t),It=o(t,"OL",{start:!0});var Wl=r(It);Ve=o(Wl,"LI",{});var Jl=r(Ve);Ko=p(Jl,"Define your credentials"),Jl.forEach(e),Wl.forEach(e),Ys=f(t),h(xt.$$.fragment,t),Rs=f(t),Ht=o(t,"OL",{start:!0});var Ql=r(Ht);Xe=o(Ql,"LI",{});var Vl=r(Xe);Wo=p(Vl,"Create your FileSystem instance"),Vl.forEach(e),Ql.forEach(e),Ms=f(t),h(Bt.$$.fragment,t),Us=f(t),I=o(t,"H3",{class:!0});var Ha=r(I);Q=o(Ha,"A",{id:!0,class:!0,href:!0});var Xl=r(Q);Ze=o(Xl,"SPAN",{});var Zl=r(Ze);h(Gt.$$.fragment,Zl),Zl.forEach(e),Xl.forEach(e),Jo=f(Ha),ts=o(Ha,"SPAN",{});var tn=r(ts);Qo=p(tn,"Azure Blob Storage"),tn.forEach(e),Ha.forEach(e),Ks=f(t),he=o(t,"OL",{});var en=r(he);es=o(en,"LI",{});var sn=r(es);Vo=p(sn,"Install the Azure Blob Storage implementation:"),sn.forEach(e),en.forEach(e),Ws=f(t),h(Yt.$$.fragment,t),Js=f(t),Rt=o(t,"OL",{start:!0});var an=r(Rt);ss=o(an,"LI",{});var on=r(ss);Xo=p(on,"Define your credentials"),on.forEach(e),an.forEach(e),Qs=f(t),h(Mt.$$.fragment,t),Vs=f(t),Ut=o(t,"OL",{start:!0});var rn=r(Ut);as=o(rn,"LI",{});var ln=r(as);Zo=p(ln,"Create your FileSystem instance"),ln.forEach(e),rn.forEach(e),Xs=f(t),h(Kt.$$.fragment,t),Zs=f(t),x=o(t,"H2",{class:!0});var Ba=r(x);V=o(Ba,"A",{id:!0,class:!0,href:!0});var nn=r(V);os=o(nn,"SPAN",{});var pn=r(os);h(Wt.$$.fragment,pn),pn.forEach(e),nn.forEach(e),tr=f(Ba),rs=o(Ba,"SPAN",{});var dn=r(rs);er=p(dn,"Load and Save your datasets using your cloud storage FileSystem"),dn.forEach(e),Ba.forEach(e),ta=f(t),H=o(t,"H3",{class:!0});var Ga=r(H);X=o(Ga,"A",{id:!0,class:!0,href:!0});var fn=r(X);ls=o(fn,"SPAN",{});var un=r(ls);h(Jt.$$.fragment,un),un.forEach(e),fn.forEach(e),sr=f(Ga),ns=o(Ga,"SPAN",{});var cn=r(ns);ar=p(cn,"Download and prepare a dataset into a cloud storage"),cn.forEach(e),Ga.forEach(e),ea=f(t),S=o(t,"P",{});var gt=r(S);or=p(gt,"You can download and prepare a dataset into your cloud storage by specifying a remote "),is=o(gt,"CODE",{});var hn=r(is);rr=p(hn,"output_dir"),hn.forEach(e),lr=p(gt," in "),ps=o(gt,"CODE",{});var mn=r(ps);nr=p(mn,"download_and_prepare"),mn.forEach(e),ir=p(gt,`.
Don\u2019t forget to use the previously defined `),ds=o(gt,"CODE",{});var gn=r(ds);pr=p(gn,"storage_options"),gn.forEach(e),dr=p(gt," containing your credentials to write into a private cloud storage."),gt.forEach(e),sa=f(t),Z=o(t,"P",{});var Ya=r(Z);fr=p(Ya,"The "),fs=o(Ya,"CODE",{});var _n=r(fs);ur=p(_n,"download_and_prepare"),_n.forEach(e),cr=p(Ya," method works in two steps:"),Ya.forEach(e),aa=f(t),tt=o(t,"OL",{});var Ra=r(tt);et=o(Ra,"LI",{});var qs=r(et);hr=p(qs,"it first downloads the raw data files (if any) in your local cache. You can set your cache directory by passing "),us=o(qs,"CODE",{});var vn=r(us);mr=p(vn,"cache_dir"),vn.forEach(e),gr=p(qs," to "),me=o(qs,"A",{href:!0});var yn=r(me);_r=p(yn,"load_dataset_builder()"),yn.forEach(e),qs.forEach(e),vr=f(Ra),cs=o(Ra,"LI",{});var $n=r(cs);yr=p($n,"then it generates the dataset in Arrow or Parquet format in your cloud storage by iterating over the raw data files."),$n.forEach(e),Ra.forEach(e),oa=f(t),st=o(t,"P",{});var Ma=r(st);$r=p(Ma,"Load a dataset builder from the Hugging Face Hub (see "),ge=o(Ma,"A",{href:!0});var wn=r(ge);wr=p(wn,"how to load from the Hugging Face Hub"),wn.forEach(e),br=p(Ma,"):"),Ma.forEach(e),ra=f(t),h(Qt.$$.fragment,t),la=f(t),at=o(t,"P",{});var Ua=r(at);Er=p(Ua,"Load a dataset builder using a loading script (see "),_e=o(Ua,"A",{href:!0});var bn=r(_e);kr=p(bn,"how to load a local loading script"),bn.forEach(e),jr=p(Ua,"):"),Ua.forEach(e),na=f(t),h(Vt.$$.fragment,t),ia=f(t),ot=o(t,"P",{});var Ka=r(ot);qr=p(Ka,"Use your own data files (see "),ve=o(Ka,"A",{href:!0});var En=r(ve);Sr=p(En,"how to load local and remote files"),En.forEach(e),Ar=p(Ka,"):"),Ka.forEach(e),pa=f(t),h(Xt.$$.fragment,t),da=f(t),rt=o(t,"P",{});var Wa=r(rt);Dr=p(Wa,"It is highly recommended to save the files as compressed Parquet files to optimize I/O by specifying "),hs=o(Wa,"CODE",{});var kn=r(hs);Tr=p(kn,'file_format="parquet"'),kn.forEach(e),Pr=p(Wa,`.
Otherwise the dataset is saved as an uncompressed Arrow file.`),Wa.forEach(e),fa=f(t),lt=o(t,"P",{});var Ja=r(lt);Cr=p(Ja,"You can also specify the size of the shards using "),ms=o(Ja,"CODE",{});var jn=r(ms);zr=p(jn,"max_shard_size"),jn.forEach(e),Lr=p(Ja," (default is 500MB):"),Ja.forEach(e),ua=f(t),h(Zt.$$.fragment,t),ca=f(t),B=o(t,"H4",{class:!0});var Qa=r(B);nt=o(Qa,"A",{id:!0,class:!0,href:!0});var qn=r(nt);gs=o(qn,"SPAN",{});var Sn=r(gs);h(te.$$.fragment,Sn),Sn.forEach(e),qn.forEach(e),Nr=f(Qa),_s=o(Qa,"SPAN",{});var An=r(_s);Or=p(An,"Dask"),An.forEach(e),Qa.forEach(e),ha=f(t),ye=o(t,"P",{});var Dn=r(ye);Fr=p(Dn,`Dask is a parallel computing library and it has a pandas-like API for working with larger than memory Parquet datasets in parallel.
Dask can use multiple threads or processes on a single machine, or a cluster of machines to process data in parallel.
Dask supports local data but also data from a cloud storage.`),Dn.forEach(e),ma=f(t),$e=o(t,"P",{});var Tn=r($e);Ir=p(Tn,"Therefore you can load a dataset saved as sharded Parquet files in Dask with"),Tn.forEach(e),ga=f(t),h(ee.$$.fragment,t),_a=f(t),it=o(t,"P",{});var Va=r(it);xr=p(Va,"You can find more about dask dataframes in their "),se=o(Va,"A",{href:!0,rel:!0});var Pn=r(se);Hr=p(Pn,"documentation"),Pn.forEach(e),Br=p(Va,"."),Va.forEach(e),va=f(t),G=o(t,"H2",{class:!0});var Xa=r(G);pt=o(Xa,"A",{id:!0,class:!0,href:!0});var Cn=r(pt);vs=o(Cn,"SPAN",{});var zn=r(vs);h(ae.$$.fragment,zn),zn.forEach(e),Cn.forEach(e),Gr=f(Xa),ys=o(Xa,"SPAN",{});var Ln=r(ys);Yr=p(Ln,"Saving serialized datasets"),Ln.forEach(e),Xa.forEach(e),ya=f(t),dt=o(t,"P",{});var Za=r(dt);Rr=p(Za,"After you have processed your dataset, you can save it to your cloud storage with "),we=o(Za,"A",{href:!0});var Nn=r(we);Mr=p(Nn,"Dataset.save_to_disk()"),Nn.forEach(e),Ur=p(Za,":"),Za.forEach(e),$a=f(t),h(oe.$$.fragment,t),wa=f(t),h(ft.$$.fragment,t),ba=f(t),Y=o(t,"H2",{class:!0});var to=r(Y);ut=o(to,"A",{id:!0,class:!0,href:!0});var On=r(ut);$s=o(On,"SPAN",{});var Fn=r($s);h(re.$$.fragment,Fn),Fn.forEach(e),On.forEach(e),Kr=f(to),ws=o(to,"SPAN",{});var In=r(ws);Wr=p(In,"Listing serialized datasets"),In.forEach(e),to.forEach(e),Ea=f(t),T=o(t,"P",{});var Ee=r(T);Jr=p(Ee,"List files from a cloud storage with your FileSystem instance "),bs=o(Ee,"CODE",{});var xn=r(bs);Qr=p(xn,"fs"),xn.forEach(e),Vr=p(Ee,", using "),Es=o(Ee,"CODE",{});var Hn=r(Es);Xr=p(Hn,"fs.ls"),Hn.forEach(e),Zr=p(Ee,":"),Ee.forEach(e),ka=f(t),h(le.$$.fragment,t),ja=f(t),R=o(t,"H3",{class:!0});var eo=r(R);ct=o(eo,"A",{id:!0,class:!0,href:!0});var Bn=r(ct);ks=o(Bn,"SPAN",{});var Gn=r(ks);h(ne.$$.fragment,Gn),Gn.forEach(e),Bn.forEach(e),tl=f(eo),js=o(eo,"SPAN",{});var Yn=r(js);el=p(Yn,"Load serialized datasets"),Yn.forEach(e),eo.forEach(e),qa=f(t),ht=o(t,"P",{});var so=r(ht);sl=p(so,"When you are ready to use your dataset again, reload it with "),be=o(so,"A",{href:!0});var Rn=r(be);al=p(Rn,"Dataset.load_from_disk()"),Rn.forEach(e),ol=p(so,":"),so.forEach(e),Sa=f(t),h(ie.$$.fragment,t),this.h()},h(){u(y,"name","hf:doc:metadata"),u(y,"content",JSON.stringify(Zn)),u(j,"id","cloud-storage"),u(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(j,"href","#cloud-storage"),u(w,"class","relative group"),u($t,"href","https://s3fs.readthedocs.io/en/latest/"),u($t,"rel","nofollow"),u(bt,"href","https://gcsfs.readthedocs.io/en/latest/"),u(bt,"rel","nofollow"),u(kt,"href","https://github.com/fsspec/adlfs"),u(kt,"rel","nofollow"),u(qt,"href","https://github.com/MarineChap/dropboxdrivefs"),u(qt,"rel","nofollow"),u(At,"href","https://github.com/intake/gdrivefs"),u(At,"rel","nofollow"),u(K,"id","set-up-your-cloud-storage-filesystem"),u(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(K,"href","#set-up-your-cloud-storage-filesystem"),u(N,"class","relative group"),u(W,"id","amazon-s3"),u(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(W,"href","#amazon-s3"),u(O,"class","relative group"),u(Ct,"start","2"),u(Lt,"start","3"),u(J,"id","google-cloud-storage"),u(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(J,"href","#google-cloud-storage"),u(F,"class","relative group"),u(It,"start","2"),u(Ht,"start","3"),u(Q,"id","azure-blob-storage"),u(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Q,"href","#azure-blob-storage"),u(I,"class","relative group"),u(Rt,"start","2"),u(Ut,"start","3"),u(V,"id","load-and-save-your-datasets-using-your-cloud-storage-filesystem"),u(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(V,"href","#load-and-save-your-datasets-using-your-cloud-storage-filesystem"),u(x,"class","relative group"),u(X,"id","download-and-prepare-a-dataset-into-a-cloud-storage"),u(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(X,"href","#download-and-prepare-a-dataset-into-a-cloud-storage"),u(H,"class","relative group"),u(me,"href","/docs/datasets/v2.8.0/en/package_reference/loading_methods#datasets.load_dataset_builder"),u(ge,"href","./loading#hugging-face-hub"),u(_e,"href","./loading#local-loading-script"),u(ve,"href","./loading#local-and-remote-files"),u(nt,"id","dask"),u(nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(nt,"href","#dask"),u(B,"class","relative group"),u(se,"href","https://docs.dask.org/en/stable/dataframe.html"),u(se,"rel","nofollow"),u(pt,"id","saving-serialized-datasets"),u(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(pt,"href","#saving-serialized-datasets"),u(G,"class","relative group"),u(we,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset.save_to_disk"),u(ut,"id","listing-serialized-datasets"),u(ut,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ut,"href","#listing-serialized-datasets"),u(Y,"class","relative group"),u(ct,"id","load-serialized-datasets"),u(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ct,"href","#load-serialized-datasets"),u(R,"class","relative group"),u(be,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Dataset.load_from_disk")},m(t,l){s(document.head,y),n(t,M,l),n(t,w,l),s(w,j),s(j,C),m(b,C,null),s(w,_t),s(w,z),s(z,L),n(t,E,l),n(t,A,l),s(A,de),s(A,ke),s(ke,ao),s(A,oo),n(t,As,l),n(t,U,l),s(U,je),s(je,vt),s(vt,qe),s(qe,ro),s(vt,lo),s(vt,Se),s(Se,no),s(U,io),s(U,k),s(k,yt),s(yt,Ae),s(Ae,po),s(yt,fo),s(yt,De),s(De,$t),s($t,uo),s(k,co),s(k,wt),s(wt,Te),s(Te,ho),s(wt,mo),s(wt,Pe),s(Pe,bt),s(bt,go),s(k,_o),s(k,Et),s(Et,Ce),s(Ce,vo),s(Et,yo),s(Et,ze),s(ze,kt),s(kt,$o),s(k,wo),s(k,jt),s(jt,Le),s(Le,bo),s(jt,Eo),s(jt,Ne),s(Ne,qt),s(qt,ko),s(k,jo),s(k,St),s(St,Oe),s(Oe,qo),s(St,So),s(St,Fe),s(Fe,At),s(At,Ao),n(t,Ds,l),n(t,fe,l),s(fe,Do),n(t,Ts,l),n(t,N,l),s(N,K),s(K,Ie),m(Dt,Ie,null),s(N,To),s(N,xe),s(xe,Po),n(t,Ps,l),n(t,O,l),s(O,W),s(W,He),m(Tt,He,null),s(O,Co),s(O,Be),s(Be,zo),n(t,Cs,l),n(t,ue,l),s(ue,Ge),s(Ge,Lo),n(t,zs,l),m(Pt,t,l),n(t,Ls,l),n(t,Ct,l),s(Ct,Ye),s(Ye,No),n(t,Ns,l),n(t,q,l),s(q,Oo),s(q,Re),s(Re,Fo),s(q,Io),s(q,Me),s(Me,xo),s(q,Ho),s(q,Ue),s(Ue,Bo),s(q,Go),n(t,Os,l),m(zt,t,l),n(t,Fs,l),n(t,Lt,l),s(Lt,Ke),s(Ke,Yo),n(t,Is,l),m(Nt,t,l),n(t,xs,l),n(t,F,l),s(F,J),s(J,We),m(Ot,We,null),s(F,Ro),s(F,Je),s(Je,Mo),n(t,Hs,l),n(t,ce,l),s(ce,Qe),s(Qe,Uo),n(t,Bs,l),m(Ft,t,l),n(t,Gs,l),n(t,It,l),s(It,Ve),s(Ve,Ko),n(t,Ys,l),m(xt,t,l),n(t,Rs,l),n(t,Ht,l),s(Ht,Xe),s(Xe,Wo),n(t,Ms,l),m(Bt,t,l),n(t,Us,l),n(t,I,l),s(I,Q),s(Q,Ze),m(Gt,Ze,null),s(I,Jo),s(I,ts),s(ts,Qo),n(t,Ks,l),n(t,he,l),s(he,es),s(es,Vo),n(t,Ws,l),m(Yt,t,l),n(t,Js,l),n(t,Rt,l),s(Rt,ss),s(ss,Xo),n(t,Qs,l),m(Mt,t,l),n(t,Vs,l),n(t,Ut,l),s(Ut,as),s(as,Zo),n(t,Xs,l),m(Kt,t,l),n(t,Zs,l),n(t,x,l),s(x,V),s(V,os),m(Wt,os,null),s(x,tr),s(x,rs),s(rs,er),n(t,ta,l),n(t,H,l),s(H,X),s(X,ls),m(Jt,ls,null),s(H,sr),s(H,ns),s(ns,ar),n(t,ea,l),n(t,S,l),s(S,or),s(S,is),s(is,rr),s(S,lr),s(S,ps),s(ps,nr),s(S,ir),s(S,ds),s(ds,pr),s(S,dr),n(t,sa,l),n(t,Z,l),s(Z,fr),s(Z,fs),s(fs,ur),s(Z,cr),n(t,aa,l),n(t,tt,l),s(tt,et),s(et,hr),s(et,us),s(us,mr),s(et,gr),s(et,me),s(me,_r),s(tt,vr),s(tt,cs),s(cs,yr),n(t,oa,l),n(t,st,l),s(st,$r),s(st,ge),s(ge,wr),s(st,br),n(t,ra,l),m(Qt,t,l),n(t,la,l),n(t,at,l),s(at,Er),s(at,_e),s(_e,kr),s(at,jr),n(t,na,l),m(Vt,t,l),n(t,ia,l),n(t,ot,l),s(ot,qr),s(ot,ve),s(ve,Sr),s(ot,Ar),n(t,pa,l),m(Xt,t,l),n(t,da,l),n(t,rt,l),s(rt,Dr),s(rt,hs),s(hs,Tr),s(rt,Pr),n(t,fa,l),n(t,lt,l),s(lt,Cr),s(lt,ms),s(ms,zr),s(lt,Lr),n(t,ua,l),m(Zt,t,l),n(t,ca,l),n(t,B,l),s(B,nt),s(nt,gs),m(te,gs,null),s(B,Nr),s(B,_s),s(_s,Or),n(t,ha,l),n(t,ye,l),s(ye,Fr),n(t,ma,l),n(t,$e,l),s($e,Ir),n(t,ga,l),m(ee,t,l),n(t,_a,l),n(t,it,l),s(it,xr),s(it,se),s(se,Hr),s(it,Br),n(t,va,l),n(t,G,l),s(G,pt),s(pt,vs),m(ae,vs,null),s(G,Gr),s(G,ys),s(ys,Yr),n(t,ya,l),n(t,dt,l),s(dt,Rr),s(dt,we),s(we,Mr),s(dt,Ur),n(t,$a,l),m(oe,t,l),n(t,wa,l),m(ft,t,l),n(t,ba,l),n(t,Y,l),s(Y,ut),s(ut,$s),m(re,$s,null),s(Y,Kr),s(Y,ws),s(ws,Wr),n(t,Ea,l),n(t,T,l),s(T,Jr),s(T,bs),s(bs,Qr),s(T,Vr),s(T,Es),s(Es,Xr),s(T,Zr),n(t,ka,l),m(le,t,l),n(t,ja,l),n(t,R,l),s(R,ct),s(ct,ks),m(ne,ks,null),s(R,tl),s(R,js),s(js,el),n(t,qa,l),n(t,ht,l),s(ht,sl),s(ht,be),s(be,al),s(ht,ol),n(t,Sa,l),m(ie,t,l),Aa=!0},p(t,[l]){const pe={};l&2&&(pe.$$scope={dirty:l,ctx:t}),ft.$set(pe)},i(t){Aa||(g(b.$$.fragment,t),g(Dt.$$.fragment,t),g(Tt.$$.fragment,t),g(Pt.$$.fragment,t),g(zt.$$.fragment,t),g(Nt.$$.fragment,t),g(Ot.$$.fragment,t),g(Ft.$$.fragment,t),g(xt.$$.fragment,t),g(Bt.$$.fragment,t),g(Gt.$$.fragment,t),g(Yt.$$.fragment,t),g(Mt.$$.fragment,t),g(Kt.$$.fragment,t),g(Wt.$$.fragment,t),g(Jt.$$.fragment,t),g(Qt.$$.fragment,t),g(Vt.$$.fragment,t),g(Xt.$$.fragment,t),g(Zt.$$.fragment,t),g(te.$$.fragment,t),g(ee.$$.fragment,t),g(ae.$$.fragment,t),g(oe.$$.fragment,t),g(ft.$$.fragment,t),g(re.$$.fragment,t),g(le.$$.fragment,t),g(ne.$$.fragment,t),g(ie.$$.fragment,t),Aa=!0)},o(t){_(b.$$.fragment,t),_(Dt.$$.fragment,t),_(Tt.$$.fragment,t),_(Pt.$$.fragment,t),_(zt.$$.fragment,t),_(Nt.$$.fragment,t),_(Ot.$$.fragment,t),_(Ft.$$.fragment,t),_(xt.$$.fragment,t),_(Bt.$$.fragment,t),_(Gt.$$.fragment,t),_(Yt.$$.fragment,t),_(Mt.$$.fragment,t),_(Kt.$$.fragment,t),_(Wt.$$.fragment,t),_(Jt.$$.fragment,t),_(Qt.$$.fragment,t),_(Vt.$$.fragment,t),_(Xt.$$.fragment,t),_(Zt.$$.fragment,t),_(te.$$.fragment,t),_(ee.$$.fragment,t),_(ae.$$.fragment,t),_(oe.$$.fragment,t),_(ft.$$.fragment,t),_(re.$$.fragment,t),_(le.$$.fragment,t),_(ne.$$.fragment,t),_(ie.$$.fragment,t),Aa=!1},d(t){e(y),t&&e(M),t&&e(w),v(b),t&&e(E),t&&e(A),t&&e(As),t&&e(U),t&&e(Ds),t&&e(fe),t&&e(Ts),t&&e(N),v(Dt),t&&e(Ps),t&&e(O),v(Tt),t&&e(Cs),t&&e(ue),t&&e(zs),v(Pt,t),t&&e(Ls),t&&e(Ct),t&&e(Ns),t&&e(q),t&&e(Os),v(zt,t),t&&e(Fs),t&&e(Lt),t&&e(Is),v(Nt,t),t&&e(xs),t&&e(F),v(Ot),t&&e(Hs),t&&e(ce),t&&e(Bs),v(Ft,t),t&&e(Gs),t&&e(It),t&&e(Ys),v(xt,t),t&&e(Rs),t&&e(Ht),t&&e(Ms),v(Bt,t),t&&e(Us),t&&e(I),v(Gt),t&&e(Ks),t&&e(he),t&&e(Ws),v(Yt,t),t&&e(Js),t&&e(Rt),t&&e(Qs),v(Mt,t),t&&e(Vs),t&&e(Ut),t&&e(Xs),v(Kt,t),t&&e(Zs),t&&e(x),v(Wt),t&&e(ta),t&&e(H),v(Jt),t&&e(ea),t&&e(S),t&&e(sa),t&&e(Z),t&&e(aa),t&&e(tt),t&&e(oa),t&&e(st),t&&e(ra),v(Qt,t),t&&e(la),t&&e(at),t&&e(na),v(Vt,t),t&&e(ia),t&&e(ot),t&&e(pa),v(Xt,t),t&&e(da),t&&e(rt),t&&e(fa),t&&e(lt),t&&e(ua),v(Zt,t),t&&e(ca),t&&e(B),v(te),t&&e(ha),t&&e(ye),t&&e(ma),t&&e($e),t&&e(ga),v(ee,t),t&&e(_a),t&&e(it),t&&e(va),t&&e(G),v(ae),t&&e(ya),t&&e(dt),t&&e($a),v(oe,t),t&&e(wa),v(ft,t),t&&e(ba),t&&e(Y),v(re),t&&e(Ea),t&&e(T),t&&e(ka),v(le,t),t&&e(ja),t&&e(R),v(ne),t&&e(qa),t&&e(ht),t&&e(Sa),v(ie,t)}}}const Zn={local:"cloud-storage",sections:[{local:"set-up-your-cloud-storage-filesystem",sections:[{local:"amazon-s3",title:"Amazon S3"},{local:"google-cloud-storage",title:"Google Cloud Storage"},{local:"azure-blob-storage",title:"Azure Blob Storage"}],title:"Set up your cloud storage FileSystem"},{local:"load-and-save-your-datasets-using-your-cloud-storage-filesystem",sections:[{local:"download-and-prepare-a-dataset-into-a-cloud-storage",sections:[{local:"dask",title:"Dask"}],title:"Download and prepare a dataset into a cloud storage"}],title:"Load and Save your datasets using your cloud storage FileSystem"},{local:"saving-serialized-datasets",title:"Saving serialized datasets"},{local:"listing-serialized-datasets",sections:[{local:"load-serialized-datasets",title:"Load serialized datasets"}],title:"Listing serialized datasets"}],title:"Cloud storage"};function ti(Ss){return Jn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ri extends Mn{constructor(y){super();Un(this,y,ti,Xn,Kn,{})}}export{ri as default,Zn as metadata};
