import{S as Kp,i as Zp,s as ed,e as s,k as d,w as m,t as r,M as td,c as o,d as a,m as f,a as l,x as _,h as n,b as u,G as t,g as p,y as g,q as w,o as v,B as y,v as ad}from"../chunks/vendor-hf-doc-builder.js";import{T as Wo}from"../chunks/Tip-hf-doc-builder.js";import{I}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as j}from"../chunks/CodeBlock-hf-doc-builder.js";function sd(O){let h,E,c,$,S;return{c(){h=s("p"),E=r(`The dataset script is optional if your dataset is in one of the following formats: CSV, JSON, JSON lines, text or Parquet.
With those formats, you should be able to load your dataset automatically with `),c=s("a"),$=r("load_dataset()"),S=r("."),this.h()},l(q){h=o(q,"P",{});var b=l(h);E=n(b,`The dataset script is optional if your dataset is in one of the following formats: CSV, JSON, JSON lines, text or Parquet.
With those formats, you should be able to load your dataset automatically with `),c=o(b,"A",{href:!0});var x=l(c);$=n(x,"load_dataset()"),x.forEach(a),S=n(b,"."),b.forEach(a),this.h()},h(){u(c,"href","/docs/datasets/v2.8.0/en/package_reference/loading_methods#datasets.load_dataset")},m(q,b){p(q,h,b),t(h,E),t(h,c),t(c,$),t(h,S)},d(q){q&&a(h)}}}function od(O){let h,E,c,$,S;return{c(){h=s("p"),E=r("To help you get started, try beginning with the dataset loading script "),c=s("a"),$=r("template"),S=r("!"),this.h()},l(q){h=o(q,"P",{});var b=l(h);E=n(b,"To help you get started, try beginning with the dataset loading script "),c=o(b,"A",{href:!0,rel:!0});var x=l(c);$=n(x,"template"),x.forEach(a),S=n(b,"!"),b.forEach(a),this.h()},h(){u(c,"href","https://github.com/huggingface/datasets/blob/main/templates/new_dataset_script.py"),u(c,"rel","nofollow")},m(q,b){p(q,h,b),t(h,E),t(h,c),t(c,$),t(h,S)},d(q){q&&a(h)}}}function ld(O){let h,E;return{c(){h=s("p"),E=r("Only use a default configuration when it makes sense. Don\u2019t set one because it may be more convenient for the user to not specify a configuration when they load your dataset. For example, multi-lingual datasets often have a separate configuration for each language. An appropriate default may be an aggregated configuration that loads all the languages of the dataset if the user doesn\u2019t request a particular one.")},l(c){h=o(c,"P",{});var $=l(h);E=n($,"Only use a default configuration when it makes sense. Don\u2019t set one because it may be more convenient for the user to not specify a configuration when they load your dataset. For example, multi-lingual datasets often have a separate configuration for each language. An appropriate default may be an aggregated configuration that loads all the languages of the dataset if the user doesn\u2019t request a particular one."),$.forEach(a)},m(c,$){p(c,h,$),t(h,E)},d(c){c&&a(h)}}}function rd(O){let h,E;return{c(){h=s("p"),E=r("If the data files live in the same folder or repository of the dataset script, you can just pass the relative paths to the files instead of URLs.")},l(c){h=o(c,"P",{});var $=l(h);E=n($,"If the data files live in the same folder or repository of the dataset script, you can just pass the relative paths to the files instead of URLs."),$.forEach(a)},m(c,$){p(c,h,$),t(h,E)},d(c){c&&a(h)}}}function nd(O){let h,E,c,$,S,q,b,x,Ko,hs,ae,cs,ht,Zo,ms,ct,el,_s,T,tl,Qt,al,sl,Yt,ol,ll,gs,Ae,ws,Se,vs,mt,rl,ys,k,Xt,nl,il,Jt,pl,dl,Wt,fl,ul,Kt,hl,cl,Zt,ml,$s,se,_l,De,gl,wl,Es,oe,qs,H,le,ea,xe,vl,ta,yl,bs,re,$l,aa,El,ql,js,ne,sa,_t,oa,bl,jl,kl,la,ie,ra,Al,Sl,gt,Dl,xl,ks,Ie,As,z,na,wt,ia,Il,Ol,Tl,pa,vt,da,Cl,Ll,Ss,yt,Nl,Ds,Oe,xs,Q,pe,fa,Te,Pl,ua,Bl,Is,C,Gl,Ce,Rl,Ul,$t,Vl,Fl,Os,de,Ml,Le,Hl,zl,Ts,Et,Ne,Ql,qt,Yl,Xl,Cs,Pe,Ls,Be,Ge,Jl,ha,Wl,Kl,Ns,Re,Ps,Ue,Ve,Zl,ca,er,tr,Bs,Fe,Gs,Y,fe,ma,Me,ar,_a,sr,Rs,L,or,ga,lr,rr,wa,nr,ir,Us,He,Vs,ue,Fs,X,he,va,ze,pr,ya,dr,Ms,bt,fr,Hs,jt,$a,ur,zs,Qe,Qs,ce,Ys,Ye,Xe,me,kt,hr,cr,At,mr,_r,gr,Je,Ea,D,wr,qa,vr,yr,ba,$r,Er,ja,qr,br,ka,jr,kr,Ar,Aa,St,Sa,Sr,Dr,Xs,_e,xr,Da,Ir,Or,Js,We,Ws,J,ge,xa,Ke,Tr,Ia,Cr,Ks,Dt,Lr,Zs,N,Oa,Nr,Pr,Ta,Br,Gr,Ca,Rr,eo,xt,Ur,to,we,La,ve,Na,Vr,Fr,Pa,Mr,Hr,zr,Ba,Ze,Qr,Ga,Yr,Xr,ao,et,so,W,ye,Ra,tt,Jr,Ua,Wr,oo,P,Kr,Va,Zr,en,Fa,tn,an,lo,$e,sn,Ma,on,ln,ro,at,no,B,rn,Ha,nn,pn,za,dn,fn,io,K,Ee,Qa,st,un,Ya,hn,po,G,cn,It,mn,_n,Ot,gn,wn,fo,Tt,vn,uo,ot,ho,Z,qe,Xa,lt,yn,Ja,$n,co,ee,be,Wa,rt,En,Ka,qn,mo,je,bn,nt,jn,kn,_o,R,An,Za,Sn,Dn,es,xn,In,go,it,wo,U,On,ts,Tn,Cn,as,Ln,Nn,vo,te,ke,ss,pt,Pn,os,Bn,yo,Ct,Gn,$o,dt,Eo,A,Rn,Lt,Un,Vn,Nt,Fn,Mn,ls,Hn,zn,rs,Qn,Yn,qo,ft,bo,Pt,Xn,jo;return q=new I({}),ae=new Wo({props:{$$slots:{default:[sd]},$$scope:{ctx:O}}}),Ae=new j({props:{code:`my_dataset/
\u251C\u2500\u2500 README.md
\u2514\u2500\u2500 my_dataset.py`,highlighted:`my<span class="hljs-emphasis">_dataset/
\u251C\u2500\u2500 README.md
\u2514\u2500\u2500 my_</span>dataset.py`}}),Se=new j({props:{code:`from datasets import load_dataset
load_dataset("path/to/my_dataset")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>load_dataset(<span class="hljs-string">&quot;path/to/my_dataset&quot;</span>)`}}),oe=new Wo({props:{$$slots:{default:[od]},$$scope:{ctx:O}}}),xe=new I({}),Ie=new j({props:{code:`datasets.Features(
    {
        "id": datasets.Value("string"),
        "title": datasets.Value("string"),
        "context": datasets.Value("string"),
        "question": datasets.Value("string"),
        "answers": datasets.Sequence(
            {
                "text": datasets.Value("string"),
                "answer_start": datasets.Value("int32"),
            }
        ),
    }
)`,highlighted:`datasets.Features(
    {
        <span class="hljs-string">&quot;id&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
        <span class="hljs-string">&quot;title&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
        <span class="hljs-string">&quot;context&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
        <span class="hljs-string">&quot;question&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
        <span class="hljs-string">&quot;answers&quot;</span>: datasets.<span class="hljs-type">Sequence</span>(
            {
                <span class="hljs-string">&quot;text&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                <span class="hljs-string">&quot;answer_start&quot;</span>: datasets.Value(<span class="hljs-string">&quot;int32&quot;</span>),
            }
        ),
    }
)`}}),Oe=new j({props:{code:`def _info(self):
    return datasets.DatasetInfo(
        description=_DESCRIPTION,
        features=datasets.Features(
            {
                "id": datasets.Value("string"),
                "title": datasets.Value("string"),
                "context": datasets.Value("string"),
                "question": datasets.Value("string"),
                "answers": datasets.features.Sequence(
                    {"text": datasets.Value("string"), "answer_start": datasets.Value("int32"),}
                ),
            }
        ),
        # No default supervised_keys (as we have to pass both question
        # and context as input).
        supervised_keys=None,
        homepage="https://rajpurkar.github.io/SQuAD-explorer/",
        citation=_CITATION,
    )`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_info</span>(<span class="hljs-params">self</span>):
    <span class="hljs-keyword">return</span> datasets.DatasetInfo(
        description=_DESCRIPTION,
        features=datasets.Features(
            {
                <span class="hljs-string">&quot;id&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                <span class="hljs-string">&quot;title&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                <span class="hljs-string">&quot;context&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                <span class="hljs-string">&quot;question&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                <span class="hljs-string">&quot;answers&quot;</span>: datasets.features.<span class="hljs-type">Sequence</span>(
                    {<span class="hljs-string">&quot;text&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>), <span class="hljs-string">&quot;answer_start&quot;</span>: datasets.Value(<span class="hljs-string">&quot;int32&quot;</span>),}
                ),
            }
        ),
        <span class="hljs-comment"># No default supervised_keys (as we have to pass both question</span>
        <span class="hljs-comment"># and context as input).</span>
        supervised_keys=<span class="hljs-literal">None</span>,
        homepage=<span class="hljs-string">&quot;https://rajpurkar.github.io/SQuAD-explorer/&quot;</span>,
        citation=_CITATION,
    )`}}),Te=new I({}),Pe=new j({props:{code:`class SuperGlueConfig(datasets.BuilderConfig):
    """BuilderConfig for SuperGLUE."""

    def __init__(self, features, data_url, citation, url, label_classes=("False", "True"), **kwargs):
        """BuilderConfig for SuperGLUE.

        Args:
        features: *list[string]*, list of the features that will appear in the
            feature dict. Should not include "label".
        data_url: *string*, url to download the zip file from.
        citation: *string*, citation for the data set.
        url: *string*, url for information about the data set.
        label_classes: *list[string]*, the list of classes for the label if the
            label is present as a string. Non-string labels will be cast to either
            'False' or 'True'.
        **kwargs: keyword arguments forwarded to super.
        """
        # Version history:
        # 1.0.2: Fixed non-nondeterminism in ReCoRD.
        # 1.0.1: Change from the pre-release trial version of SuperGLUE (v1.9) to
        #        the full release (v2.0).
        # 1.0.0: S3 (new shuffling, sharding and slicing mechanism).
        # 0.0.2: Initial version.
        super().__init__(version=datasets.Version("1.0.2"), **kwargs)
        self.features = features
        self.label_classes = label_classes
        self.data_url = data_url
        self.citation = citation
        self.url = url`,highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">SuperGlueConfig</span>(datasets.BuilderConfig):
    <span class="hljs-string">&quot;&quot;&quot;BuilderConfig for SuperGLUE.&quot;&quot;&quot;</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, features, data_url, citation, url, label_classes=(<span class="hljs-params"><span class="hljs-string">&quot;False&quot;</span>, <span class="hljs-string">&quot;True&quot;</span></span>), **kwargs</span>):
        <span class="hljs-string">&quot;&quot;&quot;BuilderConfig for SuperGLUE.

        Args:
        features: *list[string]*, list of the features that will appear in the
            feature dict. Should not include &quot;label&quot;.
        data_url: *string*, url to download the zip file from.
        citation: *string*, citation for the data set.
        url: *string*, url for information about the data set.
        label_classes: *list[string]*, the list of classes for the label if the
            label is present as a string. Non-string labels will be cast to either
            &#x27;False&#x27; or &#x27;True&#x27;.
        **kwargs: keyword arguments forwarded to super.
        &quot;&quot;&quot;</span>
        <span class="hljs-comment"># Version history:</span>
        <span class="hljs-comment"># 1.0.2: Fixed non-nondeterminism in ReCoRD.</span>
        <span class="hljs-comment"># 1.0.1: Change from the pre-release trial version of SuperGLUE (v1.9) to</span>
        <span class="hljs-comment">#        the full release (v2.0).</span>
        <span class="hljs-comment"># 1.0.0: S3 (new shuffling, sharding and slicing mechanism).</span>
        <span class="hljs-comment"># 0.0.2: Initial version.</span>
        <span class="hljs-built_in">super</span>().__init__(version=datasets.Version(<span class="hljs-string">&quot;1.0.2&quot;</span>), **kwargs)
        self.features = features
        self.label_classes = label_classes
        self.data_url = data_url
        self.citation = citation
        self.url = url`}}),Re=new j({props:{code:`class SuperGlue(datasets.GeneratorBasedBuilder):
    """The SuperGLUE benchmark."""

    BUILDER_CONFIGS = [
        SuperGlueConfig(
            name="boolq",
            description=_BOOLQ_DESCRIPTION,
            features=["question", "passage"],
            data_url="https://dl.fbaipublicfiles.com/glue/superglue/data/v2/BoolQ.zip",
            citation=_BOOLQ_CITATION,
            url="https://github.com/google-research-datasets/boolean-questions",
        ),
        ...
        ...
        SuperGlueConfig(
            name="axg",
            description=_AXG_DESCRIPTION,
            features=["premise", "hypothesis"],
            label_classes=["entailment", "not_entailment"],
            data_url="https://dl.fbaipublicfiles.com/glue/superglue/data/v2/AX-g.zip",
            citation=_AXG_CITATION,
            url="https://github.com/rudinger/winogender-schemas",
        ),`,highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">SuperGlue</span>(datasets.GeneratorBasedBuilder):
    <span class="hljs-string">&quot;&quot;&quot;The SuperGLUE benchmark.&quot;&quot;&quot;</span>

    BUILDER_CONFIGS = [
        SuperGlueConfig(
            name=<span class="hljs-string">&quot;boolq&quot;</span>,
            description=_BOOLQ_DESCRIPTION,
            features=[<span class="hljs-string">&quot;question&quot;</span>, <span class="hljs-string">&quot;passage&quot;</span>],
            data_url=<span class="hljs-string">&quot;https://dl.fbaipublicfiles.com/glue/superglue/data/v2/BoolQ.zip&quot;</span>,
            citation=_BOOLQ_CITATION,
            url=<span class="hljs-string">&quot;https://github.com/google-research-datasets/boolean-questions&quot;</span>,
        ),
        ...
        ...
        SuperGlueConfig(
            name=<span class="hljs-string">&quot;axg&quot;</span>,
            description=_AXG_DESCRIPTION,
            features=[<span class="hljs-string">&quot;premise&quot;</span>, <span class="hljs-string">&quot;hypothesis&quot;</span>],
            label_classes=[<span class="hljs-string">&quot;entailment&quot;</span>, <span class="hljs-string">&quot;not_entailment&quot;</span>],
            data_url=<span class="hljs-string">&quot;https://dl.fbaipublicfiles.com/glue/superglue/data/v2/AX-g.zip&quot;</span>,
            citation=_AXG_CITATION,
            url=<span class="hljs-string">&quot;https://github.com/rudinger/winogender-schemas&quot;</span>,
        ),`}}),Fe=new j({props:{code:`from datasets import load_dataset
dataset = load_dataset('super_glue', 'boolq')`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&#x27;super_glue&#x27;</span>, <span class="hljs-string">&#x27;boolq&#x27;</span>)`}}),Me=new I({}),He=new j({props:{code:`class NewDataset(datasets.GeneratorBasedBuilder):

VERSION = datasets.Version("1.1.0")

BUILDER_CONFIGS = [
    datasets.BuilderConfig(name="first_domain", version=VERSION, description="This part of my dataset covers a first domain"),
    datasets.BuilderConfig(name="second_domain", version=VERSION, description="This part of my dataset covers a second domain"),
]

DEFAULT_CONFIG_NAME = "first_domain"`,highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">NewDataset</span>(datasets.GeneratorBasedBuilder):

VERSION = datasets.Version(<span class="hljs-string">&quot;1.1.0&quot;</span>)

BUILDER_CONFIGS = [
    datasets.BuilderConfig(name=<span class="hljs-string">&quot;first_domain&quot;</span>, version=VERSION, description=<span class="hljs-string">&quot;This part of my dataset covers a first domain&quot;</span>),
    datasets.BuilderConfig(name=<span class="hljs-string">&quot;second_domain&quot;</span>, version=VERSION, description=<span class="hljs-string">&quot;This part of my dataset covers a second domain&quot;</span>),
]

DEFAULT_CONFIG_NAME = <span class="hljs-string">&quot;first_domain&quot;</span>`}}),ue=new Wo({props:{warning:!0,$$slots:{default:[ld]},$$scope:{ctx:O}}}),ze=new I({}),Qe=new j({props:{code:`_URL = "https://rajpurkar.github.io/SQuAD-explorer/dataset/"
_URLS = {
    "train": _URL + "train-v1.1.json",
    "dev": _URL + "dev-v1.1.json",
}`,highlighted:`_URL = <span class="hljs-string">&quot;https://rajpurkar.github.io/SQuAD-explorer/dataset/&quot;</span>
_URLS = {
    <span class="hljs-string">&quot;train&quot;</span>: _URL + <span class="hljs-string">&quot;train-v1.1.json&quot;</span>,
    <span class="hljs-string">&quot;dev&quot;</span>: _URL + <span class="hljs-string">&quot;dev-v1.1.json&quot;</span>,
}`}}),ce=new Wo({props:{$$slots:{default:[rd]},$$scope:{ctx:O}}}),We=new j({props:{code:`def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]:
    urls_to_download = self._URLS
    downloaded_files = dl_manager.download_and_extract(urls_to_download)

    return [
        datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": downloaded_files["train"]}),
        datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={"filepath": downloaded_files["dev"]}),
    ]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_split_generators</span>(<span class="hljs-params">self, dl_manager: datasets.DownloadManager</span>) -&gt; <span class="hljs-type">List</span>[datasets.SplitGenerator]:
    urls_to_download = self._URLS
    downloaded_files = dl_manager.download_and_extract(urls_to_download)

    <span class="hljs-keyword">return</span> [
        datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={<span class="hljs-string">&quot;filepath&quot;</span>: downloaded_files[<span class="hljs-string">&quot;train&quot;</span>]}),
        datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={<span class="hljs-string">&quot;filepath&quot;</span>: downloaded_files[<span class="hljs-string">&quot;dev&quot;</span>]}),
    ]`}}),Ke=new I({}),et=new j({props:{code:`def _generate_examples(self, filepath):
    """This function returns the examples in the raw (text) form."""
    logger.info("generating examples from = %s", filepath)
    with open(filepath) as f:
        squad = json.load(f)
        for article in squad["data"]:
            title = article.get("title", "").strip()
            for paragraph in article["paragraphs"]:
                context = paragraph["context"].strip()
                for qa in paragraph["qas"]:
                    question = qa["question"].strip()
                    id_ = qa["id"]

                    answer_starts = [answer["answer_start"] for answer in qa["answers"]]
                    answers = [answer["text"].strip() for answer in qa["answers"]]

                    # Features currently used are "context", "question", and "answers".
                    # Others are extracted here for the ease of future expansions.
                    yield id_, {
                        "title": title,
                        "context": context,
                        "question": question,
                        "id": id_,
                        "answers": {"answer_start": answer_starts, "text": answers,},
                    }`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_examples</span>(<span class="hljs-params">self, filepath</span>):
    <span class="hljs-string">&quot;&quot;&quot;This function returns the examples in the raw (text) form.&quot;&quot;&quot;</span>
    logger.info(<span class="hljs-string">&quot;generating examples from = %s&quot;</span>, filepath)
    <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(filepath) <span class="hljs-keyword">as</span> f:
        squad = json.load(f)
        <span class="hljs-keyword">for</span> article <span class="hljs-keyword">in</span> squad[<span class="hljs-string">&quot;data&quot;</span>]:
            title = article.get(<span class="hljs-string">&quot;title&quot;</span>, <span class="hljs-string">&quot;&quot;</span>).strip()
            <span class="hljs-keyword">for</span> paragraph <span class="hljs-keyword">in</span> article[<span class="hljs-string">&quot;paragraphs&quot;</span>]:
                context = paragraph[<span class="hljs-string">&quot;context&quot;</span>].strip()
                <span class="hljs-keyword">for</span> qa <span class="hljs-keyword">in</span> paragraph[<span class="hljs-string">&quot;qas&quot;</span>]:
                    question = qa[<span class="hljs-string">&quot;question&quot;</span>].strip()
                    id_ = qa[<span class="hljs-string">&quot;id&quot;</span>]

                    answer_starts = [answer[<span class="hljs-string">&quot;answer_start&quot;</span>] <span class="hljs-keyword">for</span> answer <span class="hljs-keyword">in</span> qa[<span class="hljs-string">&quot;answers&quot;</span>]]
                    answers = [answer[<span class="hljs-string">&quot;text&quot;</span>].strip() <span class="hljs-keyword">for</span> answer <span class="hljs-keyword">in</span> qa[<span class="hljs-string">&quot;answers&quot;</span>]]

                    <span class="hljs-comment"># Features currently used are &quot;context&quot;, &quot;question&quot;, and &quot;answers&quot;.</span>
                    <span class="hljs-comment"># Others are extracted here for the ease of future expansions.</span>
                    <span class="hljs-keyword">yield</span> id_, {
                        <span class="hljs-string">&quot;title&quot;</span>: title,
                        <span class="hljs-string">&quot;context&quot;</span>: context,
                        <span class="hljs-string">&quot;question&quot;</span>: question,
                        <span class="hljs-string">&quot;id&quot;</span>: id_,
                        <span class="hljs-string">&quot;answers&quot;</span>: {<span class="hljs-string">&quot;answer_start&quot;</span>: answer_starts, <span class="hljs-string">&quot;text&quot;</span>: answers,},
                    }`}}),tt=new I({}),at=new j({props:{code:"datasets-cli test path/to/<your-dataset-loading-script> --save_info --all_configs",highlighted:'datasets-cli test path/<span class="hljs-keyword">to</span>/&lt;your-dataset-loading-<span class="hljs-keyword">script</span>&gt; <span class="hljs-comment">--save_info --all_configs</span>'}}),st=new I({}),ot=new j({props:{code:`from datasets import load_dataset
load_dataset("<username>/my_dataset")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>load_dataset(<span class="hljs-string">&quot;&lt;username&gt;/my_dataset&quot;</span>)`}}),lt=new I({}),rt=new I({}),it=new j({props:{code:`
class MyShardedDataset(datasets.GeneratorBasedBuilder):

    def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]:
        downloaded_files = dl_manager.download([f"data/shard_{i}.jsonl" for i in range(1024)])
        return [
            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepaths": downloaded_files}),
        ]

    def _generate_examples(self, filepaths):
        # Each worker can be given a slice of the original \`filepaths\` list defined in the \`gen_kwargs\`
        # so that this code can run in parallel on several shards at the same time
        for filepath in filepaths:
            ...`,highlighted:`
<span class="hljs-keyword">class</span> <span class="hljs-title class_">MyShardedDataset</span>(datasets.GeneratorBasedBuilder):

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">_split_generators</span>(<span class="hljs-params">self, dl_manager: datasets.DownloadManager</span>) -&gt; <span class="hljs-type">List</span>[datasets.SplitGenerator]:
        downloaded_files = dl_manager.download([<span class="hljs-string">f&quot;data/shard_<span class="hljs-subst">{i}</span>.jsonl&quot;</span> <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1024</span>)])
        <span class="hljs-keyword">return</span> [
            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={<span class="hljs-string">&quot;filepaths&quot;</span>: downloaded_files}),
        ]

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_examples</span>(<span class="hljs-params">self, filepaths</span>):
        <span class="hljs-comment"># Each worker can be given a slice of the original \`filepaths\` list defined in the \`gen_kwargs\`</span>
        <span class="hljs-comment"># so that this code can run in parallel on several shards at the same time</span>
        <span class="hljs-keyword">for</span> filepath <span class="hljs-keyword">in</span> filepaths:
            ...`}}),pt=new I({}),dt=new j({props:{code:`import pyarrow as pa
pa_table = pa.Table.from_pandas(df)`,highlighted:`<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
pa_table = pa.Table.from_pandas(df)`}}),ft=new j({props:{code:`class MySuperFastDataset(datasets.ArrowBasedBuilder):

    def _generate_tables(self, filepaths):
        idx = 0
        for filepath in filepaths:
            ...
            yield idx, pa_table
            idx += 1`,highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">MySuperFastDataset</span>(datasets.ArrowBasedBuilder):

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_tables</span>(<span class="hljs-params">self, filepaths</span>):
        idx = <span class="hljs-number">0</span>
        <span class="hljs-keyword">for</span> filepath <span class="hljs-keyword">in</span> filepaths:
            ...
            <span class="hljs-keyword">yield</span> idx, pa_table
            idx += <span class="hljs-number">1</span>`}}),{c(){h=s("meta"),E=d(),c=s("h1"),$=s("a"),S=s("span"),m(q.$$.fragment),b=d(),x=s("span"),Ko=r("Create a dataset loading script"),hs=d(),m(ae.$$.fragment),cs=d(),ht=s("p"),Zo=r("Write a dataset script to load and share your own datasets. It is a Python file that defines the different configurations and splits of your dataset, as well as how to download and process the data."),ms=d(),ct=s("p"),el=r("The script can download data files from any website, or from the same dataset repository."),_s=d(),T=s("p"),tl=r("A dataset loading script should have the same name as a dataset repository or directory. For example, a repository named "),Qt=s("code"),al=r("my_dataset"),sl=r(" should contain "),Yt=s("code"),ol=r("my_dataset.py"),ll=r(" script. This way it can be loaded with:"),gs=d(),m(Ae.$$.fragment),ws=d(),m(Se.$$.fragment),vs=d(),mt=s("p"),rl=r("The following guide includes instructions for dataset scripts for how to:"),ys=d(),k=s("ul"),Xt=s("li"),nl=r("Add dataset metadata."),il=d(),Jt=s("li"),pl=r("Download data files."),dl=d(),Wt=s("li"),fl=r("Generate samples."),ul=d(),Kt=s("li"),hl=r("Generate dataset metadata."),cl=d(),Zt=s("li"),ml=r("Upload a dataset to the Hub."),$s=d(),se=s("p"),_l=r("Open the "),De=s("a"),gl=r("SQuAD dataset loading script"),wl=r(" template to follow along on how to share a dataset."),Es=d(),m(oe.$$.fragment),qs=d(),H=s("h2"),le=s("a"),ea=s("span"),m(xe.$$.fragment),vl=d(),ta=s("span"),yl=r("Add dataset attributes"),bs=d(),re=s("p"),$l=r("The first step is to add some information, or attributes, about your dataset in "),aa=s("code"),El=r("DatasetBuilder._info()"),ql=r(". The most important attributes you should specify are:"),js=d(),ne=s("ol"),sa=s("li"),_t=s("p"),oa=s("code"),bl=r("DatasetInfo.description"),jl=r(" provides a concise description of your dataset. The description informs the user what\u2019s in the dataset, how it was collected, and how it can be used for a NLP task."),kl=d(),la=s("li"),ie=s("p"),ra=s("code"),Al=r("DatasetInfo.features"),Sl=r(" defines the name and type of each column in your dataset. This will also provide the structure for each example, so it is possible to create nested subfields in a column if you want. Take a look at "),gt=s("a"),Dl=r("Features"),xl=r(" for a full list of feature types you can use."),ks=d(),m(Ie.$$.fragment),As=d(),z=s("ol"),na=s("li"),wt=s("p"),ia=s("code"),Il=r("DatasetInfo.homepage"),Ol=r(" contains the URL to the dataset homepage so users can find more details about the dataset."),Tl=d(),pa=s("li"),vt=s("p"),da=s("code"),Cl=r("DatasetInfo.citation"),Ll=r(" contains a BibTeX citation for the dataset."),Ss=d(),yt=s("p"),Nl=r("After you\u2019ve filled out all these fields in the template, it should look like the following example from the SQuAD loading script:"),Ds=d(),m(Oe.$$.fragment),xs=d(),Q=s("h3"),pe=s("a"),fa=s("span"),m(Te.$$.fragment),Pl=d(),ua=s("span"),Bl=r("Multiple configurations"),Is=d(),C=s("p"),Gl=r("In some cases, your dataset may have multiple configurations. For example, the "),Ce=s("a"),Rl=r("SuperGLUE"),Ul=r(" dataset is a collection of 5 datasets designed to evaluate language understanding tasks. \u{1F917} Datasets provides "),$t=s("a"),Vl=r("BuilderConfig"),Fl=r(" which allows you to create different configurations for the user to select from."),Os=d(),de=s("p"),Ml=r("Let\u2019s study the "),Le=s("a"),Hl=r("SuperGLUE loading script"),zl=r(" to see how you can define several configurations."),Ts=d(),Et=s("ol"),Ne=s("li"),Ql=r("Create a "),qt=s("a"),Yl=r("BuilderConfig"),Xl=r(" subclass with attributes about your dataset. These attributes can be the features of your dataset, label classes, and a URL to the data files."),Cs=d(),m(Pe.$$.fragment),Ls=d(),Be=s("ol"),Ge=s("li"),Jl=r("Create instances of your config to specify the values of the attributes of each configuration. This gives you the flexibility to specify all the name and description of each configuration. These sub-class instances should be listed under "),ha=s("code"),Wl=r("DatasetBuilder.BUILDER_CONFIGS"),Kl=r(":"),Ns=d(),m(Re.$$.fragment),Ps=d(),Ue=s("ol"),Ve=s("li"),Zl=r("Now, users can load a specific configuration of the dataset with the configuration "),ca=s("code"),er=r("name"),tr=r(":"),Bs=d(),m(Fe.$$.fragment),Gs=d(),Y=s("h3"),fe=s("a"),ma=s("span"),m(Me.$$.fragment),ar=d(),_a=s("span"),sr=r("Default configurations"),Rs=d(),L=s("p"),or=r("Users must specify a configuration name when they load a dataset with multiple configurations. Otherwise, \u{1F917} Datasets will raise a "),ga=s("code"),lr=r("ValueError"),rr=r(", and prompt the user to select a configuration name. You can avoid this by setting a default dataset configuration with the "),wa=s("code"),nr=r("DEFAULT_CONFIG_NAME"),ir=r(" attribute:"),Us=d(),m(He.$$.fragment),Vs=d(),m(ue.$$.fragment),Fs=d(),X=s("h2"),he=s("a"),va=s("span"),m(ze.$$.fragment),pr=d(),ya=s("span"),dr=r("Download data files and organize splits"),Ms=d(),bt=s("p"),fr=r("After you\u2019ve defined the attributes of your dataset, the next step is to download the data files and organize them according to their splits."),Hs=d(),jt=s("ol"),$a=s("li"),ur=r("Create a dictionary of URLs in the loading script that point to the original SQuAD data files:"),zs=d(),m(Qe.$$.fragment),Qs=d(),m(ce.$$.fragment),Ys=d(),Ye=s("ol"),Xe=s("li"),me=s("p"),kt=s("a"),hr=r("DownloadManager.download_and_extract()"),cr=r(" takes this dictionary and downloads the data files. Once the files are downloaded, use "),At=s("a"),mr=r("SplitGenerator"),_r=r(" to organize each split in the dataset. This is a simple class that contains:"),gr=d(),Je=s("ul"),Ea=s("li"),D=s("p"),wr=r("The "),qa=s("code"),vr=r("name"),yr=r(" of each split. You should use the standard split names: "),ba=s("code"),$r=r("Split.TRAIN"),Er=r(", "),ja=s("code"),qr=r("Split.TEST"),br=r(", and "),ka=s("code"),jr=r("Split.VALIDATION"),kr=r("."),Ar=d(),Aa=s("li"),St=s("p"),Sa=s("code"),Sr=r("gen_kwargs"),Dr=r(" provides the file paths to the data files to load for each split."),Xs=d(),_e=s("p"),xr=r("Your "),Da=s("code"),Ir=r("DatasetBuilder._split_generator()"),Or=r(" should look like this now:"),Js=d(),m(We.$$.fragment),Ws=d(),J=s("h2"),ge=s("a"),xa=s("span"),m(Ke.$$.fragment),Tr=d(),Ia=s("span"),Cr=r("Generate samples"),Ks=d(),Dt=s("p"),Lr=r("At this point, you have:"),Zs=d(),N=s("ul"),Oa=s("li"),Nr=r("Added the dataset attributes."),Pr=d(),Ta=s("li"),Br=r("Provided instructions for how to download the data files."),Gr=d(),Ca=s("li"),Rr=r("Organized the splits."),eo=d(),xt=s("p"),Ur=r("The next step is to actually generate the samples in each split."),to=d(),we=s("ol"),La=s("li"),ve=s("p"),Na=s("code"),Vr=r("DatasetBuilder._generate_examples"),Fr=r(" takes the file path provided by "),Pa=s("code"),Mr=r("gen_kwargs"),Hr=r(" to read and parse the data files. You need to write a function that loads the data files and extracts the columns."),zr=d(),Ba=s("li"),Ze=s("p"),Qr=r("Your function should yield a tuple of an "),Ga=s("code"),Yr=r("id_"),Xr=r(", and an example from the dataset."),ao=d(),m(et.$$.fragment),so=d(),W=s("h2"),ye=s("a"),Ra=s("span"),m(tt.$$.fragment),Jr=d(),Ua=s("span"),Wr=r("(Optional) Generate dataset metadata"),oo=d(),P=s("p"),Kr=r("Adding dataset metadata is a great way to include information about your dataset. The metadata is stored in the dataset card "),Va=s("code"),Zr=r("README.md"),en=r(" in YAML. It includes information like the number of examples required to confirm the dataset was correctly generated, and information about the dataset like its "),Fa=s("code"),tn=r("features"),an=r("."),lo=d(),$e=s("p"),sn=r("Run the following command to generate your dataset metadata in "),Ma=s("code"),on=r("README.md"),ln=r(" and make sure your new dataset loading script works correctly:"),ro=d(),m(at.$$.fragment),no=d(),B=s("p"),rn=r("If your dataset loading script passed the test, you should now have a "),Ha=s("code"),nn=r("README.md"),pn=r(" file in your dataset folder containing a "),za=s("code"),dn=r("dataset_info"),fn=r(" field with some metadata."),io=d(),K=s("h2"),Ee=s("a"),Qa=s("span"),m(st.$$.fragment),un=d(),Ya=s("span"),hn=r("Upload to the Hub"),po=d(),G=s("p"),cn=r("Once your script is ready, "),It=s("a"),mn=r("create a dataset card"),_n=r(" and "),Ot=s("a"),gn=r("upload it to the Hub"),wn=r("."),fo=d(),Tt=s("p"),vn=r("Congratulations, you can now load your dataset from the Hub! \u{1F973}"),uo=d(),m(ot.$$.fragment),ho=d(),Z=s("h2"),qe=s("a"),Xa=s("span"),m(lt.$$.fragment),yn=d(),Ja=s("span"),$n=r("Advanced features"),co=d(),ee=s("h3"),be=s("a"),Wa=s("span"),m(rt.$$.fragment),En=d(),Ka=s("span"),qn=r("Sharding"),mo=d(),je=s("p"),bn=r(`If your dataset is made of many big files, \u{1F917} Datasets automatically runs your script in parallel to make it super fast!
It can help if you have hundreds or thousands of TAR archives, or JSONL files like `),nt=s("a"),jn=r("oscar"),kn=r(" for example."),_o=d(),R=s("p"),An=r("To make it work, we consider lists of files in "),Za=s("code"),Sn=r("gen_kwargs"),Dn=r(` to be shards.
Therefore \u{1F917} Datasets can automatically spawn several workers to run `),es=s("code"),xn=r("_generate_examples"),In=r(" in parallel, and each worker is given a subset of shards to process."),go=d(),m(it.$$.fragment),wo=d(),U=s("p"),On=r("Users can also specify "),ts=s("code"),Tn=r("num_proc="),Cn=r(" in "),as=s("code"),Ln=r("load_dataset()"),Nn=r(" to specify the number of processes to use as workers."),vo=d(),te=s("h3"),ke=s("a"),ss=s("span"),m(pt.$$.fragment),Pn=d(),os=s("span"),Bn=r("ArrowBasedBuilder"),yo=d(),Ct=s("p"),Gn=r(`For some datasets it can be much faster to yield batches of data rather than examples one by one.
You can speed up the dataset generation by yielding Arrow tables directly, instead of examples.
This is especially useful if your data comes from Pandas DataFrames for example, since the conversion from Pandas to Arrow is as simple as:`),$o=d(),m(dt.$$.fragment),Eo=d(),A=s("p"),Rn=r("To yield Arrow tables instead of single examples, make your dataset builder inherit from "),Lt=s("a"),Un=r("ArrowBasedBuilder"),Vn=r(" instead of "),Nt=s("a"),Fn=r("GeneratorBasedBuilder"),Mn=r(", and use "),ls=s("code"),Hn=r("_generate_tables"),zn=r(" instead of "),rs=s("code"),Qn=r("_generate_examples"),Yn=r(":"),qo=d(),m(ft.$$.fragment),bo=d(),Pt=s("p"),Xn=r("Don\u2019t forget to keep your script memory efficient, in case users run them on machines with a low amount of RAM."),this.h()},l(e){const i=td('[data-svelte="svelte-1phssyn"]',document.head);h=o(i,"META",{name:!0,content:!0}),i.forEach(a),E=f(e),c=o(e,"H1",{class:!0});var ut=l(c);$=o(ut,"A",{id:!0,class:!0,href:!0});var ns=l($);S=o(ns,"SPAN",{});var is=l(S);_(q.$$.fragment,is),is.forEach(a),ns.forEach(a),b=f(ut),x=o(ut,"SPAN",{});var ps=l(x);Ko=n(ps,"Create a dataset loading script"),ps.forEach(a),ut.forEach(a),hs=f(e),_(ae.$$.fragment,e),cs=f(e),ht=o(e,"P",{});var ei=l(ht);Zo=n(ei,"Write a dataset script to load and share your own datasets. It is a Python file that defines the different configurations and splits of your dataset, as well as how to download and process the data."),ei.forEach(a),ms=f(e),ct=o(e,"P",{});var ti=l(ct);el=n(ti,"The script can download data files from any website, or from the same dataset repository."),ti.forEach(a),_s=f(e),T=o(e,"P",{});var Bt=l(T);tl=n(Bt,"A dataset loading script should have the same name as a dataset repository or directory. For example, a repository named "),Qt=o(Bt,"CODE",{});var ai=l(Qt);al=n(ai,"my_dataset"),ai.forEach(a),sl=n(Bt," should contain "),Yt=o(Bt,"CODE",{});var si=l(Yt);ol=n(si,"my_dataset.py"),si.forEach(a),ll=n(Bt," script. This way it can be loaded with:"),Bt.forEach(a),gs=f(e),_(Ae.$$.fragment,e),ws=f(e),_(Se.$$.fragment,e),vs=f(e),mt=o(e,"P",{});var oi=l(mt);rl=n(oi,"The following guide includes instructions for dataset scripts for how to:"),oi.forEach(a),ys=f(e),k=o(e,"UL",{});var V=l(k);Xt=o(V,"LI",{});var li=l(Xt);nl=n(li,"Add dataset metadata."),li.forEach(a),il=f(V),Jt=o(V,"LI",{});var ri=l(Jt);pl=n(ri,"Download data files."),ri.forEach(a),dl=f(V),Wt=o(V,"LI",{});var ni=l(Wt);fl=n(ni,"Generate samples."),ni.forEach(a),ul=f(V),Kt=o(V,"LI",{});var ii=l(Kt);hl=n(ii,"Generate dataset metadata."),ii.forEach(a),cl=f(V),Zt=o(V,"LI",{});var pi=l(Zt);ml=n(pi,"Upload a dataset to the Hub."),pi.forEach(a),V.forEach(a),$s=f(e),se=o(e,"P",{});var ko=l(se);_l=n(ko,"Open the "),De=o(ko,"A",{href:!0,rel:!0});var di=l(De);gl=n(di,"SQuAD dataset loading script"),di.forEach(a),wl=n(ko," template to follow along on how to share a dataset."),ko.forEach(a),Es=f(e),_(oe.$$.fragment,e),qs=f(e),H=o(e,"H2",{class:!0});var Ao=l(H);le=o(Ao,"A",{id:!0,class:!0,href:!0});var fi=l(le);ea=o(fi,"SPAN",{});var ui=l(ea);_(xe.$$.fragment,ui),ui.forEach(a),fi.forEach(a),vl=f(Ao),ta=o(Ao,"SPAN",{});var hi=l(ta);yl=n(hi,"Add dataset attributes"),hi.forEach(a),Ao.forEach(a),bs=f(e),re=o(e,"P",{});var So=l(re);$l=n(So,"The first step is to add some information, or attributes, about your dataset in "),aa=o(So,"CODE",{});var ci=l(aa);El=n(ci,"DatasetBuilder._info()"),ci.forEach(a),ql=n(So,". The most important attributes you should specify are:"),So.forEach(a),js=f(e),ne=o(e,"OL",{});var Do=l(ne);sa=o(Do,"LI",{});var mi=l(sa);_t=o(mi,"P",{});var Jn=l(_t);oa=o(Jn,"CODE",{});var _i=l(oa);bl=n(_i,"DatasetInfo.description"),_i.forEach(a),jl=n(Jn," provides a concise description of your dataset. The description informs the user what\u2019s in the dataset, how it was collected, and how it can be used for a NLP task."),Jn.forEach(a),mi.forEach(a),kl=f(Do),la=o(Do,"LI",{});var gi=l(la);ie=o(gi,"P",{});var ds=l(ie);ra=o(ds,"CODE",{});var wi=l(ra);Al=n(wi,"DatasetInfo.features"),wi.forEach(a),Sl=n(ds," defines the name and type of each column in your dataset. This will also provide the structure for each example, so it is possible to create nested subfields in a column if you want. Take a look at "),gt=o(ds,"A",{href:!0});var vi=l(gt);Dl=n(vi,"Features"),vi.forEach(a),xl=n(ds," for a full list of feature types you can use."),ds.forEach(a),gi.forEach(a),Do.forEach(a),ks=f(e),_(Ie.$$.fragment,e),As=f(e),z=o(e,"OL",{start:!0});var xo=l(z);na=o(xo,"LI",{});var yi=l(na);wt=o(yi,"P",{});var Wn=l(wt);ia=o(Wn,"CODE",{});var $i=l(ia);Il=n($i,"DatasetInfo.homepage"),$i.forEach(a),Ol=n(Wn," contains the URL to the dataset homepage so users can find more details about the dataset."),Wn.forEach(a),yi.forEach(a),Tl=f(xo),pa=o(xo,"LI",{});var Ei=l(pa);vt=o(Ei,"P",{});var Kn=l(vt);da=o(Kn,"CODE",{});var qi=l(da);Cl=n(qi,"DatasetInfo.citation"),qi.forEach(a),Ll=n(Kn," contains a BibTeX citation for the dataset."),Kn.forEach(a),Ei.forEach(a),xo.forEach(a),Ss=f(e),yt=o(e,"P",{});var bi=l(yt);Nl=n(bi,"After you\u2019ve filled out all these fields in the template, it should look like the following example from the SQuAD loading script:"),bi.forEach(a),Ds=f(e),_(Oe.$$.fragment,e),xs=f(e),Q=o(e,"H3",{class:!0});var Io=l(Q);pe=o(Io,"A",{id:!0,class:!0,href:!0});var ji=l(pe);fa=o(ji,"SPAN",{});var ki=l(fa);_(Te.$$.fragment,ki),ki.forEach(a),ji.forEach(a),Pl=f(Io),ua=o(Io,"SPAN",{});var Ai=l(ua);Bl=n(Ai,"Multiple configurations"),Ai.forEach(a),Io.forEach(a),Is=f(e),C=o(e,"P",{});var Gt=l(C);Gl=n(Gt,"In some cases, your dataset may have multiple configurations. For example, the "),Ce=o(Gt,"A",{href:!0,rel:!0});var Si=l(Ce);Rl=n(Si,"SuperGLUE"),Si.forEach(a),Ul=n(Gt," dataset is a collection of 5 datasets designed to evaluate language understanding tasks. \u{1F917} Datasets provides "),$t=o(Gt,"A",{href:!0});var Di=l($t);Vl=n(Di,"BuilderConfig"),Di.forEach(a),Fl=n(Gt," which allows you to create different configurations for the user to select from."),Gt.forEach(a),Os=f(e),de=o(e,"P",{});var Oo=l(de);Ml=n(Oo,"Let\u2019s study the "),Le=o(Oo,"A",{href:!0,rel:!0});var xi=l(Le);Hl=n(xi,"SuperGLUE loading script"),xi.forEach(a),zl=n(Oo," to see how you can define several configurations."),Oo.forEach(a),Ts=f(e),Et=o(e,"OL",{});var Ii=l(Et);Ne=o(Ii,"LI",{});var To=l(Ne);Ql=n(To,"Create a "),qt=o(To,"A",{href:!0});var Oi=l(qt);Yl=n(Oi,"BuilderConfig"),Oi.forEach(a),Xl=n(To," subclass with attributes about your dataset. These attributes can be the features of your dataset, label classes, and a URL to the data files."),To.forEach(a),Ii.forEach(a),Cs=f(e),_(Pe.$$.fragment,e),Ls=f(e),Be=o(e,"OL",{start:!0});var Ti=l(Be);Ge=o(Ti,"LI",{});var Co=l(Ge);Jl=n(Co,"Create instances of your config to specify the values of the attributes of each configuration. This gives you the flexibility to specify all the name and description of each configuration. These sub-class instances should be listed under "),ha=o(Co,"CODE",{});var Ci=l(ha);Wl=n(Ci,"DatasetBuilder.BUILDER_CONFIGS"),Ci.forEach(a),Kl=n(Co,":"),Co.forEach(a),Ti.forEach(a),Ns=f(e),_(Re.$$.fragment,e),Ps=f(e),Ue=o(e,"OL",{start:!0});var Li=l(Ue);Ve=o(Li,"LI",{});var Lo=l(Ve);Zl=n(Lo,"Now, users can load a specific configuration of the dataset with the configuration "),ca=o(Lo,"CODE",{});var Ni=l(ca);er=n(Ni,"name"),Ni.forEach(a),tr=n(Lo,":"),Lo.forEach(a),Li.forEach(a),Bs=f(e),_(Fe.$$.fragment,e),Gs=f(e),Y=o(e,"H3",{class:!0});var No=l(Y);fe=o(No,"A",{id:!0,class:!0,href:!0});var Pi=l(fe);ma=o(Pi,"SPAN",{});var Bi=l(ma);_(Me.$$.fragment,Bi),Bi.forEach(a),Pi.forEach(a),ar=f(No),_a=o(No,"SPAN",{});var Gi=l(_a);sr=n(Gi,"Default configurations"),Gi.forEach(a),No.forEach(a),Rs=f(e),L=o(e,"P",{});var Rt=l(L);or=n(Rt,"Users must specify a configuration name when they load a dataset with multiple configurations. Otherwise, \u{1F917} Datasets will raise a "),ga=o(Rt,"CODE",{});var Ri=l(ga);lr=n(Ri,"ValueError"),Ri.forEach(a),rr=n(Rt,", and prompt the user to select a configuration name. You can avoid this by setting a default dataset configuration with the "),wa=o(Rt,"CODE",{});var Ui=l(wa);nr=n(Ui,"DEFAULT_CONFIG_NAME"),Ui.forEach(a),ir=n(Rt," attribute:"),Rt.forEach(a),Us=f(e),_(He.$$.fragment,e),Vs=f(e),_(ue.$$.fragment,e),Fs=f(e),X=o(e,"H2",{class:!0});var Po=l(X);he=o(Po,"A",{id:!0,class:!0,href:!0});var Vi=l(he);va=o(Vi,"SPAN",{});var Fi=l(va);_(ze.$$.fragment,Fi),Fi.forEach(a),Vi.forEach(a),pr=f(Po),ya=o(Po,"SPAN",{});var Mi=l(ya);dr=n(Mi,"Download data files and organize splits"),Mi.forEach(a),Po.forEach(a),Ms=f(e),bt=o(e,"P",{});var Hi=l(bt);fr=n(Hi,"After you\u2019ve defined the attributes of your dataset, the next step is to download the data files and organize them according to their splits."),Hi.forEach(a),Hs=f(e),jt=o(e,"OL",{});var zi=l(jt);$a=o(zi,"LI",{});var Qi=l($a);ur=n(Qi,"Create a dictionary of URLs in the loading script that point to the original SQuAD data files:"),Qi.forEach(a),zi.forEach(a),zs=f(e),_(Qe.$$.fragment,e),Qs=f(e),_(ce.$$.fragment,e),Ys=f(e),Ye=o(e,"OL",{start:!0});var Yi=l(Ye);Xe=o(Yi,"LI",{});var Bo=l(Xe);me=o(Bo,"P",{});var fs=l(me);kt=o(fs,"A",{href:!0});var Xi=l(kt);hr=n(Xi,"DownloadManager.download_and_extract()"),Xi.forEach(a),cr=n(fs," takes this dictionary and downloads the data files. Once the files are downloaded, use "),At=o(fs,"A",{href:!0});var Ji=l(At);mr=n(Ji,"SplitGenerator"),Ji.forEach(a),_r=n(fs," to organize each split in the dataset. This is a simple class that contains:"),fs.forEach(a),gr=f(Bo),Je=o(Bo,"UL",{});var Go=l(Je);Ea=o(Go,"LI",{});var Wi=l(Ea);D=o(Wi,"P",{});var F=l(D);wr=n(F,"The "),qa=o(F,"CODE",{});var Ki=l(qa);vr=n(Ki,"name"),Ki.forEach(a),yr=n(F," of each split. You should use the standard split names: "),ba=o(F,"CODE",{});var Zi=l(ba);$r=n(Zi,"Split.TRAIN"),Zi.forEach(a),Er=n(F,", "),ja=o(F,"CODE",{});var ep=l(ja);qr=n(ep,"Split.TEST"),ep.forEach(a),br=n(F,", and "),ka=o(F,"CODE",{});var tp=l(ka);jr=n(tp,"Split.VALIDATION"),tp.forEach(a),kr=n(F,"."),F.forEach(a),Wi.forEach(a),Ar=f(Go),Aa=o(Go,"LI",{});var ap=l(Aa);St=o(ap,"P",{});var Zn=l(St);Sa=o(Zn,"CODE",{});var sp=l(Sa);Sr=n(sp,"gen_kwargs"),sp.forEach(a),Dr=n(Zn," provides the file paths to the data files to load for each split."),Zn.forEach(a),ap.forEach(a),Go.forEach(a),Bo.forEach(a),Yi.forEach(a),Xs=f(e),_e=o(e,"P",{});var Ro=l(_e);xr=n(Ro,"Your "),Da=o(Ro,"CODE",{});var op=l(Da);Ir=n(op,"DatasetBuilder._split_generator()"),op.forEach(a),Or=n(Ro," should look like this now:"),Ro.forEach(a),Js=f(e),_(We.$$.fragment,e),Ws=f(e),J=o(e,"H2",{class:!0});var Uo=l(J);ge=o(Uo,"A",{id:!0,class:!0,href:!0});var lp=l(ge);xa=o(lp,"SPAN",{});var rp=l(xa);_(Ke.$$.fragment,rp),rp.forEach(a),lp.forEach(a),Tr=f(Uo),Ia=o(Uo,"SPAN",{});var np=l(Ia);Cr=n(np,"Generate samples"),np.forEach(a),Uo.forEach(a),Ks=f(e),Dt=o(e,"P",{});var ip=l(Dt);Lr=n(ip,"At this point, you have:"),ip.forEach(a),Zs=f(e),N=o(e,"UL",{});var Ut=l(N);Oa=o(Ut,"LI",{});var pp=l(Oa);Nr=n(pp,"Added the dataset attributes."),pp.forEach(a),Pr=f(Ut),Ta=o(Ut,"LI",{});var dp=l(Ta);Br=n(dp,"Provided instructions for how to download the data files."),dp.forEach(a),Gr=f(Ut),Ca=o(Ut,"LI",{});var fp=l(Ca);Rr=n(fp,"Organized the splits."),fp.forEach(a),Ut.forEach(a),eo=f(e),xt=o(e,"P",{});var up=l(xt);Ur=n(up,"The next step is to actually generate the samples in each split."),up.forEach(a),to=f(e),we=o(e,"OL",{});var Vo=l(we);La=o(Vo,"LI",{});var hp=l(La);ve=o(hp,"P",{});var us=l(ve);Na=o(us,"CODE",{});var cp=l(Na);Vr=n(cp,"DatasetBuilder._generate_examples"),cp.forEach(a),Fr=n(us," takes the file path provided by "),Pa=o(us,"CODE",{});var mp=l(Pa);Mr=n(mp,"gen_kwargs"),mp.forEach(a),Hr=n(us," to read and parse the data files. You need to write a function that loads the data files and extracts the columns."),us.forEach(a),hp.forEach(a),zr=f(Vo),Ba=o(Vo,"LI",{});var _p=l(Ba);Ze=o(_p,"P",{});var Fo=l(Ze);Qr=n(Fo,"Your function should yield a tuple of an "),Ga=o(Fo,"CODE",{});var gp=l(Ga);Yr=n(gp,"id_"),gp.forEach(a),Xr=n(Fo,", and an example from the dataset."),Fo.forEach(a),_p.forEach(a),Vo.forEach(a),ao=f(e),_(et.$$.fragment,e),so=f(e),W=o(e,"H2",{class:!0});var Mo=l(W);ye=o(Mo,"A",{id:!0,class:!0,href:!0});var wp=l(ye);Ra=o(wp,"SPAN",{});var vp=l(Ra);_(tt.$$.fragment,vp),vp.forEach(a),wp.forEach(a),Jr=f(Mo),Ua=o(Mo,"SPAN",{});var yp=l(Ua);Wr=n(yp,"(Optional) Generate dataset metadata"),yp.forEach(a),Mo.forEach(a),oo=f(e),P=o(e,"P",{});var Vt=l(P);Kr=n(Vt,"Adding dataset metadata is a great way to include information about your dataset. The metadata is stored in the dataset card "),Va=o(Vt,"CODE",{});var $p=l(Va);Zr=n($p,"README.md"),$p.forEach(a),en=n(Vt," in YAML. It includes information like the number of examples required to confirm the dataset was correctly generated, and information about the dataset like its "),Fa=o(Vt,"CODE",{});var Ep=l(Fa);tn=n(Ep,"features"),Ep.forEach(a),an=n(Vt,"."),Vt.forEach(a),lo=f(e),$e=o(e,"P",{});var Ho=l($e);sn=n(Ho,"Run the following command to generate your dataset metadata in "),Ma=o(Ho,"CODE",{});var qp=l(Ma);on=n(qp,"README.md"),qp.forEach(a),ln=n(Ho," and make sure your new dataset loading script works correctly:"),Ho.forEach(a),ro=f(e),_(at.$$.fragment,e),no=f(e),B=o(e,"P",{});var Ft=l(B);rn=n(Ft,"If your dataset loading script passed the test, you should now have a "),Ha=o(Ft,"CODE",{});var bp=l(Ha);nn=n(bp,"README.md"),bp.forEach(a),pn=n(Ft," file in your dataset folder containing a "),za=o(Ft,"CODE",{});var jp=l(za);dn=n(jp,"dataset_info"),jp.forEach(a),fn=n(Ft," field with some metadata."),Ft.forEach(a),io=f(e),K=o(e,"H2",{class:!0});var zo=l(K);Ee=o(zo,"A",{id:!0,class:!0,href:!0});var kp=l(Ee);Qa=o(kp,"SPAN",{});var Ap=l(Qa);_(st.$$.fragment,Ap),Ap.forEach(a),kp.forEach(a),un=f(zo),Ya=o(zo,"SPAN",{});var Sp=l(Ya);hn=n(Sp,"Upload to the Hub"),Sp.forEach(a),zo.forEach(a),po=f(e),G=o(e,"P",{});var Mt=l(G);cn=n(Mt,"Once your script is ready, "),It=o(Mt,"A",{href:!0});var Dp=l(It);mn=n(Dp,"create a dataset card"),Dp.forEach(a),_n=n(Mt," and "),Ot=o(Mt,"A",{href:!0});var xp=l(Ot);gn=n(xp,"upload it to the Hub"),xp.forEach(a),wn=n(Mt,"."),Mt.forEach(a),fo=f(e),Tt=o(e,"P",{});var Ip=l(Tt);vn=n(Ip,"Congratulations, you can now load your dataset from the Hub! \u{1F973}"),Ip.forEach(a),uo=f(e),_(ot.$$.fragment,e),ho=f(e),Z=o(e,"H2",{class:!0});var Qo=l(Z);qe=o(Qo,"A",{id:!0,class:!0,href:!0});var Op=l(qe);Xa=o(Op,"SPAN",{});var Tp=l(Xa);_(lt.$$.fragment,Tp),Tp.forEach(a),Op.forEach(a),yn=f(Qo),Ja=o(Qo,"SPAN",{});var Cp=l(Ja);$n=n(Cp,"Advanced features"),Cp.forEach(a),Qo.forEach(a),co=f(e),ee=o(e,"H3",{class:!0});var Yo=l(ee);be=o(Yo,"A",{id:!0,class:!0,href:!0});var Lp=l(be);Wa=o(Lp,"SPAN",{});var Np=l(Wa);_(rt.$$.fragment,Np),Np.forEach(a),Lp.forEach(a),En=f(Yo),Ka=o(Yo,"SPAN",{});var Pp=l(Ka);qn=n(Pp,"Sharding"),Pp.forEach(a),Yo.forEach(a),mo=f(e),je=o(e,"P",{});var Xo=l(je);bn=n(Xo,`If your dataset is made of many big files, \u{1F917} Datasets automatically runs your script in parallel to make it super fast!
It can help if you have hundreds or thousands of TAR archives, or JSONL files like `),nt=o(Xo,"A",{href:!0,rel:!0});var Bp=l(nt);jn=n(Bp,"oscar"),Bp.forEach(a),kn=n(Xo," for example."),Xo.forEach(a),_o=f(e),R=o(e,"P",{});var Ht=l(R);An=n(Ht,"To make it work, we consider lists of files in "),Za=o(Ht,"CODE",{});var Gp=l(Za);Sn=n(Gp,"gen_kwargs"),Gp.forEach(a),Dn=n(Ht,` to be shards.
Therefore \u{1F917} Datasets can automatically spawn several workers to run `),es=o(Ht,"CODE",{});var Rp=l(es);xn=n(Rp,"_generate_examples"),Rp.forEach(a),In=n(Ht," in parallel, and each worker is given a subset of shards to process."),Ht.forEach(a),go=f(e),_(it.$$.fragment,e),wo=f(e),U=o(e,"P",{});var zt=l(U);On=n(zt,"Users can also specify "),ts=o(zt,"CODE",{});var Up=l(ts);Tn=n(Up,"num_proc="),Up.forEach(a),Cn=n(zt," in "),as=o(zt,"CODE",{});var Vp=l(as);Ln=n(Vp,"load_dataset()"),Vp.forEach(a),Nn=n(zt," to specify the number of processes to use as workers."),zt.forEach(a),vo=f(e),te=o(e,"H3",{class:!0});var Jo=l(te);ke=o(Jo,"A",{id:!0,class:!0,href:!0});var Fp=l(ke);ss=o(Fp,"SPAN",{});var Mp=l(ss);_(pt.$$.fragment,Mp),Mp.forEach(a),Fp.forEach(a),Pn=f(Jo),os=o(Jo,"SPAN",{});var Hp=l(os);Bn=n(Hp,"ArrowBasedBuilder"),Hp.forEach(a),Jo.forEach(a),yo=f(e),Ct=o(e,"P",{});var zp=l(Ct);Gn=n(zp,`For some datasets it can be much faster to yield batches of data rather than examples one by one.
You can speed up the dataset generation by yielding Arrow tables directly, instead of examples.
This is especially useful if your data comes from Pandas DataFrames for example, since the conversion from Pandas to Arrow is as simple as:`),zp.forEach(a),$o=f(e),_(dt.$$.fragment,e),Eo=f(e),A=o(e,"P",{});var M=l(A);Rn=n(M,"To yield Arrow tables instead of single examples, make your dataset builder inherit from "),Lt=o(M,"A",{href:!0});var Qp=l(Lt);Un=n(Qp,"ArrowBasedBuilder"),Qp.forEach(a),Vn=n(M," instead of "),Nt=o(M,"A",{href:!0});var Yp=l(Nt);Fn=n(Yp,"GeneratorBasedBuilder"),Yp.forEach(a),Mn=n(M,", and use "),ls=o(M,"CODE",{});var Xp=l(ls);Hn=n(Xp,"_generate_tables"),Xp.forEach(a),zn=n(M," instead of "),rs=o(M,"CODE",{});var Jp=l(rs);Qn=n(Jp,"_generate_examples"),Jp.forEach(a),Yn=n(M,":"),M.forEach(a),qo=f(e),_(ft.$$.fragment,e),bo=f(e),Pt=o(e,"P",{});var Wp=l(Pt);Xn=n(Wp,"Don\u2019t forget to keep your script memory efficient, in case users run them on machines with a low amount of RAM."),Wp.forEach(a),this.h()},h(){u(h,"name","hf:doc:metadata"),u(h,"content",JSON.stringify(id)),u($,"id","create-a-dataset-loading-script"),u($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($,"href","#create-a-dataset-loading-script"),u(c,"class","relative group"),u(De,"href","https://huggingface.co/datasets/squad/blob/main/squad.py"),u(De,"rel","nofollow"),u(le,"id","add-dataset-attributes"),u(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(le,"href","#add-dataset-attributes"),u(H,"class","relative group"),u(gt,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Features"),u(z,"start","3"),u(pe,"id","multiple-configurations"),u(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(pe,"href","#multiple-configurations"),u(Q,"class","relative group"),u(Ce,"href","https://huggingface.co/datasets/super_glue"),u(Ce,"rel","nofollow"),u($t,"href","/docs/datasets/v2.8.0/en/package_reference/builder_classes#datasets.BuilderConfig"),u(Le,"href","https://huggingface.co/datasets/super_glue/blob/main/super_glue.py"),u(Le,"rel","nofollow"),u(qt,"href","/docs/datasets/v2.8.0/en/package_reference/builder_classes#datasets.BuilderConfig"),u(Be,"start","2"),u(Ue,"start","3"),u(fe,"id","default-configurations"),u(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(fe,"href","#default-configurations"),u(Y,"class","relative group"),u(he,"id","download-data-files-and-organize-splits"),u(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(he,"href","#download-data-files-and-organize-splits"),u(X,"class","relative group"),u(kt,"href","/docs/datasets/v2.8.0/en/package_reference/builder_classes#datasets.DownloadManager.download_and_extract"),u(At,"href","/docs/datasets/v2.8.0/en/package_reference/builder_classes#datasets.SplitGenerator"),u(Ye,"start","2"),u(ge,"id","generate-samples"),u(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ge,"href","#generate-samples"),u(J,"class","relative group"),u(ye,"id","optional-generate-dataset-metadata"),u(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ye,"href","#optional-generate-dataset-metadata"),u(W,"class","relative group"),u(Ee,"id","upload-to-the-hub"),u(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ee,"href","#upload-to-the-hub"),u(K,"class","relative group"),u(It,"href","dataset_card"),u(Ot,"href","share"),u(qe,"id","advanced-features"),u(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(qe,"href","#advanced-features"),u(Z,"class","relative group"),u(be,"id","sharding"),u(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(be,"href","#sharding"),u(ee,"class","relative group"),u(nt,"href","https://huggingface.co/datasets/oscar/blob/main/oscar.py"),u(nt,"rel","nofollow"),u(ke,"id","arrowbasedbuilder"),u(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ke,"href","#arrowbasedbuilder"),u(te,"class","relative group"),u(Lt,"href","/docs/datasets/v2.8.0/en/package_reference/builder_classes#datasets.ArrowBasedBuilder"),u(Nt,"href","/docs/datasets/v2.8.0/en/package_reference/builder_classes#datasets.GeneratorBasedBuilder")},m(e,i){t(document.head,h),p(e,E,i),p(e,c,i),t(c,$),t($,S),g(q,S,null),t(c,b),t(c,x),t(x,Ko),p(e,hs,i),g(ae,e,i),p(e,cs,i),p(e,ht,i),t(ht,Zo),p(e,ms,i),p(e,ct,i),t(ct,el),p(e,_s,i),p(e,T,i),t(T,tl),t(T,Qt),t(Qt,al),t(T,sl),t(T,Yt),t(Yt,ol),t(T,ll),p(e,gs,i),g(Ae,e,i),p(e,ws,i),g(Se,e,i),p(e,vs,i),p(e,mt,i),t(mt,rl),p(e,ys,i),p(e,k,i),t(k,Xt),t(Xt,nl),t(k,il),t(k,Jt),t(Jt,pl),t(k,dl),t(k,Wt),t(Wt,fl),t(k,ul),t(k,Kt),t(Kt,hl),t(k,cl),t(k,Zt),t(Zt,ml),p(e,$s,i),p(e,se,i),t(se,_l),t(se,De),t(De,gl),t(se,wl),p(e,Es,i),g(oe,e,i),p(e,qs,i),p(e,H,i),t(H,le),t(le,ea),g(xe,ea,null),t(H,vl),t(H,ta),t(ta,yl),p(e,bs,i),p(e,re,i),t(re,$l),t(re,aa),t(aa,El),t(re,ql),p(e,js,i),p(e,ne,i),t(ne,sa),t(sa,_t),t(_t,oa),t(oa,bl),t(_t,jl),t(ne,kl),t(ne,la),t(la,ie),t(ie,ra),t(ra,Al),t(ie,Sl),t(ie,gt),t(gt,Dl),t(ie,xl),p(e,ks,i),g(Ie,e,i),p(e,As,i),p(e,z,i),t(z,na),t(na,wt),t(wt,ia),t(ia,Il),t(wt,Ol),t(z,Tl),t(z,pa),t(pa,vt),t(vt,da),t(da,Cl),t(vt,Ll),p(e,Ss,i),p(e,yt,i),t(yt,Nl),p(e,Ds,i),g(Oe,e,i),p(e,xs,i),p(e,Q,i),t(Q,pe),t(pe,fa),g(Te,fa,null),t(Q,Pl),t(Q,ua),t(ua,Bl),p(e,Is,i),p(e,C,i),t(C,Gl),t(C,Ce),t(Ce,Rl),t(C,Ul),t(C,$t),t($t,Vl),t(C,Fl),p(e,Os,i),p(e,de,i),t(de,Ml),t(de,Le),t(Le,Hl),t(de,zl),p(e,Ts,i),p(e,Et,i),t(Et,Ne),t(Ne,Ql),t(Ne,qt),t(qt,Yl),t(Ne,Xl),p(e,Cs,i),g(Pe,e,i),p(e,Ls,i),p(e,Be,i),t(Be,Ge),t(Ge,Jl),t(Ge,ha),t(ha,Wl),t(Ge,Kl),p(e,Ns,i),g(Re,e,i),p(e,Ps,i),p(e,Ue,i),t(Ue,Ve),t(Ve,Zl),t(Ve,ca),t(ca,er),t(Ve,tr),p(e,Bs,i),g(Fe,e,i),p(e,Gs,i),p(e,Y,i),t(Y,fe),t(fe,ma),g(Me,ma,null),t(Y,ar),t(Y,_a),t(_a,sr),p(e,Rs,i),p(e,L,i),t(L,or),t(L,ga),t(ga,lr),t(L,rr),t(L,wa),t(wa,nr),t(L,ir),p(e,Us,i),g(He,e,i),p(e,Vs,i),g(ue,e,i),p(e,Fs,i),p(e,X,i),t(X,he),t(he,va),g(ze,va,null),t(X,pr),t(X,ya),t(ya,dr),p(e,Ms,i),p(e,bt,i),t(bt,fr),p(e,Hs,i),p(e,jt,i),t(jt,$a),t($a,ur),p(e,zs,i),g(Qe,e,i),p(e,Qs,i),g(ce,e,i),p(e,Ys,i),p(e,Ye,i),t(Ye,Xe),t(Xe,me),t(me,kt),t(kt,hr),t(me,cr),t(me,At),t(At,mr),t(me,_r),t(Xe,gr),t(Xe,Je),t(Je,Ea),t(Ea,D),t(D,wr),t(D,qa),t(qa,vr),t(D,yr),t(D,ba),t(ba,$r),t(D,Er),t(D,ja),t(ja,qr),t(D,br),t(D,ka),t(ka,jr),t(D,kr),t(Je,Ar),t(Je,Aa),t(Aa,St),t(St,Sa),t(Sa,Sr),t(St,Dr),p(e,Xs,i),p(e,_e,i),t(_e,xr),t(_e,Da),t(Da,Ir),t(_e,Or),p(e,Js,i),g(We,e,i),p(e,Ws,i),p(e,J,i),t(J,ge),t(ge,xa),g(Ke,xa,null),t(J,Tr),t(J,Ia),t(Ia,Cr),p(e,Ks,i),p(e,Dt,i),t(Dt,Lr),p(e,Zs,i),p(e,N,i),t(N,Oa),t(Oa,Nr),t(N,Pr),t(N,Ta),t(Ta,Br),t(N,Gr),t(N,Ca),t(Ca,Rr),p(e,eo,i),p(e,xt,i),t(xt,Ur),p(e,to,i),p(e,we,i),t(we,La),t(La,ve),t(ve,Na),t(Na,Vr),t(ve,Fr),t(ve,Pa),t(Pa,Mr),t(ve,Hr),t(we,zr),t(we,Ba),t(Ba,Ze),t(Ze,Qr),t(Ze,Ga),t(Ga,Yr),t(Ze,Xr),p(e,ao,i),g(et,e,i),p(e,so,i),p(e,W,i),t(W,ye),t(ye,Ra),g(tt,Ra,null),t(W,Jr),t(W,Ua),t(Ua,Wr),p(e,oo,i),p(e,P,i),t(P,Kr),t(P,Va),t(Va,Zr),t(P,en),t(P,Fa),t(Fa,tn),t(P,an),p(e,lo,i),p(e,$e,i),t($e,sn),t($e,Ma),t(Ma,on),t($e,ln),p(e,ro,i),g(at,e,i),p(e,no,i),p(e,B,i),t(B,rn),t(B,Ha),t(Ha,nn),t(B,pn),t(B,za),t(za,dn),t(B,fn),p(e,io,i),p(e,K,i),t(K,Ee),t(Ee,Qa),g(st,Qa,null),t(K,un),t(K,Ya),t(Ya,hn),p(e,po,i),p(e,G,i),t(G,cn),t(G,It),t(It,mn),t(G,_n),t(G,Ot),t(Ot,gn),t(G,wn),p(e,fo,i),p(e,Tt,i),t(Tt,vn),p(e,uo,i),g(ot,e,i),p(e,ho,i),p(e,Z,i),t(Z,qe),t(qe,Xa),g(lt,Xa,null),t(Z,yn),t(Z,Ja),t(Ja,$n),p(e,co,i),p(e,ee,i),t(ee,be),t(be,Wa),g(rt,Wa,null),t(ee,En),t(ee,Ka),t(Ka,qn),p(e,mo,i),p(e,je,i),t(je,bn),t(je,nt),t(nt,jn),t(je,kn),p(e,_o,i),p(e,R,i),t(R,An),t(R,Za),t(Za,Sn),t(R,Dn),t(R,es),t(es,xn),t(R,In),p(e,go,i),g(it,e,i),p(e,wo,i),p(e,U,i),t(U,On),t(U,ts),t(ts,Tn),t(U,Cn),t(U,as),t(as,Ln),t(U,Nn),p(e,vo,i),p(e,te,i),t(te,ke),t(ke,ss),g(pt,ss,null),t(te,Pn),t(te,os),t(os,Bn),p(e,yo,i),p(e,Ct,i),t(Ct,Gn),p(e,$o,i),g(dt,e,i),p(e,Eo,i),p(e,A,i),t(A,Rn),t(A,Lt),t(Lt,Un),t(A,Vn),t(A,Nt),t(Nt,Fn),t(A,Mn),t(A,ls),t(ls,Hn),t(A,zn),t(A,rs),t(rs,Qn),t(A,Yn),p(e,qo,i),g(ft,e,i),p(e,bo,i),p(e,Pt,i),t(Pt,Xn),jo=!0},p(e,[i]){const ut={};i&2&&(ut.$$scope={dirty:i,ctx:e}),ae.$set(ut);const ns={};i&2&&(ns.$$scope={dirty:i,ctx:e}),oe.$set(ns);const is={};i&2&&(is.$$scope={dirty:i,ctx:e}),ue.$set(is);const ps={};i&2&&(ps.$$scope={dirty:i,ctx:e}),ce.$set(ps)},i(e){jo||(w(q.$$.fragment,e),w(ae.$$.fragment,e),w(Ae.$$.fragment,e),w(Se.$$.fragment,e),w(oe.$$.fragment,e),w(xe.$$.fragment,e),w(Ie.$$.fragment,e),w(Oe.$$.fragment,e),w(Te.$$.fragment,e),w(Pe.$$.fragment,e),w(Re.$$.fragment,e),w(Fe.$$.fragment,e),w(Me.$$.fragment,e),w(He.$$.fragment,e),w(ue.$$.fragment,e),w(ze.$$.fragment,e),w(Qe.$$.fragment,e),w(ce.$$.fragment,e),w(We.$$.fragment,e),w(Ke.$$.fragment,e),w(et.$$.fragment,e),w(tt.$$.fragment,e),w(at.$$.fragment,e),w(st.$$.fragment,e),w(ot.$$.fragment,e),w(lt.$$.fragment,e),w(rt.$$.fragment,e),w(it.$$.fragment,e),w(pt.$$.fragment,e),w(dt.$$.fragment,e),w(ft.$$.fragment,e),jo=!0)},o(e){v(q.$$.fragment,e),v(ae.$$.fragment,e),v(Ae.$$.fragment,e),v(Se.$$.fragment,e),v(oe.$$.fragment,e),v(xe.$$.fragment,e),v(Ie.$$.fragment,e),v(Oe.$$.fragment,e),v(Te.$$.fragment,e),v(Pe.$$.fragment,e),v(Re.$$.fragment,e),v(Fe.$$.fragment,e),v(Me.$$.fragment,e),v(He.$$.fragment,e),v(ue.$$.fragment,e),v(ze.$$.fragment,e),v(Qe.$$.fragment,e),v(ce.$$.fragment,e),v(We.$$.fragment,e),v(Ke.$$.fragment,e),v(et.$$.fragment,e),v(tt.$$.fragment,e),v(at.$$.fragment,e),v(st.$$.fragment,e),v(ot.$$.fragment,e),v(lt.$$.fragment,e),v(rt.$$.fragment,e),v(it.$$.fragment,e),v(pt.$$.fragment,e),v(dt.$$.fragment,e),v(ft.$$.fragment,e),jo=!1},d(e){a(h),e&&a(E),e&&a(c),y(q),e&&a(hs),y(ae,e),e&&a(cs),e&&a(ht),e&&a(ms),e&&a(ct),e&&a(_s),e&&a(T),e&&a(gs),y(Ae,e),e&&a(ws),y(Se,e),e&&a(vs),e&&a(mt),e&&a(ys),e&&a(k),e&&a($s),e&&a(se),e&&a(Es),y(oe,e),e&&a(qs),e&&a(H),y(xe),e&&a(bs),e&&a(re),e&&a(js),e&&a(ne),e&&a(ks),y(Ie,e),e&&a(As),e&&a(z),e&&a(Ss),e&&a(yt),e&&a(Ds),y(Oe,e),e&&a(xs),e&&a(Q),y(Te),e&&a(Is),e&&a(C),e&&a(Os),e&&a(de),e&&a(Ts),e&&a(Et),e&&a(Cs),y(Pe,e),e&&a(Ls),e&&a(Be),e&&a(Ns),y(Re,e),e&&a(Ps),e&&a(Ue),e&&a(Bs),y(Fe,e),e&&a(Gs),e&&a(Y),y(Me),e&&a(Rs),e&&a(L),e&&a(Us),y(He,e),e&&a(Vs),y(ue,e),e&&a(Fs),e&&a(X),y(ze),e&&a(Ms),e&&a(bt),e&&a(Hs),e&&a(jt),e&&a(zs),y(Qe,e),e&&a(Qs),y(ce,e),e&&a(Ys),e&&a(Ye),e&&a(Xs),e&&a(_e),e&&a(Js),y(We,e),e&&a(Ws),e&&a(J),y(Ke),e&&a(Ks),e&&a(Dt),e&&a(Zs),e&&a(N),e&&a(eo),e&&a(xt),e&&a(to),e&&a(we),e&&a(ao),y(et,e),e&&a(so),e&&a(W),y(tt),e&&a(oo),e&&a(P),e&&a(lo),e&&a($e),e&&a(ro),y(at,e),e&&a(no),e&&a(B),e&&a(io),e&&a(K),y(st),e&&a(po),e&&a(G),e&&a(fo),e&&a(Tt),e&&a(uo),y(ot,e),e&&a(ho),e&&a(Z),y(lt),e&&a(co),e&&a(ee),y(rt),e&&a(mo),e&&a(je),e&&a(_o),e&&a(R),e&&a(go),y(it,e),e&&a(wo),e&&a(U),e&&a(vo),e&&a(te),y(pt),e&&a(yo),e&&a(Ct),e&&a($o),y(dt,e),e&&a(Eo),e&&a(A),e&&a(qo),y(ft,e),e&&a(bo),e&&a(Pt)}}}const id={local:"create-a-dataset-loading-script",sections:[{local:"add-dataset-attributes",sections:[{local:"multiple-configurations",title:"Multiple configurations"},{local:"default-configurations",title:"Default configurations"}],title:"Add dataset attributes"},{local:"download-data-files-and-organize-splits",title:"Download data files and organize splits"},{local:"generate-samples",title:"Generate samples"},{local:"optional-generate-dataset-metadata",title:"(Optional) Generate dataset metadata"},{local:"upload-to-the-hub",title:"Upload to the Hub"},{local:"advanced-features",sections:[{local:"sharding",title:"Sharding"},{local:"arrowbasedbuilder",title:"ArrowBasedBuilder"}],title:"Advanced features"}],title:"Create a dataset loading script"};function pd(O){return ad(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class cd extends Kp{constructor(h){super();Zp(this,h,pd,nd,ed,{})}}export{cd as default,id as metadata};
