import{S as gt,i as xt,s as qt,e as o,k as p,w as L,t as d,M as yt,c as n,d as a,m as r,a as i,x as S,h as m,b as g,G as e,g as l,y as A,L as $t,q as C,o as B,B as N,v as jt}from"../chunks/vendor-hf-doc-builder.js";import{I as vt}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as V}from"../chunks/CodeBlock-hf-doc-builder.js";function wt(rt){let f,D,u,c,T,x,W,E,X,H,h,Z,q,tt,at,M,w,st,R,y,U,_,et,P,lt,ot,z,$,I,b,nt,O,j,Y,k,pt,G,v,J;return x=new vt({}),y=new V({props:{code:`from datasets import load_dataset
dataset = load_dataset("text", data_files={"train": ["my_text_1.txt", "my_text_2.txt"], "test": "my_test_file.txt"})

dataset = load_dataset("text", data_dir="path/to/text/dataset")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;text&quot;</span>, data_files={<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;my_text_1.txt&quot;</span>, <span class="hljs-string">&quot;my_text_2.txt&quot;</span>], <span class="hljs-string">&quot;test&quot;</span>: <span class="hljs-string">&quot;my_test_file.txt&quot;</span>})

<span class="hljs-comment"># Load from a directory</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;text&quot;</span>, data_dir=<span class="hljs-string">&quot;path/to/text/dataset&quot;</span>)`}}),$=new V({props:{code:`dataset = load_dataset("text", data_files={"train": "my_train_file.txt", "test": "my_test_file.txt"}, sample_by="paragraph")

dataset = load_dataset("text", data_files={"train": "my_train_file.txt", "test": "my_test_file.txt"}, sample_by="document")`,highlighted:`<span class="hljs-comment"># Sample by paragraph</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;text&quot;</span>, data_files={<span class="hljs-string">&quot;train&quot;</span>: <span class="hljs-string">&quot;my_train_file.txt&quot;</span>, <span class="hljs-string">&quot;test&quot;</span>: <span class="hljs-string">&quot;my_test_file.txt&quot;</span>}, sample_by=<span class="hljs-string">&quot;paragraph&quot;</span>)

<span class="hljs-comment"># Sample by document</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;text&quot;</span>, data_files={<span class="hljs-string">&quot;train&quot;</span>: <span class="hljs-string">&quot;my_train_file.txt&quot;</span>, <span class="hljs-string">&quot;test&quot;</span>: <span class="hljs-string">&quot;my_test_file.txt&quot;</span>}, sample_by=<span class="hljs-string">&quot;document&quot;</span>)`}}),j=new V({props:{code:`from datasets import load_dataset
c4_subset = load_dataset("allenai/c4", data_files="en/c4-train.0000*-of-01024.json.gz")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>c4_subset = load_dataset(<span class="hljs-string">&quot;allenai/c4&quot;</span>, data_files=<span class="hljs-string">&quot;en/c4-train.0000*-of-01024.json.gz&quot;</span>)`}}),v=new V({props:{code:'dataset = load_dataset("text", data_files="https://huggingface.co/datasets/lhoestq/test/resolve/main/some_text.txt")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;text&quot;</span>, data_files=<span class="hljs-string">&quot;https://huggingface.co/datasets/lhoestq/test/resolve/main/some_text.txt&quot;</span>)'}}),{c(){f=o("meta"),D=p(),u=o("h1"),c=o("a"),T=o("span"),L(x.$$.fragment),W=p(),E=o("span"),X=d("Load text data"),H=p(),h=o("p"),Z=d("This guide shows you how to load text datasets. To learn how to load any type of dataset, take a look at the "),q=o("a"),tt=d("general loading guide"),at=d("."),M=p(),w=o("p"),st=d("Text files are one of the most common file types for storing a dataset. By default, \u{1F917} Datasets samples a text file line by line to build the dataset."),R=p(),L(y.$$.fragment),U=p(),_=o("p"),et=d("To sample a text file by paragraph or even an entire document, use the "),P=o("code"),lt=d("sample_by"),ot=d(" parameter:"),z=p(),L($.$$.fragment),I=p(),b=o("p"),nt=d("You can also use grep patterns to load specific files:"),O=p(),L(j.$$.fragment),Y=p(),k=o("p"),pt=d("To load remote text files via HTTP, pass the URLs instead:"),G=p(),L(v.$$.fragment),this.h()},l(t){const s=yt('[data-svelte="svelte-1phssyn"]',document.head);f=n(s,"META",{name:!0,content:!0}),s.forEach(a),D=r(t),u=n(t,"H1",{class:!0});var F=i(u);c=n(F,"A",{id:!0,class:!0,href:!0});var it=i(c);T=n(it,"SPAN",{});var dt=i(T);S(x.$$.fragment,dt),dt.forEach(a),it.forEach(a),W=r(F),E=n(F,"SPAN",{});var mt=i(E);X=m(mt,"Load text data"),mt.forEach(a),F.forEach(a),H=r(t),h=n(t,"P",{});var K=i(h);Z=m(K,"This guide shows you how to load text datasets. To learn how to load any type of dataset, take a look at the "),q=n(K,"A",{class:!0,href:!0});var ft=i(q);tt=m(ft,"general loading guide"),ft.forEach(a),at=m(K,"."),K.forEach(a),M=r(t),w=n(t,"P",{});var ut=i(w);st=m(ut,"Text files are one of the most common file types for storing a dataset. By default, \u{1F917} Datasets samples a text file line by line to build the dataset."),ut.forEach(a),R=r(t),S(y.$$.fragment,t),U=r(t),_=n(t,"P",{});var Q=i(_);et=m(Q,"To sample a text file by paragraph or even an entire document, use the "),P=n(Q,"CODE",{});var ct=i(P);lt=m(ct,"sample_by"),ct.forEach(a),ot=m(Q," parameter:"),Q.forEach(a),z=r(t),S($.$$.fragment,t),I=r(t),b=n(t,"P",{});var ht=i(b);nt=m(ht,"You can also use grep patterns to load specific files:"),ht.forEach(a),O=r(t),S(j.$$.fragment,t),Y=r(t),k=n(t,"P",{});var _t=i(k);pt=m(_t,"To load remote text files via HTTP, pass the URLs instead:"),_t.forEach(a),G=r(t),S(v.$$.fragment,t),this.h()},h(){g(f,"name","hf:doc:metadata"),g(f,"content",JSON.stringify(bt)),g(c,"id","load-text-data"),g(c,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(c,"href","#load-text-data"),g(u,"class","relative group"),g(q,"class","underline decoration-sky-400 decoration-2 font-semibold"),g(q,"href","./loading")},m(t,s){e(document.head,f),l(t,D,s),l(t,u,s),e(u,c),e(c,T),A(x,T,null),e(u,W),e(u,E),e(E,X),l(t,H,s),l(t,h,s),e(h,Z),e(h,q),e(q,tt),e(h,at),l(t,M,s),l(t,w,s),e(w,st),l(t,R,s),A(y,t,s),l(t,U,s),l(t,_,s),e(_,et),e(_,P),e(P,lt),e(_,ot),l(t,z,s),A($,t,s),l(t,I,s),l(t,b,s),e(b,nt),l(t,O,s),A(j,t,s),l(t,Y,s),l(t,k,s),e(k,pt),l(t,G,s),A(v,t,s),J=!0},p:$t,i(t){J||(C(x.$$.fragment,t),C(y.$$.fragment,t),C($.$$.fragment,t),C(j.$$.fragment,t),C(v.$$.fragment,t),J=!0)},o(t){B(x.$$.fragment,t),B(y.$$.fragment,t),B($.$$.fragment,t),B(j.$$.fragment,t),B(v.$$.fragment,t),J=!1},d(t){a(f),t&&a(D),t&&a(u),N(x),t&&a(H),t&&a(h),t&&a(M),t&&a(w),t&&a(R),N(y,t),t&&a(U),t&&a(_),t&&a(z),N($,t),t&&a(I),t&&a(b),t&&a(O),N(j,t),t&&a(Y),t&&a(k),t&&a(G),N(v,t)}}}const bt={local:"load-text-data",title:"Load text data"};function kt(rt){return jt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Lt extends gt{constructor(f){super();xt(this,f,kt,wt,qt,{})}}export{Lt as default,bt as metadata};
