import{S as vl,i as wl,s as $l,e as a,k as c,w as _,t as n,M as yl,c as r,d as t,m as f,a as o,x as g,h as i,b as u,G as s,g as p,y as v,q as w,o as $,B as y,v as bl}from"../chunks/vendor-hf-doc-builder.js";import{T as pa}from"../chunks/Tip-hf-doc-builder.js";import{I as pe}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as S}from"../chunks/CodeBlock-hf-doc-builder.js";function El(I){let d,b,h,m,q;return{c(){d=a("p"),b=n("Metrics is deprecated in \u{1F917} Datasets. To learn more about how to use metrics, take a look at the library \u{1F917} "),h=a("a"),m=n("Evaluate"),q=n("! In addition to metrics, you can find more tools for evaluating models and datasets."),this.h()},l(E){d=r(E,"P",{});var j=o(d);b=i(j,"Metrics is deprecated in \u{1F917} Datasets. To learn more about how to use metrics, take a look at the library \u{1F917} "),h=r(j,"A",{href:!0,rel:!0});var P=o(h);m=i(P,"Evaluate"),P.forEach(t),q=i(j,"! In addition to metrics, you can find more tools for evaluating models and datasets."),j.forEach(t),this.h()},h(){u(h,"href","https://huggingface.co/docs/evaluate/index"),u(h,"rel","nofollow")},m(E,j){p(E,d,j),s(d,b),s(d,h),s(h,m),s(d,q)},d(E){E&&t(d)}}}function jl(I){let d,b;return{c(){d=a("p"),b=n("Metrics accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation.")},l(h){d=r(h,"P",{});var m=o(d);b=i(m,"Metrics accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation."),m.forEach(t)},m(h,m){p(h,d,m),s(d,b)},d(h){h&&t(d)}}}function ql(I){let d,b,h,m,q;return{c(){d=a("p"),b=n("Get jump started with our metric loading script "),h=a("a"),m=n("template"),q=n("!"),this.h()},l(E){d=r(E,"P",{});var j=o(d);b=i(j,"Get jump started with our metric loading script "),h=r(j,"A",{href:!0,rel:!0});var P=o(h);m=i(P,"template"),P.forEach(t),q=i(j,"!"),j.forEach(t),this.h()},h(){u(h,"href","https://github.com/huggingface/datasets/blob/main/templates/new_metric_script.py"),u(h,"rel","nofollow")},m(E,j){p(E,d,j),s(d,b),s(d,h),s(h,m),s(d,q)},d(E){E&&t(d)}}}function kl(I){let d,b;return{c(){d=a("p"),b=n("If the files are stored locally, provide a dictionary of path(s) instead of URLs.")},l(h){d=r(h,"P",{});var m=o(d);b=i(m,"If the files are stored locally, provide a dictionary of path(s) instead of URLs."),m.forEach(t)},m(h,m){p(h,d,m),s(d,b)},d(h){h&&t(d)}}}function Pl(I){let d,b,h,m,q,E,j,P,ca,es,z,ts,Re,fa,ss,Be,ua,as,A,it,da,ha,pt,ma,_a,ct,ga,rs,T,K,ft,ce,va,ut,wa,os,V,$a,He,ya,ba,ls,Y,dt,x,ze,Ea,ja,ht,qa,ka,mt,Pa,Ia,Aa,_t,L,Ke,xa,La,gt,Ca,Sa,vt,Ta,Ma,ns,Q,Oa,Ve,Da,Na,is,fe,ps,W,cs,M,G,wt,ue,Ua,$t,Ra,fs,F,Ba,Ye,Ha,za,us,J,Ka,de,Va,Ya,ds,Qe,yt,Qa,hs,he,ms,me,bt,Wa,_s,_e,gs,ge,O,Ga,Et,Fa,Ja,jt,Xa,Za,vs,ve,ws,We,$s,D,X,qt,we,er,kt,tr,ys,Z,sr,Ge,ar,rr,bs,ee,or,$e,lr,nr,Es,te,js,N,se,Pt,ye,ir,It,pr,qs,ae,cr,At,fr,ur,ks,k,xt,Fe,Lt,dr,hr,mr,Ct,Je,St,_r,gr,vr,Tt,Xe,Mt,wr,$r,yr,Ot,Ze,Dt,br,Er,Ps,et,jr,Is,be,As,U,re,Nt,Ee,qr,Ut,kr,xs,C,Pr,Rt,Ir,Ar,je,xr,Lr,Ls,tt,Bt,Cr,Cs,qe,Ss,oe,Ts,ke,st,Ht,Sr,Tr,Ms,Pe,Os,R,le,zt,Ie,Mr,Kt,Or,Ds,B,Vt,Dr,Nr,Ae,Ur,Rr,Ns,at,xe,Br,Yt,Hr,zr,Us,Le,Rs,Ce,Se,Kr,Qt,Vr,Yr,Bs,Te,Hs,H,ne,Wt,Me,Qr,Gt,Wr,zs,rt,Gr,Ks,Oe,Vs;return E=new pe({}),z=new pa({props:{warning:!0,$$slots:{default:[El]},$$scope:{ctx:I}}}),ce=new pe({}),fe=new S({props:{code:`import datasets
metric = datasets.load_metric('my_metric')
for model_input, gold_references in evaluation_dataset:
    model_predictions = model(model_inputs)
    metric.add_batch(predictions=model_predictions, references=gold_references)
final_score = metric.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> datasets
<span class="hljs-meta">&gt;&gt;&gt; </span>metric = datasets.load_metric(<span class="hljs-string">&#x27;my_metric&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> model_input, gold_references <span class="hljs-keyword">in</span> evaluation_dataset:
<span class="hljs-meta">... </span>    model_predictions = model(model_inputs)
<span class="hljs-meta">... </span>    metric.add_batch(predictions=model_predictions, references=gold_references)
<span class="hljs-meta">&gt;&gt;&gt; </span>final_score = metric.compute()`}}),W=new pa({props:{$$slots:{default:[jl]},$$scope:{ctx:I}}}),ue=new pe({}),he=new S({props:{code:`import datasets
metric = datasets.load_metric('sacrebleu')`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> datasets
<span class="hljs-meta">&gt;&gt;&gt; </span>metric = datasets.load_metric(<span class="hljs-string">&#x27;sacrebleu&#x27;</span>)`}}),_e=new S({props:{code:`print(metric.inputs_description)
`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(metric.inputs_description)
Produces BLEU scores along <span class="hljs-keyword">with</span> its sufficient statistics
<span class="hljs-keyword">from</span> a source against one <span class="hljs-keyword">or</span> more references.

Args:
    predictions: The system stream (a sequence of segments).
    references: A <span class="hljs-built_in">list</span> of one <span class="hljs-keyword">or</span> more reference streams (each a sequence of segments).
    smooth_method: The smoothing method to use. (Default: <span class="hljs-string">&#x27;exp&#x27;</span>).
    smooth_value: The smoothing value. Only valid <span class="hljs-keyword">for</span> <span class="hljs-string">&#x27;floor&#x27;</span> <span class="hljs-keyword">and</span> <span class="hljs-string">&#x27;add-k&#x27;</span>. (Defaults: floor: <span class="hljs-number">0.1</span>, add-k: <span class="hljs-number">1</span>).
    tokenize: Tokenization method to use <span class="hljs-keyword">for</span> BLEU. If <span class="hljs-keyword">not</span> provided, defaults to <span class="hljs-string">&#x27;zh&#x27;</span> <span class="hljs-keyword">for</span> Chinese, <span class="hljs-string">&#x27;ja-mecab&#x27;</span> <span class="hljs-keyword">for</span> Japanese <span class="hljs-keyword">and</span> <span class="hljs-string">&#x27;13a&#x27;</span> (mteval) otherwise.
    lowercase: Lowercase the data. If <span class="hljs-literal">True</span>, enables case-insensitivity. (Default: <span class="hljs-literal">False</span>).
    force: Insist that your tokenized <span class="hljs-built_in">input</span> <span class="hljs-keyword">is</span> actually detokenized.
...`}}),ve=new S({props:{code:'score = metric.compute(smooth_method="floor", smooth_value=0.2)',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>score = metric.compute(smooth_method=<span class="hljs-string">&quot;floor&quot;</span>, smooth_value=<span class="hljs-number">0.2</span>)'}}),we=new pe({}),te=new pa({props:{$$slots:{default:[ql]},$$scope:{ctx:I}}}),ye=new pe({}),be=new S({props:{code:`class Squad(datasets.Metric):
    def _info(self):
        return datasets.MetricInfo(
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            features=datasets.Features(
                {
                    "predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")},
                    "references": {
                        "id": datasets.Value("string"),
                        "answers": datasets.features.Sequence(
                            {
                                "text": datasets.Value("string"),
                                "answer_start": datasets.Value("int32"),
                            }
                        ),
                    },
                }
            ),
            codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
            reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
        )`,highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">Squad</span>(datasets.Metric):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">_info</span>(<span class="hljs-params">self</span>):
        <span class="hljs-keyword">return</span> datasets.MetricInfo(
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            features=datasets.Features(
                {
                    <span class="hljs-string">&quot;predictions&quot;</span>: {<span class="hljs-string">&quot;id&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>), <span class="hljs-string">&quot;prediction_text&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>)},
                    <span class="hljs-string">&quot;references&quot;</span>: {
                        <span class="hljs-string">&quot;id&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                        <span class="hljs-string">&quot;answers&quot;</span>: datasets.features.<span class="hljs-type">Sequence</span>(
                            {
                                <span class="hljs-string">&quot;text&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                                <span class="hljs-string">&quot;answer_start&quot;</span>: datasets.Value(<span class="hljs-string">&quot;int32&quot;</span>),
                            }
                        ),
                    },
                }
            ),
            codebase_urls=[<span class="hljs-string">&quot;https://rajpurkar.github.io/SQuAD-explorer/&quot;</span>],
            reference_urls=[<span class="hljs-string">&quot;https://rajpurkar.github.io/SQuAD-explorer/&quot;</span>],
        )`}}),Ee=new pe({}),qe=new S({props:{code:`CHECKPOINT_URLS = {
    "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip",
    "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip",
    "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip",
    "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip",
    "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip",
    "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip",
}`,highlighted:`CHECKPOINT_URLS = {
    <span class="hljs-string">&quot;bleurt-tiny-128&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-tiny-512&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-base-128&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-base-512&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-large-128&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-large-512&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip&quot;</span>,
}`}}),oe=new pa({props:{$$slots:{default:[kl]},$$scope:{ctx:I}}}),Pe=new S({props:{code:`def _download_and_prepare(self, dl_manager):

    # check that config name specifies a valid BLEURT model
    if self.config_name == "default":
        logger.warning(
            "Using default BLEURT-Base checkpoint for sequence maximum length 128. "
            "You can use a bigger model for better results with e.g.: datasets.load_metric('bleurt', 'bleurt-large-512')."
        )
        self.config_name = "bleurt-base-128"
    if self.config_name not in CHECKPOINT_URLS.keys():
        raise KeyError(
            f"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}"
        )

    # download the model checkpoint specified by self.config_name and set up the scorer
    model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[self.config_name])
    self.scorer = score.BleurtScorer(os.path.join(model_path, self.config_name))`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_download_and_prepare</span>(<span class="hljs-params">self, dl_manager</span>):

    <span class="hljs-comment"># check that config name specifies a valid BLEURT model</span>
    <span class="hljs-keyword">if</span> self.config_name == <span class="hljs-string">&quot;default&quot;</span>:
        logger.warning(
            <span class="hljs-string">&quot;Using default BLEURT-Base checkpoint for sequence maximum length 128. &quot;</span>
            <span class="hljs-string">&quot;You can use a bigger model for better results with e.g.: datasets.load_metric(&#x27;bleurt&#x27;, &#x27;bleurt-large-512&#x27;).&quot;</span>
        )
        self.config_name = <span class="hljs-string">&quot;bleurt-base-128&quot;</span>
    <span class="hljs-keyword">if</span> self.config_name <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> CHECKPOINT_URLS.keys():
        <span class="hljs-keyword">raise</span> KeyError(
            <span class="hljs-string">f&quot;<span class="hljs-subst">{self.config_name}</span> model not found. You should supply the name of a model checkpoint for bleurt in <span class="hljs-subst">{CHECKPOINT_URLS.keys()}</span>&quot;</span>
        )

    <span class="hljs-comment"># download the model checkpoint specified by self.config_name and set up the scorer</span>
    model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[self.config_name])
    self.scorer = score.BleurtScorer(os.path.join(model_path, self.config_name))`}}),Ie=new pe({}),Le=new S({props:{code:`def simple_accuracy(preds, labels):
    return (preds == labels).mean().item()

def acc_and_f1(preds, labels):
    acc = simple_accuracy(preds, labels)
    f1 = f1_score(y_true=labels, y_pred=preds).item()
    return {
        "accuracy": acc,
        "f1": f1,
    }

def pearson_and_spearman(preds, labels):
    pearson_corr = pearsonr(preds, labels)[0].item()
    spearman_corr = spearmanr(preds, labels)[0].item()
    return {
        "pearson": pearson_corr,
        "spearmanr": spearman_corr,
    }`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">simple_accuracy</span>(<span class="hljs-params">preds, labels</span>):
    <span class="hljs-keyword">return</span> (preds == labels).mean().item()

<span class="hljs-keyword">def</span> <span class="hljs-title function_">acc_and_f1</span>(<span class="hljs-params">preds, labels</span>):
    acc = simple_accuracy(preds, labels)
    f1 = f1_score(y_true=labels, y_pred=preds).item()
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;accuracy&quot;</span>: acc,
        <span class="hljs-string">&quot;f1&quot;</span>: f1,
    }

<span class="hljs-keyword">def</span> <span class="hljs-title function_">pearson_and_spearman</span>(<span class="hljs-params">preds, labels</span>):
    pearson_corr = pearsonr(preds, labels)[<span class="hljs-number">0</span>].item()
    spearman_corr = spearmanr(preds, labels)[<span class="hljs-number">0</span>].item()
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;pearson&quot;</span>: pearson_corr,
        <span class="hljs-string">&quot;spearmanr&quot;</span>: spearman_corr,
    }`}}),Te=new S({props:{code:`def _compute(self, predictions, references):
    if self.config_name == "cola":
        return {"matthews_correlation": matthews_corrcoef(references, predictions)}
    elif self.config_name == "stsb":
        return pearson_and_spearman(predictions, references)
    elif self.config_name in ["mrpc", "qqp"]:
        return acc_and_f1(predictions, references)
    elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
        return {"accuracy": simple_accuracy(predictions, references)}
    else:
        raise KeyError(
            "You should supply a configuration name selected in "
            '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
            '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'
        )`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_compute</span>(<span class="hljs-params">self, predictions, references</span>):
    <span class="hljs-keyword">if</span> self.config_name == <span class="hljs-string">&quot;cola&quot;</span>:
        <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;matthews_correlation&quot;</span>: matthews_corrcoef(references, predictions)}
    <span class="hljs-keyword">elif</span> self.config_name == <span class="hljs-string">&quot;stsb&quot;</span>:
        <span class="hljs-keyword">return</span> pearson_and_spearman(predictions, references)
    <span class="hljs-keyword">elif</span> self.config_name <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;mrpc&quot;</span>, <span class="hljs-string">&quot;qqp&quot;</span>]:
        <span class="hljs-keyword">return</span> acc_and_f1(predictions, references)
    <span class="hljs-keyword">elif</span> self.config_name <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;sst2&quot;</span>, <span class="hljs-string">&quot;mnli&quot;</span>, <span class="hljs-string">&quot;mnli_mismatched&quot;</span>, <span class="hljs-string">&quot;mnli_matched&quot;</span>, <span class="hljs-string">&quot;qnli&quot;</span>, <span class="hljs-string">&quot;rte&quot;</span>, <span class="hljs-string">&quot;wnli&quot;</span>, <span class="hljs-string">&quot;hans&quot;</span>]:
        <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;accuracy&quot;</span>: simple_accuracy(predictions, references)}
    <span class="hljs-keyword">else</span>:
        <span class="hljs-keyword">raise</span> KeyError(
            <span class="hljs-string">&quot;You should supply a configuration name selected in &quot;</span>
            <span class="hljs-string">&#x27;[&quot;sst2&quot;, &quot;mnli&quot;, &quot;mnli_mismatched&quot;, &quot;mnli_matched&quot;, &#x27;</span>
            <span class="hljs-string">&#x27;&quot;cola&quot;, &quot;stsb&quot;, &quot;mrpc&quot;, &quot;qqp&quot;, &quot;qnli&quot;, &quot;rte&quot;, &quot;wnli&quot;, &quot;hans&quot;]&#x27;</span>
        )`}}),Me=new pe({}),Oe=new S({props:{code:`from datasets import load_metric
metric = load_metric('PATH/TO/MY/SCRIPT.py')`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_metric
<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load_metric(<span class="hljs-string">&#x27;PATH/TO/MY/SCRIPT.py&#x27;</span>)`}}),{c(){d=a("meta"),b=c(),h=a("h1"),m=a("a"),q=a("span"),_(E.$$.fragment),j=c(),P=a("span"),ca=n("Metrics"),es=c(),_(z.$$.fragment),ts=c(),Re=a("p"),fa=n("Metrics are important for evaluating a model\u2019s predictions. In the tutorial, you learned how to compute a metric over an entire evaluation set. You have also seen how to load a metric."),ss=c(),Be=a("p"),ua=n("This guide will show you how to:"),as=c(),A=a("ul"),it=a("li"),da=n("Add predictions and references."),ha=c(),pt=a("li"),ma=n("Compute metrics using different methods."),_a=c(),ct=a("li"),ga=n("Write your own metric loading script."),rs=c(),T=a("h2"),K=a("a"),ft=a("span"),_(ce.$$.fragment),va=c(),ut=a("span"),wa=n("Add predictions and references"),os=c(),V=a("p"),$a=n("When you want to add model predictions and references to a "),He=a("a"),ya=n("Metric"),ba=n(" instance, you have two options:"),ls=c(),Y=a("ul"),dt=a("li"),x=a("p"),ze=a("a"),Ea=n("Metric.add()"),ja=n(" adds a single "),ht=a("code"),qa=n("prediction"),ka=n(" and "),mt=a("code"),Pa=n("reference"),Ia=n("."),Aa=c(),_t=a("li"),L=a("p"),Ke=a("a"),xa=n("Metric.add_batch()"),La=n(" adds a batch of "),gt=a("code"),Ca=n("predictions"),Sa=n(" and "),vt=a("code"),Ta=n("references"),Ma=n("."),ns=c(),Q=a("p"),Oa=n("Use "),Ve=a("a"),Da=n("Metric.add_batch()"),Na=n(" by passing it your model predictions, and the references the model predictions should be evaluated against:"),is=c(),_(fe.$$.fragment),ps=c(),_(W.$$.fragment),cs=c(),M=a("h2"),G=a("a"),wt=a("span"),_(ue.$$.fragment),Ua=c(),$t=a("span"),Ra=n("Compute scores"),fs=c(),F=a("p"),Ba=n("The most straightforward way to calculate a metric is to call "),Ye=a("a"),Ha=n("Metric.compute()"),za=n(". But some metrics have additional arguments that allow you to modify the metrics behavior."),us=c(),J=a("p"),Ka=n("Let\u2019s load the "),de=a("a"),Va=n("SacreBLEU"),Ya=n(" metric, and compute it with a different smoothing method."),ds=c(),Qe=a("ol"),yt=a("li"),Qa=n("Load the SacreBLEU metric:"),hs=c(),_(he.$$.fragment),ms=c(),me=a("ol"),bt=a("li"),Wa=n("Inspect the different argument methods for computing the metric:"),_s=c(),_(_e.$$.fragment),gs=c(),ge=a("ol"),O=a("li"),Ga=n("Compute the metric with the "),Et=a("code"),Fa=n("floor"),Ja=n(" method, and a different "),jt=a("code"),Xa=n("smooth_value"),Za=n(":"),vs=c(),_(ve.$$.fragment),ws=c(),We=a("a"),$s=c(),D=a("h2"),X=a("a"),qt=a("span"),_(we.$$.fragment),er=c(),kt=a("span"),tr=n("Custom metric loading script"),ys=c(),Z=a("p"),sr=n("Write a metric loading script to use your own custom metric (or one that is not on the Hub). Then you can load it as usual with "),Ge=a("a"),ar=n("load_metric()"),rr=n("."),bs=c(),ee=a("p"),or=n("To help you get started, open the "),$e=a("a"),lr=n("SQuAD metric loading script"),nr=n(" and follow along."),Es=c(),_(te.$$.fragment),js=c(),N=a("h3"),se=a("a"),Pt=a("span"),_(ye.$$.fragment),ir=c(),It=a("span"),pr=n("Add metric attributes"),qs=c(),ae=a("p"),cr=n("Start by adding some information about your metric in "),At=a("code"),fr=n("Metric._info()"),ur=n(". The most important attributes you should specify are:"),ks=c(),k=a("ol"),xt=a("li"),Fe=a("p"),Lt=a("code"),dr=n("MetricInfo.description"),hr=n(" provides a brief description about your metric."),mr=c(),Ct=a("li"),Je=a("p"),St=a("code"),_r=n("MetricInfo.citation"),gr=n(" contains a BibTex citation for the metric."),vr=c(),Tt=a("li"),Xe=a("p"),Mt=a("code"),wr=n("MetricInfo.inputs_description"),$r=n(" describes the expected inputs and outputs. It may also provide an example usage of the metric."),yr=c(),Ot=a("li"),Ze=a("p"),Dt=a("code"),br=n("MetricInfo.features"),Er=n(" defines the name and type of the predictions and references."),Ps=c(),et=a("p"),jr=n("After you\u2019ve filled out all these fields in the template, it should look like the following example from the SQuAD metric script:"),Is=c(),_(be.$$.fragment),As=c(),U=a("h3"),re=a("a"),Nt=a("span"),_(Ee.$$.fragment),qr=c(),Ut=a("span"),kr=n("Download metric files"),xs=c(),C=a("p"),Pr=n("If your metric needs to download, or retrieve local files, you will need to use the "),Rt=a("code"),Ir=n("Metric._download_and_prepare()"),Ar=n(" method. For this example, let\u2019s examine the "),je=a("a"),xr=n("BLEURT metric loading script"),Lr=n("."),Ls=c(),tt=a("ol"),Bt=a("li"),Cr=n("Provide a dictionary of URLs that point to the metric files:"),Cs=c(),_(qe.$$.fragment),Ss=c(),_(oe.$$.fragment),Ts=c(),ke=a("ol"),st=a("li"),Ht=a("code"),Sr=n("Metric._download_and_prepare()"),Tr=n(" will take the URLs and download the metric files specified:"),Ms=c(),_(Pe.$$.fragment),Os=c(),R=a("h3"),le=a("a"),zt=a("span"),_(Ie.$$.fragment),Mr=c(),Kt=a("span"),Or=n("Compute score"),Ds=c(),B=a("p"),Vt=a("code"),Dr=n("DatasetBuilder._compute"),Nr=n(" provides the actual instructions for how to compute a metric given the predictions and references. Now let\u2019s take a look at the "),Ae=a("a"),Ur=n("GLUE metric loading script"),Rr=n("."),Ns=c(),at=a("ol"),xe=a("li"),Br=n("Provide the functions for "),Yt=a("code"),Hr=n("DatasetBuilder._compute"),zr=n(" to calculate your metric:"),Us=c(),_(Le.$$.fragment),Rs=c(),Ce=a("ol"),Se=a("li"),Kr=n("Create "),Qt=a("code"),Vr=n("DatasetBuilder._compute"),Yr=n(" with instructions for what metric to calculate for each configuration:"),Bs=c(),_(Te.$$.fragment),Hs=c(),H=a("h3"),ne=a("a"),Wt=a("span"),_(Me.$$.fragment),Qr=c(),Gt=a("span"),Wr=n("Test"),zs=c(),rt=a("p"),Gr=n("Once you\u2019re finished writing your metric loading script, try to load it locally:"),Ks=c(),_(Oe.$$.fragment),this.h()},l(e){const l=yl('[data-svelte="svelte-1phssyn"]',document.head);d=r(l,"META",{name:!0,content:!0}),l.forEach(t),b=f(e),h=r(e,"H1",{class:!0});var De=o(h);m=r(De,"A",{id:!0,class:!0,href:!0});var Ft=o(m);q=r(Ft,"SPAN",{});var Jt=o(q);g(E.$$.fragment,Jt),Jt.forEach(t),Ft.forEach(t),j=f(De),P=r(De,"SPAN",{});var Xt=o(P);ca=i(Xt,"Metrics"),Xt.forEach(t),De.forEach(t),es=f(e),g(z.$$.fragment,e),ts=f(e),Re=r(e,"P",{});var to=o(Re);fa=i(to,"Metrics are important for evaluating a model\u2019s predictions. In the tutorial, you learned how to compute a metric over an entire evaluation set. You have also seen how to load a metric."),to.forEach(t),ss=f(e),Be=r(e,"P",{});var so=o(Be);ua=i(so,"This guide will show you how to:"),so.forEach(t),as=f(e),A=r(e,"UL",{});var ot=o(A);it=r(ot,"LI",{});var ao=o(it);da=i(ao,"Add predictions and references."),ao.forEach(t),ha=f(ot),pt=r(ot,"LI",{});var ro=o(pt);ma=i(ro,"Compute metrics using different methods."),ro.forEach(t),_a=f(ot),ct=r(ot,"LI",{});var oo=o(ct);ga=i(oo,"Write your own metric loading script."),oo.forEach(t),ot.forEach(t),rs=f(e),T=r(e,"H2",{class:!0});var Ys=o(T);K=r(Ys,"A",{id:!0,class:!0,href:!0});var lo=o(K);ft=r(lo,"SPAN",{});var no=o(ft);g(ce.$$.fragment,no),no.forEach(t),lo.forEach(t),va=f(Ys),ut=r(Ys,"SPAN",{});var io=o(ut);wa=i(io,"Add predictions and references"),io.forEach(t),Ys.forEach(t),os=f(e),V=r(e,"P",{});var Qs=o(V);$a=i(Qs,"When you want to add model predictions and references to a "),He=r(Qs,"A",{href:!0});var po=o(He);ya=i(po,"Metric"),po.forEach(t),ba=i(Qs," instance, you have two options:"),Qs.forEach(t),ls=f(e),Y=r(e,"UL",{});var Ws=o(Y);dt=r(Ws,"LI",{});var co=o(dt);x=r(co,"P",{});var Ne=o(x);ze=r(Ne,"A",{href:!0});var fo=o(ze);Ea=i(fo,"Metric.add()"),fo.forEach(t),ja=i(Ne," adds a single "),ht=r(Ne,"CODE",{});var uo=o(ht);qa=i(uo,"prediction"),uo.forEach(t),ka=i(Ne," and "),mt=r(Ne,"CODE",{});var ho=o(mt);Pa=i(ho,"reference"),ho.forEach(t),Ia=i(Ne,"."),Ne.forEach(t),co.forEach(t),Aa=f(Ws),_t=r(Ws,"LI",{});var mo=o(_t);L=r(mo,"P",{});var Ue=o(L);Ke=r(Ue,"A",{href:!0});var _o=o(Ke);xa=i(_o,"Metric.add_batch()"),_o.forEach(t),La=i(Ue," adds a batch of "),gt=r(Ue,"CODE",{});var go=o(gt);Ca=i(go,"predictions"),go.forEach(t),Sa=i(Ue," and "),vt=r(Ue,"CODE",{});var vo=o(vt);Ta=i(vo,"references"),vo.forEach(t),Ma=i(Ue,"."),Ue.forEach(t),mo.forEach(t),Ws.forEach(t),ns=f(e),Q=r(e,"P",{});var Gs=o(Q);Oa=i(Gs,"Use "),Ve=r(Gs,"A",{href:!0});var wo=o(Ve);Da=i(wo,"Metric.add_batch()"),wo.forEach(t),Na=i(Gs," by passing it your model predictions, and the references the model predictions should be evaluated against:"),Gs.forEach(t),is=f(e),g(fe.$$.fragment,e),ps=f(e),g(W.$$.fragment,e),cs=f(e),M=r(e,"H2",{class:!0});var Fs=o(M);G=r(Fs,"A",{id:!0,class:!0,href:!0});var $o=o(G);wt=r($o,"SPAN",{});var yo=o(wt);g(ue.$$.fragment,yo),yo.forEach(t),$o.forEach(t),Ua=f(Fs),$t=r(Fs,"SPAN",{});var bo=o($t);Ra=i(bo,"Compute scores"),bo.forEach(t),Fs.forEach(t),fs=f(e),F=r(e,"P",{});var Js=o(F);Ba=i(Js,"The most straightforward way to calculate a metric is to call "),Ye=r(Js,"A",{href:!0});var Eo=o(Ye);Ha=i(Eo,"Metric.compute()"),Eo.forEach(t),za=i(Js,". But some metrics have additional arguments that allow you to modify the metrics behavior."),Js.forEach(t),us=f(e),J=r(e,"P",{});var Xs=o(J);Ka=i(Xs,"Let\u2019s load the "),de=r(Xs,"A",{href:!0,rel:!0});var jo=o(de);Va=i(jo,"SacreBLEU"),jo.forEach(t),Ya=i(Xs," metric, and compute it with a different smoothing method."),Xs.forEach(t),ds=f(e),Qe=r(e,"OL",{});var qo=o(Qe);yt=r(qo,"LI",{});var ko=o(yt);Qa=i(ko,"Load the SacreBLEU metric:"),ko.forEach(t),qo.forEach(t),hs=f(e),g(he.$$.fragment,e),ms=f(e),me=r(e,"OL",{start:!0});var Po=o(me);bt=r(Po,"LI",{});var Io=o(bt);Wa=i(Io,"Inspect the different argument methods for computing the metric:"),Io.forEach(t),Po.forEach(t),_s=f(e),g(_e.$$.fragment,e),gs=f(e),ge=r(e,"OL",{start:!0});var Ao=o(ge);O=r(Ao,"LI",{});var lt=o(O);Ga=i(lt,"Compute the metric with the "),Et=r(lt,"CODE",{});var xo=o(Et);Fa=i(xo,"floor"),xo.forEach(t),Ja=i(lt," method, and a different "),jt=r(lt,"CODE",{});var Lo=o(jt);Xa=i(Lo,"smooth_value"),Lo.forEach(t),Za=i(lt,":"),lt.forEach(t),Ao.forEach(t),vs=f(e),g(ve.$$.fragment,e),ws=f(e),We=r(e,"A",{id:!0}),o(We).forEach(t),$s=f(e),D=r(e,"H2",{class:!0});var Zs=o(D);X=r(Zs,"A",{id:!0,class:!0,href:!0});var Co=o(X);qt=r(Co,"SPAN",{});var So=o(qt);g(we.$$.fragment,So),So.forEach(t),Co.forEach(t),er=f(Zs),kt=r(Zs,"SPAN",{});var To=o(kt);tr=i(To,"Custom metric loading script"),To.forEach(t),Zs.forEach(t),ys=f(e),Z=r(e,"P",{});var ea=o(Z);sr=i(ea,"Write a metric loading script to use your own custom metric (or one that is not on the Hub). Then you can load it as usual with "),Ge=r(ea,"A",{href:!0});var Mo=o(Ge);ar=i(Mo,"load_metric()"),Mo.forEach(t),rr=i(ea,"."),ea.forEach(t),bs=f(e),ee=r(e,"P",{});var ta=o(ee);or=i(ta,"To help you get started, open the "),$e=r(ta,"A",{href:!0,rel:!0});var Oo=o($e);lr=i(Oo,"SQuAD metric loading script"),Oo.forEach(t),nr=i(ta," and follow along."),ta.forEach(t),Es=f(e),g(te.$$.fragment,e),js=f(e),N=r(e,"H3",{class:!0});var sa=o(N);se=r(sa,"A",{id:!0,class:!0,href:!0});var Do=o(se);Pt=r(Do,"SPAN",{});var No=o(Pt);g(ye.$$.fragment,No),No.forEach(t),Do.forEach(t),ir=f(sa),It=r(sa,"SPAN",{});var Uo=o(It);pr=i(Uo,"Add metric attributes"),Uo.forEach(t),sa.forEach(t),qs=f(e),ae=r(e,"P",{});var aa=o(ae);cr=i(aa,"Start by adding some information about your metric in "),At=r(aa,"CODE",{});var Ro=o(At);fr=i(Ro,"Metric._info()"),Ro.forEach(t),ur=i(aa,". The most important attributes you should specify are:"),aa.forEach(t),ks=f(e),k=r(e,"OL",{});var ie=o(k);xt=r(ie,"LI",{});var Bo=o(xt);Fe=r(Bo,"P",{});var Fr=o(Fe);Lt=r(Fr,"CODE",{});var Ho=o(Lt);dr=i(Ho,"MetricInfo.description"),Ho.forEach(t),hr=i(Fr," provides a brief description about your metric."),Fr.forEach(t),Bo.forEach(t),mr=f(ie),Ct=r(ie,"LI",{});var zo=o(Ct);Je=r(zo,"P",{});var Jr=o(Je);St=r(Jr,"CODE",{});var Ko=o(St);_r=i(Ko,"MetricInfo.citation"),Ko.forEach(t),gr=i(Jr," contains a BibTex citation for the metric."),Jr.forEach(t),zo.forEach(t),vr=f(ie),Tt=r(ie,"LI",{});var Vo=o(Tt);Xe=r(Vo,"P",{});var Xr=o(Xe);Mt=r(Xr,"CODE",{});var Yo=o(Mt);wr=i(Yo,"MetricInfo.inputs_description"),Yo.forEach(t),$r=i(Xr," describes the expected inputs and outputs. It may also provide an example usage of the metric."),Xr.forEach(t),Vo.forEach(t),yr=f(ie),Ot=r(ie,"LI",{});var Qo=o(Ot);Ze=r(Qo,"P",{});var Zr=o(Ze);Dt=r(Zr,"CODE",{});var Wo=o(Dt);br=i(Wo,"MetricInfo.features"),Wo.forEach(t),Er=i(Zr," defines the name and type of the predictions and references."),Zr.forEach(t),Qo.forEach(t),ie.forEach(t),Ps=f(e),et=r(e,"P",{});var Go=o(et);jr=i(Go,"After you\u2019ve filled out all these fields in the template, it should look like the following example from the SQuAD metric script:"),Go.forEach(t),Is=f(e),g(be.$$.fragment,e),As=f(e),U=r(e,"H3",{class:!0});var ra=o(U);re=r(ra,"A",{id:!0,class:!0,href:!0});var Fo=o(re);Nt=r(Fo,"SPAN",{});var Jo=o(Nt);g(Ee.$$.fragment,Jo),Jo.forEach(t),Fo.forEach(t),qr=f(ra),Ut=r(ra,"SPAN",{});var Xo=o(Ut);kr=i(Xo,"Download metric files"),Xo.forEach(t),ra.forEach(t),xs=f(e),C=r(e,"P",{});var nt=o(C);Pr=i(nt,"If your metric needs to download, or retrieve local files, you will need to use the "),Rt=r(nt,"CODE",{});var Zo=o(Rt);Ir=i(Zo,"Metric._download_and_prepare()"),Zo.forEach(t),Ar=i(nt," method. For this example, let\u2019s examine the "),je=r(nt,"A",{href:!0,rel:!0});var el=o(je);xr=i(el,"BLEURT metric loading script"),el.forEach(t),Lr=i(nt,"."),nt.forEach(t),Ls=f(e),tt=r(e,"OL",{});var tl=o(tt);Bt=r(tl,"LI",{});var sl=o(Bt);Cr=i(sl,"Provide a dictionary of URLs that point to the metric files:"),sl.forEach(t),tl.forEach(t),Cs=f(e),g(qe.$$.fragment,e),Ss=f(e),g(oe.$$.fragment,e),Ts=f(e),ke=r(e,"OL",{start:!0});var al=o(ke);st=r(al,"LI",{});var eo=o(st);Ht=r(eo,"CODE",{});var rl=o(Ht);Sr=i(rl,"Metric._download_and_prepare()"),rl.forEach(t),Tr=i(eo," will take the URLs and download the metric files specified:"),eo.forEach(t),al.forEach(t),Ms=f(e),g(Pe.$$.fragment,e),Os=f(e),R=r(e,"H3",{class:!0});var oa=o(R);le=r(oa,"A",{id:!0,class:!0,href:!0});var ol=o(le);zt=r(ol,"SPAN",{});var ll=o(zt);g(Ie.$$.fragment,ll),ll.forEach(t),ol.forEach(t),Mr=f(oa),Kt=r(oa,"SPAN",{});var nl=o(Kt);Or=i(nl,"Compute score"),nl.forEach(t),oa.forEach(t),Ds=f(e),B=r(e,"P",{});var Zt=o(B);Vt=r(Zt,"CODE",{});var il=o(Vt);Dr=i(il,"DatasetBuilder._compute"),il.forEach(t),Nr=i(Zt," provides the actual instructions for how to compute a metric given the predictions and references. Now let\u2019s take a look at the "),Ae=r(Zt,"A",{href:!0,rel:!0});var pl=o(Ae);Ur=i(pl,"GLUE metric loading script"),pl.forEach(t),Rr=i(Zt,"."),Zt.forEach(t),Ns=f(e),at=r(e,"OL",{});var cl=o(at);xe=r(cl,"LI",{});var la=o(xe);Br=i(la,"Provide the functions for "),Yt=r(la,"CODE",{});var fl=o(Yt);Hr=i(fl,"DatasetBuilder._compute"),fl.forEach(t),zr=i(la," to calculate your metric:"),la.forEach(t),cl.forEach(t),Us=f(e),g(Le.$$.fragment,e),Rs=f(e),Ce=r(e,"OL",{start:!0});var ul=o(Ce);Se=r(ul,"LI",{});var na=o(Se);Kr=i(na,"Create "),Qt=r(na,"CODE",{});var dl=o(Qt);Vr=i(dl,"DatasetBuilder._compute"),dl.forEach(t),Yr=i(na," with instructions for what metric to calculate for each configuration:"),na.forEach(t),ul.forEach(t),Bs=f(e),g(Te.$$.fragment,e),Hs=f(e),H=r(e,"H3",{class:!0});var ia=o(H);ne=r(ia,"A",{id:!0,class:!0,href:!0});var hl=o(ne);Wt=r(hl,"SPAN",{});var ml=o(Wt);g(Me.$$.fragment,ml),ml.forEach(t),hl.forEach(t),Qr=f(ia),Gt=r(ia,"SPAN",{});var _l=o(Gt);Wr=i(_l,"Test"),_l.forEach(t),ia.forEach(t),zs=f(e),rt=r(e,"P",{});var gl=o(rt);Gr=i(gl,"Once you\u2019re finished writing your metric loading script, try to load it locally:"),gl.forEach(t),Ks=f(e),g(Oe.$$.fragment,e),this.h()},h(){u(d,"name","hf:doc:metadata"),u(d,"content",JSON.stringify(Il)),u(m,"id","metrics"),u(m,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(m,"href","#metrics"),u(h,"class","relative group"),u(K,"id","add-predictions-and-references"),u(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(K,"href","#add-predictions-and-references"),u(T,"class","relative group"),u(He,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Metric"),u(ze,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Metric.add"),u(Ke,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Metric.add_batch"),u(Ve,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Metric.add_batch"),u(G,"id","compute-scores"),u(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(G,"href","#compute-scores"),u(M,"class","relative group"),u(Ye,"href","/docs/datasets/v2.8.0/en/package_reference/main_classes#datasets.Metric.compute"),u(de,"href","https://huggingface.co/metrics/sacrebleu"),u(de,"rel","nofollow"),u(me,"start","2"),u(ge,"start","3"),u(We,"id","metric_script"),u(X,"id","custom-metric-loading-script"),u(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(X,"href","#custom-metric-loading-script"),u(D,"class","relative group"),u(Ge,"href","/docs/datasets/v2.8.0/en/package_reference/loading_methods#datasets.load_metric"),u($e,"href","https://github.com/huggingface/datasets/blob/main/metrics/squad/squad.py"),u($e,"rel","nofollow"),u(se,"id","add-metric-attributes"),u(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(se,"href","#add-metric-attributes"),u(N,"class","relative group"),u(re,"id","download-metric-files"),u(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(re,"href","#download-metric-files"),u(U,"class","relative group"),u(je,"href","https://github.com/huggingface/datasets/blob/main/metrics/bleurt/bleurt.py"),u(je,"rel","nofollow"),u(ke,"start","2"),u(le,"id","compute-score"),u(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(le,"href","#compute-score"),u(R,"class","relative group"),u(Ae,"href","https://github.com/huggingface/datasets/blob/main/metrics/glue/glue.py"),u(Ae,"rel","nofollow"),u(Ce,"start","2"),u(ne,"id","test"),u(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ne,"href","#test"),u(H,"class","relative group")},m(e,l){s(document.head,d),p(e,b,l),p(e,h,l),s(h,m),s(m,q),v(E,q,null),s(h,j),s(h,P),s(P,ca),p(e,es,l),v(z,e,l),p(e,ts,l),p(e,Re,l),s(Re,fa),p(e,ss,l),p(e,Be,l),s(Be,ua),p(e,as,l),p(e,A,l),s(A,it),s(it,da),s(A,ha),s(A,pt),s(pt,ma),s(A,_a),s(A,ct),s(ct,ga),p(e,rs,l),p(e,T,l),s(T,K),s(K,ft),v(ce,ft,null),s(T,va),s(T,ut),s(ut,wa),p(e,os,l),p(e,V,l),s(V,$a),s(V,He),s(He,ya),s(V,ba),p(e,ls,l),p(e,Y,l),s(Y,dt),s(dt,x),s(x,ze),s(ze,Ea),s(x,ja),s(x,ht),s(ht,qa),s(x,ka),s(x,mt),s(mt,Pa),s(x,Ia),s(Y,Aa),s(Y,_t),s(_t,L),s(L,Ke),s(Ke,xa),s(L,La),s(L,gt),s(gt,Ca),s(L,Sa),s(L,vt),s(vt,Ta),s(L,Ma),p(e,ns,l),p(e,Q,l),s(Q,Oa),s(Q,Ve),s(Ve,Da),s(Q,Na),p(e,is,l),v(fe,e,l),p(e,ps,l),v(W,e,l),p(e,cs,l),p(e,M,l),s(M,G),s(G,wt),v(ue,wt,null),s(M,Ua),s(M,$t),s($t,Ra),p(e,fs,l),p(e,F,l),s(F,Ba),s(F,Ye),s(Ye,Ha),s(F,za),p(e,us,l),p(e,J,l),s(J,Ka),s(J,de),s(de,Va),s(J,Ya),p(e,ds,l),p(e,Qe,l),s(Qe,yt),s(yt,Qa),p(e,hs,l),v(he,e,l),p(e,ms,l),p(e,me,l),s(me,bt),s(bt,Wa),p(e,_s,l),v(_e,e,l),p(e,gs,l),p(e,ge,l),s(ge,O),s(O,Ga),s(O,Et),s(Et,Fa),s(O,Ja),s(O,jt),s(jt,Xa),s(O,Za),p(e,vs,l),v(ve,e,l),p(e,ws,l),p(e,We,l),p(e,$s,l),p(e,D,l),s(D,X),s(X,qt),v(we,qt,null),s(D,er),s(D,kt),s(kt,tr),p(e,ys,l),p(e,Z,l),s(Z,sr),s(Z,Ge),s(Ge,ar),s(Z,rr),p(e,bs,l),p(e,ee,l),s(ee,or),s(ee,$e),s($e,lr),s(ee,nr),p(e,Es,l),v(te,e,l),p(e,js,l),p(e,N,l),s(N,se),s(se,Pt),v(ye,Pt,null),s(N,ir),s(N,It),s(It,pr),p(e,qs,l),p(e,ae,l),s(ae,cr),s(ae,At),s(At,fr),s(ae,ur),p(e,ks,l),p(e,k,l),s(k,xt),s(xt,Fe),s(Fe,Lt),s(Lt,dr),s(Fe,hr),s(k,mr),s(k,Ct),s(Ct,Je),s(Je,St),s(St,_r),s(Je,gr),s(k,vr),s(k,Tt),s(Tt,Xe),s(Xe,Mt),s(Mt,wr),s(Xe,$r),s(k,yr),s(k,Ot),s(Ot,Ze),s(Ze,Dt),s(Dt,br),s(Ze,Er),p(e,Ps,l),p(e,et,l),s(et,jr),p(e,Is,l),v(be,e,l),p(e,As,l),p(e,U,l),s(U,re),s(re,Nt),v(Ee,Nt,null),s(U,qr),s(U,Ut),s(Ut,kr),p(e,xs,l),p(e,C,l),s(C,Pr),s(C,Rt),s(Rt,Ir),s(C,Ar),s(C,je),s(je,xr),s(C,Lr),p(e,Ls,l),p(e,tt,l),s(tt,Bt),s(Bt,Cr),p(e,Cs,l),v(qe,e,l),p(e,Ss,l),v(oe,e,l),p(e,Ts,l),p(e,ke,l),s(ke,st),s(st,Ht),s(Ht,Sr),s(st,Tr),p(e,Ms,l),v(Pe,e,l),p(e,Os,l),p(e,R,l),s(R,le),s(le,zt),v(Ie,zt,null),s(R,Mr),s(R,Kt),s(Kt,Or),p(e,Ds,l),p(e,B,l),s(B,Vt),s(Vt,Dr),s(B,Nr),s(B,Ae),s(Ae,Ur),s(B,Rr),p(e,Ns,l),p(e,at,l),s(at,xe),s(xe,Br),s(xe,Yt),s(Yt,Hr),s(xe,zr),p(e,Us,l),v(Le,e,l),p(e,Rs,l),p(e,Ce,l),s(Ce,Se),s(Se,Kr),s(Se,Qt),s(Qt,Vr),s(Se,Yr),p(e,Bs,l),v(Te,e,l),p(e,Hs,l),p(e,H,l),s(H,ne),s(ne,Wt),v(Me,Wt,null),s(H,Qr),s(H,Gt),s(Gt,Wr),p(e,zs,l),p(e,rt,l),s(rt,Gr),p(e,Ks,l),v(Oe,e,l),Vs=!0},p(e,[l]){const De={};l&2&&(De.$$scope={dirty:l,ctx:e}),z.$set(De);const Ft={};l&2&&(Ft.$$scope={dirty:l,ctx:e}),W.$set(Ft);const Jt={};l&2&&(Jt.$$scope={dirty:l,ctx:e}),te.$set(Jt);const Xt={};l&2&&(Xt.$$scope={dirty:l,ctx:e}),oe.$set(Xt)},i(e){Vs||(w(E.$$.fragment,e),w(z.$$.fragment,e),w(ce.$$.fragment,e),w(fe.$$.fragment,e),w(W.$$.fragment,e),w(ue.$$.fragment,e),w(he.$$.fragment,e),w(_e.$$.fragment,e),w(ve.$$.fragment,e),w(we.$$.fragment,e),w(te.$$.fragment,e),w(ye.$$.fragment,e),w(be.$$.fragment,e),w(Ee.$$.fragment,e),w(qe.$$.fragment,e),w(oe.$$.fragment,e),w(Pe.$$.fragment,e),w(Ie.$$.fragment,e),w(Le.$$.fragment,e),w(Te.$$.fragment,e),w(Me.$$.fragment,e),w(Oe.$$.fragment,e),Vs=!0)},o(e){$(E.$$.fragment,e),$(z.$$.fragment,e),$(ce.$$.fragment,e),$(fe.$$.fragment,e),$(W.$$.fragment,e),$(ue.$$.fragment,e),$(he.$$.fragment,e),$(_e.$$.fragment,e),$(ve.$$.fragment,e),$(we.$$.fragment,e),$(te.$$.fragment,e),$(ye.$$.fragment,e),$(be.$$.fragment,e),$(Ee.$$.fragment,e),$(qe.$$.fragment,e),$(oe.$$.fragment,e),$(Pe.$$.fragment,e),$(Ie.$$.fragment,e),$(Le.$$.fragment,e),$(Te.$$.fragment,e),$(Me.$$.fragment,e),$(Oe.$$.fragment,e),Vs=!1},d(e){t(d),e&&t(b),e&&t(h),y(E),e&&t(es),y(z,e),e&&t(ts),e&&t(Re),e&&t(ss),e&&t(Be),e&&t(as),e&&t(A),e&&t(rs),e&&t(T),y(ce),e&&t(os),e&&t(V),e&&t(ls),e&&t(Y),e&&t(ns),e&&t(Q),e&&t(is),y(fe,e),e&&t(ps),y(W,e),e&&t(cs),e&&t(M),y(ue),e&&t(fs),e&&t(F),e&&t(us),e&&t(J),e&&t(ds),e&&t(Qe),e&&t(hs),y(he,e),e&&t(ms),e&&t(me),e&&t(_s),y(_e,e),e&&t(gs),e&&t(ge),e&&t(vs),y(ve,e),e&&t(ws),e&&t(We),e&&t($s),e&&t(D),y(we),e&&t(ys),e&&t(Z),e&&t(bs),e&&t(ee),e&&t(Es),y(te,e),e&&t(js),e&&t(N),y(ye),e&&t(qs),e&&t(ae),e&&t(ks),e&&t(k),e&&t(Ps),e&&t(et),e&&t(Is),y(be,e),e&&t(As),e&&t(U),y(Ee),e&&t(xs),e&&t(C),e&&t(Ls),e&&t(tt),e&&t(Cs),y(qe,e),e&&t(Ss),y(oe,e),e&&t(Ts),e&&t(ke),e&&t(Ms),y(Pe,e),e&&t(Os),e&&t(R),y(Ie),e&&t(Ds),e&&t(B),e&&t(Ns),e&&t(at),e&&t(Us),y(Le,e),e&&t(Rs),e&&t(Ce),e&&t(Bs),y(Te,e),e&&t(Hs),e&&t(H),y(Me),e&&t(zs),e&&t(rt),e&&t(Ks),y(Oe,e)}}}const Il={local:"metrics",sections:[{local:"add-predictions-and-references",title:"Add predictions and references"},{local:"compute-scores",title:"Compute scores"},{local:"custom-metric-loading-script",sections:[{local:"add-metric-attributes",title:"Add metric attributes"},{local:"download-metric-files",title:"Download metric files"},{local:"compute-score",title:"Compute score"},{local:"test",title:"Test"}],title:"Custom metric loading script"}],title:"Metrics"};function Al(I){return bl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Tl extends vl{constructor(d){super();wl(this,d,Al,Pl,$l,{})}}export{Tl as default,Il as metadata};
