draft V0: test with sample input

README.md

@@ -38,7 +38,7 @@ If you encounter problem on the space, don't hesitate to restart it to remove th
 
 # Code logic for more complex edits
 
-You'll find 
+You'll find
 - the main table' columns names and properties in `src/display/utils.py`
 - the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -1,103 +1,58 @@
-import subprocess
 import gradio as gr
 import pandas as pd
 
-# from apscheduler.schedulers.background import BackgroundScheduler
-
-# from huggingface_hub import snapshot_download
-
-from src.about import (
-    # CITATION_BUTTON_LABEL,
-    # CITATION_BUTTON_TEXT,
-    # EVALUATION_QUEUE_TEXT,
+from src.about import (  # CITATION_BUTTON_LABEL,; CITATION_BUTTON_TEXT,; EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
 )
 from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
+from src.display.utils import (  # EVAL_TYPES,; WeightType,; BENCHMARK_COLS,; EVAL_COLS,; NUMERIC_INTERVALS,; ModelType,; Precision,
     COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    NUMERIC_INTERVALS,
     TYPES,
     AutoEvalColumn,
-    ModelType,
     fields,
-    WeightType,
-    Precision,
 )
 
 # from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, REPO_ID
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
+from src.envs import CRM_RESULTS_PATH
+from src.populate import get_leaderboard_df_crm
 
-# try:
-#     print(EVAL_REQUESTS_PATH)
-#     snapshot_download(
-#         repo_id=QUEUE_REPO,
-#         local_dir=EVAL_REQUESTS_PATH,
-#         repo_type="dataset",
-#         tqdm_class=None,
-#         etag_timeout=30,
-#         token=TOKEN,
-#     )
-# except Exception:
-#     restart_space()
-# try:
-#     print(EVAL_RESULTS_PATH)
-#     snapshot_download(
-#         repo_id=RESULTS_REPO,
-#         local_dir=EVAL_RESULTS_PATH,
-#         repo_type="dataset",
-#         tqdm_class=None,
-#         etag_timeout=30,
-#         token=TOKEN,
-#     )
-# except Exception:
-#     restart_space()
+original_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS)
 
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+# raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# leaderboard_df = leaderboard_df.style.format({"accuracy_metric_average": "{0:.2f}"})
 
 
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
-    query: str,
+    accuracy_method_query: str,
+    # type_query: list,
+    # precision_query: str,
+    # size_query: list,
+    # show_deleted: bool,
+    # query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
+    # filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    # filtered_df = filter_queries(query, filtered_df)
+    filtered_df = filter_accuracy_method_func(hidden_df, accuracy_method_query)
     df = select_columns(filtered_df, columns)
     return df
 
 
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
+def filter_accuracy_method_func(df: pd.DataFrame, accuracy_method_query: str) -> pd.DataFrame:
+    return df[df["Accuracy Method"] == accuracy_method_query]
+
+
+# def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+#     return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
 
 
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
         AutoEvalColumn.model.name,
     ]
     # We use COLS to maintain sorting
@@ -105,44 +60,45 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     return filtered_df
 
 
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-            )
+# def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
+#     final_df = []
+#     if query != "":
+#         queries = [q.strip() for q in query.split(";")]
+#         for _q in queries:
+#             _q = _q.strip()
+#             if _q != "":
+#                 temp_filtered_df = search_table(filtered_df, _q)
+#                 if len(temp_filtered_df) > 0:
+#                     final_df.append(temp_filtered_df)
+#         if len(final_df) > 0:
+#             filtered_df = pd.concat(final_df)
+#             filtered_df = filtered_df.drop_duplicates(
+#                 subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+#             )
 
-    return filtered_df
+#     return filtered_df
 
 
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+# def filter_models(
+#     df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+# ) -> pd.DataFrame:
+#     # Show all models
+#     filtered_df = df
+#     # if show_deleted:
+#     #     filtered_df = df
+#     # else:  # Show only still on the hub models
+#     #     filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True]
 
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+#     type_emoji = [t[0] for t in type_query]
+#     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+#     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
+#     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
+#     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+#     mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+#     filtered_df = filtered_df.loc[mask]
 
-    return filtered_df
+#     return filtered_df
 
 
 demo = gr.Blocks(css=custom_css)
@@ -154,12 +110,12 @@ with demo:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
+                    # with gr.Row():
+                    #     search_bar = gr.Textbox(
+                    #         placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                    #         show_label=False,
+                    #         elem_id="search-bar",
+                    #     )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
                             choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden],
@@ -172,32 +128,89 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                # with gr.Column(min_width=320):
+                #     # with gr.Box(elem_id="box-filter"):
+                #     filter_columns_type = gr.CheckboxGroup(
+                #         label="Model types",
+                #         choices=[t.to_str() for t in ModelType],
+                #         value=[t.to_str() for t in ModelType],
+                #         interactive=True,
+                #         elem_id="filter-columns-type",
+                #     )
+                # filter_columns_precision = gr.CheckboxGroup(
+                #     label="Precision",
+                #     choices=[i.value.name for i in Precision],
+                #     value=[i.value.name for i in Precision],
+                #     interactive=True,
+                #     elem_id="filter-columns-precision",
+                # )
+                # filter_columns_size = gr.CheckboxGroup(
+                #     label="Model sizes (in billions of parameters)",
+                #     choices=list(NUMERIC_INTERVALS.keys()),
+                #     value=list(NUMERIC_INTERVALS.keys()),
+                #     interactive=True,
+                #     elem_id="filter-columns-size",
+                # )
+            with gr.Row():
+                with gr.Column():
+                    filter_use_case_type = gr.CheckboxGroup(
+                        choices=["Summary", "Generation"],
+                        value=["Summary", "Generation"],
+                        label="Use Case Type",
+                        info="",
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_use_case = gr.Dropdown(
+                        choices=list(original_df["Use Case Name"].unique()),
+                        # value=list(original_df["Use Case Name"].unique()),
+                        label="Use Case",
+                        info="",
+                        multiselect=True,
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_metric_area = gr.CheckboxGroup(
+                        choices=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
+                        value=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
+                        label="Metric Area",
+                        info="",
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_accuracy_method = gr.Radio(
+                        choices=["Manual", "Auto"],
+                        value="Manual",
+                        label="Accuracy Method",
+                        info="accuracy method",
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_accuracy_threshold = gr.Number(
+                        value="3",
+                        label="Accuracy Threshold",
+                        info="",
                         interactive=True,
-                        elem_id="filter-columns-type",
                     )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
+                with gr.Column():
+                    filter_llm = gr.CheckboxGroup(
+                        choices=list(original_df["Model Name"].unique()),
+                        value=list(original_df["Model Name"].unique()),
+                        label="Model Name",
+                        info="",
                         interactive=True,
-                        elem_id="filter-columns-precision",
                     )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
+                with gr.Column():
+                    filter_llm_provider = gr.CheckboxGroup(
+                        choices=list(original_df["LLM Provider"].unique()),
+                        value=list(original_df["LLM Provider"].unique()),
+                        label="LLM Provider",
+                        info="",
                         interactive=True,
-                        elem_id="filter-columns-size",
                     )
 
             leaderboard_table = gr.components.Dataframe(
@@ -216,36 +229,38 @@ with demo:
                 datatype=TYPES,
                 visible=False,
             )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
+            # search_bar.submit(
+            #     update_table,
+            #     [
+            #         hidden_leaderboard_table_for_search,
+            #         shown_columns,
+            #         filter_columns_type,
+            #         filter_columns_precision,
+            #         filter_columns_size,
+            #         deleted_models_visibility,
+            #         search_bar,
+            #     ],
+            #     leaderboard_table,
+            # )
             for selector in [
                 shown_columns,
-                filter_columns_type,
-                filter_columns_precision,
-                filter_columns_size,
-                deleted_models_visibility,
+                filter_accuracy_method,
+                # filter_columns_type,
+                # filter_columns_precision,
+                # filter_columns_size,
+                # deleted_models_visibility,
             ]:
                 selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
                         shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
-                        search_bar,
+                        filter_accuracy_method,
+                        # filter_columns_type,
+                        # filter_columns_precision,
+                        # filter_columns_size,
+                        # deleted_models_visibility,
+                        # search_bar,
                     ],
                     leaderboard_table,
                     queue=True,

crm-results/hf_leaderboard_accuracy.csv

@@ -0,0 +1,250 @@
+Use Case Name,Use Case Type,Accuracy Method,Model Name,Model Version,LLM Provider,Instruction Following,Completeness,Conciseness,Factuality,Average
+Service: Conversation summary,Summary,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.9834791059280854,3.9193391642371234,3.847424684159378,3.871720116618076,3.9054907677356656
+Service: Conversation summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.9961127308066082,3.9844509232264333,3.9300291545189503,3.9669582118561713,3.9693877551020407
+Service: Conversation summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.998056365403304,3.9737609329446064,3.8104956268221573,3.9310009718172982,3.9283284742468414
+Service: Conversation summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,4.0,3.9805636540330416,3.9067055393586005,3.9504373177842567,3.9594266277939747
+Service: Conversation summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.998056365403304,3.992225461613217,3.881438289601555,3.9591836734693877,3.957725947521866
+Service: Conversation summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.999028182701652,3.993197278911565,3.6997084548104957,3.9591836734693877,3.912779397473275
+Service: Conversation summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,4.0,3.9951409135082603,3.8328474246841595,3.954324586977648,3.945578231292517
+Service: Conversation summary,Summary,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.9941690962099123,3.836734693877551,3.9047619047619047,3.880466472303207,3.9040330417881437
+Service: Conversation summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.991253644314869,3.9407191448007777,3.7862001943634596,3.8746355685131197,3.898202137998057
+Service: Conversation summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.999028182701652,3.990281827016521,3.9280855199222544,3.9591836734693877,3.9691448007774537
+Service: Conversation summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.938775510204082,3.938775510204082,3.6530612244897958,3.7560738581146746,3.8216715257531586
+Service: Conversation summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.9961127308066082,3.935860058309038,3.9834791059280854,3.9310009718172982,3.961613216715257
+Service: Conversation summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.998056365403304,3.989310009718173,3.7657920310981536,3.9640427599611274,3.9293002915451893
+Service: Conversation summary,Summary,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.9961127308066082,3.9727891156462585,3.943634596695821,3.9494655004859087,3.9655004859086493
+Service: Conversation summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,4.0,3.9961127308066082,3.565597667638484,3.922254616132167,3.8709912536443145
+Service: Conversation summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.927113702623907,3.924198250728863,3.327502429543246,3.7764820213799806,3.738824101068999
+Service: Conversation summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.987366375121477,3.9825072886297375,3.3751214771622933,3.9115646258503403,3.814139941690962
+Service: Conversation summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.997084548104956,3.9280855199222544,3.7755102040816326,3.9037900874635567,3.9011175898931
+Sales: Email Generation,Generation,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,1.9811320754716981,2.018867924528302,1.8867924528301887,2.2452830188679247,2.0330188679245285
+Sales: Email Generation,Generation,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.207547169811321,3.0377358490566038,3.0754716981132075,3.188679245283019,3.1273584905660377
+Sales: Email Generation,Generation,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.7358490566037736,3.0754716981132075,3.5849056603773586,3.660377358490566,3.5141509433962264
+Sales: Email Generation,Generation,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.4150943396226414,2.9245283018867925,3.5283018867924527,3.4716981132075473,3.334905660377359
+Sales: Email Generation,Generation,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,2.5283018867924527,2.811320754716981,2.5283018867924527,3.0377358490566038,2.7264150943396226
+Sales: Email Generation,Generation,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.6792452830188678,3.018867924528302,3.547169811320755,3.7358490566037736,3.4952830188679247
+Sales: Email Generation,Generation,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,2.358490566037736,2.452830188679245,2.2641509433962264,2.7547169811320753,2.4575471698113205
+Sales: Email Generation,Generation,Auto,XGen 22B,XGen 22B (1228),Salesforce,2.339622641509434,2.2830188679245285,2.3962264150943398,2.509433962264151,2.3820754716981134
+Sales: Email Generation,Generation,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.849056603773585,3.018867924528302,3.69811320754717,3.811320754716981,3.5943396226415096
+Sales: Email Generation,Generation,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.849056603773585,3.0,3.7358490566037736,3.811320754716981,3.599056603773585
+Sales: Email Generation,Generation,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.056603773584906,3.0,3.509433962264151,3.452830188679245,3.2547169811320753
+Sales: Email Generation,Generation,Auto,GPT4-o,GPT4-o,OpenAI,3.830188679245283,3.0377358490566038,3.4716981132075473,3.792452830188679,3.533018867924528
+Sales: Email Generation,Generation,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,2.9245283018867925,2.69811320754717,2.7547169811320753,3.0377358490566038,2.8537735849056607
+Sales: Email Generation,Generation,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.7735849056603774,3.0,3.69811320754717,3.6792452830188678,3.537735849056604
+Sales: Email Generation,Generation,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.056603773584906,2.69811320754717,3.056603773584906,3.4339622641509435,3.061320754716981
+Sales: Email Generation,Generation,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.547169811320755,3.0,3.30188679245283,3.7358490566037736,3.3962264150943398
+Sales: Email Generation,Generation,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,2.169811320754717,2.490566037735849,2.150943396226415,2.6226415094339623,2.3584905660377355
+Sales: Email Generation,Generation,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.150943396226415,2.8867924528301887,3.150943396226415,3.4716981132075473,3.165094339622642
+Service: Reply Recommendations,Generation,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.3333333333333335,2.7222222222222223,2.9444444444444446,2.7777777777777777,2.9444444444444446
+Service: Reply Recommendations,Generation,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.7222222222222223,3.5,2.611111111111111,3.1666666666666665,3.25
+Service: Reply Recommendations,Generation,Auto,GPT4-o,GPT4-o,OpenAI,3.9444444444444446,3.111111111111111,3.8333333333333335,3.388888888888889,3.5694444444444446
+Service: Reply Recommendations,Generation,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.9444444444444446,3.3333333333333335,3.6666666666666665,3.2222222222222223,3.541666666666667
+Service: Reply Recommendations,Generation,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.9444444444444446,3.111111111111111,3.5555555555555554,3.111111111111111,3.4305555555555554
+Service: Reply Recommendations,Generation,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.7777777777777777,3.1666666666666665,3.2777777777777777,3.111111111111111,3.333333333333333
+Service: Reply Recommendations,Generation,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.611111111111111,3.1666666666666665,2.888888888888889,3.0,3.1666666666666665
+Service: Reply Recommendations,Generation,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.2777777777777777,3.1666666666666665,2.5555555555555554,3.0555555555555554,3.013888888888889
+Service: Reply Recommendations,Generation,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.5555555555555554,2.8333333333333335,3.111111111111111,3.2777777777777777,3.1944444444444446
+Service: Reply Recommendations,Generation,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.7222222222222223,3.111111111111111,3.2777777777777777,3.0,3.2777777777777777
+Service: Reply Recommendations,Generation,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,4.0,3.2777777777777777,3.888888888888889,3.1666666666666665,3.583333333333333
+Service: Reply Recommendations,Generation,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,4.0,3.5555555555555554,3.388888888888889,3.3333333333333335,3.5694444444444446
+Service: Reply Recommendations,Generation,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.888888888888889,3.1666666666666665,3.611111111111111,3.2222222222222223,3.4722222222222223
+Service: Reply Recommendations,Generation,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.7777777777777777,2.888888888888889,3.8333333333333335,3.2777777777777777,3.4444444444444446
+Service: Reply Recommendations,Generation,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.888888888888889,3.0555555555555554,3.7777777777777777,3.1666666666666665,3.472222222222222
+Service: Reply Recommendations,Generation,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.9444444444444446,3.1666666666666665,3.5,3.2222222222222223,3.458333333333333
+Service: Reply Recommendations,Generation,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.8333333333333335,3.2222222222222223,3.2777777777777777,3.0,3.333333333333333
+Service: Reply Recommendations,Generation,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.888888888888889,3.1666666666666665,3.2777777777777777,3.2777777777777777,3.4027777777777777
+Sales & Service: Update CRM Info,Generation,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.4,3.4242424242424243,3.4451219512195124,3.466666666666667,3.434007760532151
+Sales & Service: Update CRM Info,Generation,Auto,GPT4-o,GPT4-o,OpenAI,3.8848484848484848,3.6484848484848484,3.915151515151515,3.909090909090909,3.83939393939394
+Sales & Service: Update CRM Info,Generation,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.581818181818182,3.533333333333333,3.709090909090909,3.6787878787878787,3.625757575757576
+Sales & Service: Update CRM Info,Generation,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.709090909090909,3.6363636363636362,3.806060606060606,3.7333333333333334,3.7212121212121216
+Sales & Service: Update CRM Info,Generation,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.824242424242424,3.775757575757576,3.83030303030303,3.806060606060606,3.809090909090909
+Sales & Service: Update CRM Info,Generation,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.618181818181818,3.624242424242424,3.7151515151515153,3.6666666666666665,3.656060606060606
+Sales & Service: Update CRM Info,Generation,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.7636363636363637,3.624242424242424,3.757575757575758,3.812121212121212,3.7393939393939393
+Sales & Service: Update CRM Info,Generation,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.7333333333333334,3.6787878787878787,3.7818181818181817,3.7454545454545456,3.734848484848485
+Sales & Service: Update CRM Info,Generation,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,2.9696969696969697,3.0545454545454547,2.8,3.096969696969697,2.9803030303030305
+Sales & Service: Update CRM Info,Generation,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.806060606060606,3.7515151515151515,3.8181818181818183,3.812121212121212,3.796969696969697
+Sales & Service: Update CRM Info,Generation,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,2.909090909090909,2.896969696969697,2.8666666666666667,2.903030303030303,2.893939393939394
+Sales & Service: Update CRM Info,Generation,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.703030303030303,3.6606060606060606,3.7515151515151515,3.7515151515151515,3.716666666666667
+Sales & Service: Update CRM Info,Generation,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.7454545454545456,3.606060606060606,3.812121212121212,3.7636363636363637,3.731818181818182
+Sales & Service: Update CRM Info,Generation,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.478787878787879,3.3575757575757574,3.5393939393939395,3.503030303030303,3.4696969696969697
+Sales & Service: Update CRM Info,Generation,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,2.787878787878788,3.242424242424242,2.3333333333333335,3.3575757575757574,2.93030303030303
+Sales & Service: Update CRM Info,Generation,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,1.6606060606060606,1.6848484848484848,1.6424242424242423,1.6787878787878787,1.6666666666666667
+Sales & Service: Update CRM Info,Generation,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.6666666666666665,3.6363636363636362,3.690909090909091,3.690909090909091,3.6712121212121214
+Sales & Service: Update CRM Info,Generation,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,2.981818181818182,3.139393939393939,3.012121212121212,3.1575757575757577,3.0727272727272728
+Service: Live Chat Summary,Summary,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.542,3.479,3.303,3.381,3.42625
+Service: Live Chat Summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.965,3.436,3.946,3.641,3.747
+Service: Live Chat Summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.715,3.331,3.714,3.6236236236236237,3.595905905905906
+Service: Live Chat Summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.882,3.4004004004004003,3.904,3.718,3.7261001001001
+Service: Live Chat Summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.927,3.566,3.607,3.588,3.6720000000000006
+Service: Live Chat Summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.945,3.725,3.666,3.453,3.69725
+Service: Live Chat Summary,Summary,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.703,3.247,3.689,3.605,3.561
+Service: Live Chat Summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.694,3.158,3.75,3.593,3.54875
+Service: Live Chat Summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.903,3.309,3.947,3.765,3.731
+Service: Live Chat Summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.744,3.497,3.533,3.502,3.569
+Service: Live Chat Summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.992,3.822,3.914,3.587,3.82875
+Service: Live Chat Summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.835,3.164164164164164,3.891,3.717,3.651791041041041
+Service: Live Chat Summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.862,3.492,3.864,3.574,3.698
+Service: Live Chat Summary,Summary,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.724,3.635,3.567,3.396,3.5805
+Service: Live Chat Summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.93,3.7,3.772,3.681,3.7707500000000005
+Service: Live Chat Summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.942,3.353,3.831,3.492,3.6544999999999996
+Service: Live Chat Summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.866,3.379,3.846,3.609,3.6750000000000003
+Service: Live Chat Summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.595,3.129129129129129,3.709,3.504,3.4842822822822823
+Service: Email Summary,Summary,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.7755102040816326,3.7857142857142856,3.4285714285714284,3.561224489795918,3.6377551020408165
+Service: Email Summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.9381443298969074,3.9587628865979383,3.8762886597938144,3.752577319587629,3.8814432989690726
+Service: Email Summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.6122448979591835,3.5306122448979593,3.4285714285714284,3.377551020408163,3.4872448979591835
+Service: Email Summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.989795918367347,3.979591836734694,3.9693877551020407,3.673469387755102,3.9030612244897958
+Service: Email Summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.979591836734694,3.9591836734693877,3.673469387755102,3.857142857142857,3.86734693877551
+Service: Email Summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,4.0,4.0,3.857142857142857,3.693877551020408,3.8877551020408165
+Service: Email Summary,Summary,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.5670103092783507,3.268041237113402,3.5463917525773194,3.288659793814433,3.4175257731958766
+Service: Email Summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.7755102040816326,3.8979591836734695,3.5816326530612246,3.479591836734694,3.683673469387755
+Service: Email Summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,4.0,3.989795918367347,3.979591836734694,3.622448979591837,3.8979591836734695
+Service: Email Summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.9591836734693877,3.836734693877551,3.7448979591836733,3.3877551020408165,3.732142857142857
+Service: Email Summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.989795918367347,4.0,3.8469387755102042,3.795918367346939,3.908163265306123
+Service: Email Summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.989795918367347,3.86734693877551,3.9693877551020407,3.5918367346938775,3.854591836734694
+Service: Email Summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,4.0,3.979591836734694,3.8469387755102042,3.795918367346939,3.9056122448979593
+Service: Email Summary,Summary,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.989795918367347,3.9183673469387754,3.938775510204082,3.5510204081632653,3.849489795918367
+Service: Email Summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,4.0,3.979591836734694,3.86734693877551,3.693877551020408,3.885204081632653
+Service: Email Summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.8877551020408165,3.8979591836734695,3.520408163265306,3.642857142857143,3.737244897959184
+Service: Email Summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.979591836734694,3.9591836734693877,3.7346938775510203,3.673469387755102,3.836734693877551
+Service: Email Summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.377551020408163,3.6530612244897958,3.36734693877551,3.377551020408163,3.443877551020408
+Service: Knowledge creation from Case Info,Generation,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.1875,3.0625,2.8125,3.0,3.015625
+Service: Knowledge creation from Case Info,Generation,Auto,GPT4-o,GPT4-o,OpenAI,3.625,3.6875,3.1875,3.3125,3.453125
+Service: Knowledge creation from Case Info,Generation,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.125,3.125,2.8125,3.125,3.046875
+Service: Knowledge creation from Case Info,Generation,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.5625,3.4375,3.25,3.1875,3.359375
+Service: Knowledge creation from Case Info,Generation,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.5625,3.25,3.1875,3.125,3.28125
+Service: Knowledge creation from Case Info,Generation,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.5625,3.1875,3.125,3.0,3.21875
+Service: Knowledge creation from Case Info,Generation,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.25,3.1875,3.0,3.1875,3.15625
+Service: Knowledge creation from Case Info,Generation,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.5625,3.375,3.0625,3.3125,3.328125
+Service: Knowledge creation from Case Info,Generation,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,2.4375,2.3125,2.0625,3.0625,2.46875
+Service: Knowledge creation from Case Info,Generation,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.0625,3.0625,2.8125,3.125,3.015625
+Service: Knowledge creation from Case Info,Generation,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.6875,3.375,3.1875,3.3125,3.390625
+Service: Knowledge creation from Case Info,Generation,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.5625,3.4375,3.3125,3.25,3.390625
+Service: Knowledge creation from Case Info,Generation,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.375,3.1875,2.9375,3.125,3.15625
+Service: Knowledge creation from Case Info,Generation,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.4375,3.125,3.125,3.1875,3.21875
+Service: Knowledge creation from Case Info,Generation,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.5625,3.4375,3.0,3.1875,3.296875
+Service: Knowledge creation from Case Info,Generation,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.5625,3.375,2.9375,3.0625,3.234375
+Service: Knowledge creation from Case Info,Generation,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.4375,3.375,3.125,3.25,3.296875
+Service: Knowledge creation from Case Info,Generation,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.4375,3.3125,3.0,3.125,3.21875
+Sales: Email Summary,Summary,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.3877551020408165,3.4285714285714284,3.193877551020408,3.377551020408163,3.346938775510204
+Sales: Email Summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.9693877551020407,3.8979591836734695,3.86734693877551,3.8877551020408165,3.9056122448979593
+Sales: Email Summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.8979591836734695,3.663265306122449,3.9183673469387754,3.5714285714285716,3.7627551020408165
+Sales: Email Summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.979591836734694,3.836734693877551,3.9591836734693877,3.663265306122449,3.8596938775510203
+Sales: Email Summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.989795918367347,3.7653061224489797,3.9183673469387754,3.857142857142857,3.88265306122449
+Sales: Email Summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.8979591836734695,3.7551020408163267,3.795918367346939,3.7244897959183674,3.793367346938776
+Sales: Email Summary,Summary,Auto,XGen 22B,XGen 22B (1228),Salesforce,2.979381443298969,2.8969072164948453,3.142857142857143,3.2783505154639174,3.0743740795287184
+Sales: Email Summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.6020408163265305,3.510204081632653,3.663265306122449,3.357142857142857,3.5331632653061225
+Sales: Email Summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.9693877551020407,3.8979591836734695,3.9591836734693877,3.642857142857143,3.86734693877551
+Sales: Email Summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,2.9690721649484537,3.2783505154639174,2.649484536082474,3.154639175257732,3.0128865979381443
+Sales: Email Summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,4.0,3.9591836734693877,3.9285714285714284,3.826530612244898,3.928571428571429
+Sales: Email Summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.9591836734693877,3.693877551020408,3.9693877551020407,3.7857142857142856,3.8520408163265305
+Sales: Email Summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,4.0,3.938775510204082,3.9183673469387754,3.826530612244898,3.920918367346939
+Sales: Email Summary,Summary,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.9285714285714284,3.663265306122449,3.9183673469387754,3.7653061224489797,3.8188775510204085
+Sales: Email Summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,4.0,3.806122448979592,3.9693877551020407,3.7346938775510203,3.877551020408163
+Sales: Email Summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.948453608247423,3.948453608247423,3.7653061224489797,3.6288659793814433,3.822769829581317
+Sales: Email Summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.9591836734693877,3.816326530612245,3.8469387755102042,3.5306122448979593,3.788265306122449
+Sales: Email Summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.836734693877551,3.642857142857143,3.7346938775510203,3.5306122448979593,3.686224489795918
+Service: Call Summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.95,3.925,3.925,3.75,3.8875
+Service: Call Summary,Summary,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.95,3.85,3.9,3.775,3.8687500000000004
+Service: Call Summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.95,3.925,3.95,3.925,3.9375
+Service: Call Summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.95,3.95,3.95,3.975,3.9562500000000003
+Service: Call Summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.95,3.95,3.95,3.9,3.9375000000000004
+Service: Call Summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.925,3.925,3.775,3.8,3.85625
+Service: Call Summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.95,3.925,3.95,3.925,3.9375
+Service: Call Summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.75,3.85,3.675,3.6,3.7187499999999996
+Service: Call Summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,4.0,3.875,3.975,3.95,3.95
+Service: Call Summary,Summary,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.95,3.8,3.95,3.65,3.8375
+Service: Call Summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.95,3.925,3.95,3.875,3.925
+Service: Call Summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,4.0,4.0,4.0,3.95,3.9875
+Service: Call Summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.975,3.925,3.95,3.825,3.91875
+Service: Call Summary,Summary,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.95,3.925,4.0,3.95,3.95625
+Service: Call Summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.95,3.875,3.95,3.85,3.90625
+Service: Call Summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.95,3.925,3.95,3.9,3.93125
+Service: Call Summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.95,3.9,3.925,3.825,3.8999999999999995
+Service: Call Summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.95,3.875,3.95,3.875,3.9125
+Sales: Call Summary,Summary,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.0526315789473686,2.8947368421052633,2.473684210526316,2.789473684210526,2.8026315789473686
+Sales: Call Summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.9473684210526314,3.210526315789474,3.473684210526316,3.9473684210526314,3.6447368421052633
+Sales: Call Summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.3157894736842106,3.0,3.1052631578947367,3.3684210526315788,3.1973684210526314
+Sales: Call Summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.3684210526315788,2.789473684210526,3.263157894736842,3.6842105263157894,3.276315789473684
+Sales: Call Summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.473684210526316,3.0526315789473686,3.3157894736842106,3.6315789473684212,3.368421052631579
+Sales: Call Summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.6842105263157894,3.0526315789473686,2.8421052631578947,3.789473684210526,3.3421052631578942
+Sales: Call Summary,Summary,Auto,XGen 22B,XGen 22B (1228),Salesforce,2.736842105263158,2.6315789473684212,2.5789473684210527,2.789473684210526,2.6842105263157894
+Sales: Call Summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.1052631578947367,2.6315789473684212,3.210526315789474,3.210526315789474,3.0394736842105265
+Sales: Call Summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.263157894736842,2.8421052631578947,3.473684210526316,3.4210526315789473,3.2499999999999996
+Sales: Call Summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,2.473684210526316,2.3157894736842106,2.1052631578947367,1.7894736842105263,2.1710526315789473
+Sales: Call Summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.736842105263158,3.0,3.3157894736842106,4.0,3.513157894736842
+Sales: Call Summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.3684210526315788,2.888888888888889,3.8421052631578947,3.8421052631578947,3.485380116959064
+Sales: Call Summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.8947368421052633,3.0,3.5789473684210527,3.9473684210526314,3.605263157894737
+Sales: Call Summary,Summary,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.6315789473684212,3.0526315789473686,3.0526315789473686,3.736842105263158,3.3684210526315788
+Sales: Call Summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.1578947368421053,2.8947368421052633,2.6315789473684212,2.9473684210526314,2.9078947368421053
+Sales: Call Summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.4210526315789473,3.1052631578947367,3.210526315789474,3.5555555555555554,3.323099415204678
+Sales: Call Summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.263157894736842,3.0,2.6315789473684212,3.263157894736842,3.0394736842105265
+Sales: Call Summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,2.9473684210526314,2.736842105263158,3.0,3.0526315789473686,2.9342105263157894
+Service: Live Chat Insights,Summary,Auto,Gemini  Pro 1,Gemini  Pro 1,Google,3.97265625,3.8359375,3.87109375,3.76953125,3.8623046875
+Service: Live Chat Insights,Summary,Auto,GPT4-o,GPT4-o,OpenAI,4.0,3.91796875,3.9765625,3.91015625,3.951171875
+Service: Live Chat Insights,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.97265625,3.8203125,3.8515625,3.859375,3.8759765625
+Service: Live Chat Insights,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.99609375,3.96875,3.94140625,3.85546875,3.9404296875
+Service: Live Chat Insights,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,4.0,3.98046875,3.796875,3.8828125,3.9150390625
+Service: Live Chat Insights,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.984375,3.9375,3.8671875,3.87109375,3.9150390625
+Service: Live Chat Insights,Summary,Auto,XGen 22B,XGen 22B (1228),Salesforce,3.9765625,3.4375,3.921875,3.74609375,3.7705078125
+Service: Live Chat Insights,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.97265625,3.6875,3.90625,3.71484375,3.8203125
+Service: Live Chat Insights,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.9921875,3.93359375,3.97265625,3.85546875,3.9384765625
+Service: Live Chat Insights,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.9375,3.8671875,3.484375,3.51171875,3.7001953125
+Service: Live Chat Insights,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.9921875,3.9375,3.93359375,3.91015625,3.943359375
+Service: Live Chat Insights,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.96875,3.71875,4.0,3.90234375,3.8974609375
+Service: Live Chat Insights,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,4.0,3.98046875,3.82421875,3.9140625,3.9296875
+Service: Live Chat Insights,Summary,Auto,Gemini  Pro 1.5,Gemini  Pro 1.5,Google,3.98828125,3.88671875,3.9375,3.77734375,3.8974609375
+Service: Live Chat Insights,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.9921875,3.96484375,3.62890625,3.8359375,3.85546875
+Service: Live Chat Insights,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.9921875,3.875,3.7734375,3.8125,3.86328125
+Service: Live Chat Insights,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.96875,3.953125,3.6015625,3.8046875,3.83203125
+Service: Live Chat Insights,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.9765625,3.7734375,3.92578125,3.80859375,3.87109375
+Service: Reply Recommendations,Generation,Manual,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.0935185185185183,3.22037037037037,3.443518518518519,3.0453703703703705,3.2006944444444443
+Service: Reply Recommendations,Generation,Manual,GPT 4 Turbo,gpt-4-0613,OpenAI,3.525925925925926,3.3203703703703704,3.5129629629629635,3.52962962962963,3.4722222222222228
+Service: Reply Recommendations,Generation,Manual,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.015740740740741,3.0203703703703706,3.2111111111111112,2.837037037037037,3.021064814814815
+Service: Reply Recommendations,Generation,Manual,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.175,2.733333333333333,2.977777777777778,3.0962962962962965,2.995601851851852
+Service: Reply Recommendations,Generation,Manual,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,2.5185185185185186,2.4657407407407406,2.660185185185185,2.4314814814814816,2.5189814814814815
+Service: Reply Recommendations,Generation,Manual,XGen 22B,XGen 22B (1228),Salesforce,2.892592592592593,2.9074074074074074,3.1231481481481485,2.486111111111111,2.852314814814815
+Service: Reply Recommendations,Generation,Manual,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.3435185185185192,3.1203703703703702,3.184259259259259,3.28425925925926,3.233101851851852
+Service: Reply Recommendations,Generation,Manual,Cohere Command Text,cohere.command-text-v14,Cohere AI,2.3768518518518515,2.8157407407407415,2.813888888888889,2.641666666666667,2.6620370370370376
+Service: Reply Recommendations,Generation,Manual,Gemini  Pro 1,Gemini  Pro 1,Google,2.851851851851852,2.761111111111111,2.2962962962962963,3.2379629629629627,2.7868055555555555
+Service: Reply Recommendations,Generation,Manual,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.2796296296296297,3.2101851851851846,3.230555555555556,3.021296296296297,3.185416666666667
+Service: Reply Recommendations,Generation,Manual,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.087037037037037,3.1166666666666667,3.249074074074074,2.888888888888889,3.0854166666666667
+Sales: Email Generation,Generation,Manual,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.037735849056604,2.515723270440251,3.204402515723271,2.811320754716981,2.892295597484277
+Sales: Email Generation,Generation,Manual,GPT 4 Turbo,gpt-4-0613,OpenAI,3.2169811320754724,3.067610062893082,2.9088050314465415,3.248427672955976,3.1104559748427683
+Sales: Email Generation,Generation,Manual,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.1163522012578615,2.5345911949685536,2.9937106918238996,2.8899371069182394,2.8836477987421385
+Sales: Email Generation,Generation,Manual,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.0817610062893084,2.1132075471698113,2.7106918238993716,2.880503144654088,2.6965408805031448
+Sales: Email Generation,Generation,Manual,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,2.9056603773584904,1.9182389937106918,2.883647798742138,2.60062893081761,2.5770440251572326
+Sales: Email Generation,Generation,Manual,XGen 22B,XGen 22B (1228),Salesforce,3.0031446540880506,2.1540880503144657,2.90880503144654,2.7044025157232707,2.6926100628930816
+Sales: Email Generation,Generation,Manual,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.0880503144654092,2.5283018867924527,3.0220125786163528,3.1635220125786163,2.9504716981132075
+Sales: Email Generation,Generation,Manual,Cohere Command Text,cohere.command-text-v14,Cohere AI,2.878930817610063,2.1635220125786163,2.858490566037736,2.7106918238993707,2.6529088050314464
+Sales: Email Generation,Generation,Manual,Gemini  Pro 1,Gemini  Pro 1,Google,3.1163522012578633,2.5911949685534585,2.757861635220126,2.9622641509433967,2.8569182389937113
+Sales: Email Generation,Generation,Manual,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.0943396226415096,3.0062893081761004,3.2232704402515724,3.1415094339622636,3.116352201257861
+Sales: Email Generation,Generation,Manual,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.213836477987421,3.0062893081761013,2.9937106918238996,3.0534591194968557,3.066823899371069
+Sales: Email Generation,Generation,Manual,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.169871794871795,2.4839743589743586,2.9519230769230775,2.7660256410256405,2.8429487179487176
+Sales: Email Generation,Generation,Manual,GPT4-o,GPT4-o,OpenAI,3.286163522012579,3.2389937106918247,3.2955974842767293,3.2421383647798745,3.265723270440252
+Sales: Email Generation,Generation,Manual,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.1729559748427683,3.09119496855346,3.172955974842768,3.2012578616352205,3.159591194968554
+Service: Call Summary,Summary,Manual,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.5694444444444446,3.3541666666666665,3.7948717948717947,3.4193548387096775,3.534459436173146
+Service: Call Summary,Summary,Manual,GPT 4 Turbo,gpt-4-0613,OpenAI,3.4743589743589745,3.4871794871794872,3.5128205128205128,3.4615384615384617,3.483974358974359
+Service: Call Summary,Summary,Manual,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,2.725,2.6875,2.782051282051282,2.675,2.717387820512821
+Service: Call Summary,Summary,Manual,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,2.58974358974359,2.7125,2.7435897435897436,2.6538461538461537,2.674919871794872
+Service: Call Summary,Summary,Manual,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,2.75,2.775,2.8684210526315788,2.7,2.773355263157895
+Service: Call Summary,Summary,Manual,XGen 22B,XGen 22B (1228),Salesforce,2.7,2.8125,2.9615384615384617,2.7,2.7935096153846155
+Service: Call Summary,Summary,Manual,Claude 3 Haiku,Claude 3 Haiku,Anthropic,2.525,2.6,2.7222222222222223,2.5875,2.6086805555555554
+Service: Call Summary,Summary,Manual,Cohere Command Text,cohere.command-text-v14,Cohere AI,2.35,2.4125,2.527027027027027,2.375,2.416131756756757
+Service: Call Summary,Summary,Manual,Gemini  Pro 1,Gemini  Pro 1,Google,2.2051282051282053,2.3684210526315788,2.442857142857143,2.217948717948718,2.3085887796414113
+Service: Call Summary,Summary,Manual,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,2.7435897435897436,2.769230769230769,2.9285714285714284,2.6666666666666665,2.777014652014652
+Service: Call Summary,Summary,Manual,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,2.5135135135135136,2.6052631578947367,2.6818181818181817,2.4871794871794872,2.57194358510148
+Service: Live Chat Insights,Summary,Manual,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.72,3.7,3.7,3.74,3.7150000000000003
+Service: Live Chat Insights,Summary,Manual,GPT 4 Turbo,gpt-4-0613,OpenAI,3.72,3.72,3.04,3.68,3.54
+Service: Live Chat Insights,Summary,Manual,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.56,3.6,3.1,3.68,3.485
+Service: Live Chat Insights,Summary,Manual,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.56,3.64,3.2,3.68,3.52
+Service: Live Chat Insights,Summary,Manual,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.56,3.54,3.2,3.6,3.475
+Service: Live Chat Insights,Summary,Manual,XGen 22B,XGen 22B (1228),Salesforce,3.54,3.6,3.52,3.64,3.575
+Service: Live Chat Insights,Summary,Manual,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.64,3.6,2.92,3.64,3.45
+Service: Live Chat Insights,Summary,Manual,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.26,3.36,3.0,3.56,3.295
+Service: Live Chat Insights,Summary,Manual,Gemini  Pro 1,Gemini  Pro 1,Google,3.42,3.5,3.46,3.58,3.4899999999999998
+Service: Live Chat Insights,Summary,Manual,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.7,3.64,3.24,3.7,3.5700000000000003
+Service: Live Chat Insights,Summary,Manual,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.66,3.62,3.14,3.66,3.52
+Service: Live Chat Insights,Summary,Manual,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.46,3.56,3.58,3.56,3.54
+Service: Live Chat Insights,Summary,Manual,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.56,3.64,3.68,3.66,3.6350000000000002
+Service: Live Chat Insights,Summary,Manual,GPT4-o,GPT4-o,OpenAI,3.78,3.78,3.78,3.78,3.78
+Service: Live Chat Insights,Summary,Manual,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.72,3.74,3.6,3.72,3.6950000000000003

requirements.txt

@@ -1,3 +1,4 @@
+accelerate
 APScheduler
 black
 click
@@ -5,14 +6,13 @@ datasets
 gradio
 gradio_client
 huggingface-hub>=0.18.0
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
 matplotlib
 numpy
 pandas
 python-dateutil
 requests
+sentencepiece
+tokenizers>=0.15.0
 tqdm
 transformers
-tokenizers>=0.15.0
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
-accelerate
-sentencepiece

src/about.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from enum import Enum
 
+
 @dataclass
 class Task:
     benchmark: str
@@ -11,29 +12,27 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
     task0 = Task("anli_r1", "acc", "ANLI")
     task1 = Task("logiqa", "acc_norm", "LogiQA")
 
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
 
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">🤗 LLM Leaderboard for CRM</h1>
+<h3>Assess which LLMs are accurate enough or need fine-tuning, and weigh this versus tradeoffs of speed, costs, and trust and safety. This is based on human manual and automated evaluation with real operational CRM data per use case.</h3>
+"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+LLM_BENCHMARKS_TEXT = """
 
 """
 

src/display/css_html_js.py

@@ -33,7 +33,7 @@ custom_css = """
     background: none;
     border: none;
 }
- 
+
 #search-bar {
     padding: 0px;
 }
@@ -77,7 +77,7 @@ table th:first-child {
 #filter_type label > .wrap{
     width: 103px;
 }
-#filter_type label > .wrap .wrap-inner{  
+#filter_type label > .wrap .wrap-inner{
     padding: 2px;
 }
 #filter_type label > .wrap .wrap-inner input{

src/display/utils.py

@@ -5,6 +5,7 @@ import pandas as pd
 
 from src.about import Tasks
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
@@ -20,29 +21,66 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(
+    ["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)]
+)
+auto_eval_column_dict.append(
+    ["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True, never_hidden=True)]
+)
+auto_eval_column_dict.append(["use_case_name", ColumnContent, ColumnContent("Use Case Name", "markdown", True)])
+auto_eval_column_dict.append(["use_case_type", ColumnContent, ColumnContent("Use Case Type", "markdown", True)])
+
+auto_eval_column_dict.append(["accuracy_method", ColumnContent, ColumnContent("Accuracy Method", "markdown", True)])
+# Accuracy metrics
+auto_eval_column_dict.append(["accuracy_metric_average", ColumnContent, ColumnContent("Average", "markdown", True)])
+auto_eval_column_dict.append(
+    [
+        "accuracy_metric_instruction_following",
+        ColumnContent,
+        ColumnContent("Instruction Following", "markdown", True),
+    ]
+)
+auto_eval_column_dict.append(
+    ["accuracy_metric_completeness", ColumnContent, ColumnContent("Completeness", "markdown", True)]
+)
+auto_eval_column_dict.append(
+    ["accuracy_metric_conciseness", ColumnContent, ColumnContent("Conciseness", "markdown", True)]
+)
+auto_eval_column_dict.append(
+    ["accuracy_metric_factuality", ColumnContent, ColumnContent("Factuality", "markdown", True)]
+)
+# Speed (Latency) metrics
+
+
+# Cost metrics
+
+# Trust & Safety metrics
+
+
+# Scores
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# for task in Tasks:
+#     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -53,12 +91,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -83,18 +122,20 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     float32 = ModelDetails("float32")
-    #qt_8bit = ModelDetails("8bit")
-    #qt_4bit = ModelDetails("4bit")
-    #qt_GPTQ = ModelDetails("GPTQ")
+    # qt_8bit = ModelDetails("8bit")
+    # qt_4bit = ModelDetails("4bit")
+    # qt_GPTQ = ModelDetails("GPTQ")
     Unknown = ModelDetails("?")
 
     def from_str(precision):
@@ -104,14 +145,15 @@ class Precision(Enum):
             return Precision.bfloat16
         if precision in ["float32"]:
             return Precision.float32
-        #if precision in ["8bit"]:
+        # if precision in ["8bit"]:
         #    return Precision.qt_8bit
-        #if precision in ["4bit"]:
+        # if precision in ["4bit"]:
         #    return Precision.qt_4bit
-        #if precision in ["GPTQ", "None"]:
+        # if precision in ["GPTQ", "None"]:
         #    return Precision.qt_GPTQ
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]

src/envs.py

@@ -4,9 +4,9 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("TOKEN")  # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "demo-leaderboard-backend"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
@@ -14,11 +14,14 @@ QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+
+CRM_RESULTS_PATH = os.path.join(CACHE_PATH, "crm-results")
+
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 

src/leaderboard/read_evals.py

@@ -1,6 +1,5 @@
 import glob
 import json
-import math
 import os
 from dataclasses import dataclass
 
@@ -8,28 +7,29 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
+from src.display.utils import AutoEvalColumn, ModelType, Precision, Tasks, WeightType
+
+# from src.submission.check_validity import is_model_on_hub
 
 
 @dataclass
 class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
+
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
     license: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = False
 
     @classmethod
@@ -57,14 +57,14 @@ class EvalResult:
             result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
+        # still_on_hub, _, model_config = is_model_on_hub(
+        #     full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        # )
         architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+        # if model_config is not None:
+        #     architectures = getattr(model_config, "architectures", None)
+        #     if architectures:
+        #         architecture = ";".join(architectures)
 
         # Extract results available in this file (some results are split in several files)
         results = {}
@@ -85,10 +85,10 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
+            precision=precision,
+            revision=config.get("model_sha", ""),
+            still_on_hub=False,
+            architecture=architecture,
         )
 
     def update_with_request_file(self, requests_path):
@@ -105,7 +105,9 @@ class EvalResult:
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(
+                f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}"
+            )
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
@@ -146,10 +148,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
+            if req_content["status"] in ["FINISHED"] and req_content["precision"] == precision.split(".")[-1]:
                 request_file = tmp_request_file
     return request_file
 
@@ -188,7 +187,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict()  # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             continue

src/populate.py

@@ -8,6 +8,19 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
+def get_leaderboard_df_crm(crm_results_path: str, cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    leaderboard_accuracy_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_accuracy.csv"))
+    # leaderboard_accuracy_df = leaderboard_accuracy_df.sort_values(
+    #     by=[AutoEvalColumn.accuracy_metric_average.name], ascending=False
+    # )
+    # print(leaderboard_accuracy_df)
+    # print(leaderboard_accuracy_df.columns)
+    # print(leaderboard_accuracy_df["Model Name"].nunique())
+    leaderboard_accuracy_df = leaderboard_accuracy_df[cols].round(decimals=2)
+    return leaderboard_accuracy_df
+
+
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )