works ish

app.py

@@ -12,6 +12,14 @@ from leaderboard.css import custom_css
 from leaderboard.md import *
 from leaderboard.utils import load_all_data
 
+# get v1 data
+rb_orig_snapshot = pd.read_csv("leaderboard/final-rbv1-data.csv")
+# rename column "Unnamed: 0" to ""
+rb_orig_snapshot = rb_orig_snapshot.rename(columns={"Unnamed: 0": ""})
+# rb_orig_snapshot = rb_orig_snapshot.drop(columns=["Unnamed: 0", ''])
+rb_orig_snapshot.reset_index(drop=True, inplace=True)
+
+# import ipdb; ipdb.set_trace()
 #######################################################
 #                     Setup                           # 
 #######################################################
@@ -152,11 +160,10 @@ rewardbench_data_avg = avg_over_rewardbench_v2(rewardbench_data).sort_values(by=
 rewardbench_data = prep_df(rewardbench_data)
 rewardbench_data_avg = prep_df(rewardbench_data_avg).rename(columns={"Average": "Score"})
 
-col_types_rewardbench = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data.columns) - 1)
-col_types_rewardbench_avg = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
 
-# get v1 data
-rb_orig_snapshot = pd.read_csv("leaderboard/final-rbv1-data.csv")
+
+col_types_rewardbench = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
+col_types_rewardbench_v1 = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rb_orig_snapshot.columns) - 1)
 
 ###########################################
 # Leaderboard Helpers & Setting           #
@@ -297,6 +304,11 @@ total_models = len(
         rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False
     ).values
 )
+total_models_v1 = len(
+    regex_table(
+        rb_orig_snapshot.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False
+    ).values
+)
 assets = Path("leaderboard").resolve()  # absolute dir with the image
 
 # Using a string for a predefined color
@@ -352,7 +364,7 @@ with gr.Blocks(theme=theme, css=custom_css) as app:
                         # reference data
                         rewardbench_table_hidden = gr.Dataframe(
                             rewardbench_data_avg.values,
-                            datatype=col_types_rewardbench_avg,
+                            datatype=col_types_rewardbench_v1,
                             headers=rewardbench_data_avg.columns.tolist(),
                             visible=False,
                         )
@@ -362,7 +374,7 @@ with gr.Blocks(theme=theme, css=custom_css) as app:
                                 "",
                                 ["Seq. Classifiers", "Custom Classifiers", "Generative"],
                             ),
-                            datatype=col_types_rewardbench_avg,
+                            datatype=col_types_rewardbench_v1,
                             headers=rewardbench_data_avg.columns.tolist(),
                             elem_id="rewardbench_dataframe_avg",
                             max_height=800,  # 800 px ≈ ~25 rows on default row-height
@@ -385,42 +397,42 @@ with gr.Blocks(theme=theme, css=custom_css) as app:
                     button_data.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display])
         with gr.TabItem("RewardBench", scale=1.5):
             with gr.Row():
-                gr.Markdown(CAPTION_V1.format(str(total_models)))
+                gr.Markdown(CAPTION_V1.format(str(total_models_v1)))
             with gr.Tabs(elem_id="inner-tabs", elem_classes="tabs-small") as tabs:
                 with gr.TabItem("Leaderboard"):
                     pass
-                    # with gr.Row():
-                    #     search_1 = gr.Textbox(
-                    #         label="Model Search (delimit with , )",
-                    #         placeholder="Model Search (delimit with , )",
-                    #         show_label=False,
-                    #     )
-                    #     model_types_1 = gr.CheckboxGroup(
-                    #         ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
-                    #         value=["Seq. Classifiers", "Custom Classifiers", "Generative"],
-                    #         label="Model Types",
-                    #         show_label=False,
-                    #         #  info="Which model types to include.",
-                    #     )
-                    # with gr.Row():
-                    #     # reference data
-                    #     rewardbench_table_hidden = gr.Dataframe(
-                    #         rewardbench_data_avg.values,
-                    #         datatype=col_types_rewardbench_avg,
-                    #         headers=rewardbench_data_avg.columns.tolist(),
-                    #         visible=False,
-                    #     )
-                    #     rewardbench_table = gr.Dataframe(
-                    #         regex_table(
-                    #             rewardbench_data_avg.copy(),
-                    #             "",
-                    #             ["Seq. Classifiers", "Custom Classifiers", "Generative"],
-                    #         ),
-                    #         datatype=col_types_rewardbench_avg,
-                    #         headers=rewardbench_data_avg.columns.tolist(),
-                    #         elem_id="rewardbench_dataframe_avg",
-                    #         max_height=800,  # 800 px ≈ ~25 rows on default row-height
-                    #     )
+                    with gr.Row():
+                        search_1_v1 = gr.Textbox(
+                            label="Model Search (delimit with , )",
+                            placeholder="Model Search (delimit with , )",
+                            show_label=False,
+                        )
+                        model_types_1_v1 = gr.CheckboxGroup(
+                            ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
+                            value=["Seq. Classifiers", "Custom Classifiers", "Generative"],
+                            label="Model Types",
+                            show_label=False,
+                            #  info="Which model types to include.",
+                        )
+                    with gr.Row():
+                        # reference data
+                        rewardbench_table_hidden_v1 = gr.Dataframe(
+                            rb_orig_snapshot.values,
+                            datatype=col_types_rewardbench,
+                            headers=rb_orig_snapshot.columns.tolist(),
+                            visible=False,
+                        )
+                        rewardbench_table_v1 = gr.Dataframe(
+                            regex_table(
+                                rb_orig_snapshot.copy(),
+                                "",
+                                ["Seq. Classifiers", "Custom Classifiers", "Generative"],
+                            ),
+                            datatype=col_types_rewardbench,
+                            headers=rb_orig_snapshot.columns.tolist(),
+                            elem_id="rewardbench_dataframe_avg_v1",
+                            max_height=800,  # 800 px ≈ ~25 rows on default row-height
+                        )
                 with gr.TabItem("About"):
                     with gr.Row():
                         gr.Markdown(ABOUT_TEXT_V1)
@@ -433,19 +445,20 @@ with gr.Blocks(theme=theme, css=custom_css) as app:
                         button_data_v1 = gr.Button("Show Random Sample")
 
                     with gr.Row():
-                        sample_display = gr.Markdown("{sampled data loads here}")
-
-                    button_data_v1.click(fn=random_sample_v1, inputs=[subset_selector_v1], outputs=[sample_display])
+                        sample_display_v1 = gr.Markdown("{sampled data loads here}")
 
+                    button_data_v1.click(fn=random_sample_v1, inputs=[subset_selector_v1], outputs=[sample_display_v1])
 
 
     search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
-    # search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
+    search_1_v1.change(regex_table, inputs=[rewardbench_table_hidden_v1, search_1_v1, model_types_1_v1], outputs=rewardbench_table_v1)
 
     model_types_1.change(
         regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table
     )
-    # model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
+    model_types_1_v1.change(
+        regex_table, inputs=[rewardbench_table_hidden_v1, search_1_v1, model_types_1_v1], outputs=rewardbench_table_v1
+    )
 
     with gr.Row():
         with gr.Accordion("📚 Citation", open=False):

leaderboard/md.py

@@ -112,7 +112,7 @@ CAPTION_V2 = f"""The *new version* of RewardBench that is based on unseen human
 
 CAPTION_V1 = """The original RewardBench -- the first reward model evaluation.
 
-**Note**: This leaderboard is frozen and will not be updated. The final version of the evaluation results are available [here](TODO).
+**Note**: This leaderboard is frozen and will not be updated. The final version of the evaluation results are available in the source for this application.
 
 ⚠️ Many of the top models were trained on unintentionally contaminated, AI-generated data, for more information, see this [gist](https://gist.github.com/natolambert/1aed306000c13e0e8c5bc17c1a5dd300).
 """

leaderboard/utils.py

@@ -6,7 +6,10 @@ import numpy as np
 import pandas as pd
 from datasets import load_dataset
 
-UNVERIFIED_MODELS = [
+UNVERIFIED_MODELS = []
+CONTAMINATED_MODELS = []
+
+UNVERIFIED_MODELS_V1 = [
     "nvidia/Nemotron-4-340B-Reward",
     "nvidia/Llama3-70B-SteerLM-RM",
     "Cohere May 2024",
@@ -24,7 +27,8 @@ UNVERIFIED_MODELS = [
     "nvidia/Llama-3.1-Nemotron-70B-Reward",
 ]
 
-CONTAMINATED_MODELS = [
+# No longer used
+CONTAMINATED_MODELS_V1 = [
     "Skywork/Skywork-Reward-Gemma-2-27B",
     "Skywork/Skywork-Critic-Llama-3.1-70B",
     "LxzGordon/URM-LLaMa-3.1-8B",
@@ -39,7 +43,6 @@ CONTAMINATED_MODELS = [
     "Ray2333/GRM-Gemma-2B-rewardmodel-ft",
 ]
 
-
 # From Open LLM Leaderboard
 def model_hyperlink(link, model_name):
     # if model_name is above 50 characters, return first 47 characters and "..."