Apply 8 categories from FinBen paper

app.py

@@ -64,20 +64,29 @@ leaderboard_df = original_df.copy()
 def update_table(
     hidden_df: pd.DataFrame,
     columns_info: list,
-    columns_eval: list,
-    columns_metadata: list,
-    columns_popularity: list,
-    columns_revision: list,
+    columns_IE: list,
+    columns_TA: list,
+    columns_QA: list,
+    columns_TG: list,
+    columns_RM: list,
+    columns_FO: list,
+    columns_DM: list,
+    columns_spanish: list,
+    columns_other: list,
     type_query: list,
     precision_query: list,
     size_query: list,
     show_deleted: bool,
     query: str,
 ):
+    # Combine all column selections
+    selected_columns = (
+        columns_info + columns_IE + columns_TA + columns_QA + columns_TG +
+        columns_RM + columns_FO + columns_DM + columns_spanish + columns_other
+    )
+    # Filter models based on queries
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     filtered_df = filter_queries(query, filtered_df)
-    # Combine all column selections
-    selected_columns = columns_info + columns_eval + columns_metadata + columns_popularity + columns_revision
     df = select_columns(filtered_df, selected_columns)
     return df
 
@@ -91,13 +100,18 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
         AutoEvalColumn.model_type_symbol.name,
         AutoEvalColumn.model.name,
     ]
+
+    # Ensure no duplicates when never_hidden and displayed_by_default are both True
+    unique_columns = set(always_here_cols + columns)
+
     # We use COLS to maintain sorting
     filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
+        [c for c in COLS if c in df.columns and c in unique_columns]
     ]
     return filtered_df
 
 
+
 def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
     final_df = []
     if query != "":
@@ -138,7 +152,7 @@ def filter_models(
     return filtered_df
 
 def uncheck_all():
-    return [], [], [], [], []
+    return [], [], [], [], [], [], [], [], [], []
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -164,32 +178,67 @@ with demo:
                                     label="Model Information",
                                     interactive=True,
                                 )
-                            with gr.Tab("Evaluation Scores"):
-                                shown_columns_eval = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Evaluation Scores"],
-                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Evaluation Scores"],
-                                    label="Evaluation Scores",
+                            with gr.Tab("Information Extraction (IE)"):
+                                shown_columns_IE = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Information Extraction (IE)"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Information Extraction (IE)"],
+                                    label="Information Extraction (IE)",
+                                    interactive=True,
+                                )
+                            with gr.Tab("Textual Analysis (TA)"):
+                                shown_columns_TA = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Textual Analysis (TA)"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Textual Analysis (TA)"],
+                                    label="Textual Analysis (TA)",
+                                    interactive=True,
+                                )
+                            with gr.Tab("Question Answering (QA)"):
+                                shown_columns_QA = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Question Answering (QA)"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Question Answering (QA)"],
+                                    label="Question Answering (QA)",
+                                    interactive=True,
+                                )
+                            with gr.Tab("Text Generation (TG)"):
+                                shown_columns_TG = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Text Generation (TG)"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Text Generation (TG)"],
+                                    label="Text Generation (TG)",
+                                    interactive=True,
+                                )
+                            with gr.Tab("Risk Management (RM)"):
+                                shown_columns_RM = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Risk Management (RM)"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Risk Management (RM)"],
+                                    label="Risk Management (RM)",
                                     interactive=True,
                                 )
-                            with gr.Tab("Model Metadata"):
-                                shown_columns_metadata = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Model Metadata"],
-                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Model Metadata"],
-                                    label="Model Metadata",
+                            with gr.Tab("Forecasting (FO)"):
+                                shown_columns_FO = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Forecasting (FO)"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Forecasting (FO)"],
+                                    label="Forecasting (FO)",
                                     interactive=True,
                                 )
-                            with gr.Tab("Popularity Metrics"):
-                                shown_columns_popularity = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Popularity Metrics"],
-                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Popularity Metrics"],
-                                    label="Popularity Metrics",
+                            with gr.Tab("Decision-Making (DM)"):
+                                shown_columns_DM = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Decision-Making (DM)"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Decision-Making (DM)"],
+                                    label="Decision-Making (DM)",
                                     interactive=True,
                                 )
-                            with gr.Tab("Revision and Availability"):
-                                shown_columns_revision = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Revision and Availability"],
-                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Revision and Availability"],
-                                    label="Revision and Availability",
+                            with gr.Tab("Spanish"):
+                                shown_columns_spanish = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Spanish"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Spanish"],
+                                    label="Spanish",
+                                    interactive=True,
+                                )
+                            with gr.Tab("Other"):
+                                shown_columns_other = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if c.category == "Other"],
+                                    value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and c.category == "Other"],
+                                    label="Other",
                                     interactive=True,
                                 )
                     with gr.Row():
@@ -199,10 +248,16 @@ with demo:
                             inputs=[],
                             outputs=[
                                 shown_columns_info,
-                                shown_columns_eval,
-                                shown_columns_metadata,
-                                shown_columns_popularity,
-                                shown_columns_revision
+                                shown_columns_IE,
+                                shown_columns_TA,
+                                shown_columns_QA,
+                                shown_columns_TG,
+                                shown_columns_RM,
+                                shown_columns_FO,
+                                shown_columns_DM,
+                                shown_columns_spanish,
+                                shown_columns_other,
+
                             ],
                         )
                     with gr.Row():
@@ -236,16 +291,17 @@ with demo:
             leaderboard_table = gr.Dataframe(
                 value=leaderboard_df[
                     [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                    + [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default]
+                    + [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.never_hidden]
                 ],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                        + [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+                        + [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.never_hidden],
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
             )
 
+
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.Dataframe(
                 value=original_df[COLS],
@@ -258,10 +314,15 @@ with demo:
                 inputs=[
                     hidden_leaderboard_table_for_search,
                     shown_columns_info,
-                    shown_columns_eval,
-                    shown_columns_metadata,
-                    shown_columns_popularity,
-                    shown_columns_revision,
+                    shown_columns_IE,
+                    shown_columns_TA,
+                    shown_columns_QA,
+                    shown_columns_TG,
+                    shown_columns_RM,
+                    shown_columns_FO,
+                    shown_columns_DM,
+                    shown_columns_spanish,
+                    shown_columns_other,
                     filter_columns_type,
                     filter_columns_precision,
                     filter_columns_size,
@@ -271,8 +332,16 @@ with demo:
                 outputs=leaderboard_table,
             )
             for selector in [
-                shown_columns_info, shown_columns_eval, shown_columns_metadata, 
-                shown_columns_popularity, shown_columns_revision, 
+                shown_columns_info,
+                shown_columns_IE,
+                shown_columns_TA,
+                shown_columns_QA,
+                shown_columns_TG,
+                shown_columns_RM,
+                shown_columns_FO,
+                shown_columns_DM,
+                shown_columns_spanish,
+                shown_columns_other,
                 filter_columns_type, filter_columns_precision, 
                 filter_columns_size, deleted_models_visibility
             ]:
@@ -281,10 +350,15 @@ with demo:
                     inputs=[
                         hidden_leaderboard_table_for_search,
                         shown_columns_info,
-                        shown_columns_eval,
-                        shown_columns_metadata,
-                        shown_columns_popularity,
-                        shown_columns_revision,
+                        shown_columns_IE,
+                        shown_columns_TA,
+                        shown_columns_QA,
+                        shown_columns_TG,
+                        shown_columns_RM,
+                        shown_columns_FO,
+                        shown_columns_DM,
+                        shown_columns_spanish,
+                        shown_columns_other,
                         filter_columns_type,
                         filter_columns_precision,
                         filter_columns_size,

src/about.py

@@ -7,46 +7,46 @@ class Task:
     benchmark: str
     metric: str
     col_name: str
+    category: str
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    task0 = Task("FPB", "F1", "FPB")
-    task2 = Task("FiQA-SA", "F1", "FiQA-SA")
-    task3 = Task("TSA", "RMSE", "TSA")
-    task4 = Task("Headlines", "AvgF1", "Headlines")
-    task5 = Task("FOMC", "F1", "FOMC")
-    task7 = Task("FinArg-ACC", "MicroF1", "FinArg-ACC")
-    task8 = Task("FinArg-ARC", "MicroF1", "FinArg-ARC")
-    task9 = Task("MultiFin", "MicroF1", "Multifin")
-    task10 = Task("MA", "MicroF1", "MA")
-    task11 = Task("MLESG", "MicroF1", "MLESG")
-    task12 = Task("NER", "EntityF1", "NER")
-    task13 = Task("FINER-ORD", "EntityF1", "FINER-ORD")
-    task14 = Task("FinRED", "F1", "FinRED")
-    task15 = Task("SC", "F1", "SC")
-    task16 = Task("CD", "F1", "CD")
-    task17 = Task("FinQA", "EmAcc", "FinQA")
-    task18 = Task("TATQA", "EmAcc", "TATQA")
-    task19 = Task("ConvFinQA", "EmAcc", "ConvFinQA")
-    task20 = Task("FNXL", "EntityF1", "FNXL")
-    task21 = Task("FSRL", "EntityF1", "FSRL")
-    task22 = Task("EDTSUM", "Rouge-1", "EDTSUM")
-    task25 = Task("ECTSUM", "Rouge-1", "ECTSUM")
-    task28 = Task("BigData22", "Acc", "BigData22")
-    task30 = Task("ACL18", "Acc", "ACL18")
-    task32 = Task("CIKM18", "Acc", "CIKM18")
-    task34 = Task("German", "MCC", "German")
-    task36 = Task("Australian", "MCC", "Australian")
-    task38 = Task("LendingClub", "MCC", "LendingClub")
-    task40 = Task("ccf", "MCC", "ccf")
-    task42 = Task("ccfraud", "MCC", "ccfraud")
-    task44 = Task("polish", "MCC", "polish")
-    task46 = Task("taiwan", "MCC", "taiwan")
-    task48 = Task("portoseguro", "MCC", "portoseguro")
-    task50 = Task("travelinsurance", "MCC", "travelinsurance")
+    task0 = Task("FPB", "F1", "FPB", category="Spanish")
+    task2 = Task("FiQA-SA", "F1", "FiQA-SA", category="Textual Analysis (TA)")
+    task3 = Task("TSA", "RMSE", "TSA", category="Textual Analysis (TA)")
+    task4 = Task("Headlines", "AvgF1", "Headlines", category="Textual Analysis (TA)")
+    task5 = Task("FOMC", "F1", "FOMC", category="Forecasting (FO)")
+    task7 = Task("FinArg-ACC", "MicroF1", "FinArg-ACC", category="Textual Analysis (TA)")
+    task8 = Task("FinArg-ARC", "MicroF1", "FinArg-ARC", category="Textual Analysis (TA)")
+    task9 = Task("MultiFin", "MicroF1", "Multifin", category="Textual Analysis (TA)")
+    task10 = Task("MA", "MicroF1", "MA", category="Textual Analysis (TA)")
+    task11 = Task("MLESG", "MicroF1", "MLESG", category="Textual Analysis (TA)")
+    task12 = Task("NER", "EntityF1", "NER", category="Information Extraction (IE)")
+    task13 = Task("FINER-ORD", "EntityF1", "FINER-ORD", category="Information Extraction (IE)")
+    task14 = Task("FinRED", "F1", "FinRED", category="Information Extraction (IE)")
+    task15 = Task("SC", "F1", "SC", category="Spanish")
+    task16 = Task("CD", "F1", "CD", category="Spanish")
+    task17 = Task("FinQA", "EmAcc", "FinQA", category="Question Answering (QA)")
+    task18 = Task("TATQA", "EmAcc", "TATQA", category="Question Answering (QA)")
+    task19 = Task("ConvFinQA", "EmAcc", "ConvFinQA", category="Question Answering (QA)")
+    task20 = Task("FNXL", "EntityF1", "FNXL", category="Information Extraction (IE)")
+    task21 = Task("FSRL", "EntityF1", "FSRL", category="Information Extraction (IE)")
+    task22 = Task("EDTSUM", "Rouge-1", "EDTSUM", category="Text Generation (TG)")
+    task25 = Task("ECTSUM", "Rouge-1", "ECTSUM", category="Text Generation (TG)")
+    task28 = Task("BigData22", "Acc", "BigData22", category="Risk Management (RM)")
+    task30 = Task("ACL18", "Acc", "ACL18", category="Decision-Making (DM)")
+    task32 = Task("CIKM18", "Acc", "CIKM18", category="Decision-Making (DM)")
+    task34 = Task("German", "MCC", "German", category="Decision-Making (DM)")
+    task36 = Task("Australian", "MCC", "Australian", category="Decision-Making (DM)")
+    task38 = Task("LendingClub", "MCC", "LendingClub", category="Risk Management (RM)")
+    task40 = Task("ccf", "MCC", "ccf", category="Risk Management (RM)")
+    task42 = Task("ccfraud", "MCC", "ccfraud", category="Risk Management (RM)")
+    task44 = Task("polish", "MCC", "polish", category="Risk Management (RM)")
+    task46 = Task("taiwan", "MCC", "taiwan", category="Risk Management (RM)")
+    task48 = Task("portoseguro", "MCC", "portoseguro", category="Risk Management (RM)")
+    task50 = Task("travelinsurance", "MCC", "travelinsurance", category="Risk Management (RM)")
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/display/utils.py

@@ -27,26 +27,19 @@ auto_eval_column_dict = []
 # Model Information
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, category="Model Information", never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, category="Model Information", never_hidden=True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True, category="Model Information")])
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False, category="Model Information")])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False, category="Model Information")])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, category="Model Information", hidden=True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False, category="Model Information")])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False, category="Model Information")])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False, category="Model Information")])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False, category="Model Information")])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, category="Model Information")])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, category="Model Information", hidden=False)])
 
-# Evaluation Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True, category="Evaluation Scores")])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, category="Evaluation Scores")])
-
-# Model Metadata
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, category="Model Metadata", hidden=True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False, category="Model Metadata")])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False, category="Model Metadata")])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False, category="Model Metadata")])
-
-# Popularity Metrics
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False, category="Popularity Metrics")])
-
-# Revision and Availability
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, category="Revision and Availability")])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, category="Revision and Availability", hidden=False)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, category=task.value.category)])
 
 # We use make_dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)