Update app.py

app.py

@@ -1,294 +1,278 @@
 import os
 import json
 import datetime
-import smtplib
-from email.mime.text import MIMEText
-from email.mime.multipart import MIMEMultipart
 from email.utils import parseaddr
 import numpy as np
 import gradio as gr
 import pandas as pd
 from datasets import load_dataset
+from evaluation.evaluator import question_scorer as eval_scorer
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
+from content import format_error, format_warning, format_log, TITLE
+
+# Placeholder for the question_scorer function
+def question_scorer(prediction, gold_answer):
+    acc, has_ans = eval_scorer(prediction, gold_answer)
+    return acc, has_ans
 
-TOKEN = os.environ.get("TOKEN", None)
-SUBMISSION_DATASET = "KoLMogorov-Test/submissions"
-VERSION = "v1"
 
+# Constants and Configuration
+TOKEN = os.environ.get("TOKEN", None)
+OWNER = "Ori"
+DATA_DATASET = f"Ori/AssistantBench_V1.0"
+RESULTS_DATASET = f"Ori/results"
+SUBMISSION_DATASET = f"AssistantBench/submissions"
+LEADERBOARD_PATH = f"{OWNER}/leaderboard"
 api = HfApi()
 
-        
-def format_error(msg):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
+YEAR_VERSION = "default"
+
+os.makedirs("scored", exist_ok=True)
+
+# Load datasets
+eval_results = load_dataset(RESULTS_DATASET, token=TOKEN, download_mode="force_redownload", trust_remote_code=True)
+gold_results = load_dataset(DATA_DATASET, token=TOKEN, trust_remote_code=True)
+
+gold_answers = {split: {row["id"]: row["answer"] for row in gold_results[split]} for split in ["test"]}
+gold_difficulties = {split: {row["id"]: row["difficulty"] for row in gold_results[split]} for split in ["test"]}
+
 
-def format_warning(msg):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+# Function to get dataframe from results
+def get_dataframe_from_results(eval_results, split):
+    local_df = eval_results[split]
+    df = pd.DataFrame(local_df)
+    df = df.sort_values(by=["Accuracy"], ascending=False)
+    numeric_cols = [c for c in local_df.column_names if "score" in c]
+    df[numeric_cols] = df[numeric_cols].multiply(100).round(decimals=2)
+    return df
 
-def format_log(msg):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
+# Update function to format dataframe
+def format_dataframe(df):
+    df["Accuracy"] = df["Accuracy"].apply(lambda x: f"**{x:.2f}**")
+    if "URL" in df.columns:
+        df["Model Name"] = df.apply(lambda row: f"[{row['Model Name']}]({row['URL']})", axis=1)
+        df = df.drop(columns=["URL"])
+    #df = df.rename(columns={"Model Family": "Base Model"})
+    df = df[["Model Name", "Accuracy",  "Answer rate", "Precision", "EM", "Accuracy (easy)", "Accuracy (medium)", "Accuracy (hard)", "Base Model", "Organization"]]
+    return df
 
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
+eval_results = load_dataset(RESULTS_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", trust_remote_code=True)
+eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")
+eval_dataframe_test = format_dataframe(eval_dataframe_test)
 
 # Function to restart the space
 def restart_space():
-    return
+    api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)
+
 
 TYPES = ["markdown", "markdown", "number", "number", "number", "number", "number", "number", "str", "str"]
 
 
-def save_files_for_eval(
-    model_name: str,
-    organization: str,
-    organization_email: str,
-    path_to_file,
-    prior_file,
-    results_json,
-    modality: str,
-    split: str,
-    token: str,
+# Function to add a new evaluation
+def add_new_eval(
+        model_name: str,
+        model_family: str,
+        url: str,
+        path_to_file: str,
+        organization: str,
+        mail: str,
 ):
-    # Validate inputs
-    if not model_name:
-        return format_error("Model name is required.")
-    if not organization and organization_email:
-        return format_error("Organization and email  required.")
-    
-    # Get the current date for the directory name
-    date_str = datetime.datetime.today().strftime('%Y-%m-%d')
-    
-    # Define the base path in the repository (remove leading slash)
-    base_path_in_repo = f"{organization}/{model_name}/{VERSION}_{split}_{modality}_{date_str}"
-    
-    # List of files to upload
-    files_to_upload = [
-        (path_to_file, "prediction.jsonl"),
-        (prior_file, "decoder.py"),
-        (results_json, "results.json")
-    ]
-    
-    # Upload each file
-    for file, filename in files_to_upload:
-        if file is not None:
-            path_in_repo = f"{base_path_in_repo}/{filename}"
-            api.upload_file(
-                repo_id=SUBMISSION_DATASET,
-                path_or_fileobj=file.name,
-                path_in_repo=path_in_repo,
-                repo_type="dataset",
-                token=token
-            )
-        else:
-            return format_error(f"File {filename} is missing.")
+    _, parsed_mail = parseaddr(mail)
+    if "@" not in parsed_mail:
+        return format_warning("Please provide a valid email address.")
+
+    print("Adding new eval")
+
+    if model_name.lower() in set(
+            [m.lower() for m in eval_results["test"]["Model Name"]]) and organization.lower() in set(
+            [o.lower() for o in eval_results["test"]["Organization"]]):
+        return format_warning("This model has already been submitted.")
+
+    if path_to_file is None:
+        return format_warning("Please attach a file.")
+
+    api.upload_file(
+        repo_id=SUBMISSION_DATASET,
+        path_or_fileobj=path_to_file.name,
+        path_in_repo=f"{organization}/{model_name}/{YEAR_VERSION}_test_raw_{datetime.datetime.today()}.jsonl",
+        repo_type="dataset",
+        token=TOKEN
+    )
+
+    file_path = path_to_file.name
+    scores = 0
+    num_questions = 0
+
+    difficulty_scores = {"Easy": 0, "Medium": 0, "Hard": 0}
+    difficulty_counts = {"Easy": 0, "Medium": 0, "Hard": 0}
+
+    all_scores = list()
+
+    with open(f"scored/{organization}_{model_name}.jsonl", "w") as scored_file:
+        with open(file_path, 'r') as f:
+            submitted_ids = set()
+            for ix, line in enumerate(f):
+                try:
+                    task = json.loads(line)
+                except Exception:
+                    return format_error(f"Line {ix} is incorrectly formatted. Please fix it and resubmit your file.")
+
+                if "answer" not in task:
+                    return format_error(
+                        f"Line {ix} contains no answer key. Please fix it and resubmit your file.")
+
+                answer = task["answer"]
+                task_id = task["id"]
+                if task_id not in gold_answers["test"]:
+                    return format_error(
+                        f"{task_id} not found in test set. Are you sure you submitted the correct file?")
+
+                score, has_ans = question_scorer(task['answer'], gold_answers["test"][task_id])
+                difficulty = gold_difficulties["test"][task_id]
+
+                scored_file.write(
+                    json.dumps({
+                        "id": task_id,
+                        "model_answer": answer,
+                        "score": score,
+                        "has_ans": has_ans
+                    }) + "\n"
+                )
+
+                all_scores.append({"score": score, "has_ans": has_ans, "model_answer": answer, 'id': task_id})
+                submitted_ids.add(task["id"])
+                scores += score
+                num_questions += 1
+                difficulty_scores[difficulty] += score
+                difficulty_counts[difficulty] += 1
+
+    # Check if all gold answer IDs are present in the submission
+    missing_ids = set(gold_answers["test"].keys()) - submitted_ids
+    if missing_ids:
+        return format_error(f"Submission is missing the following IDs: {', '.join(missing_ids)}")
+
+    accuracy_easy = difficulty_scores["Easy"] / difficulty_counts["Easy"] if difficulty_counts["Easy"] > 0 else 0
+    accuracy_medium = difficulty_scores["Medium"] / difficulty_counts["Medium"] if difficulty_counts["Medium"] > 0 else 0
+    accuracy_hard = difficulty_scores["Hard"] / difficulty_counts["Hard"] if difficulty_counts["Hard"] > 0 else 0
+
+    api.upload_file(
+        repo_id=SUBMISSION_DATASET,
+        path_or_fileobj=f"scored/{organization}_{model_name}.jsonl",
+        path_in_repo=f"{organization}/{model_name}/{YEAR_VERSION}_test_scored_{datetime.datetime.today()}.jsonl",
+        repo_type="dataset",
+        token=TOKEN
+    )
+
+    accuracy = float("{:.1f}".format(np.average([x["score"] for x in all_scores]) * 100))
+    coverage = float("{:.1f}".format(np.average([x["has_ans"] for x in all_scores]) * 100))
+    em = float("{:.1f}".format(np.average([1 if x["score"] == 1 else 0 for x in all_scores]) * 100))
+    precision = float("{:.1f}".format(np.average([x["score"] for x in all_scores if x["has_ans"] == 1]) * 100))
+    accuracy_easy = float("{:.1f}".format(accuracy_easy * 100))
+    accuracy_medium = float("{:.1f}".format(accuracy_medium * 100))
+    accuracy_hard = float("{:.1f}".format(accuracy_hard * 100))
+
+    eval_entry = {
+        "Model Name": model_name,
+        "Base Model": model_family,
+        "URL": url,
+        "Organization": organization,
+        "Accuracy": accuracy,
+        "Accuracy (easy)": accuracy_easy,
+        "Accuracy (medium)": accuracy_medium,
+        "Accuracy (hard)": accuracy_hard,
+        "Answer rate": coverage,
+        "Precision": precision,
+        "EM": em
+    }
+    eval_results["test"] = eval_results["test"].add_item(eval_entry)
+
+    eval_results.push_to_hub(RESULTS_DATASET, config_name=YEAR_VERSION, token=TOKEN)
+
+    return format_log(
+        f"Model {model_name} submitted by {organization} successfully.\nPlease wait a few hours and refresh the leaderboard to see your score displayed.")
+
 
-    return format_log("All files uploaded successfully. Please email us at [email protected] to verify your submission.")
+# Function to refresh the results
+def refresh():
+    eval_results = load_dataset(RESULTS_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", trust_remote_code=True)
+    eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")
+    eval_dataframe_test = format_dataframe(eval_dataframe_test)
+    return eval_dataframe_test
 
 
 # Gradio interface
 demo = gr.Blocks()
 with demo:
-    gr.HTML("""<h1 id="space-title">The KoLMogorov-Test: Can CodeLMs lead to the next breakthrough in data compression?</h1>""")
-    gr.HTML("""The Kolmogorov complexity of a sequence is the length of the shortest computer program that produces the sequence. <br><br>The aim of the KoLMogorov-Test (KT) is to empirically evaluate the ability of CodeLMs to detect patterns in and compress sequences by writing short programs that output them.
-<h2>The Task</h2>
-Given a sequence, the CodeLM is asked to produce a short python program that outputs the sequence. The programs are compressed by a user-defined compressor before submission, and code length is measured in compressed form. In order to evaluate the correctness of a program, it is first decoded using a user-provided decoder and then executed. The decoder size counts towards the compressed length, and may contain e.g. a library of helper functions (DSL). <br><br>
-
-The length of python and its standard library are not counted, but in order to prevent cheating the programs must pass a stringent check. Programs cannot import arbitrary modules, use advanced language features, access the internet, etc. <br><br>
-
-KT currently includes six modalities - text, DNA, three encodings of audio data (MFCC, 16-bit, and 8-bit), and synthetic sequences produced by random programs. Two dataset sizes are available: a small one with 1MB per modality, and a large one with 1GB.""")
-
-    gr.HTML("""<h2>Getting Started</h2>
-            Download the data from <a href="https://www.example.com" target="_blank">here</a>. 
-            <br><br>We provide code to reproduce experiments from the paper including our DSL and evaluation code at the <a href="https://www.example.com" target="_blank">GitHub repo</a>.
-    <br><br>You are allowed to:<br><br>
-    <ul>
-        <li>Split the original sequence to subsequent sub-sequences.</li>
-        <li>Use the standard python library including Gzip.</li>
-        <li>Create new DSLs.</li>
-        <li>Create new priors to encode the programs.</li>
-    </ul>
-
-    <br>With the following restrictions:<br><br>
-    <ul>
-        <li>To prevent cases where compression of the data is performed by external code, we prevent access to the internet and usage of previous compression algorithms, excluding Gzip which is viewed as a strong baseline. </li>
-        <li>We return the compression rate with and without the cost of the decoder. When reporting results, we consider the additional code as negligible if it is &lt100KB and does not scale with the length of the sequence. When reporting results for the 1GB seqeunces, please always report results including the decoder.</li>
-    </ul>
-                    
-    <br>If you have any questions, please email us at <a href="mailto:thekolmogorovtest@gmail.com">[email protected]</a>.
-
-""")
-
-    gr.HTML("""<h2>Making a New Submission</h2>""")
-            
-    gr.Markdown("""To make a new submission, upload three files - a file with the encoded program, a file that decodes the programs to executable code, and a file with the expected results. Let's consider a toy example for the sequence [5, 10, 13, 14, 16, 5, 5, 5].""")
-    
-    gr.Markdown("""The programs file is a jsonl that follows the following format. We recommend submitting a single program for the whole sequence:
-            ```
-            {"sub_sequence_start_index": 0, "sub_sequence_end_index": 7 (the length of the sequence), "encoded_program": "H4sIAI7Te2cC/8svLSkoLVGwVYg21VEwNABiYyA2AWIzHQVTCIoFAMxnYTIlAAAA (An encoding of the program whose execution results in the input sequence. This is the Gzip encoding of the program that returns the input sequence)."}
-            ```
-
-            For simplicity, we allow splitting to subsequent sub-sequences:
-            ```
-            {"sub_sequence_start_index": 0, "sub_sequence_end_index": k, "encoded_program": "An encoding of the program whose execution results in the subsequence between indeses [0, n]."}
-            {"sub_sequence_start_index": k+1, "sub_sequence_end_index": k+1+j, "encoded_program": "An encoding of the program whose execution results in the subsequence between indeses [n+1, n+1+m]."}
-            ...
-            {"sub_sequence_start_index": n, "sub_sequence_end_index": len(sequence), "encoded_program": "An encoding of the program whose execution results in the subsequence between indeses [n, len(sequence)]."}
-            ```
-        """)
-
-    gr.Markdown("""For decoding, upload a python file that implements the 'decode' method, which receives as input an programs from the programs file and returns the executable python program. Executing the decoded program must result in the input sequence. For example, this is a decoder that decompresses using Gzip. 
-            ```
-            import gzip
-            import base64
-
-            def decode(program):
-                return gzip.decompress(base64.b64decode(program)).decode('utf-8')
-            ```
-        """)
-    
-    gr.Markdown("""For the results, upload a Json file that matches the Result object returned from the <a href="https://www.example.com" target="_blank">official evaluation script</a>. If you are interested in verification of your results, please <a href="mailto:[email protected]">email us</a> after submitting, we will then verify the results.
-            ```
-            {
-                "compressed_programs_size": 64,
-                "decoder_size": 118,
-                "compressed_size": 182,
-                "compression_rate_without_decoder": 8.0,
-                "compression_rate": 22.75,
-                "accuracy": 1,
-                "gold_data_size": 8,
-                "first_error": null
-            }
-            ```
-        """)
-    
-    with gr.Accordion(""):
+    gr.HTML("<h1>AssistantBench</h1>")
+    gr.Markdown("""
+        AssistantBench aims to evaluate the ability of web agents to assist with real and time-consuming tasks.
+        For more information, please check out our paper or the official website.
+        To download AssistantBench, press [here](https://huggingface.co/datasets/AssistantBench/AssistantBench).
+    """)
+
+    gr.HTML("<h2>AssistantBench Leaderboard</h2>")
+    with gr.Tab("Results: Test"):
+        leaderboard_table_test = gr.Dataframe(
+            value=eval_dataframe_test, datatype=TYPES, interactive=False,
+            column_widths=["20%"]
+        )
+
+    refresh_button = gr.Button("Refresh")
+    refresh_button.click(
+        refresh,
+        inputs=[],
+        outputs=[
+            leaderboard_table_test,
+        ],
+    )
+
+    gr.HTML("<h2>Making a New Submission</h2>")
+    with gr.Accordion("Submit a new model for evaluation"):
+        with gr.Row():
+            gr.Markdown("""
+                To make a new submission, upload a predictions file. Our scoring function can be found [here](https://huggingface.co/spaces/AssistantBench/leaderboard/blob/main/scorer.py). We support JSONL files with the following format:
+                ```
+                {"id": "task_id_1", "answer": "Answer 1 from your model"}
+                {"id": "task_id_2", "answer": "Answer 2 from your model"}
+                ```
+            """)
         with gr.Row():
             with gr.Column():
                 model_name_textbox = gr.Textbox(label="Model Name")
+                model_family_textbox = gr.Textbox(label="Base Model")
+                url_textbox = gr.Textbox(label="URL to Model Information")
+            with gr.Column():
                 organization = gr.Textbox(label="Organization")
                 mail = gr.Textbox(
-                    label="Contact Email (will be stored privately)"
-                )
-                split_dropdown = gr.Dropdown(
-                    choices=["Small (1MB)", "Large (1GB)"],
-                    label="Size",
-                )
-                modality_dropdown = gr.Dropdown(
-                    choices=["Text", "Dna", "Audio-MFCC", "Audio-16-Bit", "Audio-8-Bit", "Synthetic"],
-                    label="Modality",
-                )
-                prior_dropdown = gr.Dropdown(
-                    choices=["Custom (attached file)", "None"],
-                    label="Decoding",
-                )
-                url_textbox = gr.Textbox(label="URL to Project Information",
-                                         interactive=True,)
-                model_family_textbox = gr.Textbox(label="Base Model")
+                    label="Contact Email (will be stored privately & used if there is an issue with your submission)")
+                file_output = gr.File()
 
-            with gr.Column():
-                file_output = gr.File(label="Programs File")
-                prior_file = gr.File(label="Decoder File")
-                results_json = gr.File(label="Results Json")
-                
-        submit_button = gr.Button("Submit")
+        submit_button = gr.Button("Submit Eval")
         submission_result = gr.Markdown()
-
-    submit_button.click(
-        lambda model_name, split, model_family, url, path_to_file, organization, mail, prior_file, results_json, prior_dropdown, modality_dropdown: (
-            save_files_for_eval(
-                model_name,
+        submit_button.click(
+            add_new_eval,
+            [
+                model_name_textbox,
+                model_family_textbox,
+                url_textbox,
+                file_output,
                 organization,
-                mail,
-                path_to_file,
-                prior_file,
-                results_json,
-                modality_dropdown,
-                split,
-                TOKEN  # Ensure TOKEN is defined and accessible
-            )
-        ),
-        [
-            model_name_textbox,
-            split_dropdown,
-            model_family_textbox,
-            url_textbox,
-            file_output,
-            organization,
-            mail,
-            prior_file,
-            results_json,
-            prior_dropdown,
-            modality_dropdown
-        ],
-        submission_result,
-    )
+                mail
+            ],
+            submission_result,
+        )
 
-    data = {
-        "Text": {
-            "Model": ["Gzip", "GPT4-o", "LLAMA-3.1-405B", "LLAMA-3.1-70B", "LLAMA-3.1-8B"],
-            "Pass@1": [100, 69.5, 33.3, 18.0, 8.5],
-            "Precision": [1.0, 1.34, 2.18, 2.78, 2.48],
-            "Compression Rate": [0.357, "n/a", "n/a", "n/a", "n/a"],
-            "Verified": ['✓', '✓', '✓', '✓', '✓'],
-        },
-        "DNA": {
-            "Model": ["Gzip", "GPT4-o", "LLAMA-3.1-405B", "LLAMA-3.1-70B", "LLAMA-3.1-8B"],
-            "Pass@1": [100, 54.2, 6.5, 9.6, 1.4],
-            "Precision": [1.0, 1.94, 3.17, 3.17, 3.12],
-            "Compression Rate": [0.714, "n/a", "n/a", "n/a", "n/a"],
-            "Verified": ['✓', '✓', '✓', '✓', '✓'],
-        },
-        "Audio-8-bit": {
-            "Model": ["Gzip", "GPT4-o", "LLAMA-3.1-405B", "LLAMA-3.1-70B", "LLAMA-3.1-8B"],
-            "Pass@1": [100, 36.4, 15.0, 10.1, 3.9],
-            "Precision": [1.0, 1.43, 1.66, 1.67, 1.74],
-            "Compression Rate": [0.398, "n/a", "n/a", "n/a", "n/a"],
-            "Verified": ['✓', '✓', '✓', '✓', '✓'],
-        },
-        "Audio-16-bit": {
-            "Model": ["Gzip", "GPT4-o", "LLAMA-3.1-405B", "LLAMA-3.1-70B", "LLAMA-3.1-8B"],
-            "Pass@1": [100, 69.5, 35.6, 18.0, 5.9],
-            "Precision": [1.0, 1.34, 1.66, 1.96, 1.54],
-            "Compression Rate": [0.920, "n/a", "n/a", "n/a", "n/a"],
-            "Verified": ['✓', '✓', '✓', '✓', '✓'],
-        },
-        "Audio-MFCC": {
-            "Model": ["Gzip", "GPT4-o", "LLAMA-3.1-405B", "LLAMA-3.1-70B", "LLAMA-3.1-8B"],
-            "Pass@1": [100, 83.8, 29.6, 24.2, 8.8],
-            "Precision": [1.0, 1.33, 1.58, 1.56, 1.51],
-            "Compression Rate": [0.903, "n/a", "n/a", "n/a", "n/a"],
-            "Verified": ['✓', '✓', '✓', '✓', '✓'],
-        },
-        "Synthetic": {
-            "Model": ["SeqCoder-8B + Gzip", "Gzip", "GPT4-o", "LLAMA-3.1-405B", "LLAMA-3.1-70B", "LLAMA-3.1-8B"],
-            "Pass@1": [100, 100, 44.7, 24.8, 22.5, 3.7],
-            "Precision": [0.64, 1.0, 1.65, 2.06, 2.18, 2.34],
-            "Compression Rate": [0.38, 0.593, "n/a", "n/a", "n/a", "n/a"],
-            "Verified": ['✓', '✓', '✓', '✓', '✓', '✓'],
-        },
-    }
-    
-    
-    gr.HTML("<h2>KT Leaderboard - Can you beat Gzip on KT?</h2>")
-    for k,v in data.items():
-        with gr.Tab(k):
-            leaderboard_table_test = gr.Dataframe(
-                value=pd.DataFrame(data[k]),
-                interactive=False,
-            )
-     
-    
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
-            citation_text = """@inproceedings{
-anonymous2024the,
-title={The Ko{LM}ogorov Test: Compression by Code Generation},
-author={Anonymous},
-booktitle={Submitted to The Thirteenth International Conference on Learning Representations},
-year={2024},
-url={https://openreview.net/forum?id=C45YqeBDUM},
-note={under review}
+            citation_text = """@article{yoran-etal-2024-assistantbench,
+    title={AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?}, 
+    author={Ori Yoran and Samuel Amouyal and Chaitanya Malaviya and Ben Bogin and Ofir Press and Jonathan Berant},
+    year={2024},
+    eprint={?},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
 }"""
             citation_button = gr.Textbox(
                 value=citation_text,
@@ -299,9 +283,9 @@ note={under review}
             )
 
     gr.HTML(
-        "<p>We would like to thank the GAIA team for sharing the source code for their leaderboard which was used as a template and HuggingFace for hosting the leaderboard.</p>")
+        "<p>We would like to thank the GAIA team for sharing the source code for their leaderboard which we used as a template and HuggingFace for hosting the leaderboard.</p>")
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
-demo.launch(debug=True)
+demo.launch(debug=True)