app.py

import gradio as gr
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
import pandas as pd
from huggingface_hub import HfApi, create_repo
from datasets import Dataset, load_dataset
import os
import html
from src.about import (
    CITATION_BUTTON_LABEL,
    CITATION_BUTTON_TEXT,
    EVALUATION_QUEUE_TEXT,
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    TITLE,
)
from src.display.css_html_js import custom_css

HF_TOKEN = os.getenv('HF_TOKEN')
if not HF_TOKEN:
    raise ValueError("HF_TOKEN environment variable not found")
api = HfApi(token=HF_TOKEN)
DATASET_NAME = "airletters-leaderboard-results"


INITIAL_DATA = {
    "name": [
        "ViT-B/16", "MaxViT-T", "ResNet-200", "ResNeXt-101", "SE-ResNeXt-26", 
        "ResNet-50", "VideoMAE (16)", "ResNet-101 + LSTM", "ResNet-50 + LSTM",
        "ResNext-152 3D", "Strided Inflated EfficientNet 3D", "ResNext-50 3D",
        "ResNext-101 3D", "ResNext-200 3D", "Video-LLaVA (w/o contrast class)",
        "VideoLLaMA2 (w/o contrast class)", "Video-LLaVA", "VideoLLaMA2",
        "Human Performance (10 videos/class)"
    ],
    "url": ["https://arxiv.org/abs/2410.02921"] * 19,
    "top1_accuracy": [
        7.49, 7.56, 11.44, 13.09, 13.29, 13.87, 57.96, 58.45, 63.24,
        65.77, 65.97, 66.54, 69.74, 71.20, 2.53, 2.47, 7.29, 7.58, 96.67
    ],
    "organization": ["AirLetters Authors"] * 19,
    "model_type": [
        "Image", "Image", "Image", "Image", "Image",
        "Image", "Video", "Video", "Video",
        "Video", "Video", "Video", "Video", "Video",
        "Vision Language", "Vision Language", "Vision Language", "Vision Language",
        "Human Evaluation"
    ]
}

def initialize_dataset():
    try:
        dataset = load_dataset(f"rishitdagli/{DATASET_NAME}", split="train", token=HF_TOKEN, download_mode="force_redownload")
        df = dataset.to_pandas()
        if 'model_url' in df.columns:
            df = df.map(lambda row: {"name": model_hyperlink(row["model_url"], row["name"])})
            df = df.drop('model_url', axis=1)
            dataset = Dataset.from_pandas(df)
            dataset.push_to_hub(DATASET_NAME, token=HF_TOKEN)
    except Exception as e:
        print(f"Creating new dataset due to: {str(e)}")
        df = pd.DataFrame(INITIAL_DATA)
        dataset = Dataset.from_pandas(df)
        try:
            create_repo(DATASET_NAME, repo_type="dataset", token=HF_TOKEN)
        except Exception as e:
            print(f"Repo might already exist: {str(e)}")
        dataset.push_to_hub(DATASET_NAME, token=HF_TOKEN)
    return dataset

def calculate_accuracy(test_file, submitted_file):
    test = pd.read_csv(test_file)
    test2 = pd.read_csv(submitted_file)
    test.columns = test.columns.str.strip()
    test2.columns = test2.columns.str.strip()
    merged = pd.merge(test, test2, on="filename", how="left", suffixes=("_true", "_pred"))
    merged["label_pred"] = merged["label_pred"].fillna("")
    correct_predictions = (merged["label_true"] == merged["label_pred"]).sum()
    total_entries = len(merged)
    return (correct_predictions / total_entries) * 100

def model_hyperlink(link, model_name):
    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'

def update_leaderboard(name, organization, model_type, model_url, csv_file):
    top1_acc = calculate_accuracy("test.csv", csv_file)
 
    dataset = load_dataset(f"rishitdagli/{DATASET_NAME}", split="train", token=HF_TOKEN)
    df = dataset.to_pandas()
        
    new_row = {
        'name': name,
        'url': model_url,
        'organization': organization,
        'model_type': model_type,
        'top1_accuracy': top1_acc,
    }
    
    df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True)
    df = df.sort_values('top1_accuracy', ascending=False)
    df = df.reset_index(drop=True)
    
    new_dataset = Dataset.from_pandas(df)
    new_dataset.push_to_hub(DATASET_NAME, token=HF_TOKEN)
    
    return "Successfully updated leaderboard"

def init_leaderboard(dataframe):
    return Leaderboard(
        value=dataframe,
        datatype=["markdown", "str", "str", "number"],
        select_columns=SelectColumns(
            default_selection=["name", "organization", "model_type", "top1_accuracy"],
            cant_deselect=["name", "top1_accuracy"],
            label="Select Columns to Display",
        ),
        search_columns=["name", "organization"],
        filter_columns=[
            ColumnFilter("model_type", type="checkboxgroup", label="Model Type"),
        ],
    )

def refresh():
    dataset = load_dataset(f"rishitdagli/{DATASET_NAME}", split="train", token=HF_TOKEN, download_mode="force_redownload")
    dataset = dataset.map(lambda row: {"name": model_hyperlink(row["url"], row["name"])})
    df = dataset.to_pandas()
    return df

def create_interface():
    demo = gr.Blocks(css=custom_css)
    
    with demo:
        gr.HTML(TITLE)
        gr.Video("30fps.mp4", autoplay=True, width=900, loop=True, include_audio=False)
        
        with gr.Tabs(elem_classes="tab-buttons") as tabs:
            with gr.TabItem("🏅 Leaderboard", elem_id="leaderboard-tab"):
                dataset = initialize_dataset()
                dataset = dataset.map(lambda row: {"name": model_hyperlink(row["url"], row["name"])})
                df = dataset.to_pandas()
                leaderboard = init_leaderboard(df)

                refresh_button = gr.Button("Refresh")
                refresh_button.click(
                    refresh,
                    inputs=[],
                    outputs=[
                        leaderboard,
                    ],
                )
                
            with gr.TabItem("📝 About", elem_id="about-tab"):
                gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
                
            with gr.TabItem("🚀 Submit", elem_id="submit-tab"):
                gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
                
                with gr.Column():
                    name = gr.Textbox(label="Model Name")
                    model_url = gr.Textbox(label="Model URL", placeholder="https://huggingface.co/...")
                    org = gr.Textbox(label="Organization")
                    model_type = gr.Dropdown(
                        choices=["Image", "Video", "Vision Language", "Tracking", "Other"],
                        label="Model Type"
                    )
                    csv_file = gr.File(label="Results CSV", type="filepath")
                    submit_btn = gr.Button("Submit")
                    result = gr.Textbox(label="Result")
                    
                    submit_btn.click(
                        update_leaderboard,
                        inputs=[name, org, model_type, model_url, csv_file],
                        outputs=[result]
                    )
        with gr.Row():
            with gr.Accordion("📙 Citation", open=False):
                citation_button = gr.Textbox(
                    value=CITATION_BUTTON_TEXT,
                    label=CITATION_BUTTON_LABEL,
                    lines=7,
                    elem_id="citation-button",
                    show_copy_button=True,
                )
    
    return demo

if __name__ == "__main__":
    demo = create_interface()
    demo.queue().launch()