app.py

import gradio as gr
from codecarbon import EmissionsTracker
import os
import json
from datetime import datetime
import requests
from huggingface_hub import HfApi
import tempfile
from dotenv import load_dotenv

# Load environment variables
load_dotenv()

# Get environment variables
HF_TOKEN = os.getenv("HF_TOKEN")
if not HF_TOKEN:
    print("Warning: HF_TOKEN not found in environment variables. Submissions will not work.")

api = HfApi(token=HF_TOKEN)


DEFAULT_PARAMS = {
    "text":{
        "dataset_name": "QuotaClimat/frugalaichallenge-text-train",
        "test_size": 0.2,  # must be between 0 and 1
        "test_seed": 42,   # must be non-negative
    },
    "image":{
        "dataset_name": "pyronear/pyro-sdis",
        "test_size": 0.2,  # must be between 0 and 1
        "test_seed": 42,   # must be non-negative
    },
    "audio":{
        "dataset_name": "rfcx/frugalai",
        "test_size": 0.2,  # must be between 0 and 1
        "test_seed": 42,   # must be non-negative
    }
}


def evaluate_model(task: str, space_url: str):
    """
    Evaluate a model through its API endpoint
    """
    # username = space_url.split("/")[0]

    if "localhost" in space_url:
        api_url = f"{space_url}/{task}"
    else:

        try:
            info_space = api.space_info(repo_id=space_url)
        except:
            return None, None, None, gr.Warning(f"Space '{space_url}' not found, it needs to be in the format username/space-name")

        
        host = info_space.host
        api_url = f"{host}/{task}"
    
    try:
        # Make API call to the space
        params = DEFAULT_PARAMS[task]
        response = requests.post(api_url, json=params)
        if response.status_code != 200:
            return None, None, None, gr.Warning(f"API call failed with status {response.status_code}")
        
        results = response.json()
        
        # Check for required keys based on task
        base_required_keys = {
            "username", "space_url", "submission_timestamp", "model_description",
            "energy_consumed_wh", "emissions_gco2eq", "emissions_data",
            "api_route", "dataset_config"
        }

        # Add task-specific accuracy keys
        if task == "image":
            accuracy_keys = {"classification_accuracy", "mean_iou"}
        else:  # text and audio
            accuracy_keys = {"accuracy"}
        
        required_keys = base_required_keys | accuracy_keys
        
        missing_keys = required_keys - set(results.keys())
        if missing_keys:
            return None, None, None, gr.Warning(f"API response missing required keys: {', '.join(missing_keys)}")
        
        # Return appropriate accuracy metric based on task
        if task == "image":
            accuracy = results["classification_accuracy"]  # For display in UI
        else:
            accuracy = results["accuracy"]
        
        return (
            accuracy,
            results["emissions_gco2eq"],
            results["energy_consumed_wh"],
            results
        )
        
    except Exception as e:
        return None, None, None, gr.Warning(str(e))


def submit_results(task: str, results_json):
    if not results_json:
        return gr.Warning("No results to submit")
    
    if not HF_TOKEN:
        return gr.Warning("HF_TOKEN not found. Please set up your Hugging Face token.")
    
    try:
        results_str = json.dumps(results_json)
        
        # Create a temporary file with the results
        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as f:
            f.write(results_str)
            temp_path = f.name
        
        # Upload to the appropriate dataset based on task
        api = HfApi(token=HF_TOKEN)
        path_in_repo = f"submissions/{results_json['username']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
        
        dataset_mapping = {
            "text": "frugal-ai-challenge/public-leaderboard-text",
            "image": "frugal-ai-challenge/public-leaderboard-image",
            "audio": "frugal-ai-challenge/public-leaderboard-audio"
        }
        
        api.upload_file(
            path_or_fileobj=temp_path,
            path_in_repo=path_in_repo,
            repo_id=dataset_mapping[task],
            repo_type="dataset",
            token=HF_TOKEN
        )
        
        # Clean up
        os.unlink(temp_path)
        
        return gr.Info("Results submitted successfully to the leaderboard! 🎉")
    except Exception as e:
        return gr.Warning(f"Error submitting results: {str(e)}")

# Create the demo interface
with gr.Blocks() as demo:

    gr.Image("./logo.png", show_label=False, container=False)
    gr.Markdown("""
    # Frugal AI Challenge - Submission Portal
    Submit your model results for any of the three tasks: Text, Image, or Audio classification.
    """)
    
    with gr.Tabs():


        with gr.Tab("Instructions"):

            gr.Markdown("""
To submit your results in one of the three tasks, please follow the steps below:
                        
## Prepare your model submission
1. Duplicate the template of the submission API by duplicating this space https://huggingface.co/spaces/frugal-ai-challenge/submission-template on your own Hugging Face account.
2. In ``tasks/text.py``, ``tasks/image.py``, or ``tasks/audio.py``, modify the ``evaluate_model`` function to replace the baseline by your model loading and inference within the inference pass where the energy consumption and emissions are tracked.
3. Eventually complete the requirements and/or any necessaries dependencies in your space.
4. Write down your model card in the ``README.md`` file.
5. Deploy your space (FastAPI) and verify that it works.
6. (Optional) You can change the Space hardware to use any GPU directly on Hugging Face.
                        
## Submit your model to the leaderboard in the ``Model Submission`` tab
When your API is deployed : 

0. Fill out the [submission form](https://framaforms.org/2025-frugal-ai-challenge-submission-form-1736883260-0) with all the details regarding your team and project.
1. Select the task you want to submit your model to
2. Enter the Space URL of your API
3. (Optional) Precise the API route (default is ``/text``, ``/image``, or ``/audio``)
4. Step 1 - Evaluate model: Click on the button to evaluate your model. This will run you model on your API, computes the accuracy on the test set (20% of the train set), and track the energy consumption and emissions.
5. Step 2 - Submit to leaderboard: Click on the button to submit your results to the leaderboard. This will upload the results to the leaderboard dataset and update the leaderboard.
6. You can see the leaderboards at 
    - Text - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-text
    - Image - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-image
    - Audio - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-audio

## About
> You can find more information about the Frugal AI Challenge 2025 on the [Frugal AI Challenge website](https://frugalaichallenge.org/).
> Or directly on the organization page on Hugging Face: [Frugal AI Challenge](https://huggingface.co/frugal-ai-challenge)
                        
This portal is a submission portal for the Frugal AI Challenge 2025. It is a simple interface to evaluate and submit your model to the leaderboard.
The challenge is organized by Hugging Face, Data For Good, and the French Ministry of Environment. 

The goal of the Frugal AI Challenge is to encourage both academic and industry actors to keep efficiency in mind when deploying AI models. By tracking both energy consumption and performance for different AI tasks, we can incentivize frugality in AI deployment while also addressing real-world challenges.
""")

        # Text Classification Task
        with gr.Tab("📜 Text Classification"):
            with gr.Row():
                text_space_url = gr.Textbox(
                    label="Space URL",
                    placeholder="username/your-space",
                    lines=1
                )
                text_route = gr.Textbox(
                    label="API route (Advanced)",
                    value="/text",
                    lines=1
                )
            
            with gr.Row():
                with gr.Column(scale=1):
                    text_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
                with gr.Column(scale=1):
                    text_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary")
                    
            with gr.Row():
                text_accuracy = gr.Number(label="Accuracy", precision=4)
                text_energy = gr.Number(label="Energy Consumed (Wh)", precision=12)
                text_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12)
            with gr.Row():
                text_results_json = gr.JSON(label="Detailed Results", visible=True)
        
        # Image Classification Task
        with gr.Tab("🎥 Image Classification"):
            with gr.Row():
                image_space_url = gr.Textbox(
                    label="Space URL",
                    placeholder="username/your-space",
                    lines=1
                )
                image_route = gr.Textbox(
                    label="API route (Advanced)",
                    value="/image",
                    lines=1
                )
            
            with gr.Row():
                with gr.Column(scale=1):
                    image_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
                with gr.Column(scale=1):
                    image_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary")
                    
            with gr.Row():
                image_accuracy = gr.Number(label="Accuracy", precision=4)
                image_energy = gr.Number(label="Energy Consumed (Wh)", precision=12)
                image_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12)
            with gr.Row():
                image_results_json = gr.JSON(label="Detailed Results", visible=True)
        
        # Audio Classification Task
        with gr.Tab("🔊 Audio Classification"):
            with gr.Row():
                audio_space_url = gr.Textbox(
                    label="Space URL",
                    placeholder="username/your-space",
                    lines=1
                )
                audio_route = gr.Textbox(
                    label="API route (Advanced)",
                    value="/audio",
                    lines=1
                )
            
            with gr.Row():
                with gr.Column(scale=1):
                    audio_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
                with gr.Column(scale=1):
                    audio_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary")
                    
            with gr.Row():
                audio_accuracy = gr.Number(label="Accuracy", precision=4)
                audio_energy = gr.Number(label="Energy Consumed (Wh)", precision=12)
                audio_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12)
            with gr.Row():
                audio_results_json = gr.JSON(label="Detailed Results", visible=True)

    # Set up event handlers
    text_evaluate_btn.click(
        lambda url, route: evaluate_model(route.strip("/"), url),
        inputs=[text_space_url, text_route],
        outputs=[text_accuracy, text_emissions, text_energy, text_results_json]
    )
    
    image_evaluate_btn.click(
        lambda url, route: evaluate_model(route.strip("/"), url),
        inputs=[image_space_url, image_route],
        outputs=[image_accuracy, image_emissions, image_energy, image_results_json]
    )
    
    audio_evaluate_btn.click(
        lambda url, route: evaluate_model(route.strip("/"), url),
        inputs=[audio_space_url, audio_route],
        outputs=[audio_accuracy, audio_emissions, audio_energy, audio_results_json]
    )
    
    text_submit_btn.click(
        lambda results: submit_results("text", results),
        inputs=[text_results_json],
        outputs=None
    )
    
    image_submit_btn.click(
        lambda results: submit_results("image", results),
        inputs=[image_results_json],
        outputs=None
    )
    
    audio_submit_btn.click(
        lambda results: submit_results("audio", results),
        inputs=[audio_results_json],
        outputs=None
    )

if __name__ == "__main__":
    demo.launch()