Update app.py

app.py

@@ -1,129 +1,96 @@
 import gradio as gr
 from codecarbon import EmissionsTracker
-from datasets import load_dataset
-import numpy as np
-from sklearn.metrics import accuracy_score
-import random
 import os
 import json
 from datetime import datetime
+import requests
 from huggingface_hub import HfApi
-from huggingface_hub import upload_file
 import tempfile
 from dotenv import load_dotenv
-import spaces
 
-# Use dotenv to load the environment variables
+# Load environment variables
 load_dotenv()
 
 # Get environment variables
-HF_TOKEN = os.getenv("HF_TOKEN_TEXT")
-DATASET_NAME = os.getenv("DATASET_NAME", "QuotaClimat/frugalaichallenge-text-train")  # Default to public dataset
-TEST_SIZE = float(os.getenv("TEST_SIZE", "0.2"))  # Default to 20% test size
-TEST_SEED = int(os.getenv("TEST_SEED", "42"))  # Default seed for reproducibility
-
-print(f"Using dataset: {DATASET_NAME}")
-print(f"Test split size: {TEST_SIZE}")
-
+HF_TOKEN = os.getenv("HF_TOKEN")
 if not HF_TOKEN:
     print("Warning: HF_TOKEN not found in environment variables. Submissions will not work.")
 
-# Initialize carbon emissions tracker with CodeCarbon
-tracker = EmissionsTracker(allow_multiple_runs=True)
-
-#--------------------------------------------------------------------------------------------
-# FUNCTION TO UPDATE WITH YOUR MODEL SUBMISSION
-#--------------------------------------------------------------------------------------------
-
-@spaces.GPU
-def evaluate(model_description):
-    # Get space info
-    username, space_url = get_space_info()
-    
-    # Initialize tracker
-    tracker.start()
-    tracker.start_task("inference")
-
-    #--------------------------------------------------------------------------------------------
-    # YOUR MODEL INFERENCE CODE HERE
-    # Update the code below to replace the random baseline by your model inference within the inference pass where the energy consumption and emissions are tracked.
-    #--------------------------------------------------------------------------------------------   
-    
-    # Make random predictions
-    true_labels = test_dataset["label"]
-    predictions = [random.randint(0, 7) for _ in range(len(true_labels))]
-
-    #--------------------------------------------------------------------------------------------
-    # YOUR MODEL INFERENCE STOPS HERE
-    #--------------------------------------------------------------------------------------------   
-
-    # Stop tracking emissions
-    emissions_data = tracker.stop_task()    
-
-    # Calculate accuracy
-    accuracy = accuracy_score(true_labels, predictions)
-    
-    # Prepare complete results
-    results = {
-        "username": username,
-        "space_url": space_url,
-        "submission_timestamp": datetime.now().isoformat(),
-        "model_description": model_description if model_description else "No description provided",
-        "accuracy": float(accuracy),
-        "energy_consumed_wh": emissions_data.energy_consumed * 1000,
-        "emissions_gco2eq": emissions_data.emissions * 1000,
-        "emissions_data": clean_emissions_data(emissions_data)
+api = HfApi(token=HF_TOKEN)
+
+
+DEFAULT_PARAMS = {
+    "text":{
+        "dataset_name": "QuotaClimat/frugalaichallenge-text-train",
+        "test_size": 0.2,  # must be between 0 and 1
+        "test_seed": 42,   # must be non-negative
+    },
+    "image":{
+        "dataset_name": "QuotaClimat/frugalaichallenge-image-train",
+        "test_size": 0.2,  # must be between 0 and 1
+        "test_seed": 42,   # must be non-negative
+    },
+    "audio":{
+        "dataset_name": "QuotaClimat/frugalaichallenge-audio-train",
+        "test_size": 0.2,  # must be between 0 and 1
+        "test_seed": 42,   # must be non-negative
     }
-    
-    # Return both summary and detailed results
-    return [
-        accuracy,
-        emissions_data.emissions * 1000,
-        emissions_data.energy_consumed * 1000,
-        json.dumps(results, indent=2)
-    ]
+}
 
 
-#--------------------------------------------------------------------------------------------
-# HELPER FUNCTIONS
-#--------------------------------------------------------------------------------------------
+def evaluate_model(task: str, space_url: str):
+    """
+    Evaluate a model through its API endpoint
+    """
+    # username = space_url.split("/")[0]
 
-# Function to get space username and URL
-def get_space_info():
-    space_name = os.getenv("SPACE_ID", "")
-    if space_name:
-        try:
-            username = space_name.split("/")[0]
-            space_url = f"https://huggingface.co/spaces/{space_name}"
-            return username, space_url
-        except Exception as e:
-            print(f"Error getting space info: {e}")
-    return "local-user", "local-development"
+    if "localhost" in space_url:
+        api_url = f"{space_url}/{task}"
+    else:
+        api_url = f"https://{space_url.lower().replace('/', '-')}.hf.space/{task}"
 
-def clean_emissions_data(emissions_data):
-    """Remove unwanted fields from emissions data"""
-    data_dict = emissions_data.__dict__
-    fields_to_remove = ['timestamp', 'project_name', 'experiment_id', 'latitude', 'longitude']
-    return {k: v for k, v in data_dict.items() if k not in fields_to_remove}
+        # Check if the space exists, will raise an error if it doesn't
+        api.space_info(repo_id=space_url)
+    
+    try:
+        # Make API call to the space
+        params = DEFAULT_PARAMS[task]
+        response = requests.post(api_url, json=params)
+        if response.status_code != 200:
+            return None, None, None, gr.Warning(f"API call failed with status {response.status_code}")
+        
+        results = response.json()
+        
+        # Check for required keys
+        required_keys = {
+            "username", "space_url", "submission_timestamp", "model_description", 
+            "accuracy", "energy_consumed_wh", "emissions_gco2eq", "emissions_data",
+            "api_route", "dataset_config"
+        }
+        
+        missing_keys = required_keys - set(results.keys())
+        if missing_keys:
+            return None, None, None, gr.Warning(f"API response missing required keys: {', '.join(missing_keys)}")
+        
+        return (
+            results["accuracy"],
+            results["emissions_gco2eq"],
+            results["energy_consumed_wh"],
+            results
+        )
+        
+    except Exception as e:
+        return None, None, None, gr.Warning(str(e))
 
 
-def submit_results(results_json):
+def submit_results(task: str, results_json):
     if not results_json:
         return gr.Warning("No results to submit")
     
-    # Check if we're in a Space or have admin dev rights
-    space_name = os.getenv("SPACE_ID")
-    is_admin_dev = os.getenv("ADMIN_DEV") == "true"
-    
-    if not space_name and not is_admin_dev:
-        message = "You cannot submit your model locally, you need to deploy it as a Hugging Face Space first, and then submit it."
-        return gr.Warning(message)
-    
     if not HF_TOKEN:
         return gr.Warning("HF_TOKEN not found. Please set up your Hugging Face token.")
     
     try:
-        # results_json is already a dict from gr.JSON
         results_str = json.dumps(results_json)
         
         # Create a temporary file with the results
@@ -131,13 +98,20 @@ def submit_results(results_json):
             f.write(results_str)
             temp_path = f.name
         
-        # Upload to the dataset
+        # Upload to the appropriate dataset based on task
         api = HfApi(token=HF_TOKEN)
         path_in_repo = f"submissions/{results_json['username']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+        
+        dataset_mapping = {
+            "text": "frugal-ai-challenge/public-leaderboard-text",
+            "image": "frugal-ai-challenge/public-leaderboard-image",
+            "audio": "frugal-ai-challenge/public-leaderboard-audio"
+        }
+        
         api.upload_file(
             path_or_fileobj=temp_path,
             path_in_repo=path_in_repo,
-            repo_id="frugal-ai-challenge/public-leaderboard-text",
+            repo_id=dataset_mapping[task],
             repo_type="dataset",
             token=HF_TOKEN
         )
@@ -148,69 +122,43 @@ def submit_results(results_json):
         return gr.Info("Results submitted successfully to the leaderboard! 🎉")
     except Exception as e:
         return gr.Warning(f"Error submitting results: {str(e)}")
-    
-
-#--------------------------------------------------------------------------------------------
-# DATASET PREPARATION
-#--------------------------------------------------------------------------------------------
-
-# Define the label mapping
-LABEL_MAPPING = {
-    "0_not_relevant": 0,  # No relevant claim detected
-    "1_not_happening": 1,  # Global warming is not happening
-    "2_not_human": 2,  # Not caused by humans
-    "3_not_bad": 3,  # Not bad or beneficial
-    "4_solutions_harmful_unnecessary": 4,  # Solutions harmful/unnecessary
-    "5_science_unreliable": 5,  # Science is unreliable
-    "6_proponents_biased": 6,  # Proponents are biased
-    "7_fossil_fuels_needed": 7  # Fossil fuels are needed
-}
-
-# Load and prepare the dataset
-print("Loading dataset...")
-dataset = load_dataset(DATASET_NAME)
-
-# Convert string labels to integers
-dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]})
-
-# Split dataset
-train_test = dataset["train"].train_test_split(test_size=TEST_SIZE, seed=TEST_SEED)
-train_dataset = train_test["train"]
-test_dataset = train_test["test"]
-
-#--------------------------------------------------------------------------------------------
-# GRADIO INTERFACE
-#--------------------------------------------------------------------------------------------
 
 # Create the demo interface
 with gr.Blocks() as demo:
 
     gr.Image("./logo.png", show_label=False, container=False)
-
     gr.Markdown("""
-# 📜 Frugal AI Challenge - Text task - Submission portal
-## Climate Disinformation Classification
-""")
-
+    # Frugal AI Challenge - Submission Portal
+    Submit your model results for any of the three tasks: Text, Image, or Audio classification.
+    """)
+    
     with gr.Tabs():
 
+
         with gr.Tab("Instructions"):
 
             gr.Markdown("""
-To submit your results, please follow the steps below:
+To submit your results in one of the three tasks, please follow the steps below:
                         
 ## Prepare your model submission
-1. Clone the space of this portal on your own Hugging Face account.
-2. Modify the ``evaluate`` function to replace the baseline by your model loading and inference within the inference pass where the energy consumption and emissions are tracked.
+1. Duplicate the template of the submission API by duplicating this space on your own Hugging Face account.
+2. In ``tasks/text.py``, ``tasks/image.py``, or ``tasks/audio.py``, modify the ``evaluate_model`` function to replace the baseline by your model loading and inference within the inference pass where the energy consumption and emissions are tracked.
 3. Eventually complete the requirements and/or any necessaries dependencies in your space.
-4. Write down your model card in the ``modelcard.md`` file.
-5. Deploy your space and verify that it works.
+4. Write down your model card in the ``README.md`` file.
+5. Deploy your space (FastAPI) and verify that it works.
 6. (Optional) You can change the Space hardware to use any GPU directly on Hugging Face.
                         
 ## Submit your model to the leaderboard in the ``Model Submission`` tab
-7. Step 1 - Evaluate model: Click on the button to evaluate your model. This will run you model, computes the accuracy on the test set (20% of the train set), and track the energy consumption and emissions.
-8. Step 2 - Submit to leaderboard: Click on the button to submit your results to the leaderboard. This will upload the results to the leaderboard dataset and update the leaderboard.
-9. You can see the leaderboard at https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-text
+When your API is deployed : 
+1. Select the task you want to submit your model to
+2. Enter the Space URL of your API
+3. (Optional) Precise the API route (default is ``/text``, ``/image``, or ``/audio``)
+4. Step 1 - Evaluate model: Click on the button to evaluate your model. This will run you model on your API, computes the accuracy on the test set (20% of the train set), and track the energy consumption and emissions.
+5. Step 2 - Submit to leaderboard: Click on the button to submit your results to the leaderboard. This will upload the results to the leaderboard dataset and update the leaderboard.
+6. You can see the leaderboards at 
+    - Text - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-text
+    - Image - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-image
+    - Audio - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-audio
 
 ## About
 > You can find more information about the Frugal AI Challenge 2025 on the [Frugal AI Challenge website](https://frugal-ai-challenge.org/).
@@ -222,50 +170,123 @@ The challenge is organized by Hugging Face, Data For Good, and the French Minist
 The goal of the Frugal AI Challenge is to encourage both academic and industry actors to keep efficiency in mind when deploying AI models. By tracking both energy consumption and performance for different AI tasks, we can incentivize frugality in AI deployment while also addressing real-world challenges.
 """)
 
-        with gr.Tab("Model Submission"):
-            
+        # Text Classification Task
+        with gr.Tab("📜 Text Classification"):
             with gr.Row():
-                model_description = gr.Textbox(
-                    label="Model Description (one sentence)",
-                    placeholder="Describe your model in one sentence...",
-                    value="Random baseline",
-                    lines=2
+                text_space_url = gr.Textbox(
+                    label="Space URL",
+                    placeholder="username/your-space",
+                    lines=1
+                )
+                text_route = gr.Textbox(
+                    label="API route (Advanced)",
+                    value="/text",
+                    lines=1
                 )
             
             with gr.Row():
                 with gr.Column(scale=1):
-                    evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
+                    text_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
                 with gr.Column(scale=1):
-                    submit_btn = gr.Button("2. Submit to leaderboard", variant="primary", size="lg")
+                    text_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary")
                     
             with gr.Row():
-                accuracy_output = gr.Number(label="Accuracy", precision=4)
-                energy_output = gr.Number(label="Energy Consumed (Wh)", precision=12)
-                emissions_output = gr.Number(label="Emissions (gCO2eq)", precision=12)
+                text_accuracy = gr.Number(label="Accuracy", precision=4)
+                text_energy = gr.Number(label="Energy Consumed (Wh)", precision=12)
+                text_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12)
             with gr.Row():
-                results_json = gr.JSON(label="Detailed Results", visible=True)
+                text_results_json = gr.JSON(label="Detailed Results", visible=True)
         
-            evaluate_btn.click(
-                evaluate,
-                inputs=[model_description],
-                outputs=[accuracy_output, emissions_output, energy_output, results_json]
-            )
+        # Image Classification Task
+        with gr.Tab("🎥 Image Classification"):
+            with gr.Row():
+                image_space_url = gr.Textbox(
+                    label="Space URL",
+                    placeholder="username/your-space",
+                    lines=1
+                )
+                image_route = gr.Textbox(
+                    label="API route (Advanced)",
+                    value="/image",
+                    lines=1
+                )
             
-            submit_btn.click(
-                submit_results,
-                inputs=[results_json],
-                outputs=None  # No need for output component with popups
-            )
+            with gr.Row():
+                with gr.Column(scale=1):
+                    image_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
+                with gr.Column(scale=1):
+                    image_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary")
+                    
+            with gr.Row():
+                image_accuracy = gr.Number(label="Accuracy", precision=4)
+                image_energy = gr.Number(label="Energy Consumed (Wh)", precision=12)
+                image_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12)
+            with gr.Row():
+                image_results_json = gr.JSON(label="Detailed Results", visible=True)
         
-        with gr.Tab("Model Card"):
-            with open("README.md", "r") as f:
-                content = f.read()
-                # Remove the YAML header (content between --- markers)
-                if content.startswith("---"):
-                    second_marker = content.find("---", 3)
-                    if second_marker != -1:
-                        content = content[second_marker + 3:].strip()
-            gr.Markdown(content)
+        # Audio Classification Task
+        with gr.Tab("🔊 Audio Classification"):
+            with gr.Row():
+                audio_space_url = gr.Textbox(
+                    label="Space URL",
+                    placeholder="username/your-space",
+                    lines=1
+                )
+                audio_route = gr.Textbox(
+                    label="API route (Advanced)",
+                    value="/audio",
+                    lines=1
+                )
+            
+            with gr.Row():
+                with gr.Column(scale=1):
+                    audio_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary")
+                with gr.Column(scale=1):
+                    audio_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary")
+                    
+            with gr.Row():
+                audio_accuracy = gr.Number(label="Accuracy", precision=4)
+                audio_energy = gr.Number(label="Energy Consumed (Wh)", precision=12)
+                audio_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12)
+            with gr.Row():
+                audio_results_json = gr.JSON(label="Detailed Results", visible=True)
+
+    # Set up event handlers
+    text_evaluate_btn.click(
+        lambda url, route: evaluate_model(route.strip("/"), url),
+        inputs=[text_space_url, text_route],
+        outputs=[text_accuracy, text_emissions, text_energy, text_results_json]
+    )
+    
+    image_evaluate_btn.click(
+        lambda url, route: evaluate_model(route.strip("/"), url),
+        inputs=[image_space_url, image_route],
+        outputs=[image_accuracy, image_emissions, image_energy, image_results_json]
+    )
+    
+    audio_evaluate_btn.click(
+        lambda url, route: evaluate_model(route.strip("/"), url),
+        inputs=[audio_space_url, audio_route],
+        outputs=[audio_accuracy, audio_emissions, audio_energy, audio_results_json]
+    )
+    
+    text_submit_btn.click(
+        lambda results: submit_results("text", results),
+        inputs=[text_results_json],
+        outputs=None
+    )
+    
+    image_submit_btn.click(
+        lambda results: submit_results("image", results),
+        inputs=[image_results_json],
+        outputs=None
+    )
+    
+    audio_submit_btn.click(
+        lambda results: submit_results("audio", results),
+        inputs=[audio_results_json],
+        outputs=None
+    )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()