moved writing results to csv to tasks api

app.py

@@ -15,6 +15,8 @@ import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 import boto3
+import json
+from plotly.subplots import make_subplots
 
 fs = fsspec.filesystem(
         's3',
@@ -29,6 +31,8 @@ s3_client = boto3.client(
     )
 
 SAVE_PATH = f"s3://{os.getenv('AWS_BUCKET_NAME')}/{os.getenv('RESULTS_KEY')}"
+ELO_JSON_PATH = f"s3://{os.getenv('AWS_BUCKET_NAME')}/{os.getenv('ELO_JSON_PATH')}"
+ELO_CSV_PATH = f"s3://{os.getenv('AWS_BUCKET_NAME')}/{os.getenv('ELO_CSV_KEY')}"
 EMAIL_PATH = f"s3://{os.getenv('AWS_BUCKET_NAME')}/{os.getenv('EMAILS_KEY')}"
 TEMP_DIR = f"s3://{os.getenv('AWS_BUCKET_NAME')}/{os.getenv('AUDIOS_KEY')}"
 CREATE_TASK_URL = os.getenv("CREATE_TASK_URL")
@@ -57,87 +61,41 @@ class ResultWriter:
             'Ori Apex_duration', 'Ori Apex XT_duration', 'deepgram_duration', 'Ori Swift_duration', 'Ori Prime_duration','azure_score','azure_appearance','azure_duration'
         ]
 
+        self.models = ['Ori Apex', 'Ori Apex XT', 'deepgram', 'Ori Swift', 'Ori Prime', 'azure']
+
         if not fs.exists(save_path):
             print("CSV File not found in s3 bucket creating a new one",save_path)
             with fs.open(save_path, 'wb') as f:
                 df = pd.DataFrame(columns=self.headers)
                 df.to_csv(f, index=False)
 
-    def expected_rating(self, rating_a, rating_b):
-        return 1 / (1 + 10 ** ((rating_a - rating_b) / 400))
-
+    def write_result(self,
+                    user_email,
+                    audio_path,
+                    option_1_duration_info,
+                    option_2_duration_info,
+                    winner_model=None,
+                    loser_model=None,
+                    both_preferred=False,
+                    none_preferred=False
+                    ):
 
-    def updateElo(self,rating_a, rating_b, outcome, baseKFactor):
-        # Calculate the rating difference
-        ratingDiff = abs(rating_a - rating_b)
-
-        if ratingDiff > 25:
-            kFactor = baseKFactor + 10
-        else:
-            kFactor = baseKFactor
-
-        expectedA = self.expected_rating(rating_a, rating_b)
-        expectedB = self.expected_rating(rating_b, rating_a)
-
-        if outcome == 'win':
-            scoreA = 1
-            scoreB = 0
-        elif outcome == 'tie':
-            scoreA = 0.5
-            scoreB = 0.5
-        else:  # no_result
-            # No change in ratings for no result
-            return rating_a, rating_b
-
-        newRatingA = rating_a + kFactor * (scoreA - expectedA)
-        newRatingB = rating_b + kFactor * (scoreB - expectedB)
-
-        return round(newRatingA, 3), round(newRatingB, 3)
-
-    def write_result(self,user_email ,audio_path,option_1_duration_info,option_2_duration_info ,winner_model=None, loser_model=None, both_preferred=False, none_preferred=False):
-        result = {
-            'email': user_email,
-            'path': audio_path,
-            'Ori Apex_score': 0, 'Ori Apex XT_score': 0, 'deepgram_score': 0, 'Ori Swift_score': 0, 'Ori Prime_score': 0,
-            'Ori Apex_appearance': 0, 'Ori Apex XT_appearance': 0, 'deepgram_appearance': 0, 'Ori Swift_appearance': 0, 'Ori Prime_appearance': 0,
-            'Ori Apex_duration':0, 'Ori Apex XT_duration':0, 'deepgram_duration':0, 'Ori Swift_duration':0, 'Ori Prime_duration':0,'azure_score':0,'azure_appearance':0,'azure_duration':0
+        payload = {
+            "task":"write_result",
+            "payload":{
+                "winner_model":winner_model,
+                "loser_model":loser_model,
+                "both_preferred":both_preferred,
+                "none_preferred":none_preferred,
+                "user_email":user_email,
+                "audio_path":audio_path,
+                "option_1_duration_info":option_1_duration_info,
+                "option_2_duration_info":option_2_duration_info
+            }
         }
 
-        if winner_model:
-            result[f'{winner_model}_appearance'] = 1
-
-        if loser_model:
-            result[f'{loser_model}_appearance'] = 1
-
-        if both_preferred:
-            if winner_model:
-                result[f'{winner_model}_score'] = 1
-            if loser_model:
-                result[f'{loser_model}_score'] = 1
-        elif not none_preferred and winner_model:
-            result[f'{winner_model}_score'] = 1
+        send_task(payload)
 
-        if option_1_duration_info and option_1_duration_info[0]:
-            duration_key, duration_value = option_1_duration_info[0]
-            if duration_key in self.headers:
-                result[duration_key] = float(duration_value)
-
-        if option_2_duration_info and option_2_duration_info[0]:
-            duration_key, duration_value = option_2_duration_info[0]
-            if duration_key in self.headers:
-                result[duration_key] = float(duration_value)
-
-        self.write_to_s3(result)
-
-    def write_to_s3(self,result):
-        with fs.open(self.save_path, 'rb') as f:
-            df = pd.read_csv(f)
-
-        records = df.to_dict('records')
-        records.append(result)
-        df = pd.DataFrame(records)
-        with fs.open(self.save_path, 'wb') as f:
-            df.to_csv(f, index=False)
 
 
 def decode_audio_array(base64_string):
@@ -433,6 +391,33 @@ def create_head_to_head_matrix(df):
 
     return fig
 
+def create_elo_chart(df):
+    fig = make_subplots(rows=1, cols=1,
+                    subplot_titles=('ELO Rating Progression'),
+                    row_heights=[0.7])
+
+    for column in df.columns:
+        fig.add_trace(
+            go.Scatter(
+                x=list(range(len(df))),
+                y=df[column],
+                name=column,
+                mode='lines+markers'
+            ),
+            row=1, col=1
+        )
+
+    fig.update_layout(
+        title='Model ELO Ratings Analysis',
+        showlegend=True,
+        hovermode='x unified'
+    )
+
+    fig.update_xaxes(title_text='Match Number', row=1, col=1)
+    fig.update_xaxes(title_text='Models', row=2, col=1)
+
+    return fig
+
 def create_metric_container(label, value, full_name=None):
     container = st.container()
     with container:
@@ -447,12 +432,36 @@ def on_refresh_click():
     with fs.open(SAVE_PATH, 'rb') as f:
         st.session_state.df = pd.read_csv(f)
 
+    try:
+        with fs.open(ELO_JSON_PATH,'r') as f:
+            st.session_state.elo_json = json.load(f)
+    except Exception:
+        st.session_state.elo_json = None
+
+    try:
+        with fs.open(ELO_CSV_PATH,'rb') as f:
+            st.session_state.elo_df = pd.read_csv(f)
+    except Exception:
+        st.session_state.elo_df = None
+
 def dashboard():
     st.title('Model Arena Scoreboard')
 
     if "df" not in st.session_state:
         with fs.open(SAVE_PATH, 'rb') as f:
             st.session_state.df = pd.read_csv(f)
+    if "elo_json" not in st.session_state:
+        try:
+            with fs.open(ELO_JSON_PATH,'r') as f:
+                st.session_state.elo_json = json.load(f)
+        except Exception:
+            st.session_state.elo_json = None
+    if "elo_df" not in st.session_state:
+        try:
+            with fs.open(ELO_CSV_PATH,'rb') as f:
+                st.session_state.elo_df = pd.read_csv(f)
+        except Exception:
+            st.session_state.elo_df = None
 
     st.button("Refresh",on_click=on_refresh_click)
 
@@ -487,7 +496,8 @@ def dashboard():
         with col1:
             create_metric_container("Total Matches", len(st.session_state.df))
 
-        best_model = max(metrics.items(), key=lambda x: x[1]['win_rate'])[0]
+        # best_model = max(metrics.items(), key=lambda x: x[1]['win_rate'])[0]
+        best_model = max(st.session_state.elo_json.items(), key=lambda x: x[1])[0] if st.session_state.elo_json else max(metrics.items(), key=lambda x: x[1]['win_rate'])[0]
         with col2:
             create_metric_container(
                 "Best Model",
@@ -517,6 +527,12 @@ def dashboard():
         appearance_chart = create_appearance_chart(metrics)
         st.plotly_chart(appearance_chart, use_container_width=True)
 
+        if st.session_state.elo_json and st.session_state.elo_df:
+            st.header('Elo Ratings')
+            st.dataframe(pd.DataFrame(st.session_state.elo_json,index=[0]),use_container_width=True)
+            elo_progression_chart = create_elo_chart(st.session_state.elo_df)
+            st.plotly_chart(elo_progression_chart, use_container_width=True)
+
         st.header('Head-to-Head Analysis')
         matrix_chart = create_head_to_head_matrix(st.session_state.df)
         st.plotly_chart(matrix_chart, use_container_width=True)