Commit
·
60132ec
1
Parent(s):
7095a34
bugfixes and added auth
Browse files- .gitignore +1 -1
- app.py +80 -73
- results/audios/temp +0 -0
- results/results.csv +1 -0
.gitignore
CHANGED
@@ -1,7 +1,7 @@
|
|
|
|
1 |
*.json
|
2 |
mapping
|
3 |
*.ipynb
|
4 |
test.py
|
5 |
-
results/
|
6 |
.notebook/
|
7 |
__pycache__/
|
|
|
1 |
+
.env
|
2 |
*.json
|
3 |
mapping
|
4 |
*.ipynb
|
5 |
test.py
|
|
|
6 |
.notebook/
|
7 |
__pycache__/
|
app.py
CHANGED
@@ -25,12 +25,13 @@ if not os.path.exists("results"):
|
|
25 |
os.mkdir("results")
|
26 |
|
27 |
if not os.path.exists(SAVE_PATH):
|
28 |
-
open(SAVE_PATH,"w")
|
|
|
29 |
|
30 |
if not os.path.exists(TEMP_DIR):
|
31 |
os.mkdir(TEMP_DIR)
|
32 |
|
33 |
-
CREATE_TASK_URL =
|
34 |
|
35 |
def decode_audio_array(base64_string):
|
36 |
bytes_data = base64.b64decode(base64_string)
|
@@ -41,7 +42,10 @@ def decode_audio_array(base64_string):
|
|
41 |
return audio_array
|
42 |
|
43 |
def send_task(payload):
|
44 |
-
|
|
|
|
|
|
|
45 |
response = response.json()
|
46 |
|
47 |
if payload["task"] == "transcribe_with_fastapi":
|
@@ -591,76 +595,79 @@ def dashboard():
|
|
591 |
st.title('Model Arena Scoreboard')
|
592 |
|
593 |
df = pd.read_csv(SAVE_PATH)
|
594 |
-
|
595 |
-
|
596 |
-
|
597 |
-
|
598 |
-
|
599 |
-
|
600 |
-
|
601 |
-
|
602 |
-
|
603 |
-
|
|
|
604 |
|
605 |
-
|
606 |
-
|
607 |
-
|
608 |
-
|
609 |
-
|
610 |
-
|
611 |
-
|
612 |
-
|
613 |
-
|
614 |
-
|
615 |
-
|
616 |
-
|
617 |
-
|
618 |
-
|
619 |
-
|
620 |
-
|
621 |
-
|
622 |
-
|
623 |
-
|
624 |
-
|
625 |
-
|
626 |
-
|
627 |
-
|
628 |
-
|
629 |
-
|
630 |
-
|
631 |
-
|
632 |
-
|
633 |
-
|
634 |
-
|
635 |
-
|
636 |
-
|
637 |
-
|
638 |
-
|
639 |
-
|
640 |
-
|
641 |
-
|
642 |
-
|
643 |
-
|
644 |
-
|
645 |
-
|
646 |
-
|
647 |
-
|
648 |
-
|
649 |
-
|
650 |
-
|
651 |
-
|
652 |
-
|
653 |
-
|
654 |
-
|
655 |
-
|
656 |
-
|
657 |
-
|
658 |
-
|
659 |
-
|
660 |
-
|
661 |
-
|
662 |
-
|
663 |
-
|
|
|
|
|
664 |
else:
|
665 |
st.write('You have not entered your email and name yet')
|
666 |
st.write('Please Navigate to login page in the dropdown menu')
|
@@ -732,7 +739,7 @@ def validate_name(name):
|
|
732 |
return re.match(pattern, name) is not None
|
733 |
|
734 |
def create_login_page():
|
735 |
-
st.title("Welcome to the
|
736 |
|
737 |
if 'logged_in' not in st.session_state:
|
738 |
st.session_state.logged_in = False
|
|
|
25 |
os.mkdir("results")
|
26 |
|
27 |
if not os.path.exists(SAVE_PATH):
|
28 |
+
with open(SAVE_PATH,"w") as f:
|
29 |
+
f.write("""email,path,Ori Apex_score,Ori Apex XT_score,deepgram_score,Ori Swift_score,Ori Prime_score,Ori Apex_appearance,Ori Apex XT_appearance,deepgram_appearance,Ori Swift_appearance,Ori Prime_appearance,Ori Apex_duration,Ori Apex XT_duration,deepgram_duration,Ori Swift_duration,Ori Prime_duration,azure_score,azure_appearance,azure_duration\n""")
|
30 |
|
31 |
if not os.path.exists(TEMP_DIR):
|
32 |
os.mkdir(TEMP_DIR)
|
33 |
|
34 |
+
CREATE_TASK_URL = os.getenv("CREATE_TASK_URL")
|
35 |
|
36 |
def decode_audio_array(base64_string):
|
37 |
bytes_data = base64.b64decode(base64_string)
|
|
|
42 |
return audio_array
|
43 |
|
44 |
def send_task(payload):
|
45 |
+
header = {
|
46 |
+
"Authorization": f"Bearer {os.getenv('CREATE_TASK_API_KEY')}"
|
47 |
+
}
|
48 |
+
response = requests.post(CREATE_TASK_URL,json=payload,headers=header)
|
49 |
response = response.json()
|
50 |
|
51 |
if payload["task"] == "transcribe_with_fastapi":
|
|
|
595 |
st.title('Model Arena Scoreboard')
|
596 |
|
597 |
df = pd.read_csv(SAVE_PATH)
|
598 |
+
if len(df) != 0:
|
599 |
+
metrics = calculate_metrics(df)
|
600 |
+
|
601 |
+
MODEL_DESCRIPTIONS = {
|
602 |
+
"Ori Prime": "Foundational, large, and stable.",
|
603 |
+
"Ori Swift": "Lighter and faster than Ori Prime.",
|
604 |
+
"Ori Apex": "The top-performing model, fast and stable.",
|
605 |
+
"Ori Apex XT": "Enhanced with more training, though slightly less stable than Ori Apex.",
|
606 |
+
"DG" : "Deepgram Nova-2 API",
|
607 |
+
"Azure" : "Azure Speech Services API"
|
608 |
+
}
|
609 |
|
610 |
+
st.header('Model Descriptions')
|
611 |
+
|
612 |
+
cols = st.columns(2)
|
613 |
+
for idx, (model, description) in enumerate(MODEL_DESCRIPTIONS.items()):
|
614 |
+
with cols[idx % 2]:
|
615 |
+
st.markdown(f"""
|
616 |
+
<div style='padding: 1rem; border: 1px solid #e1e4e8; border-radius: 6px; margin-bottom: 1rem;'>
|
617 |
+
<h3 style='margin: 0; margin-bottom: 0.5rem;'>{model}</h3>
|
618 |
+
<p style='margin: 0; color: #6e7681;'>{description}</p>
|
619 |
+
</div>
|
620 |
+
""", unsafe_allow_html=True)
|
621 |
+
|
622 |
+
st.header('Overall Performance')
|
623 |
+
|
624 |
+
col1, col2, col3= st.columns(3)
|
625 |
+
|
626 |
+
with col1:
|
627 |
+
create_metric_container("Total Matches", len(df))
|
628 |
+
|
629 |
+
best_model = max(metrics.items(), key=lambda x: x[1]['win_rate'])[0]
|
630 |
+
with col2:
|
631 |
+
create_metric_container(
|
632 |
+
"Best Model",
|
633 |
+
get_model_abbreviation(best_model),
|
634 |
+
full_name=best_model
|
635 |
+
)
|
636 |
+
|
637 |
+
most_appearances = max(metrics.items(), key=lambda x: x[1]['appearances'])[0]
|
638 |
+
with col3:
|
639 |
+
create_metric_container(
|
640 |
+
"Most Used",
|
641 |
+
get_model_abbreviation(most_appearances),
|
642 |
+
full_name=most_appearances
|
643 |
+
)
|
644 |
+
|
645 |
+
st.header('Win Rates')
|
646 |
+
win_rate_chart = create_win_rate_chart(metrics)
|
647 |
+
st.plotly_chart(win_rate_chart, use_container_width=True)
|
648 |
+
|
649 |
+
st.header('Appearance Distribution')
|
650 |
+
appearance_chart = create_appearance_chart(metrics)
|
651 |
+
st.plotly_chart(appearance_chart, use_container_width=True)
|
652 |
+
|
653 |
+
st.header('Head-to-Head Analysis')
|
654 |
+
matrix_chart = create_head_to_head_matrix(df)
|
655 |
+
st.plotly_chart(matrix_chart, use_container_width=True)
|
656 |
+
|
657 |
+
st.header('Detailed Metrics')
|
658 |
+
metrics_df = pd.DataFrame.from_dict(metrics, orient='index')
|
659 |
+
metrics_df['win_rate'] = metrics_df['win_rate'].round(2)
|
660 |
+
metrics_df.drop(["avg_response_time","response_time_std"],axis=1,inplace=True)
|
661 |
+
# metrics_df['avg_response_time'] = metrics_df['avg_response_time'].round(3)
|
662 |
+
metrics_df.index = [get_model_abbreviation(model) for model in metrics_df.index]
|
663 |
+
st.dataframe(metrics_df)
|
664 |
+
|
665 |
+
st.header('Full Dataframe')
|
666 |
+
df = df.drop('path', axis=1)
|
667 |
+
df = df.drop(['Ori Apex_duration', 'Ori Apex XT_duration', 'deepgram_duration', 'Ori Swift_duration', 'Ori Prime_duration','azure_duration','email'],axis=1)
|
668 |
+
st.dataframe(df)
|
669 |
+
else:
|
670 |
+
st.write("No Data to show")
|
671 |
else:
|
672 |
st.write('You have not entered your email and name yet')
|
673 |
st.write('Please Navigate to login page in the dropdown menu')
|
|
|
739 |
return re.match(pattern, name) is not None
|
740 |
|
741 |
def create_login_page():
|
742 |
+
st.title("Welcome to the Speech-To-Text Arena")
|
743 |
|
744 |
if 'logged_in' not in st.session_state:
|
745 |
st.session_state.logged_in = False
|
results/audios/temp
ADDED
File without changes
|
results/results.csv
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
email,path,Ori Apex_score,Ori Apex XT_score,deepgram_score,Ori Swift_score,Ori Prime_score,Ori Apex_appearance,Ori Apex XT_appearance,deepgram_appearance,Ori Swift_appearance,Ori Prime_appearance,Ori Apex_duration,Ori Apex XT_duration,deepgram_duration,Ori Swift_dur
|