Spaces:
Sleeping
Sleeping
evaluation; minor UI change
#3
by
floydchow7
- opened
- app.py +34 -31
- instructions.py +9 -0
- utils.py +5 -1
app.py
CHANGED
@@ -82,7 +82,7 @@ def handle_create_parallel(task, human_input, session_manager, api_key, identifi
|
|
82 |
session_manager.update(session_index, ai_initial_answer, 'ai_initial_answer')
|
83 |
session_manager.update(session_index, final_answer, 'merged_final_answer')
|
84 |
session_manager.update(session_index, identification_code, 'user_identification_code')
|
85 |
-
return ai_initial_answer,
|
86 |
|
87 |
def handle_create_reverse_sequential(task, session_manager, api_key, identification_code):
|
88 |
cooperate_style = "reverse_sequential"
|
@@ -99,7 +99,8 @@ def handle_modify_reverse_sequential(session_index, modification_suggestions, se
|
|
99 |
if word_limit_validation(modification_suggestions):
|
100 |
final_answer = word_limit_validation(modification_suggestions)
|
101 |
else:
|
102 |
-
final_answer =
|
|
|
103 |
session_manager.update(session_index, final_answer, 'final_answer')
|
104 |
return final_answer, session_index
|
105 |
|
@@ -151,10 +152,21 @@ if __name__ == "__main__":
|
|
151 |
|
152 |
session_manager = SessionManager()
|
153 |
|
154 |
-
with gr.Blocks(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
155 |
with gr.Row():
|
156 |
identification_code = gr.Textbox(label="Enter your identification code")
|
157 |
login_button = gr.Button("Login")
|
|
|
|
|
158 |
login_status = gr.Textbox(label="Next Tasks", interactive=False)
|
159 |
group = gr.State()
|
160 |
|
@@ -205,16 +217,13 @@ if __name__ == "__main__":
|
|
205 |
outputs=[ai_output, session_index]
|
206 |
)
|
207 |
|
208 |
-
ai_output.change(
|
209 |
-
fn = lambda session_index: on_textbox_change(session_index, session_manager, service, SHEET_ID1),
|
210 |
-
inputs = [session_index]
|
211 |
-
)
|
212 |
|
213 |
# evaluate same for every group
|
214 |
-
evaluate_btn = gr.Button("Evaluate")
|
215 |
-
|
216 |
-
|
217 |
-
|
|
|
218 |
fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
|
219 |
inputs=[session_index],
|
220 |
outputs=[evaluation_result]
|
@@ -235,8 +244,6 @@ if __name__ == "__main__":
|
|
235 |
outputs=[save_result]
|
236 |
)
|
237 |
|
238 |
-
with gr.Accordion(label="Appendix: AI instructions", open=False):
|
239 |
-
gr.Markdown(GPT_PROMPT_sequential)
|
240 |
|
241 |
with group_b_content:
|
242 |
with gr.Row():
|
@@ -270,15 +277,11 @@ if __name__ == "__main__":
|
|
270 |
outputs=[final_answer, session_index]
|
271 |
)
|
272 |
|
273 |
-
final_answer.change(
|
274 |
-
fn = lambda session_index: on_textbox_change(session_index, session_manager, service, SHEET_ID3),
|
275 |
-
inputs = [session_index]
|
276 |
-
)
|
277 |
|
278 |
-
evaluate_btn = gr.Button("Evaluate")
|
279 |
-
evaluation_result = gr.Textbox(label="Evaluation Result")
|
280 |
|
281 |
-
|
282 |
fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
|
283 |
inputs=[session_index],
|
284 |
outputs=[evaluation_result]
|
@@ -298,8 +301,6 @@ if __name__ == "__main__":
|
|
298 |
outputs=[save_result]
|
299 |
)
|
300 |
|
301 |
-
with gr.Accordion(label="Appendix: AI instructions", open=False):
|
302 |
-
gr.Markdown(GPT_PROMPT_reverse_sequential)
|
303 |
|
304 |
|
305 |
with group_c_content:
|
@@ -311,25 +312,29 @@ if __name__ == "__main__":
|
|
311 |
create_btn = gr.Button("Submit & See AI Output")
|
312 |
with gr.Row():
|
313 |
ai_initial_output = gr.Textbox(label="AI Output")
|
|
|
|
|
|
|
314 |
final_output = gr.Textbox(label="Final Merged Output")
|
315 |
session_index = gr.Number(label="Session Index", visible=False)
|
316 |
|
317 |
create_btn.click(
|
318 |
fn=lambda task, human_input, id: handle_create_parallel(task, human_input, session_manager, api_key, id),
|
319 |
inputs=[description, human_input, identification_code],
|
320 |
-
outputs=[ai_initial_output,
|
321 |
)
|
322 |
|
323 |
-
|
324 |
-
fn
|
325 |
-
inputs = [session_index]
|
|
|
326 |
)
|
327 |
|
328 |
|
329 |
-
evaluate_btn = gr.Button("Evaluate")
|
330 |
-
evaluation_result = gr.Textbox(label="Evaluation Result")
|
331 |
|
332 |
-
|
333 |
fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
|
334 |
inputs=[session_index],
|
335 |
outputs=[evaluation_result]
|
@@ -349,7 +354,5 @@ if __name__ == "__main__":
|
|
349 |
outputs=[save_result]
|
350 |
)
|
351 |
|
352 |
-
with gr.Accordion(label="Appendix: AI instructions", open=False):
|
353 |
-
gr.Markdown(GPT_PROMPT_parallel)
|
354 |
|
355 |
app.launch(share=True)
|
|
|
82 |
session_manager.update(session_index, ai_initial_answer, 'ai_initial_answer')
|
83 |
session_manager.update(session_index, final_answer, 'merged_final_answer')
|
84 |
session_manager.update(session_index, identification_code, 'user_identification_code')
|
85 |
+
return ai_initial_answer, session_index
|
86 |
|
87 |
def handle_create_reverse_sequential(task, session_manager, api_key, identification_code):
|
88 |
cooperate_style = "reverse_sequential"
|
|
|
99 |
if word_limit_validation(modification_suggestions):
|
100 |
final_answer = word_limit_validation(modification_suggestions)
|
101 |
else:
|
102 |
+
final_answer = modification_suggestions
|
103 |
+
#final_answer = modify_with_suggestion(session['task'], modification_suggestions, api_key)
|
104 |
session_manager.update(session_index, final_answer, 'final_answer')
|
105 |
return final_answer, session_index
|
106 |
|
|
|
152 |
|
153 |
session_manager = SessionManager()
|
154 |
|
155 |
+
with gr.Blocks(fill_width=True,
|
156 |
+
css = """
|
157 |
+
.gradio-container {
|
158 |
+
background: url('https://platform.vox.com/wp-content/uploads/sites/2/chorus/uploads/chorus_asset/file/9376591/blade3.jpg');
|
159 |
+
background-size: cover;
|
160 |
+
background-attachment: fixed;
|
161 |
+
background-position: center;
|
162 |
+
}
|
163 |
+
""") as app:
|
164 |
+
title = gr.HTML("<h1> Human-AI Ensemble </h1>")
|
165 |
with gr.Row():
|
166 |
identification_code = gr.Textbox(label="Enter your identification code")
|
167 |
login_button = gr.Button("Login")
|
168 |
+
experiment_notes = gr.Textbox(label ="Notes for the participants",
|
169 |
+
value = notes_for_participants())
|
170 |
login_status = gr.Textbox(label="Next Tasks", interactive=False)
|
171 |
group = gr.State()
|
172 |
|
|
|
217 |
outputs=[ai_output, session_index]
|
218 |
)
|
219 |
|
|
|
|
|
|
|
|
|
220 |
|
221 |
# evaluate same for every group
|
222 |
+
#evaluate_btn = gr.Button("Evaluate", visible=False)
|
223 |
+
# Evaluate without showing
|
224 |
+
evaluation_result = gr.Textbox(label="Evaluation Result", visible=False)
|
225 |
+
|
226 |
+
ai_output.change(
|
227 |
fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
|
228 |
inputs=[session_index],
|
229 |
outputs=[evaluation_result]
|
|
|
244 |
outputs=[save_result]
|
245 |
)
|
246 |
|
|
|
|
|
247 |
|
248 |
with group_b_content:
|
249 |
with gr.Row():
|
|
|
277 |
outputs=[final_answer, session_index]
|
278 |
)
|
279 |
|
|
|
|
|
|
|
|
|
280 |
|
281 |
+
#evaluate_btn = gr.Button("Evaluate")
|
282 |
+
evaluation_result = gr.Textbox(label="Evaluation Result", visible=False)
|
283 |
|
284 |
+
final_answer.change(
|
285 |
fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
|
286 |
inputs=[session_index],
|
287 |
outputs=[evaluation_result]
|
|
|
301 |
outputs=[save_result]
|
302 |
)
|
303 |
|
|
|
|
|
304 |
|
305 |
|
306 |
with group_c_content:
|
|
|
312 |
create_btn = gr.Button("Submit & See AI Output")
|
313 |
with gr.Row():
|
314 |
ai_initial_output = gr.Textbox(label="AI Output")
|
315 |
+
with gr.Row():
|
316 |
+
merge_btn = gr.Button("Merge Your Input")
|
317 |
+
with gr.Row():
|
318 |
final_output = gr.Textbox(label="Final Merged Output")
|
319 |
session_index = gr.Number(label="Session Index", visible=False)
|
320 |
|
321 |
create_btn.click(
|
322 |
fn=lambda task, human_input, id: handle_create_parallel(task, human_input, session_manager, api_key, id),
|
323 |
inputs=[description, human_input, identification_code],
|
324 |
+
outputs=[ai_initial_output, session_index]
|
325 |
)
|
326 |
|
327 |
+
merge_btn.click(
|
328 |
+
fn= lambda session_index : display_merged_output(session_index, session_manager),
|
329 |
+
inputs = [session_index],
|
330 |
+
outputs=[final_output]
|
331 |
)
|
332 |
|
333 |
|
334 |
+
#evaluate_btn = gr.Button("Evaluate")
|
335 |
+
evaluation_result = gr.Textbox(label="Evaluation Result", visible=False)
|
336 |
|
337 |
+
final_output.change(
|
338 |
fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
|
339 |
inputs=[session_index],
|
340 |
outputs=[evaluation_result]
|
|
|
354 |
outputs=[save_result]
|
355 |
)
|
356 |
|
|
|
|
|
357 |
|
358 |
app.launch(share=True)
|
instructions.py
CHANGED
@@ -43,6 +43,15 @@ def group_c_instructions():
|
|
43 |
def invalid_group():
|
44 |
return "Invalid group, please check your identification code."
|
45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
46 |
def default_task_discription():
|
47 |
TASK_DESCRIPTION = """
|
48 |
Given Airbnb's current business model below, please propose three ideas to make it comply with the 17 SDGs.
|
|
|
43 |
def invalid_group():
|
44 |
return "Invalid group, please check your identification code."
|
45 |
|
46 |
+
def notes_for_participants():
|
47 |
+
NOTES = """
|
48 |
+
You will receive basic compensation 💰 for a valid response of at least 50 words.
|
49 |
+
|
50 |
+
Your final answer will be evaluated by an AI. If it ranks in the top 30 of 600 participants, you’ll earn an extra 🎁 $15.
|
51 |
+
"""
|
52 |
+
|
53 |
+
return NOTES
|
54 |
+
|
55 |
def default_task_discription():
|
56 |
TASK_DESCRIPTION = """
|
57 |
Given Airbnb's current business model below, please propose three ideas to make it comply with the 17 SDGs.
|
utils.py
CHANGED
@@ -89,4 +89,8 @@ def get_evaluation_with_gpt(task_description, text, api_key=None):
|
|
89 |
f"Inimitability: [Score]\n"
|
90 |
f"Alignment: [Score]\n"
|
91 |
)
|
92 |
-
return generate_text_with_gpt(prompt, api_key)
|
|
|
|
|
|
|
|
|
|
89 |
f"Inimitability: [Score]\n"
|
90 |
f"Alignment: [Score]\n"
|
91 |
)
|
92 |
+
return generate_text_with_gpt(prompt, api_key)
|
93 |
+
|
94 |
+
def display_merged_output(session_index, session_manager):
|
95 |
+
session = session_manager.get_session(session_index)
|
96 |
+
return session['merged_final_answer']
|