evaluation; minor UI change

#3
by floydchow7 - opened
Files changed (3) hide show
  1. app.py +34 -31
  2. instructions.py +9 -0
  3. utils.py +5 -1
app.py CHANGED
@@ -82,7 +82,7 @@ def handle_create_parallel(task, human_input, session_manager, api_key, identifi
82
  session_manager.update(session_index, ai_initial_answer, 'ai_initial_answer')
83
  session_manager.update(session_index, final_answer, 'merged_final_answer')
84
  session_manager.update(session_index, identification_code, 'user_identification_code')
85
- return ai_initial_answer, final_answer, session_index
86
 
87
  def handle_create_reverse_sequential(task, session_manager, api_key, identification_code):
88
  cooperate_style = "reverse_sequential"
@@ -99,7 +99,8 @@ def handle_modify_reverse_sequential(session_index, modification_suggestions, se
99
  if word_limit_validation(modification_suggestions):
100
  final_answer = word_limit_validation(modification_suggestions)
101
  else:
102
- final_answer = modify_with_suggestion(session['task'], modification_suggestions, api_key)
 
103
  session_manager.update(session_index, final_answer, 'final_answer')
104
  return final_answer, session_index
105
 
@@ -151,10 +152,21 @@ if __name__ == "__main__":
151
 
152
  session_manager = SessionManager()
153
 
154
- with gr.Blocks() as app:
 
 
 
 
 
 
 
 
 
155
  with gr.Row():
156
  identification_code = gr.Textbox(label="Enter your identification code")
157
  login_button = gr.Button("Login")
 
 
158
  login_status = gr.Textbox(label="Next Tasks", interactive=False)
159
  group = gr.State()
160
 
@@ -205,16 +217,13 @@ if __name__ == "__main__":
205
  outputs=[ai_output, session_index]
206
  )
207
 
208
- ai_output.change(
209
- fn = lambda session_index: on_textbox_change(session_index, session_manager, service, SHEET_ID1),
210
- inputs = [session_index]
211
- )
212
 
213
  # evaluate same for every group
214
- evaluate_btn = gr.Button("Evaluate")
215
- evaluation_result = gr.Textbox(label="Evaluation Result")
216
-
217
- evaluate_btn.click(
 
218
  fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
219
  inputs=[session_index],
220
  outputs=[evaluation_result]
@@ -235,8 +244,6 @@ if __name__ == "__main__":
235
  outputs=[save_result]
236
  )
237
 
238
- with gr.Accordion(label="Appendix: AI instructions", open=False):
239
- gr.Markdown(GPT_PROMPT_sequential)
240
 
241
  with group_b_content:
242
  with gr.Row():
@@ -270,15 +277,11 @@ if __name__ == "__main__":
270
  outputs=[final_answer, session_index]
271
  )
272
 
273
- final_answer.change(
274
- fn = lambda session_index: on_textbox_change(session_index, session_manager, service, SHEET_ID3),
275
- inputs = [session_index]
276
- )
277
 
278
- evaluate_btn = gr.Button("Evaluate")
279
- evaluation_result = gr.Textbox(label="Evaluation Result")
280
 
281
- evaluate_btn.click(
282
  fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
283
  inputs=[session_index],
284
  outputs=[evaluation_result]
@@ -298,8 +301,6 @@ if __name__ == "__main__":
298
  outputs=[save_result]
299
  )
300
 
301
- with gr.Accordion(label="Appendix: AI instructions", open=False):
302
- gr.Markdown(GPT_PROMPT_reverse_sequential)
303
 
304
 
305
  with group_c_content:
@@ -311,25 +312,29 @@ if __name__ == "__main__":
311
  create_btn = gr.Button("Submit & See AI Output")
312
  with gr.Row():
313
  ai_initial_output = gr.Textbox(label="AI Output")
 
 
 
314
  final_output = gr.Textbox(label="Final Merged Output")
315
  session_index = gr.Number(label="Session Index", visible=False)
316
 
317
  create_btn.click(
318
  fn=lambda task, human_input, id: handle_create_parallel(task, human_input, session_manager, api_key, id),
319
  inputs=[description, human_input, identification_code],
320
- outputs=[ai_initial_output, final_output, session_index]
321
  )
322
 
323
- ai_initial_output.change(
324
- fn = lambda session_index: on_textbox_change(session_index, session_manager, service, SHEET_ID2),
325
- inputs = [session_index]
 
326
  )
327
 
328
 
329
- evaluate_btn = gr.Button("Evaluate")
330
- evaluation_result = gr.Textbox(label="Evaluation Result")
331
 
332
- evaluate_btn.click(
333
  fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
334
  inputs=[session_index],
335
  outputs=[evaluation_result]
@@ -349,7 +354,5 @@ if __name__ == "__main__":
349
  outputs=[save_result]
350
  )
351
 
352
- with gr.Accordion(label="Appendix: AI instructions", open=False):
353
- gr.Markdown(GPT_PROMPT_parallel)
354
 
355
  app.launch(share=True)
 
82
  session_manager.update(session_index, ai_initial_answer, 'ai_initial_answer')
83
  session_manager.update(session_index, final_answer, 'merged_final_answer')
84
  session_manager.update(session_index, identification_code, 'user_identification_code')
85
+ return ai_initial_answer, session_index
86
 
87
  def handle_create_reverse_sequential(task, session_manager, api_key, identification_code):
88
  cooperate_style = "reverse_sequential"
 
99
  if word_limit_validation(modification_suggestions):
100
  final_answer = word_limit_validation(modification_suggestions)
101
  else:
102
+ final_answer = modification_suggestions
103
+ #final_answer = modify_with_suggestion(session['task'], modification_suggestions, api_key)
104
  session_manager.update(session_index, final_answer, 'final_answer')
105
  return final_answer, session_index
106
 
 
152
 
153
  session_manager = SessionManager()
154
 
155
+ with gr.Blocks(fill_width=True,
156
+ css = """
157
+ .gradio-container {
158
+ background: url('https://platform.vox.com/wp-content/uploads/sites/2/chorus/uploads/chorus_asset/file/9376591/blade3.jpg');
159
+ background-size: cover;
160
+ background-attachment: fixed;
161
+ background-position: center;
162
+ }
163
+ """) as app:
164
+ title = gr.HTML("<h1> Human-AI Ensemble </h1>")
165
  with gr.Row():
166
  identification_code = gr.Textbox(label="Enter your identification code")
167
  login_button = gr.Button("Login")
168
+ experiment_notes = gr.Textbox(label ="Notes for the participants",
169
+ value = notes_for_participants())
170
  login_status = gr.Textbox(label="Next Tasks", interactive=False)
171
  group = gr.State()
172
 
 
217
  outputs=[ai_output, session_index]
218
  )
219
 
 
 
 
 
220
 
221
  # evaluate same for every group
222
+ #evaluate_btn = gr.Button("Evaluate", visible=False)
223
+ # Evaluate without showing
224
+ evaluation_result = gr.Textbox(label="Evaluation Result", visible=False)
225
+
226
+ ai_output.change(
227
  fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
228
  inputs=[session_index],
229
  outputs=[evaluation_result]
 
244
  outputs=[save_result]
245
  )
246
 
 
 
247
 
248
  with group_b_content:
249
  with gr.Row():
 
277
  outputs=[final_answer, session_index]
278
  )
279
 
 
 
 
 
280
 
281
+ #evaluate_btn = gr.Button("Evaluate")
282
+ evaluation_result = gr.Textbox(label="Evaluation Result", visible=False)
283
 
284
+ final_answer.change(
285
  fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
286
  inputs=[session_index],
287
  outputs=[evaluation_result]
 
301
  outputs=[save_result]
302
  )
303
 
 
 
304
 
305
 
306
  with group_c_content:
 
312
  create_btn = gr.Button("Submit & See AI Output")
313
  with gr.Row():
314
  ai_initial_output = gr.Textbox(label="AI Output")
315
+ with gr.Row():
316
+ merge_btn = gr.Button("Merge Your Input")
317
+ with gr.Row():
318
  final_output = gr.Textbox(label="Final Merged Output")
319
  session_index = gr.Number(label="Session Index", visible=False)
320
 
321
  create_btn.click(
322
  fn=lambda task, human_input, id: handle_create_parallel(task, human_input, session_manager, api_key, id),
323
  inputs=[description, human_input, identification_code],
324
+ outputs=[ai_initial_output, session_index]
325
  )
326
 
327
+ merge_btn.click(
328
+ fn= lambda session_index : display_merged_output(session_index, session_manager),
329
+ inputs = [session_index],
330
+ outputs=[final_output]
331
  )
332
 
333
 
334
+ #evaluate_btn = gr.Button("Evaluate")
335
+ evaluation_result = gr.Textbox(label="Evaluation Result", visible=False)
336
 
337
+ final_output.change(
338
  fn=lambda session_index: evaluate_interaction(session_index, session_manager, api_key),
339
  inputs=[session_index],
340
  outputs=[evaluation_result]
 
354
  outputs=[save_result]
355
  )
356
 
 
 
357
 
358
  app.launch(share=True)
instructions.py CHANGED
@@ -43,6 +43,15 @@ def group_c_instructions():
43
  def invalid_group():
44
  return "Invalid group, please check your identification code."
45
 
 
 
 
 
 
 
 
 
 
46
  def default_task_discription():
47
  TASK_DESCRIPTION = """
48
  Given Airbnb's current business model below, please propose three ideas to make it comply with the 17 SDGs.
 
43
  def invalid_group():
44
  return "Invalid group, please check your identification code."
45
 
46
+ def notes_for_participants():
47
+ NOTES = """
48
+ You will receive basic compensation 💰 for a valid response of at least 50 words.
49
+
50
+ Your final answer will be evaluated by an AI. If it ranks in the top 30 of 600 participants, you’ll earn an extra 🎁 $15.
51
+ """
52
+
53
+ return NOTES
54
+
55
  def default_task_discription():
56
  TASK_DESCRIPTION = """
57
  Given Airbnb's current business model below, please propose three ideas to make it comply with the 17 SDGs.
utils.py CHANGED
@@ -89,4 +89,8 @@ def get_evaluation_with_gpt(task_description, text, api_key=None):
89
  f"Inimitability: [Score]\n"
90
  f"Alignment: [Score]\n"
91
  )
92
- return generate_text_with_gpt(prompt, api_key)
 
 
 
 
 
89
  f"Inimitability: [Score]\n"
90
  f"Alignment: [Score]\n"
91
  )
92
+ return generate_text_with_gpt(prompt, api_key)
93
+
94
+ def display_merged_output(session_index, session_manager):
95
+ session = session_manager.get_session(session_index)
96
+ return session['merged_final_answer']