import openai import os import gradio as gr from enum import Enum from dataclasses import dataclass, asdict, field from typing import List, Optional, Union, Dict, Any import json api_key = os.getenv("OPEN_AI_KEY") # Define the COCOClass enum class COCOClass(Enum): person = 0 bicycle = 1 car = 2 motorcycle = 3 airplane = 4 bus = 5 train = 6 truck = 7 boat = 8 traffic_light = 9 fire_hydrant = 10 stop_sign = 11 parking_meter = 12 bench = 13 bird = 14 cat = 15 dog = 16 horse = 17 sheep = 18 cow = 19 elephant = 20 bear = 21 zebra = 22 giraffe = 23 backpack = 24 umbrella = 25 handbag = 26 tie = 27 suitcase = 28 frisbee = 29 skis = 30 snowboard = 31 sports_ball = 32 kite = 33 baseball_bat = 34 baseball_glove = 35 skateboard = 36 surfboard = 37 tennis_racket = 38 bottle = 39 wine_glass = 40 cup = 41 fork = 42 knife = 43 spoon = 44 bowl = 45 banana = 46 apple = 47 sandwich = 48 orange = 49 broccoli = 50 carrot = 51 hot_dog = 52 pizza = 53 donut = 54 cake = 55 chair = 56 couch = 57 potted_plant = 58 bed = 59 dining_table = 60 toilet = 61 tv = 62 laptop = 63 mouse = 64 remote = 65 keyboard = 66 cell_phone = 67 microwave = 68 oven = 69 toaster = 70 sink = 71 refrigerator = 72 book = 73 clock = 74 vase = 75 scissors = 76 teddy_bear = 77 hair_drier = 78 toothbrush = 79 # Define data classes @dataclass class VehicleProps: brand: Optional[str] = None type: Optional[COCOClass] = None # Should be a vehicle class plate: Optional[str] = None @dataclass class PersonProps: face_images: Optional[List[str]] = field(default_factory=list) age: Optional[int] = None race: Optional[str] = None # Should be one of the specified races gender: Optional[str] = None # Male or Female top_color: Optional[str] = None # Changed from shirt_color bottom_color: Optional[str] = None @dataclass class Activity: prompt: Optional[str] = None type: Optional[str] = None # "full_screen" or "square" @dataclass class Investigation: target: COCOClass images: List[str] activity: Optional[Activity] = None complex_appearance: Optional[str] = None props: Optional[Union[VehicleProps, PersonProps]] = None primary_color: Optional[str] = None secondary_color: Optional[str] = None # Default system message (moved to a global variable) DEFAULT_SYSTEM_MESSAGE = """ You are a helpful assistant that extracts structured information from text descriptions. Your task is to parse the following text prompt and extract information to populate an Investigation JSON object as per the definitions provided. Definitions: Investigation: {{ "target": A COCO class name (from the COCOClass enum), "images": List of image URLs, "activity": {{ "prompt": A description of an activity, e.g., "crossing the street", "crossing red light", "holding a gun", "type": Either "full_screen" or "square" - "full_screen": When the activity requires the full scene for context (e.g., "seeing a movie"). - "square": When the activity context can be understood from a close-up image (e.g., "holding a cat"). }}, "complex_appearance": Description of appearance details that do not fit into other fields, e.g., "has a hat with Nike logo" or "Tattoo on left arm", "props": Either VehicleProps or PersonProps (only if the target is vehicle or person), "primary_color": Primary color mentioned in the prompt, "secondary_color": Secondary color mentioned in the prompt }} VehicleProps: {{ "brand": Vehicle brand, e.g., "Mercedes", "type": COCO class name of vehicles (e.g., "truck"), "plate": License plate number, e.g., "123AB" }} PersonProps: {{ "face_images": List of face image URLs, "age": Age as a number, "race": Race or ethnicity (one of: asian, white, middle eastern, indian, latino, black), "gender": Gender (Male or Female), "top_color": Color of the top garment (e.g., shirt, blouse), # Changed from shirt_color "bottom_color": Color of the bottom garment (pants, skirt, etc.) }} COCOClass Enum: {{ {', '.join([f'"{member.name}"' for member in COCOClass])} }} Important Notes: 1. The output JSON should be as minimal as possible. Do not include fields like 'primary_color' or 'secondary_color' if they are not mentioned in the prompt. 2. Be especially careful with 'activity' and 'complex_appearance' fields. Use them only if the prompt has data that does not fit elsewhere in the JSON. For example: - "a guy with red shirt" -> Map 'red shirt' to 'top_color' in PersonProps. - "a guy with a black hat" -> Since there isn't any field for 'hat', include "black hat" in 'complex_appearance'. 3. Avoid using 'complex_appearance' and 'activity' fields unless absolutely necessary. 4. Do not include undefined fields or fields not mentioned in the prompt. 5. Use the COCOClass enum for the target class name. Now, process the following prompt: '''prompt_text''' Provide the Investigation JSON object, including only the relevant fields based on the prompt. Do not include any explanations. """ # Function to process the prompt def process_prompt(prompt_text: str, images: List[str], face_images: List[str], system_message: Optional[str] = None, user_message: Optional[str] = None, temperature: float = 0.0) -> Optional[Dict[str, Any]]: client = openai.OpenAI(api_key=api_key) # Default user message if not user_message: user_message = "" # Prepare messages for the API messages = [] if system_message.strip(): messages.append({"role": "system", "content": system_message.replace("prompt_text", prompt_text)}) if user_message.strip(): messages.append({"role": "user", "content": user_message.replace("prompt_text", prompt_text)}) response = client.chat.completions.create( model="gpt-4o", messages=messages, response_format={ "type": "json_object" }, temperature=temperature, max_tokens=1000, ) # Extract the content content = response.choices[0].message.content # Parse the JSON output try: investigation_data = json.loads(content) except json.JSONDecodeError as e: print("Error parsing JSON:", e) print("OpenAI response:", content) return None # Construct the Investigation object investigation = parse_investigation(investigation_data, images, face_images) # Convert the Investigation object to dictionary if investigation: investigation_dict = asdict(investigation) # Convert enums to their names investigation_dict['target'] = investigation.target.name if investigation.props: if isinstance(investigation.props, VehicleProps) and investigation.props.type: investigation_dict['props']['type'] = investigation.props.type.name elif isinstance(investigation.props, PersonProps): pass # No enums in PersonProps return investigation_dict else: return None # Function to parse the Investigation data def parse_investigation(data: Dict[str, Any], images: List[str], face_images: List[str]) -> Optional[Investigation]: # Parse target target_name = data.get('target') try: target_enum = COCOClass[target_name] except KeyError: print(f"Invalid COCO class name: {target_name}") return None # Parse activity activity_data = data.get('activity') if activity_data: activity = Activity( prompt=activity_data.get('prompt'), type=activity_data.get('type') ) else: activity = None # Parse props props_data = data.get('props') props = None if props_data: if 'face_images' in props_data: # PersonProps props = PersonProps( face_images=face_images, age=props_data.get('age'), race=props_data.get('race'), gender=props_data.get('gender'), top_color=props_data.get('top_color'), # Changed from shirt_color bottom_color=props_data.get('bottom_color') ) elif 'brand' in props_data: # VehicleProps vehicle_type_name = props_data.get('type') if vehicle_type_name: try: vehicle_type_enum = COCOClass[vehicle_type_name] except KeyError: print(f"Invalid vehicle type: {vehicle_type_name}") vehicle_type_enum = None else: vehicle_type_enum = None props = VehicleProps( brand=props_data.get('brand'), type=vehicle_type_enum, plate=props_data.get('plate') ) # Construct the Investigation object investigation = Investigation( target=target_enum, images=images, activity=activity, complex_appearance=data.get('complex_appearance'), props=props, primary_color=data.get('primary_color'), secondary_color=data.get('secondary_color') ) return investigation # Gradio app def gradio_app(prompts_text, system_message, user_message, temperature): # Split prompts by commas and strip whitespace prompts = [p.strip() for p in prompts_text.split('\n') if p.strip()] images = ["http://example.com/image1.jpg", "http://example.com/image2.jpg"] face_images = ["http://example.com/face1.jpg"] results = [] for p in prompts: investigation_dict = process_prompt( prompt_text=p, images=images, face_images=face_images, system_message=system_message if system_message else None, user_message=user_message if user_message else None, temperature=temperature if temperature else 0.0 ) results.append(f'{p}\n') if investigation_dict: results.append(json.dumps(investigation_dict, indent=4)) else: results.append("Failed to process prompt.") return "\n\n".join(results) if __name__ == "__main__": # Default values default_prompts = "\n".join([ "A red sports car with a license plate reading 'FAST123'.", "An elderly woman wearing a green dress and a pearl necklace.", "A cyclist in a yellow jersey riding a blue bicycle.", "A group of people playing frisbee in the park.", "A man with a large tattoo of a dragon on his right arm.", "A black and white cat sitting on a red couch.", "A delivery truck with the 'FedEx' logo on the side.", "A child holding a red balloon shaped like a dog.", "A person wearing a hoodie with the text 'OpenAI' on it.", "A woman in a blue swimsuit swimming in the ocean." ]) default_system_message = DEFAULT_SYSTEM_MESSAGE.replace("{{prompt_text}}", "{prompt_text}") # Prepare for formatting default_user_message = "" # Optional user message default_temperature = 0.0 # Default temperature # Create Gradio interface iface = gr.Interface( fn=gradio_app, inputs=[ gr.Textbox(lines=5, label="List of Prompts (comma-separated)", value=default_prompts), gr.Textbox(lines=20, label="System Message (optional)", value=default_system_message), gr.Textbox(lines=5, label="User Message (optional)", value=default_user_message), gr.Slider(minimum=0, maximum=1, step=0.1, label="Temperature", value=default_temperature) ], outputs="text", title="OpenAI Prompt Engineering Tester", description="Test different prompts and messages with the OpenAI API." ) # Launch the app iface.launch()