update

.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+*.pyc
+/logs
+*.jsonl

README.md

@@ -1,10 +1,10 @@
 ---
-title: Demo
-emoji: 🚀
-colorFrom: indigo
-colorTo: blue
+title: Beginner Demo
+emoji: 👀
+colorFrom: purple
+colorTo: green
 sdk: streamlit
-sdk_version: 1.37.1
+sdk_version: 1.36.0
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -0,0 +1,21 @@
+import home
+import streamlit as st
+
+st.set_page_config(layout="wide")
+st.sidebar.image('assets/img/textgrad_logo_1x.png', width=200)
+st.sidebar.title("TextGrad Playground")
+st.sidebar.markdown("## Menu")
+
+PAGES = {
+    "🏠 Home": home,
+}
+
+page = st.sidebar.radio("", list(PAGES.keys()))
+st.sidebar.markdown("## Links")
+
+st.sidebar.markdown("""
+                        📖 [Paper](https://arxiv.org/abs/2406.07496)
+                        👨‍💻 [Code](https://github.com/zou-group/textgrad)
+                        ⚙️ [API documentation](https://textgrad.readthedocs.io/en/latest/index.html)""")
+
+PAGES[page].app()

examples/code_editor.json

@@ -0,0 +1,7 @@
+{
+    "default_initial_solution": "def longest_increasing_subsequence(nums):\n    n = len(nums)\n    dp = [1] * n\n    \n    for i in range(1, n):\n        for j in range(i):\n            if nums[i] > nums[j]:\n                dp[i] = max(dp[i], dp[j] + 1)\n    \n    max_length = max(dp)\n    lis = []\n    \n    for i in range(n - 1, -1, -1):\n        if dp[i] == max_length:\n            lis.append(nums[i])\n            max_length -= 1\n    \n    return len(lis[::-1])",
+    "default_target_solution": "import bisect\n\ndef longest_increasing_subsequence(nums):\n    if not nums:\n        return 0\n    \n    tails = []\n    \n    for num in nums:\n        pos = bisect.bisect_left(tails, num)\n        if pos == len(tails):\n            tails.append(num)\n        else:\n            tails[pos] = num\n    \n    return len(tails)",
+    "default_loss_system_prompt": "You are a smart language model that evaluates code snippets. You do not solve problems or propose new code snippets, only evaluate existing solutions critically and give very concise feedback.",
+    "default_problem_description": "Longest Increasing Subsequence (LIS)\n\nProblem Statement:\nGiven a sequence of integers, find the length of the longest subsequence that is strictly increasing. A subsequence is a sequence that can be derived from another sequence by deleting some or no elements without changing the order of the remaining elements.\n\nInput:\nThe input consists of a list of integers representing the sequence.\n\nOutput:\nThe output should be an integer representing the length of the longest increasing subsequence.",
+    "instruction": "Think about the problem and the code snippet. Does the code solve the problem? What is the runtime complexity?"
+}

examples/code_editor_scripts.py

@@ -0,0 +1,137 @@
+import streamlit as st
+from streamlit_elements import elements, mui, editor, dashboard
+from stqdm import stqdm
+import textgrad as tg
+import os
+
+class CodeEditor:
+    def __init__(self, data) -> None:
+        self.data = data
+        self.llm_engine = tg.get_engine("gpt-4o")
+        print("="*50, "init", "="*50)
+        self.loss_value = ""
+        self.code_gradients = ""
+        if 'iteration' not in st.session_state:
+            st.session_state.iteration = 0
+        if 'results' not in st.session_state:
+            st.session_state.results = []
+        tg.set_backward_engine(self.llm_engine, override=True)
+        
+
+    def load_layout(self):
+        col1, col2 = st.columns([1, 1])
+        with col1:
+            self.problem = st.text_area("Problem description:", self.data["default_problem_description"], height=300)
+        with col2:
+            self.loss_system_prompt = st.text_area("Loss system prompt:", self.data["default_loss_system_prompt"], height=150)
+            self.instruction = st.text_area("Instruction for formatted LLM call:", self.data["instruction"], height=100)
+
+        if "code_content" not in st.session_state:
+            st.session_state.code_content = self.data["default_initial_solution"]
+
+        def update_code_content(value):
+            st.session_state.code_content = value
+        
+
+        col1, col2 = st.columns(2)
+        with col1:
+            with elements("monaco_editors_1"):
+                mui.Typography("Initial Solution:", sx={"fontSize": "20px", "fontWeight": "bold"})
+                editor.Monaco(
+                    height=300,
+                    defaultLanguage="python",
+                    defaultValue=st.session_state.code_content,
+                    onChange=update_code_content
+                )
+        with col2:
+            with elements("monaco_editors_2"):
+                mui.Typography("Current Solution:", sx={"fontSize": "20px", "fontWeight": "bold"})
+                editor.Monaco(
+                    height=300,
+                    defaultLanguage="python",
+                    value=st.session_state.code_content,
+                    options={"readOnly": True}  # Make the editor read-only
+                )
+
+
+            # format_string = f"{instruction}\nProblem: {problem}\nCurrent Code: {st.session_state.code_content}"
+            # mui.Typography(format_string)
+            
+            # mui.Typography("Final Snippet vs. Current Solution:", sx={"fontSize": "20px", "fontWeight": "bold"})
+            # editor.MonacoDiff(
+            #     original=self.data["default_target_solution"],
+            #     modified=st.session_state.code_content,
+            #     height=300,
+            # )
+        
+
+
+    def _run(self):
+        # Code is the variable of interest we want to optimize -- so requires_grad=True
+        solution = st.session_state.code_content
+        code = tg.Variable(value=solution,
+                        requires_grad=True,
+                        role_description="code instance to optimize")
+
+        # We are not interested in optimizing the problem -- so requires_grad=False
+        problem = tg.Variable(self.problem, 
+                            requires_grad=False, 
+                            role_description="the coding problem")
+
+        # Let TGD know to update code!
+        optimizer = tg.TGD(parameters=[code])
+
+
+        instruction = self.instruction
+        llm_engine = self.llm_engine
+        loss_system_prompt = self.loss_system_prompt
+        loss_system_prompt = tg.Variable(loss_system_prompt, requires_grad=False, role_description="system prompt to the loss function")
+
+        format_string = "{instruction}\nProblem: {{problem}}\nCurrent Code: {{code}}"
+        format_string = format_string.format(instruction=self.instruction)
+
+        fields = {"problem": None, "code": None}
+        formatted_llm_call = tg.autograd.FormattedLLMCall(engine=self.llm_engine,
+                                                        format_string=format_string,
+                                                        fields=fields,
+                                                        system_prompt=loss_system_prompt)
+        # Finally, the loss function
+        def loss_fn(problem: tg.Variable, code: tg.Variable) -> tg.Variable:
+            inputs = {"problem": problem, "code": code}
+            
+            return formatted_llm_call(inputs=inputs,
+                                    response_role_description=f"evaluation of the {code.get_role_description()}")
+        loss = loss_fn(problem, code)
+        self.loss_value = loss.value
+        self.graph = loss.generate_graph()
+
+        loss.backward()
+        self.gradients = code.gradients
+        optimizer.step() # Let's update the code
+
+        st.session_state.code_content = code.value
+
+    def show_results(self):
+        self._run()
+        st.session_state.iteration += 1
+        st.session_state.results.append({
+            'iteration': st.session_state.iteration,
+            'loss_value': self.loss_value,
+            'gradients': self.gradients
+        })
+
+        tabs = st.tabs([f"Iteration {i+1}" for i in range(st.session_state.iteration)])
+
+        for i, tab in enumerate(tabs):
+            with tab:
+                result = st.session_state.results[i]
+                st.markdown(f"Current iteration: **{result['iteration']}**")
+                col1, col2 = st.columns([1, 1])
+                with col1:
+                    st.markdown("## Loss value")
+                    st.markdown(result['loss_value'])
+                with col2:
+                    st.markdown("## Code gradients")
+                    for j, g in enumerate(result['gradients']):
+                        st.markdown(f"### Gradient {j}")
+                        st.markdown(g.value)

examples/example_imageQA_scripts.py

@@ -0,0 +1,99 @@
+import streamlit as st
+from streamlit_elements import elements, mui, editor, dashboard
+from stqdm import stqdm
+import textgrad as tg
+import os
+from PIL import Image
+from textgrad.autograd import MultimodalLLMCall
+from textgrad.loss import ImageQALoss
+from io import BytesIO
+
+class ImageQA:
+    def __init__(self, data) -> None:
+        self.data = data
+        self.llm_engine = tg.get_engine("gpt-4o")
+        print("="*50, "init", "="*50)
+        self.loss_value = ""
+        self.gradients = ""
+        if 'iteration' not in st.session_state:
+            st.session_state.iteration = 0
+        st.session_state.results = []
+        tg.set_backward_engine(self.llm_engine, override=True)
+
+    def load_layout(self):
+        st.markdown(f"**This is a solution optimization for image QA.**")
+        col1, col2 = st.columns([1, 1])
+        with col1:
+            uploaded_file = st.file_uploader("Upload an image", type=["png", "jpg", "jpeg"])
+            if uploaded_file is not None:
+                image = Image.open(uploaded_file)
+                st.image(image, caption="Uploaded Image")
+            else:
+                image_url = self.data["image_URL"]
+                image = Image.open(image_url)
+                st.image(image_url, caption="Default: MathVista image")
+
+            img_byte_arr = BytesIO()
+            image.save(img_byte_arr, format='PNG')  # You can choose the format you want
+            img_byte_arr = img_byte_arr.getvalue()
+            self.image_variable = tg.Variable(img_byte_arr, role_description="image to answer a question about", requires_grad=False)
+        with col2:
+            question_text = st.text_area("Question:", self.data["question_text"], height=150)
+            self.question_variable = tg.Variable(question_text, role_description="question", requires_grad=False)
+            self.evaluation_instruction_text = st.text_area("Evaluation instruction:", self.data["evaluation_instruction"], height=100)
+        
+        self.loss_fn = ImageQALoss(
+                    evaluation_instruction=self.evaluation_instruction_text,
+                    engine="gpt-4o",
+                )
+        if "current_response" not in st.session_state:
+            st.session_state.current_response = ""
+
+
+    def _run(self):
+        # Set up the textgrad variables
+        self.response = MultimodalLLMCall("gpt-4o")([
+                self.image_variable,
+                self.question_variable
+            ])
+        
+        optimizer = tg.TGD(parameters=[self.response])
+
+        loss = self.loss_fn(question=self.question_variable, image=self.image_variable, response=self.response)
+        self.loss_value = loss.value
+        # self.graph = loss.generate_graph()
+
+        loss.backward()
+        self.gradients = self.response.gradients
+
+        optimizer.step() # Let's update the response
+        st.session_state.current_response = self.response.value
+
+    def show_results(self):
+        self._run()
+        st.session_state.iteration += 1
+        st.session_state.results.append({
+            'iteration': st.session_state.iteration,
+            'loss_value': self.loss_value,
+            'response': self.response.value,
+            'gradients': self.gradients
+        })
+
+        tabs = st.tabs([f"Iteration {i+1}" for i in range(st.session_state.iteration)])
+
+        for i, tab in enumerate(tabs):
+            with tab:
+                result = st.session_state.results[i]
+                st.markdown(f"Current iteration: **{result['iteration']}**")
+                st.markdown("## Current solution:")
+                st.markdown(result['response'])
+
+                col1, col2 = st.columns([1, 1])
+                with col1:
+                    st.markdown("## Loss value")
+                    st.markdown(result['loss_value'])
+                with col2:
+                    st.markdown("## Code gradients")
+                    for j, g in enumerate(result['gradients']):
+                        st.markdown(f"### Gradient")
+                        st.markdown(g.value)

examples/example_math.json

@@ -0,0 +1,4 @@
+{
+    "default_initial_solution": "To solve the equation 3x^2 - 7x + 2 = 0, we use the quadratic formula:\nx = (-b ± √(b^2 - 4ac)) / 2a\na = 3, b = -7, c = 2\nx = (7 ± √((-7)^2 + 4(3)(2))) / 6\nx = (7 ± √73) / 6\nThe solutions are:\nx1 = (7 + √73)\nx2 = (7 - √73)",
+    "default_loss_system_prompt": "You will evaluate a solution to a math question.\nDo not attempt to solve it yourself, do not give a solution, only identify errors. Be super concise."
+}

examples/example_math_scripts.py

@@ -0,0 +1,80 @@
+import streamlit as st
+from streamlit_elements import elements, mui, editor, dashboard
+from stqdm import stqdm
+import textgrad as tg
+import os
+
+
+class MathSolution:
+    def __init__(self, data) -> None:
+        self.data = data
+        self.llm_engine = tg.get_engine("gpt-4o")
+        print("="*50, "init", "="*50)
+        self.loss_value = ""
+        self.gradients = ""
+        if 'iteration' not in st.session_state:
+            st.session_state.iteration = 0
+        st.session_state.results = []
+        tg.set_backward_engine(self.llm_engine, override=True)
+
+    def load_layout(self):
+        col1, col2 = st.columns([1, 1])
+        with col1:
+            self.initial_solution = st.text_area("Initial solution:", self.data["default_initial_solution"], height=300)
+        with col2:
+            self.loss_system_prompt = st.text_area("Loss system prompt:", self.data["default_loss_system_prompt"], height=300)
+
+        if "current_solution" not in st.session_state:
+            st.session_state.current_solution = self.data["default_initial_solution"]
+
+
+    def _run(self):
+        # Set up the textgrad variables
+        current_solution = st.session_state.current_solution
+
+        self.response = tg.Variable(current_solution,
+                            requires_grad=True,
+                            role_description="solution to the math question")
+
+        loss_fn = tg.TextLoss(tg.Variable(self.loss_system_prompt,
+                                        requires_grad=False,
+                                        role_description="system prompt"))
+        optimizer = tg.TGD([self.response])
+
+        loss = loss_fn(self.response)
+        self.loss_value = loss.value
+        self.graph = loss.generate_graph()
+
+        loss.backward()
+        self.gradients = self.response.gradients
+
+        optimizer.step() # Let's update the solution
+        st.session_state.current_solution = self.response.value
+
+    def show_results(self):
+        self._run()
+        st.session_state.iteration += 1
+        st.session_state.results.append({
+            'iteration': st.session_state.iteration,
+            'loss_value': self.loss_value,
+            'response': self.response.value,
+            'gradients': self.gradients
+        })
+
+        tabs = st.tabs([f"Iteration {i+1}" for i in range(st.session_state.iteration)])
+
+        for i, tab in enumerate(tabs):
+            with tab:
+                result = st.session_state.results[i]
+                st.markdown(f"Current iteration: **{result['iteration']}**")
+                st.markdown("## Current solution:")
+                st.markdown(result['response'])
+                col1, col2 = st.columns([1, 1])
+                with col1:
+                    st.markdown("## Loss value")
+                    st.markdown(result['loss_value'])
+                with col2:
+                    st.markdown("## Code gradients")
+                    for j, g in enumerate(result['gradients']):
+                        st.markdown(f"### Gradient")
+                        st.markdown(g.value)

examples/mathvista.json

@@ -0,0 +1,5 @@
+{
+    "image_URL": "assets/img/mathvista/math_chart.png",
+    "question_text": "What is the sum of the accuracies of the algorithms fat and acre?",
+    "evaluation_instruction": "Does this seem like a complete and good answer for the image? Criticize heavily."
+}

home.py

@@ -0,0 +1,132 @@
+from pathlib import Path
+import streamlit as st
+import streamlit.components.v1 as components
+from PIL import Image
+import base64
+import textgrad as tg
+from stqdm import stqdm
+import json
+import os
+
+def read_markdown_file(markdown_file):
+    return Path(markdown_file).read_text()
+
+def render_svg(svg_filename):
+    with open(svg_filename,"r") as f:
+        lines = f.readlines()
+        svg=''.join(lines)
+    """Renders the given svg string."""
+    b64 = base64.b64encode(svg.encode('utf-8')).decode("utf-8")
+    html = r'<img src="data:image/svg+xml;base64,%s"/>' % b64
+    st.write(html, unsafe_allow_html=True)
+
+def load_json(file_path):
+    with open(file_path, "r") as f:
+        return json.load(f)
+
+def app():
+    st.title('TextGrad: Automatic ''Differentiation'' via Text')
+    st.image('assets/img/textgrad_logo_1x.png', width=200)
+
+    st.markdown("### Examples")
+    examples = {
+        "Code Editor": load_json("examples/code_editor.json"),
+        "Example Math Solution": load_json("examples/example_math.json"),
+        # "🏞️ Image QA": load_json("examples/mathvista.json"),
+    }
+    
+    col1, col2 = st.columns([1,1])
+    with col1:
+        # Display clickable examples
+        option = st.selectbox(
+            "Select an example to use:",
+            examples.keys()
+            )
+
+        # Initialize session state if not already done
+        st.session_state.data = examples[option]
+        # if st.button("Clear example selection"):
+        #     st.session_state.data = ""
+        #     option = "None"
+
+        # Initialize session state if not already done
+        if "selected_option" not in st.session_state:
+            st.session_state.selected_option = None
+            st.session_state.object = None
+
+        # Check if the selected option has changed
+        if st.session_state.selected_option != option:
+            st.session_state.data = examples[option]
+            st.session_state.selected_option = option
+            st.session_state.object_initialized = False
+
+        # Display the selected option
+        st.markdown(f"**Example selected:** {option}")
+        
+    with col2:
+        valid_api_key = st.session_state.get('valid_api_key', False)
+        form = st.form(key='api_key')
+        OPENAI_API_KEY = form.text_input(label='Enter OpenAI API Key. Keys will not be stored', type = 'password')
+        submit_button = form.form_submit_button(label='Submit')
+
+        if submit_button:
+            os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+           
+            try:
+                engine = tg.get_engine("gpt-4o")
+                response = engine.generate("hello, what model is this?")
+                st.session_state.valid_api_key = True
+                valid_api_key = True
+            except:
+                st.error("Please enter a valid OpenAI API key.")
+                st.stop()
+
+    
+    if valid_api_key:
+        # Initialize the object when the option is selected
+        if not st.session_state.object_initialized:
+            if option == "Example Math Solution":
+                from examples import example_math_scripts
+                st.session_state.iteration = 0
+                st.session_state.object = example_math_scripts.MathSolution(data=st.session_state.get("data", ""))
+            elif option == "Code Editor":
+                from examples import code_editor_scripts
+                st.session_state.iteration = 0
+                st.session_state.object = code_editor_scripts.CodeEditor(data=st.session_state.get("data", ""))
+            # elif option == "🏞️ Image QA":
+            #     from examples import example_imageQA_scripts
+            #     st.session_state.iteration = 0
+            #     st.session_state.object = example_imageQA_scripts.ImageQA(data=st.session_state.get("data", ""))
+            st.session_state.object_initialized = True
+
+        # Ensure object is loaded
+        st.session_state.object.load_layout()
+
+        # Run TextGrad
+        # Add custom CSS to style the button
+        # st.markdown("""
+        #     <style>
+        #     .large-button {
+        #         font-size: 24px;
+        #         padding: 16px 32px;
+        #         background-color: #007bff;
+        #         color: white;
+        #         border: none;
+        #         border-radius: 4px;
+        #         cursor: pointer;
+        #     }
+        #     .large-button:hover {
+        #         background-color: #0056b3;
+        #     }
+        #     </style>
+        #     """, unsafe_allow_html=True)
+
+        # # Create a larger button using HTML
+        # if st.markdown('<button class="large-button">Run 1 iteration of TextGrad</button>', unsafe_allow_html=True):
+        if st.button("Run 1 iteration of TextGrad"):
+            st.session_state.object.show_results()
+
+
+    st.markdown("""---""")
+    st.markdown('### Disclaimer')
+    st.markdown("Do not get addicted to TextGrad. It is a powerful tool that can be used for good or evil. Use it responsibly.")

requirements.txt

@@ -0,0 +1,6 @@
+git+https://github.com/zou-group/textgrad.git
+stqdm
+streamlit_elements
+pillow
+streamlit
+IPython