Upload 1040 files

.dockerignore

@@ -0,0 +1,5 @@
+frontend/node_modules
+config.toml
+.envrc
+.env
+.git

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ipynb linguist-vendored

.gitignore

@@ -0,0 +1,235 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+./lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+requirements.txt
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+frontend/.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+.env.bak
+venv.bak/
+*venv/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+.vscode/
+.cursorignore
+
+# evaluation
+evaluation/evaluation_outputs
+evaluation/outputs
+evaluation/swe_bench/eval_workspace*
+evaluation/SWE-bench/data
+evaluation/webarena/scripts/webarena_env.sh
+evaluation/bird/data
+evaluation/gaia/data
+evaluation/gorilla/data
+evaluation/toolqa/data
+evaluation/scienceagentbench/benchmark
+evaluation/commit0_bench/repos
+
+# openhands resolver
+output/
+
+# frontend
+
+# dependencies
+frontend/.pnp
+frontend/bun.lockb
+frontend/yarn.lock
+.pnp.js
+
+# testing
+frontend/coverage
+test_results*
+/_test_files_tmp/
+
+# production
+frontend/build
+frontend/dist
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+logs
+
+# agent
+.envrc
+/workspace
+/_test_workspace
+/debug
+cache
+
+# configuration
+config.toml
+config.toml_
+config.toml.bak
+
+# swe-bench-eval
+image_build_logs
+run_instance_logs
+
+runtime_*.tar
+
+# docker build
+containers/runtime/Dockerfile
+containers/runtime/project.tar.gz
+containers/runtime/code
+**/node_modules/

.nvmrc

@@ -0,0 +1 @@
+22

.openhands/microagents/repo.md

@@ -0,0 +1,42 @@
+---
+name: repo
+type: repo
+agent: CodeActAgent
+---
+This repository contains the code for OpenHands, an automated AI software engineer. It has a Python backend
+(in the `openhands` directory) and React frontend (in the `frontend` directory).
+
+## General Setup:
+To set up the entire repo, including frontend and backend, run `make build`.
+You don't need to do this unless the user asks you to, or if you're trying to run the entire application.
+
+Before pushing any changes, you should ensure that any lint errors or simple test errors have been fixed.
+
+* If you've made changes to the backend, you should run `pre-commit run --all-files --config ./dev_config/python/.pre-commit-config.yaml`
+* If you've made changes to the frontend, you should run `cd frontend && npm run lint:fix && npm run build ; cd ..`
+
+If either command fails, it may have automatically fixed some issues. You should fix any issues that weren't automatically fixed,
+then re-run the command to ensure it passes.
+
+## Repository Structure
+Backend:
+- Located in the `openhands` directory
+- Testing:
+  - All tests are in `tests/unit/test_*.py`
+  - To test new code, run `poetry run pytest tests/unit/test_xxx.py` where `xxx` is the appropriate file for the current functionality
+  - Write all tests with pytest
+
+Frontend:
+- Located in the `frontend` directory
+- Prerequisites: A recent version of NodeJS / NPM
+- Setup: Run `npm install` in the frontend directory
+- Testing:
+  - Run tests: `npm run test`
+  - To run specific tests: `npm run test -- -t "TestName"`
+- Building:
+  - Build for production: `npm run build`
+- Environment Variables:
+  - Set in `frontend/.env` or as environment variables
+  - Available variables: VITE_BACKEND_HOST, VITE_USE_TLS, VITE_INSECURE_SKIP_VERIFY, VITE_FRONTEND_PORT
+- Internationalization:
+  - Generate i18n declaration file: `npm run make-i18n`

CODE_OF_CONDUCT.md

@@ -0,0 +1,147 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people.
+* Being respectful of differing opinions, viewpoints, and experiences.
+* Giving and gracefully accepting constructive feedback.
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience.
+* Focusing on what is best not just for us as individuals, but for the overall
+  community.
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind.
+* Trolling, insulting or derogatory comments, and personal or political attacks.
+* Public or private harassment.
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission.
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting.
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[email protected].
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+### Slack and Discord Etiquettes
+
+These Slack and Discord etiquette guidelines are designed to foster an inclusive, respectful, and productive environment for all community members. By following these best practices, we ensure effective communication and collaboration while minimizing disruptions. Let’s work together to build a supportive and welcoming community!
+
+- Communicate respectfully and professionally, avoiding sarcasm or harsh language, and remember that tone can be difficult to interpret in text.
+- Use threads for specific discussions to keep channels organized and easier to follow.
+- Tag others only when their input is critical or urgent, and use @here, @channel or @everyone sparingly to minimize disruptions.
+- Be patient, as open-source contributors and maintainers often have other commitments and may need time to respond.
+- Post questions or discussions in the most relevant channel (e.g., for [slack - #general](https://app.slack.com/client/T06P212QSEA/C06P5NCGSFP) for general topics, [slack - #questions](https://openhands-ai.slack.com/archives/C06U8UTKSAD) for queries/questions, [discord - #general](https://discord.com/channels/1222935860639563850/1222935861386018885)).
+- When asking for help or raising issues, include necessary details like links, screenshots, or clear explanations to provide context.
+- Keep discussions in public channels whenever possible to allow others to benefit from the conversation, unless the matter is sensitive or private.
+- Always adhere to [our standards](https://github.com/All-Hands-AI/OpenHands/blob/main/CODE_OF_CONDUCT.md#our-standards) to ensure a welcoming and collaborative environment.
+- If you choose to mute a channel, consider setting up alerts for topics that still interest you to stay engaged. For Slack, Go to Settings → Notifications → My Keywords to add specific keywords that will notify you when mentioned. For example, if you're here for discussions about LLMs, mute the channel if it’s too busy, but set notifications to alert you only when “LLMs” appears in messages. Also for Discord, go to the channel notifications and choose the option that best describes your need.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

COMMUNITY.md

@@ -0,0 +1,43 @@
+# 🙌 The OpenHands Community
+
+The OpenHands community is built around the belief that (1) AI and AI agents are going to fundamentally change the way
+we build software, and (2) if this is true, we should do everything we can to make sure that the benefits provided by
+such powerful technology are accessible to everyone.
+
+If this resonates with you, we'd love to have you join us in our quest!
+
+## 🤝 How to Join
+
+Check out our [How to Join the Community section.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-join-the-community)
+
+## 💪 Becoming a Contributor
+
+We welcome contributions from everyone! Whether you're a developer, a researcher, or simply enthusiastic about advancing
+the field of software engineering with AI, there are many ways to get involved:
+
+- **Code Contributions:** Help us develop new core functionality, improve our agents, improve the frontend and other
+interfaces, or anything else that would help make OpenHands better.
+- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in
+evaluating the models, or suggest improvements.
+- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
+
+For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
+
+## Code of Conduct
+
+We have a [Code of Conduct](./CODE_OF_CONDUCT.md) that we expect all contributors to adhere to.
+Long story short, we are aiming for an open, welcoming, diverse, inclusive, and healthy community.
+All contributors are expected to contribute to building this sort of community.
+
+## 🛠️ Becoming a Maintainer
+
+For contributors who have made significant and sustained contributions to the project, there is a possibility of joining
+the maintainer team. The process for this is as follows:
+
+1. Any contributor who has made sustained and high-quality contributions to the codebase can be nominated by any
+maintainer. If you feel that you may qualify you can reach out to any of the maintainers that have reviewed your PRs and ask if you can be nominated.
+2. Once a maintainer nominates a new maintainer, there will be a discussion period among the maintainers for at least 3 days.
+3. If no concerns are raised the nomination will be accepted by acclamation, and if concerns are raised there will be a discussion and possible vote.
+
+Note that just making many PRs does not immediately imply that you will become a maintainer. We will be looking
+at sustained high-quality contributions over a period of time, as well as good teamwork and adherence to our [Code of Conduct](./CODE_OF_CONDUCT.md).

CONTRIBUTING.md

@@ -0,0 +1,123 @@
+# Contributing
+
+Thanks for your interest in contributing to OpenHands! We welcome and appreciate contributions.
+
+## Understanding OpenHands's CodeBase
+
+To understand the codebase, please refer to the README in each module:
+- [frontend](./frontend/README.md)
+- [evaluation](./evaluation/README.md)
+- [openhands](./openhands/README.md)
+   - [agenthub](./openhands/agenthub/README.md)
+   - [server](./openhands/server/README.md)
+
+## Setting up Your Development Environment
+
+We have a separate doc [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) that tells you how to set up a development workflow.
+
+## How Can I Contribute?
+
+There are many ways that you can contribute:
+
+1. **Download and use** OpenHands, and send [issues](https://github.com/All-Hands-AI/OpenHands/issues) when you encounter something that isn't working or a feature that you'd like to see.
+2. **Send feedback** after each session by [clicking the thumbs-up thumbs-down buttons](https://docs.all-hands.dev/modules/usage/feedback), so we can see where things are working and failing, and also build an open dataset for training code agents.
+3. **Improve the Codebase** by sending [PRs](#sending-pull-requests-to-openhands) (see details below). In particular, we have some [good first issues](https://github.com/All-Hands-AI/OpenHands/labels/good%20first%20issue) that may be ones to start on.
+
+## What Can I Build?
+Here are a few ways you can help improve the codebase.
+
+#### UI/UX
+We're always looking to improve the look and feel of the application. If you've got a small fix
+for something that's bugging you, feel free to open up a PR that changes the [`./frontend`](./frontend) directory.
+
+If you're looking to make a bigger change, add a new UI element, or significantly alter the style
+of the application, please open an issue first, or better, join the #frontend channel in our Slack
+to gather consensus from our design team first.
+
+#### Improving the agent
+Our main agent is the CodeAct agent. You can [see its prompts here](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/agenthub/codeact_agent).
+
+Changes to these prompts, and to the underlying behavior in Python, can have a huge impact on user experience.
+You can try modifying the prompts to see how they change the behavior of the agent as you use the app
+locally, but we will need to do an end-to-end evaluation of any changes here to ensure that the agent
+is getting better over time.
+
+We use the [SWE-bench](https://www.swebench.com/) benchmark to test our agent. You can join the #evaluation
+channel in Slack to learn more.
+
+#### Adding a new agent
+You may want to experiment with building new types of agents. You can add an agent to [`openhands/agenthub`](./openhands/agenthub)
+to help expand the capabilities of OpenHands.
+
+#### Adding a new runtime
+The agent needs a place to run code and commands. When you run OpenHands on your laptop, it uses a Docker container
+to do this by default. But there are other ways of creating a sandbox for the agent.
+
+If you work for a company that provides a cloud-based runtime, you could help us add support for that runtime
+by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/base.py).
+
+#### Testing
+When you write code, it is also good to write tests. Please navigate to the [`./tests`](./tests) folder to see existing test suites.
+At the moment, we have two kinds of tests: [`unit`](./tests/unit) and [`integration`](./evaluation/integration_tests). Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project.
+
+## Sending Pull Requests to OpenHands
+
+You'll need to fork our repository to send us a Pull Request. You can learn more
+about how to fork a GitHub repo and open a PR with your changes in [this article](https://medium.com/swlh/forks-and-pull-requests-how-to-contribute-to-github-repos-8843fac34ce8).
+
+### Pull Request title
+As described [here](https://github.com/commitizen/conventional-commit-types/blob/master/index.json), a valid PR title should begin with one of the following prefixes:
+
+- `feat`: A new feature
+- `fix`: A bug fix
+- `docs`: Documentation only changes
+- `style`: Changes that do not affect the meaning of the code (white space, formatting, missing semicolons, etc.)
+- `refactor`: A code change that neither fixes a bug nor adds a feature
+- `perf`: A code change that improves performance
+- `test`: Adding missing tests or correcting existing tests
+- `build`: Changes that affect the build system or external dependencies (example scopes: gulp, broccoli, npm)
+- `ci`: Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs)
+- `chore`: Other changes that don't modify src or test files
+- `revert`: Reverts a previous commit
+
+For example, a PR title could be:
+- `refactor: modify package path`
+- `feat(frontend): xxxx`, where `(frontend)` means that this PR mainly focuses on the frontend component.
+
+You may also check out previous PRs in the [PR list](https://github.com/All-Hands-AI/OpenHands/pulls).
+
+### Pull Request description
+- If your PR is small (such as a typo fix), you can go brief.
+- If it contains a lot of changes, it's better to write more details.
+
+If your changes are user-facing (e.g. a new feature in the UI, a change in behavior, or a bugfix)
+please include a short message that we can add to our changelog.
+
+## How to Make Effective Contributions
+
+### Opening Issues
+
+If you notice any bugs or have any feature requests please open them via the [issues page](https://github.com/All-Hands-AI/OpenHands/issues). We will triage based on how critical the bug is or how potentially useful the improvement is, discuss, and implement the ones that the community has interest/effort for.
+
+Further, if you see an issue you like, please leave a "thumbs-up" or a comment, which will help us prioritize.
+
+### Making Pull Requests
+
+We're generally happy to consider all pull requests with the evaluation process varying based on the type of change:
+
+#### For Small Improvements
+
+Small improvements with few downsides are typically reviewed and approved quickly.
+One thing to check when making changes is to ensure that all continuous integration tests pass, which you can check before getting a review.
+
+#### For Core Agent Changes
+
+We need to be more careful with changes to the core agent, as it is imperative to maintain high quality. These PRs are evaluated based on three key metrics:
+
+1. **Accuracy**
+2. **Efficiency**
+3. **Code Complexity**
+
+If it improves accuracy, efficiency, or both with only a minimal change to code quality, that's great we're happy to merge it in!
+If there are bigger tradeoffs (e.g. helping efficiency a lot and hurting accuracy a little) we might want to put it behind a feature flag.
+Either way, please feel free to discuss on github issues or slack, and we will give guidance and preliminary feedback.

CREDITS.md

@@ -0,0 +1,312 @@
+# Credits
+
+## Contributors
+
+We would like to thank all the [contributors](https://github.com/All-Hands-AI/OpenHands/graphs/contributors) who have helped make OpenHands possible. We greatly appreciate your dedication and hard work.
+
+## Open Source Projects
+
+OpenHands includes and adapts the following open source projects. We are grateful for their contributions to the open source community:
+
+#### [SWE Agent](https://github.com/princeton-nlp/swe-agent)
+   - License: MIT License
+   - Description: Adapted for use in OpenHands's agent hub
+
+#### [Aider](https://github.com/paul-gauthier/aider)
+   - License: Apache License 2.0
+   - Description: AI pair programming tool. OpenHands has adapted and integrated its linter module for code-related tasks in [`agentskills utilities`](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/agent_skills/utils/aider)
+
+#### [BrowserGym](https://github.com/ServiceNow/BrowserGym)
+   - License: Apache License 2.0
+   - Description: Adapted in implementing the browsing agent
+
+
+### Reference Implementations for Evaluation Benchmarks
+OpenHands integrates code of the reference implementations for the following agent evaluation benchmarks:
+
+#### [HumanEval](https://github.com/openai/human-eval)
+   - License: MIT License
+
+#### [DSP](https://github.com/microsoft/DataScienceProblems)
+   - License: MIT License
+
+#### [HumanEvalPack](https://github.com/bigcode-project/bigcode-evaluation-harness)
+   - License: Apache License 2.0
+
+#### [AgentBench](https://github.com/THUDM/AgentBench)
+   - License: Apache License 2.0
+
+#### [SWE-Bench](https://github.com/princeton-nlp/SWE-bench)
+   - License: MIT License
+
+#### [BIRD](https://bird-bench.github.io/)
+   - License: MIT License
+   - Dataset: CC-BY-SA 4.0
+
+#### [Gorilla APIBench](https://github.com/ShishirPatil/gorilla)
+   - License: Apache License 2.0
+
+#### [GPQA](https://github.com/idavidrein/gpqa)
+   - License: MIT License
+
+#### [ProntoQA](https://github.com/asaparov/prontoqa)
+   - License: Apache License 2.0
+
+
+## Open Source licenses
+
+### MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+### BSD 3-Clause License
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+### Apache License 2.0
+
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+
+
+### Non-Open Source Reference Implementations:
+
+#### [MultiPL-E](https://github.com/nuprl/MultiPL-E)
+   - License: BSD 3-Clause License with Machine Learning Restriction
+
+BSD 3-Clause License with Machine Learning Restriction
+
+Copyright (c) 2022, Northeastern University, Oberlin College, Roblox Inc,
+Stevens Institute of Technology, University of Massachusetts Amherst, and
+Wellesley College.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+4.  The contents of this repository may not be used as training data for any
+    machine learning model, including but not limited to neural networks.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Development.md

@@ -0,0 +1,128 @@
+# Development Guide
+This guide is for people working on OpenHands and editing the source code.
+If you wish to contribute your changes, check out the [CONTRIBUTING.md](https://github.com/All-Hands-AI/OpenHands/blob/main/CONTRIBUTING.md) on how to clone and setup the project initially before moving on.
+Otherwise, you can clone the OpenHands project directly.
+
+## Start the Server for Development
+### 1. Requirements
+* Linux, Mac OS, or [WSL on Windows](https://learn.microsoft.com/en-us/windows/wsl/install)  [Ubuntu >= 22.04]
+* [Docker](https://docs.docker.com/engine/install/) (For those on MacOS, make sure to allow the default Docker socket to be used from advanced settings!)
+* [Python](https://www.python.org/downloads/) = 3.12
+* [NodeJS](https://nodejs.org/en/download/package-manager) >= 20.x
+* [Poetry](https://python-poetry.org/docs/#installing-with-the-official-installer) >= 1.8
+* OS-specific dependencies:
+  - Ubuntu: build-essential => `sudo apt-get install build-essential`
+  - WSL: netcat => `sudo apt-get install netcat`
+
+Make sure you have all these dependencies installed before moving on to `make build`.
+
+#### Develop without sudo access
+If you want to develop without system admin/sudo access to upgrade/install `Python` and/or `NodeJs`, you can use `conda` or `mamba` to manage the packages for you:
+
+```bash
+# Download and install Mamba (a faster version of conda)
+curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+bash Miniforge3-$(uname)-$(uname -m).sh
+
+# Install Python 3.12, nodejs, and poetry
+mamba install python=3.12
+mamba install conda-forge::nodejs
+mamba install conda-forge::poetry
+```
+
+### 2. Build and Setup The Environment
+Begin by building the project which includes setting up the environment and installing dependencies. This step ensures that OpenHands is ready to run on your system:
+
+```bash
+make build
+```
+
+### 3. Configuring the Language Model
+OpenHands supports a diverse array of Language Models (LMs) through the powerful [litellm](https://docs.litellm.ai) library.
+By default, we've chosen Claude Sonnet 3.5 as our go-to model, but the world is your oyster! You can unleash the
+potential of any other LM that piques your interest.
+
+To configure the LM of your choice, run:
+
+   ```bash
+   make setup-config
+   ```
+
+   This command will prompt you to enter the LLM API key, model name, and other variables ensuring that OpenHands is tailored to your specific needs. Note that the model name will apply only when you run headless. If you use the UI, please set the model in the UI.
+
+   Note: If you have previously run OpenHands using the docker command, you may have already set some environmental variables in your terminal. The final configurations are set from highest to lowest priority:
+   Environment variables > config.toml variables > default variables
+
+**Note on Alternative Models:**
+See [our documentation](https://docs.all-hands.dev/modules/usage/llms) for recommended models.
+
+### 4. Running the application
+#### Option A: Run the Full Application
+Once the setup is complete, this command starts both the backend and frontend servers, allowing you to interact with OpenHands:
+```bash
+make run
+```
+
+#### Option B: Individual Server Startup
+- **Start the Backend Server:** If you prefer, you can start the backend server independently to focus on backend-related tasks or configurations.
+    ```bash
+    make start-backend
+    ```
+
+- **Start the Frontend Server:** Similarly, you can start the frontend server on its own to work on frontend-related components or interface enhancements.
+    ```bash
+    make start-frontend
+    ```
+
+### 6. LLM Debugging
+If you encounter any issues with the Language Model (LM) or you're simply curious, export DEBUG=1 in the environment and restart the backend.
+OpenHands will log the prompts and responses in the logs/llm/CURRENT_DATE directory, allowing you to identify the causes.
+
+### 7. Help
+Need help or info on available targets and commands? Use the help command for all the guidance you need with OpenHands.
+```bash
+make help
+ ```
+
+### 8. Testing
+To run tests, refer to the following:
+#### Unit tests
+
+```bash
+poetry run pytest ./tests/unit/test_*.py
+```
+
+### 9. Add or update dependency
+1. Add your dependency in `pyproject.toml` or use `poetry add xxx`.
+2. Update the poetry.lock file via `poetry lock --no-update`.
+
+### 9. Use existing Docker image
+To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image by
+setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
+
+Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.20-nikolaik`
+
+## Develop inside Docker container
+
+TL;DR
+
+```bash
+make docker-dev
+```
+
+See more details [here](./containers/dev/README.md).
+
+If you are just interested in running `OpenHands` without installing all the required tools on your host.
+
+```bash
+make docker-run
+```
+
+If you do not have `make` on your host, run:
+
+```bash
+cd ./containers/dev
+./dev.sh
+```
+
+You do need [Docker](https://docs.docker.com/engine/install/) installed on your host though.

Dockerfile

@@ -0,0 +1,98 @@
+ARG OPENHANDS_BUILD_VERSION=dev
+FROM node:21.7.2-bookworm-slim AS frontend-builder
+
+WORKDIR /app
+
+COPY ./frontend/package.json frontend/package-lock.json ./
+RUN npm install -g [email protected]
+RUN npm ci
+
+COPY ./frontend ./
+RUN npm run build
+
+FROM python:3.12.3-slim AS backend-builder
+
+WORKDIR /app
+ENV PYTHONPATH='/app'
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+RUN apt-get update -y \
+    && apt-get install -y curl make git build-essential \
+    && python3 -m pip install poetry==1.8.2  --break-system-packages
+
+COPY ./pyproject.toml ./poetry.lock ./
+RUN touch README.md
+RUN export POETRY_CACHE_DIR && poetry install --without evaluation,llama-index --no-root && rm -rf $POETRY_CACHE_DIR
+
+FROM python:3.12.3-slim AS openhands-app
+
+WORKDIR /app
+
+ARG OPENHANDS_BUILD_VERSION #re-declare for this section
+
+ENV RUN_AS_OPENHANDS=true
+# A random number--we need this to be different from the user's UID on the host machine
+ENV OPENHANDS_USER_ID=42420
+ENV SANDBOX_LOCAL_RUNTIME_URL=http://host.docker.internal
+ENV USE_HOST_NETWORK=false
+ENV WORKSPACE_BASE=/opt/workspace_base
+ENV OPENHANDS_BUILD_VERSION=$OPENHANDS_BUILD_VERSION
+ENV SANDBOX_USER_ID=0
+ENV FILE_STORE=local
+ENV FILE_STORE_PATH=/.openhands-state
+RUN mkdir -p $FILE_STORE_PATH
+RUN mkdir -p $WORKSPACE_BASE
+
+RUN apt-get update -y \
+    && apt-get install -y curl ssh sudo
+
+# Default is 1000, but OSX is often 501
+RUN sed -i 's/^UID_MIN.*/UID_MIN 499/' /etc/login.defs
+# Default is 60000, but we've seen up to 200000
+RUN sed -i 's/^UID_MAX.*/UID_MAX 1000000/' /etc/login.defs
+
+RUN groupadd app
+RUN useradd -l -m -u $OPENHANDS_USER_ID -s /bin/bash openhands && \
+    usermod -aG app openhands && \
+    usermod -aG sudo openhands && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+RUN chown -R openhands:app /app && chmod -R 770 /app
+RUN sudo chown -R openhands:app $WORKSPACE_BASE && sudo chmod -R 770 $WORKSPACE_BASE
+USER openhands
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH" \
+    PYTHONPATH='/app'
+
+COPY --chown=openhands:app --chmod=770 --from=backend-builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+RUN playwright install --with-deps chromium
+
+COPY --chown=openhands:app --chmod=770 ./microagents ./microagents
+COPY --chown=openhands:app --chmod=770 ./openhands ./openhands
+COPY --chown=openhands:app --chmod=777 ./openhands/runtime/plugins ./openhands/runtime/plugins
+COPY --chown=openhands:app --chmod=770 ./openhands/agenthub ./openhands/agenthub
+COPY --chown=openhands:app ./pyproject.toml ./pyproject.toml
+COPY --chown=openhands:app ./poetry.lock ./poetry.lock
+COPY --chown=openhands:app ./README.md ./README.md
+COPY --chown=openhands:app ./MANIFEST.in ./MANIFEST.in
+COPY --chown=openhands:app ./LICENSE ./LICENSE
+
+# This is run as "openhands" user, and will create __pycache__ with openhands:openhands ownership
+RUN python openhands/core/download.py # No-op to download assets
+# Add this line to set group ownership of all files/directories not already in "app" group
+# openhands:openhands -> openhands:app
+RUN find /app \! -group app -exec chgrp app {} +
+
+COPY --chown=openhands:app --chmod=770 --from=frontend-builder /app/build ./frontend/build
+COPY --chown=openhands:app --chmod=770 ./containers/app/entrypoint.sh /app/entrypoint.sh
+
+USER root
+
+WORKDIR /app
+
+ENTRYPOINT ["/app/entrypoint.sh"]
+CMD ["uvicorn", "openhands.server.listen:app", "--host", "0.0.0.0", "--port", "3000"]

ISSUE_TRIAGE.md

@@ -0,0 +1,25 @@
+# Issue Triage
+These are the procedures and guidelines on how issues are triaged in this repo by the maintainers.
+
+## General
+* Most issues must be tagged with **enhancement** or **bug**.
+* Issues may be tagged with what it relates to (**backend**, **frontend**, **agent quality**, etc.).
+
+## Severity
+* **Low**: Minor issues or affecting single user.
+* **Medium**: Affecting multiple users.
+* **Critical**: Affecting all users or potential security issues.
+
+## Effort
+* Issues may be estimated with effort required (**small effort**, **medium effort**, **large effort**).
+
+## Difficulty
+* Issues with low implementation difficulty may be tagged with **good first issue**.
+
+## Not Enough Information
+* User is asked to provide more information (logs, how to reproduce, etc.) when the issue is not clear.
+* If an issue is unclear and the author does not provide more information or respond to a request, the issue may be closed as **not planned** (Usually after a week).
+
+## Multiple Requests/Fixes in One Issue
+* These issues will be narrowed down to one request/fix so the issue is more easily tracked and fixed.
+* Issues may be broken down into multiple issues if required.

LICENSE

@@ -0,0 +1,25 @@
+The MIT License (MIT)
+=====================
+
+Copyright © 2023
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the “Software”), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -0,0 +1,5 @@
+# Exclude all Python bytecode files
+global-exclude *.pyc
+
+# Exclude Python cache directories
+global-exclude __pycache__

Makefile

@@ -0,0 +1,332 @@
+SHELL=/bin/bash
+# Makefile for OpenHands project
+
+# Variables
+BACKEND_HOST ?= "127.0.0.1"
+BACKEND_PORT = 3000
+BACKEND_HOST_PORT = "$(BACKEND_HOST):$(BACKEND_PORT)"
+FRONTEND_PORT = 3001
+DEFAULT_WORKSPACE_DIR = "./workspace"
+DEFAULT_MODEL = "gpt-4o"
+CONFIG_FILE = config.toml
+PRE_COMMIT_CONFIG_PATH = "./dev_config/python/.pre-commit-config.yaml"
+PYTHON_VERSION = 3.12
+
+# ANSI color codes
+GREEN=$(shell tput -Txterm setaf 2)
+YELLOW=$(shell tput -Txterm setaf 3)
+RED=$(shell tput -Txterm setaf 1)
+BLUE=$(shell tput -Txterm setaf 6)
+RESET=$(shell tput -Txterm sgr0)
+
+# Build
+build:
+	@echo "$(GREEN)Building project...$(RESET)"
+	@$(MAKE) -s check-dependencies
+	@$(MAKE) -s install-python-dependencies
+	@$(MAKE) -s install-frontend-dependencies
+	@$(MAKE) -s install-pre-commit-hooks
+	@$(MAKE) -s build-frontend
+	@echo "$(GREEN)Build completed successfully.$(RESET)"
+
+check-dependencies:
+	@echo "$(YELLOW)Checking dependencies...$(RESET)"
+	@$(MAKE) -s check-system
+	@$(MAKE) -s check-python
+	@$(MAKE) -s check-npm
+	@$(MAKE) -s check-nodejs
+ifeq ($(INSTALL_DOCKER),)
+	@$(MAKE) -s check-docker
+endif
+	@$(MAKE) -s check-poetry
+	@echo "$(GREEN)Dependencies checked successfully.$(RESET)"
+
+check-system:
+	@echo "$(YELLOW)Checking system...$(RESET)"
+	@if [ "$(shell uname)" = "Darwin" ]; then \
+		echo "$(BLUE)macOS detected.$(RESET)"; \
+	elif [ "$(shell uname)" = "Linux" ]; then \
+		if [ -f "/etc/manjaro-release" ]; then \
+			echo "$(BLUE)Manjaro Linux detected.$(RESET)"; \
+		else \
+			echo "$(BLUE)Linux detected.$(RESET)"; \
+		fi; \
+	elif [ "$$(uname -r | grep -i microsoft)" ]; then \
+		echo "$(BLUE)Windows Subsystem for Linux detected.$(RESET)"; \
+	else \
+		echo "$(RED)Unsupported system detected. Please use macOS, Linux, or Windows Subsystem for Linux (WSL).$(RESET)"; \
+		exit 1; \
+	fi
+
+check-python:
+	@echo "$(YELLOW)Checking Python installation...$(RESET)"
+	@if command -v python$(PYTHON_VERSION) > /dev/null; then \
+		echo "$(BLUE)$(shell python$(PYTHON_VERSION) --version) is already installed.$(RESET)"; \
+	else \
+		echo "$(RED)Python $(PYTHON_VERSION) is not installed. Please install Python $(PYTHON_VERSION) to continue.$(RESET)"; \
+		exit 1; \
+	fi
+
+check-npm:
+	@echo "$(YELLOW)Checking npm installation...$(RESET)"
+	@if command -v npm > /dev/null; then \
+		echo "$(BLUE)npm $(shell npm --version) is already installed.$(RESET)"; \
+	else \
+		echo "$(RED)npm is not installed. Please install Node.js to continue.$(RESET)"; \
+		exit 1; \
+	fi
+
+check-nodejs:
+	@echo "$(YELLOW)Checking Node.js installation...$(RESET)"
+	@if command -v node > /dev/null; then \
+		NODE_VERSION=$(shell node --version | sed -E 's/v//g'); \
+		IFS='.' read -r -a NODE_VERSION_ARRAY <<< "$$NODE_VERSION"; \
+		if [ "$${NODE_VERSION_ARRAY[0]}" -ge 20 ]; then \
+			echo "$(BLUE)Node.js $$NODE_VERSION is already installed.$(RESET)"; \
+		else \
+			echo "$(RED)Node.js 20.x or later is required. Please install Node.js 20.x or later to continue.$(RESET)"; \
+			exit 1; \
+		fi; \
+	else \
+		echo "$(RED)Node.js is not installed. Please install Node.js to continue.$(RESET)"; \
+		exit 1; \
+	fi
+
+check-docker:
+	@echo "$(YELLOW)Checking Docker installation...$(RESET)"
+	@if command -v docker > /dev/null; then \
+		echo "$(BLUE)$(shell docker --version) is already installed.$(RESET)"; \
+	else \
+		echo "$(RED)Docker is not installed. Please install Docker to continue.$(RESET)"; \
+		exit 1; \
+	fi
+
+check-poetry:
+	@echo "$(YELLOW)Checking Poetry installation...$(RESET)"
+	@if command -v poetry > /dev/null; then \
+		POETRY_VERSION=$(shell poetry --version 2>&1 | sed -E 's/Poetry \(version ([0-9]+\.[0-9]+\.[0-9]+)\)/\1/'); \
+		IFS='.' read -r -a POETRY_VERSION_ARRAY <<< "$$POETRY_VERSION"; \
+		if [ $${POETRY_VERSION_ARRAY[0]} -gt 1 ] || ([ $${POETRY_VERSION_ARRAY[0]} -eq 1 ] && [ $${POETRY_VERSION_ARRAY[1]} -ge 8 ]); then \
+			echo "$(BLUE)$(shell poetry --version) is already installed.$(RESET)"; \
+		else \
+			echo "$(RED)Poetry 1.8 or later is required. You can install poetry by running the following command, then adding Poetry to your PATH:"; \
+			echo "$(RED) curl -sSL https://install.python-poetry.org | python$(PYTHON_VERSION) -$(RESET)"; \
+			echo "$(RED)More detail here: https://python-poetry.org/docs/#installing-with-the-official-installer$(RESET)"; \
+			exit 1; \
+		fi; \
+	else \
+		echo "$(RED)Poetry is not installed. You can install poetry by running the following command, then adding Poetry to your PATH:"; \
+		echo "$(RED) curl -sSL https://install.python-poetry.org | python$(PYTHON_VERSION) -$(RESET)"; \
+		echo "$(RED)More detail here: https://python-poetry.org/docs/#installing-with-the-official-installer$(RESET)"; \
+		exit 1; \
+	fi
+
+install-python-dependencies:
+	@echo "$(GREEN)Installing Python dependencies...$(RESET)"
+	@if [ -z "${TZ}" ]; then \
+		echo "Defaulting TZ (timezone) to UTC"; \
+		export TZ="UTC"; \
+	fi
+	poetry env use python$(PYTHON_VERSION)
+	@if [ "$(shell uname)" = "Darwin" ]; then \
+		echo "$(BLUE)Installing chroma-hnswlib...$(RESET)"; \
+		export HNSWLIB_NO_NATIVE=1; \
+		poetry run pip install chroma-hnswlib; \
+	fi
+	@poetry install --without llama-index
+	@if [ -f "/etc/manjaro-release" ]; then \
+		echo "$(BLUE)Detected Manjaro Linux. Installing Playwright dependencies...$(RESET)"; \
+		poetry run pip install playwright; \
+		poetry run playwright install chromium; \
+	else \
+		if [ ! -f cache/playwright_chromium_is_installed.txt ]; then \
+			echo "Running playwright install --with-deps chromium..."; \
+			poetry run playwright install --with-deps chromium; \
+			mkdir -p cache; \
+			touch cache/playwright_chromium_is_installed.txt; \
+		else \
+			echo "Setup already done. Skipping playwright installation."; \
+		fi \
+	fi
+	@echo "$(GREEN)Python dependencies installed successfully.$(RESET)"
+
+install-frontend-dependencies:
+	@echo "$(YELLOW)Setting up frontend environment...$(RESET)"
+	@echo "$(YELLOW)Detect Node.js version...$(RESET)"
+	@cd frontend && node ./scripts/detect-node-version.js
+	echo "$(BLUE)Installing frontend dependencies with npm...$(RESET)"
+	@cd frontend && npm install
+	@echo "$(GREEN)Frontend dependencies installed successfully.$(RESET)"
+
+install-pre-commit-hooks:
+	@echo "$(YELLOW)Installing pre-commit hooks...$(RESET)"
+	@git config --unset-all core.hooksPath || true
+	@poetry run pre-commit install --config $(PRE_COMMIT_CONFIG_PATH)
+	@echo "$(GREEN)Pre-commit hooks installed successfully.$(RESET)"
+
+lint-backend:
+	@echo "$(YELLOW)Running linters...$(RESET)"
+	@poetry run pre-commit run --files openhands/**/* agenthub/**/* evaluation/**/* --show-diff-on-failure --config $(PRE_COMMIT_CONFIG_PATH)
+
+lint-frontend:
+	@echo "$(YELLOW)Running linters for frontend...$(RESET)"
+	@cd frontend && npm run lint
+
+lint:
+	@$(MAKE) -s lint-frontend
+	@$(MAKE) -s lint-backend
+
+test-frontend:
+	@echo "$(YELLOW)Running tests for frontend...$(RESET)"
+	@cd frontend && npm run test
+
+test:
+	@$(MAKE) -s test-frontend
+
+build-frontend:
+	@echo "$(YELLOW)Building frontend...$(RESET)"
+	@cd frontend && npm run build
+
+# Start backend
+start-backend:
+	@echo "$(YELLOW)Starting backend...$(RESET)"
+	@poetry run uvicorn openhands.server.listen:app --host $(BACKEND_HOST) --port $(BACKEND_PORT) --reload --reload-exclude "./workspace"
+
+# Start frontend
+start-frontend:
+	@echo "$(YELLOW)Starting frontend...$(RESET)"
+	@cd frontend && VITE_BACKEND_HOST=$(BACKEND_HOST_PORT) VITE_FRONTEND_PORT=$(FRONTEND_PORT) npm run dev -- --port $(FRONTEND_PORT) --host $(BACKEND_HOST)
+
+# Common setup for running the app (non-callable)
+_run_setup:
+	@if [ "$(OS)" = "Windows_NT" ]; then \
+		echo "$(RED) Windows is not supported, use WSL instead!$(RESET)"; \
+		exit 1; \
+	fi
+	@mkdir -p logs
+	@echo "$(YELLOW)Starting backend server...$(RESET)"
+	@poetry run uvicorn openhands.server.listen:app --host $(BACKEND_HOST) --port $(BACKEND_PORT) &
+	@echo "$(YELLOW)Waiting for the backend to start...$(RESET)"
+	@until nc -z localhost $(BACKEND_PORT); do sleep 0.1; done
+	@echo "$(GREEN)Backend started successfully.$(RESET)"
+
+# Run the app (standard mode)
+run:
+	@echo "$(YELLOW)Running the app...$(RESET)"
+	@$(MAKE) -s _run_setup
+	@$(MAKE) -s start-frontend
+	@echo "$(GREEN)Application started successfully.$(RESET)"
+
+# Run the app (in docker)
+docker-run: WORKSPACE_BASE ?= $(PWD)/workspace
+docker-run:
+	@if [ -f /.dockerenv ]; then \
+		echo "Running inside a Docker container. Exiting..."; \
+		exit 0; \
+	else \
+		echo "$(YELLOW)Running the app in Docker $(OPTIONS)...$(RESET)"; \
+		export WORKSPACE_BASE=${WORKSPACE_BASE}; \
+		export SANDBOX_USER_ID=$(shell id -u); \
+		export DATE=$(shell date +%Y%m%d%H%M%S); \
+		docker compose up $(OPTIONS); \
+	fi
+
+# Run the app (WSL mode)
+run-wsl:
+	@echo "$(YELLOW)Running the app in WSL mode...$(RESET)"
+	@$(MAKE) -s _run_setup
+	@cd frontend && echo "$(BLUE)Starting frontend with npm (WSL mode)...$(RESET)" && npm run dev_wsl -- --port $(FRONTEND_PORT)
+	@echo "$(GREEN)Application started successfully in WSL mode.$(RESET)"
+
+# Setup config.toml
+setup-config:
+	@echo "$(YELLOW)Setting up config.toml...$(RESET)"
+	@$(MAKE) setup-config-prompts
+	@mv $(CONFIG_FILE).tmp $(CONFIG_FILE)
+	@echo "$(GREEN)Config.toml setup completed.$(RESET)"
+
+setup-config-prompts:
+	@echo "[core]" > $(CONFIG_FILE).tmp
+
+	@read -p "Enter your workspace directory (as absolute path) [default: $(DEFAULT_WORKSPACE_DIR)]: " workspace_dir; \
+	 workspace_dir=$${workspace_dir:-$(DEFAULT_WORKSPACE_DIR)}; \
+	 echo "workspace_base=\"$$workspace_dir\"" >> $(CONFIG_FILE).tmp
+
+	@echo "" >> $(CONFIG_FILE).tmp
+
+	@echo "[llm]" >> $(CONFIG_FILE).tmp
+	@read -p "Enter your LLM model name, used for running without UI. Set the model in the UI after you start the app. (see https://docs.litellm.ai/docs/providers for full list) [default: $(DEFAULT_MODEL)]: " llm_model; \
+	 llm_model=$${llm_model:-$(DEFAULT_MODEL)}; \
+	 echo "model=\"$$llm_model\"" >> $(CONFIG_FILE).tmp
+
+	@read -p "Enter your LLM api key: " llm_api_key; \
+	 echo "api_key=\"$$llm_api_key\"" >> $(CONFIG_FILE).tmp
+
+	@read -p "Enter your LLM base URL [mostly used for local LLMs, leave blank if not needed - example: http://localhost:5001/v1/]: " llm_base_url; \
+	 if [[ ! -z "$$llm_base_url" ]]; then echo "base_url=\"$$llm_base_url\"" >> $(CONFIG_FILE).tmp; fi
+
+	@echo "Enter your LLM Embedding Model"; \
+		echo "Choices are:"; \
+		echo "  - openai"; \
+		echo "  - azureopenai"; \
+		echo "  - Embeddings available only with OllamaEmbedding:"; \
+		echo "    - llama2"; \
+		echo "    - mxbai-embed-large"; \
+		echo "    - nomic-embed-text"; \
+		echo "    - all-minilm"; \
+		echo "    - stable-code"; \
+		echo "    - bge-m3"; \
+		echo "    - bge-large"; \
+		echo "    - paraphrase-multilingual"; \
+		echo "    - snowflake-arctic-embed"; \
+		echo "  - Leave blank to default to 'BAAI/bge-small-en-v1.5' via huggingface"; \
+		read -p "> " llm_embedding_model; \
+		echo "embedding_model=\"$$llm_embedding_model\"" >> $(CONFIG_FILE).tmp; \
+		if [ "$$llm_embedding_model" = "llama2" ] || [ "$$llm_embedding_model" = "mxbai-embed-large" ] || [ "$$llm_embedding_model" = "nomic-embed-text" ] || [ "$$llm_embedding_model" = "all-minilm" ] || [ "$$llm_embedding_model" = "stable-code" ]; then \
+			read -p "Enter the local model URL for the embedding model (will set llm.embedding_base_url): " llm_embedding_base_url; \
+				echo "embedding_base_url=\"$$llm_embedding_base_url\"" >> $(CONFIG_FILE).tmp; \
+		elif [ "$$llm_embedding_model" = "azureopenai" ]; then \
+			read -p "Enter the Azure endpoint URL (will overwrite llm.base_url): " llm_base_url; \
+				echo "base_url=\"$$llm_base_url\"" >> $(CONFIG_FILE).tmp; \
+			read -p "Enter the Azure LLM Embedding Deployment Name: " llm_embedding_deployment_name; \
+				echo "embedding_deployment_name=\"$$llm_embedding_deployment_name\"" >> $(CONFIG_FILE).tmp; \
+			read -p "Enter the Azure API Version: " llm_api_version; \
+				echo "api_version=\"$$llm_api_version\"" >> $(CONFIG_FILE).tmp; \
+		fi
+
+
+# Develop in container
+docker-dev:
+	@if [ -f /.dockerenv ]; then \
+		echo "Running inside a Docker container. Exiting..."; \
+		exit 0; \
+	else \
+		echo "$(YELLOW)Build and run in Docker $(OPTIONS)...$(RESET)"; \
+		./containers/dev/dev.sh $(OPTIONS); \
+	fi
+
+# Clean up all caches
+clean:
+	@echo "$(YELLOW)Cleaning up caches...$(RESET)"
+	@rm -rf openhands/.cache
+	@echo "$(GREEN)Caches cleaned up successfully.$(RESET)"
+
+# Help
+help:
+	@echo "$(BLUE)Usage: make [target]$(RESET)"
+	@echo "Targets:"
+	@echo "  $(GREEN)build$(RESET)               - Build project, including environment setup and dependencies."
+	@echo "  $(GREEN)lint$(RESET)                - Run linters on the project."
+	@echo "  $(GREEN)setup-config$(RESET)        - Setup the configuration for OpenHands by providing LLM API key,"
+	@echo "                        LLM Model name, and workspace directory."
+	@echo "  $(GREEN)start-backend$(RESET)       - Start the backend server for the OpenHands project."
+	@echo "  $(GREEN)start-frontend$(RESET)      - Start the frontend server for the OpenHands project."
+	@echo "  $(GREEN)run$(RESET)                 - Run the OpenHands application, starting both backend and frontend servers."
+	@echo "                        Backend Log file will be stored in the 'logs' directory."
+	@echo "  $(GREEN)docker-dev$(RESET)          - Build and run the OpenHands application in Docker."
+	@echo "  $(GREEN)docker-run$(RESET)          - Run the OpenHands application, starting both backend and frontend servers in Docker."
+	@echo "  $(GREEN)help$(RESET)                - Display this help message, providing information on available targets."
+
+# Phony targets
+.PHONY: build check-dependencies check-python check-npm check-docker check-poetry install-python-dependencies install-frontend-dependencies install-pre-commit-hooks lint start-backend start-frontend run run-wsl setup-config setup-config-prompts help
+.PHONY: docker-dev docker-run

build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+cp pyproject.toml poetry.lock openhands
+poetry build -v

config.sh

@@ -0,0 +1,4 @@
+DOCKER_REGISTRY=ghcr.io
+DOCKER_ORG=all-hands-ai
+DOCKER_IMAGE=openhands
+DOCKER_BASE_DIR="."

config.template.toml

@@ -0,0 +1,290 @@
+###################### OpenHands Configuration Example ######################
+#
+# All settings have default values, so you only need to uncomment and
+# modify what you want to change
+# The fields within each section are sorted in alphabetical order.
+#
+##############################################################################
+
+#################################### Core ####################################
+# General core configurations
+##############################################################################
+[core]
+# API key for E2B
+#e2b_api_key = ""
+
+# API key for Modal
+#modal_api_token_id = ""
+#modal_api_token_secret = ""
+
+# Base path for the workspace
+workspace_base = "./workspace"
+
+# Cache directory path
+#cache_dir = "/tmp/cache"
+
+# Reasoning effort for o1 models (low, medium, high, or not set)
+#reasoning_effort = "medium"
+
+# Debugging enabled
+#debug = false
+
+# Disable color in terminal output
+#disable_color = false
+
+# Enable saving and restoring the session when run from CLI
+#enable_cli_session = false
+
+# Path to store trajectories, can be a folder or a file
+# If it's a folder, the session id will be used as the file name
+#save_trajectory_path="./trajectories"
+
+# Path to replay a trajectory, must be a file path
+# If provided, trajectory will be loaded and replayed before the
+# agent responds to any user instruction
+#replay_trajectory_path = ""
+
+# File store path
+#file_store_path = "/tmp/file_store"
+
+# File store type
+#file_store = "memory"
+
+# List of allowed file extensions for uploads
+#file_uploads_allowed_extensions = [".*"]
+
+# Maximum file size for uploads, in megabytes
+#file_uploads_max_file_size_mb = 0
+
+# Maximum budget per task, 0.0 means no limit
+#max_budget_per_task = 0.0
+
+# Maximum number of iterations
+#max_iterations = 100
+
+# Path to mount the workspace in the sandbox
+#workspace_mount_path_in_sandbox = "/workspace"
+
+# Path to mount the workspace
+#workspace_mount_path = ""
+
+# Path to rewrite the workspace mount path to
+#workspace_mount_rewrite = ""
+
+# Run as openhands
+#run_as_openhands = true
+
+# Runtime environment
+#runtime = "eventstream"
+
+# Name of the default agent
+#default_agent = "CodeActAgent"
+
+# JWT secret for authentication
+#jwt_secret = ""
+
+# Restrict file types for file uploads
+#file_uploads_restrict_file_types = false
+
+# List of allowed file extensions for uploads
+#file_uploads_allowed_extensions = [".*"]
+
+#################################### LLM #####################################
+# Configuration for LLM models (group name starts with 'llm')
+# use 'llm' for the default LLM config
+##############################################################################
+[llm]
+# AWS access key ID
+#aws_access_key_id = ""
+
+# AWS region name
+#aws_region_name = ""
+
+# AWS secret access key
+#aws_secret_access_key = ""
+
+# API key to use (For Headless / CLI only -  In Web this is overridden by Session Init)
+api_key = "your-api-key"
+
+# API base URL (For Headless / CLI only -  In Web this is overridden by Session Init)
+#base_url = ""
+
+# API version
+#api_version = ""
+
+# Cost per input token
+#input_cost_per_token = 0.0
+
+# Cost per output token
+#output_cost_per_token = 0.0
+
+# Custom LLM provider
+#custom_llm_provider = ""
+
+# Embedding API base URL
+#embedding_base_url = ""
+
+# Embedding deployment name
+#embedding_deployment_name = ""
+
+# Embedding model to use
+embedding_model = "local"
+
+# Maximum number of characters in an observation's content
+#max_message_chars = 10000
+
+# Maximum number of input tokens
+#max_input_tokens = 0
+
+# Maximum number of output tokens
+#max_output_tokens = 0
+
+# Model to use. (For Headless / CLI only -  In Web this is overridden by Session Init)
+model = "gpt-4o"
+
+# Number of retries to attempt when an operation fails with the LLM.
+# Increase this value to allow more attempts before giving up
+#num_retries = 8
+
+# Maximum wait time (in seconds) between retry attempts
+# This caps the exponential backoff to prevent excessively long
+#retry_max_wait = 120
+
+# Minimum wait time (in seconds) between retry attempts
+# This sets the initial delay before the first retry
+#retry_min_wait = 15
+
+# Multiplier for exponential backoff calculation
+# The wait time increases by this factor after each failed attempt
+# A value of 2.0 means each retry waits twice as long as the previous one
+#retry_multiplier = 2.0
+
+# Drop any unmapped (unsupported) params without causing an exception
+#drop_params = false
+
+# Modify params for litellm to do transformations like adding a default message, when a message is empty.
+# Note: this setting is global, unlike drop_params, it cannot be overridden in each call to litellm.
+#modify_params = true
+
+# Using the prompt caching feature if provided by the LLM and supported
+#caching_prompt = true
+
+# Base URL for the OLLAMA API
+#ollama_base_url = ""
+
+# Temperature for the API
+#temperature = 0.0
+
+# Timeout for the API
+#timeout = 0
+
+# Top p for the API
+#top_p = 1.0
+
+# If model is vision capable, this option allows to disable image processing (useful for cost reduction).
+#disable_vision = true
+
+# Custom tokenizer to use for token counting
+# https://docs.litellm.ai/docs/completion/token_usage
+#custom_tokenizer = ""
+
+# Whether to use native tool calling if supported by the model. Can be true, false, or None by default, which chooses the model's default behavior based on the evaluation.
+# ATTENTION: Based on evaluation, enabling native function calling may lead to worse results
+# in some scenarios. Use with caution and consider testing with your specific use case.
+# https://github.com/All-Hands-AI/OpenHands/pull/4711
+#native_tool_calling = None
+
+[llm.gpt4o-mini]
+api_key = "your-api-key"
+model = "gpt-4o"
+
+
+#################################### Agent ###################################
+# Configuration for agents (group name starts with 'agent')
+# Use 'agent' for the default agent config
+# otherwise, group name must be `agent.<agent_name>` (case-sensitive), e.g.
+# agent.CodeActAgent
+##############################################################################
+[agent]
+
+# whether the browsing tool is enabled
+codeact_enable_browsing = true
+
+# whether the LLM draft editor is enabled
+codeact_enable_llm_editor = false
+
+# whether the IPython tool is enabled
+codeact_enable_jupyter = true
+
+# Name of the micro agent to use for this agent
+#micro_agent_name = ""
+
+# Memory enabled
+#memory_enabled = false
+
+# Memory maximum threads
+#memory_max_threads = 3
+
+# LLM config group to use
+#llm_config = 'your-llm-config-group'
+
+# Whether to use prompt extension (e.g., microagent, repo/runtime info) at all
+#enable_prompt_extensions = true
+
+# List of microagents to disable
+#disabled_microagents = []
+
+[agent.RepoExplorerAgent]
+# Example: use a cheaper model for RepoExplorerAgent to reduce cost, especially
+# useful when an agent doesn't demand high quality but uses a lot of tokens
+llm_config = 'gpt3'
+
+#################################### Sandbox ###################################
+# Configuration for the sandbox
+##############################################################################
+[sandbox]
+# Sandbox timeout in seconds
+#timeout = 120
+
+# Sandbox user ID
+#user_id = 1000
+
+# Container image to use for the sandbox
+#base_container_image = "nikolaik/python-nodejs:python3.12-nodejs22"
+
+# Use host network
+#use_host_network = false
+
+# runtime extra build args
+#runtime_extra_build_args = ["--network=host", "--add-host=host.docker.internal:host-gateway"]
+
+# Enable auto linting after editing
+#enable_auto_lint = false
+
+# Whether to initialize plugins
+#initialize_plugins = true
+
+# Extra dependencies to install in the runtime image
+#runtime_extra_deps = ""
+
+# Environment variables to set at the launch of the runtime
+#runtime_startup_env_vars = {}
+
+# BrowserGym environment to use for evaluation
+#browsergym_eval_env = ""
+
+#################################### Security ###################################
+# Configuration for security features
+##############################################################################
+[security]
+
+# Enable confirmation mode (For Headless / CLI only -  In Web this is overridden by Session Init)
+#confirmation_mode = false
+
+# The security analyzer to use (For Headless / CLI only -  In Web this is overridden by Session Init)
+#security_analyzer = ""
+
+#################################### Eval ####################################
+# Configuration for the evaluation, please refer to the specific evaluation
+# plugin for the available options
+##############################################################################

containers/README.md

@@ -0,0 +1,12 @@
+# Docker Containers
+
+Each folder here contains a Dockerfile, and a config.sh describing how to build
+the images and where to push them. These images are built and pushed in GitHub Actions
+by the `ghcr.yml` workflow.
+
+## Building Manually
+
+```bash
+docker build -f containers/app/Dockerfile -t openhands .
+docker build -f containers/sandbox/Dockerfile -t sandbox .
+```

containers/app/config.sh

@@ -0,0 +1,4 @@
+DOCKER_REGISTRY=ghcr.io
+DOCKER_ORG=all-hands-ai
+DOCKER_IMAGE=openhands
+DOCKER_BASE_DIR="."

containers/build.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+set -eo pipefail
+
+# Initialize variables with default values
+image_name=""
+org_name=""
+push=0
+load=0
+tag_suffix=""
+
+# Function to display usage information
+usage() {
+    echo "Usage: $0 -i <image_name> [-o <org_name>] [--push] [--load] [-t <tag_suffix>]"
+    echo "  -i: Image name (required)"
+    echo "  -o: Organization name"
+    echo "  --push: Push the image"
+    echo "  --load: Load the image"
+    echo "  -t: Tag suffix"
+    exit 1
+}
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -i) image_name="$2"; shift 2 ;;
+        -o) org_name="$2"; shift 2 ;;
+        --push) push=1; shift ;;
+        --load) load=1; shift ;;
+        -t) tag_suffix="$2"; shift 2 ;;
+        *) usage ;;
+    esac
+done
+# Check if required arguments are provided
+if [[ -z "$image_name" ]]; then
+    echo "Error: Image name is required."
+    usage
+fi
+
+echo "Building: $image_name"
+tags=()
+
+OPENHANDS_BUILD_VERSION="dev"
+
+cache_tag_base="buildcache"
+cache_tag="$cache_tag_base"
+
+if [[ -n $RELEVANT_SHA ]]; then
+  git_hash=$(git rev-parse --short "$RELEVANT_SHA")
+  tags+=("$git_hash")
+  tags+=("$RELEVANT_SHA")
+fi
+
+if [[ -n $GITHUB_REF_NAME ]]; then
+  # check if ref name is a version number
+  if [[ $GITHUB_REF_NAME =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    major_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1)
+    minor_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1,2)
+    tags+=("$major_version" "$minor_version")
+    tags+=("latest")
+  fi
+  sanitized_ref_name=$(echo "$GITHUB_REF_NAME" | sed 's/[^a-zA-Z0-9.-]\+/-/g')
+  OPENHANDS_BUILD_VERSION=$sanitized_ref_name
+  sanitized_ref_name=$(echo "$sanitized_ref_name" | tr '[:upper:]' '[:lower:]') # lower case is required in tagging
+  tags+=("$sanitized_ref_name")
+  cache_tag+="-${sanitized_ref_name}"
+fi
+
+if [[ -n $tag_suffix ]]; then
+  cache_tag+="-${tag_suffix}"
+  for i in "${!tags[@]}"; do
+    tags[$i]="${tags[$i]}-$tag_suffix"
+  done
+fi
+
+echo "Tags: ${tags[@]}"
+
+if [[ "$image_name" == "openhands" ]]; then
+  dir="./containers/app"
+elif [[ "$image_name" == "runtime" ]]; then
+  dir="./containers/runtime"
+else
+  dir="./containers/$image_name"
+fi
+
+if [[ (! -f "$dir/Dockerfile") && "$image_name" != "runtime" ]]; then
+  # Allow runtime to be built without a Dockerfile
+  echo "No Dockerfile found"
+  exit 1
+fi
+if [[ ! -f "$dir/config.sh" ]]; then
+  echo "No config.sh found for Dockerfile"
+  exit 1
+fi
+
+source "$dir/config.sh"
+
+if [[ -n "$org_name" ]]; then
+  DOCKER_ORG="$org_name"
+fi
+
+# If $DOCKER_IMAGE_SOURCE_TAG is set, add it to the tags
+if [[ -n "$DOCKER_IMAGE_SOURCE_TAG" ]]; then
+  tags+=("$DOCKER_IMAGE_SOURCE_TAG")
+fi
+# If $DOCKER_IMAGE_TAG is set, add it to the tags
+if [[ -n "$DOCKER_IMAGE_TAG" ]]; then
+  tags+=("$DOCKER_IMAGE_TAG")
+fi
+
+DOCKER_REPOSITORY="$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE"
+DOCKER_REPOSITORY=${DOCKER_REPOSITORY,,} # lowercase
+echo "Repo: $DOCKER_REPOSITORY"
+echo "Base dir: $DOCKER_BASE_DIR"
+
+args=""
+for tag in "${tags[@]}"; do
+  args+=" -t $DOCKER_REPOSITORY:$tag"
+done
+
+if [[ $push -eq 1 ]]; then
+  args+=" --push"
+  args+=" --cache-to=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag,mode=max"
+fi
+
+if [[ $load -eq 1 ]]; then
+  args+=" --load"
+fi
+
+echo "Args: $args"
+
+# Modify the platform selection based on --load flag
+if [[ $load -eq 1 ]]; then
+  # When loading, build only for the current platform
+  platform=$(docker version -f '{{.Server.Os}}/{{.Server.Arch}}')
+else
+  # For push or without load, build for multiple platforms
+  platform="linux/amd64,linux/arm64"
+fi
+
+echo "Building for platform(s): $platform"
+
+docker buildx build \
+  $args \
+  --build-arg OPENHANDS_BUILD_VERSION="$OPENHANDS_BUILD_VERSION" \
+  --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag \
+  --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag_base-main \
+  --platform $platform \
+  --provenance=false \
+  -f "$dir/Dockerfile" \
+  "$DOCKER_BASE_DIR"
+
+# If load was requested, print the loaded images
+if [[ $load -eq 1 ]]; then
+  echo "Local images built:"
+  docker images "$DOCKER_REPOSITORY" --format "{{.Repository}}:{{.Tag}}"
+fi

containers/dev/Dockerfile

@@ -0,0 +1,124 @@
+# syntax=docker/dockerfile:1
+
+###
+FROM ubuntu:22.04 AS dind
+
+# https://docs.docker.com/engine/install/ubuntu/
+RUN apt-get update && apt-get install -y \
+	ca-certificates \
+	curl \
+	&& install -m 0755 -d /etc/apt/keyrings \
+	&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \
+	&& chmod a+r /etc/apt/keyrings/docker.asc \
+	&& echo \
+		"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+		$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+RUN apt-get update && apt-get install -y \
+	docker-ce \
+	docker-ce-cli \
+	containerd.io \
+	docker-buildx-plugin \
+	docker-compose-plugin \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& apt-get clean \
+	&& apt-get autoremove -y
+
+###
+FROM dind AS openhands
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+#
+RUN apt-get update && apt-get install -y \
+	bash \
+    build-essential \
+    curl \
+	git \
+	git-lfs \
+	software-properties-common \
+	make \
+    netcat \
+    sudo \
+	wget \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& apt-get clean \
+	&& apt-get autoremove -y
+
+# https://github.com/cli/cli/blob/trunk/docs/install_linux.md
+RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+	&& chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+	&& apt-get update && apt-get -y install \
+    gh \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt-get clean \
+  && apt-get autoremove -y
+
+# Python 3.12
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.12 python3.12-venv python3.12-dev python3-pip \
+    && ln -s /usr/bin/python3.12 /usr/bin/python
+
+# NodeJS >= 18.17.1
+RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
+    && apt-get install -y nodejs
+
+# Poetry >= 1.8
+RUN curl -fsSL https://install.python-poetry.org | python3.12 - \
+    && ln -s ~/.local/bin/poetry /usr/local/bin/poetry
+
+#
+RUN <<EOF
+#!/bin/bash
+printf "#!/bin/bash
+set +x
+uname -a
+docker --version
+gh --version | head -n 1
+git --version
+#
+python --version
+echo node `node --version`
+echo npm `npm --version`
+poetry --version
+netcat -h 2>&1 | head -n 1
+" > /version.sh
+chmod a+x /version.sh
+EOF
+
+###
+FROM openhands AS dev
+
+RUN apt-get update && apt-get install -y \
+	dnsutils \
+	file \
+	iproute2 \
+	jq \
+	lsof \
+	ripgrep \
+	silversearcher-ag \
+	vim \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& apt-get clean \
+	&& apt-get autoremove -y
+
+WORKDIR /app
+
+# cache build dependencies
+RUN \
+  --mount=type=bind,source=./,target=/app/ \
+  <<EOF
+#!/bin/bash
+make -s clean
+make -s check-dependencies
+make -s install-python-dependencies
+
+# NOTE
+# node_modules are .dockerignore-d therefore not mountable
+# make -s install-frontend-dependencies
+EOF
+
+#
+CMD ["bash"]

containers/dev/README.md

@@ -0,0 +1,57 @@
+# Develop in Docker
+
+> [!WARNING]
+> This is not officially supported and may not work.
+
+Install [Docker](https://docs.docker.com/engine/install/) on your host machine and run:
+
+```bash
+make docker-dev
+# same as:
+cd ./containers/dev
+./dev.sh
+```
+
+It could take some time if you are running for the first time as Docker will pull all the  tools required for building OpenHands. The next time you run again, it should be instant.
+
+## Build and run
+
+If everything goes well, you should be inside a container after Docker finishes building the `openhands:dev` image similar to the following:
+
+```bash
+Build and run in Docker ...
+root@93fc0005fcd2:/app#
+```
+
+You may now proceed with the normal [build and run](../../Development.md) workflow as if you were on the host.
+
+## Make changes
+
+The source code on the host is mounted as `/app` inside docker. You may edit the files as usual either inside the Docker container or on your host with your favorite IDE/editors.
+
+The following are also mapped as readonly from your host:
+
+```yaml
+# host credentials
+- $HOME/.git-credentials:/root/.git-credentials:ro
+- $HOME/.gitconfig:/root/.gitconfig:ro
+- $HOME/.npmrc:/root/.npmrc:ro
+```
+
+## VSCode
+
+Alternatively, if you use VSCode, you could also [attach to the running container](https://code.visualstudio.com/docs/devcontainers/attach-container).
+
+See details for [developing in docker](https://code.visualstudio.com/docs/devcontainers/containers) or simply ask `OpenHands` ;-)
+
+## Rebuild dev image
+
+You could optionally pass additional options to the build script.
+
+```bash
+make docker-dev OPTIONS="--build"
+# or
+./containers/dev/dev.sh --build
+```
+
+See [docker compose run](https://docs.docker.com/reference/cli/docker/compose/run/) for more options.

containers/dev/compose.yml

@@ -0,0 +1,38 @@
+#
+services:
+  dev:
+    privileged: true
+    build:
+      context: ${OPENHANDS_WORKSPACE:-../../}
+      dockerfile: ./containers/dev/Dockerfile
+    image: openhands:dev
+    container_name: openhands-dev
+    environment:
+      - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
+      - SANDBOX_API_HOSTNAME=host.docker.internal
+      #
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.20-nikolaik}
+      - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
+      - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
+    ports:
+      - "3000:3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${WORKSPACE_BASE:-$PWD/workspace}:/opt/workspace_base
+      # source code
+      - ${OPENHANDS_WORKSPACE:-../../}:/app
+      # host credentials
+      - $HOME/.git-credentials:/root/.git-credentials:ro
+      - $HOME/.gitconfig:/root/.gitconfig:ro
+      - $HOME/.npmrc:/root/.npmrc:ro
+      # cache
+      - cache-data:/root/.cache
+    pull_policy: never
+    stdin_open: true
+    tty: true
+
+##
+volumes:
+  cache-data:

containers/dev/dev.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -o pipefail
+
+function get_docker() {
+    echo "Docker is required to build and run OpenHands."
+    echo "https://docs.docker.com/get-started/get-docker/"
+    exit 1
+}
+
+function check_tools() {
+	command -v docker &>/dev/null || get_docker
+}
+
+function exit_if_indocker() {
+    if [ -f /.dockerenv ]; then
+        echo "Running inside a Docker container. Exiting..."
+        exit 1
+    fi
+}
+
+#
+exit_if_indocker
+
+check_tools
+
+##
+OPENHANDS_WORKSPACE=$(git rev-parse --show-toplevel)
+
+cd "$OPENHANDS_WORKSPACE/containers/dev/" || exit 1
+
+##
+export BACKEND_HOST="0.0.0.0"
+#
+export SANDBOX_USER_ID=$(id -u)
+export WORKSPACE_BASE=${WORKSPACE_BASE:-$OPENHANDS_WORKSPACE/workspace}
+
+docker compose run --rm --service-ports "$@" dev
+
+##

containers/e2b-sandbox/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:22.04
+
+# install basic packages
+RUN apt-get update && apt-get install -y \
+    curl \
+    wget \
+    git \
+    vim \
+    nano \
+    unzip \
+    zip \
+    python3 \
+    python3-pip \
+    python3-venv \
+    python3-dev \
+    build-essential \
+    openssh-server \
+    sudo \
+    && rm -rf /var/lib/apt/lists/*

containers/e2b-sandbox/README.md

@@ -0,0 +1,15 @@
+# How to build custom E2B sandbox for OpenHands
+
+[E2B](https://e2b.dev) is an [open-source](https://github.com/e2b-dev/e2b) secure cloud environment (sandbox) made for running AI-generated code and agents. E2B offers [Python](https://pypi.org/project/e2b/) and [JS/TS](https://www.npmjs.com/package/e2b) SDK to spawn and control these sandboxes.
+
+
+1. Install the CLI with NPM.
+    ```sh
+    npm install -g @e2b/cli@latest
+    ```
+    Full CLI API is [here](https://e2b.dev/docs/cli/installation).
+
+1. Build the sandbox
+  ```sh
+  e2b template build --dockerfile ./Dockerfile --name "openhands"
+  ```

containers/e2b-sandbox/e2b.toml

@@ -0,0 +1,14 @@
+# This is a config for E2B sandbox template.
+# You can use 'template_id' (785n69crgahmz0lkdw9h) or 'template_name (openhands) from this config to spawn a sandbox:
+
+# Python SDK
+# from e2b import Sandbox
+# sandbox = Sandbox(template='openhands')
+
+# JS SDK
+# import { Sandbox } from 'e2b'
+# const sandbox = await Sandbox.create({ template: 'openhands' })
+
+dockerfile = "Dockerfile"
+template_name = "openhands"
+template_id = "785n69crgahmz0lkdw9h"

containers/runtime/README.md

@@ -0,0 +1,12 @@
+# Dynamically constructed Dockerfile
+
+This folder builds a runtime image (sandbox), which will use a dynamically generated `Dockerfile`
+that depends on the `base_image` **AND** a [Python source distribution](https://docs.python.org/3.10/distutils/sourcedist.html) that is based on the current commit of `openhands`.
+
+The following command will generate a `Dockerfile` file for `nikolaik/python-nodejs:python3.12-nodejs22` (the default base image), an updated `config.sh` and the runtime source distribution files/folders into `containers/runtime`:
+
+```bash
+poetry run python3 openhands/runtime/utils/runtime_build.py \
+    --base_image nikolaik/python-nodejs:python3.12-nodejs22 \
+    --build_folder containers/runtime
+```

containers/runtime/config.sh

@@ -0,0 +1,7 @@
+DOCKER_REGISTRY=ghcr.io
+DOCKER_ORG=all-hands-ai
+DOCKER_BASE_DIR="./containers/runtime"
+DOCKER_IMAGE=runtime
+# These variables will be appended by the runtime_build.py script
+# DOCKER_IMAGE_TAG=
+# DOCKER_IMAGE_SOURCE_TAG=

dev_config/python/.pre-commit-config.yaml

@@ -0,0 +1,43 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: docs/modules/python
+      - id: end-of-file-fixer
+        exclude: docs/modules/python
+      - id: check-yaml
+      - id: debug-statements
+
+  - repo: https://github.com/tox-dev/pyproject-fmt
+    rev: 1.7.0
+    hooks:
+      - id: pyproject-fmt
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: v0.16
+    hooks:
+      - id: validate-pyproject
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.4.1
+    hooks:
+      # Run the linter.
+      - id: ruff
+        entry: ruff check --config dev_config/python/ruff.toml
+        types_or: [python, pyi, jupyter]
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format
+        entry: ruff format --config dev_config/python/ruff.toml
+        types_or: [python, pyi, jupyter]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.9.0
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          [types-requests, types-setuptools, types-pyyaml, types-toml]
+        entry: mypy --config-file dev_config/python/mypy.ini openhands/
+        always_run: true
+        pass_filenames: false

dev_config/python/mypy.ini

@@ -0,0 +1,9 @@
+[mypy]
+warn_unused_configs = True
+ignore_missing_imports = True
+check_untyped_defs = True
+explicit_package_bases = True
+warn_unreachable = True
+warn_redundant_casts = True
+no_implicit_optional = True
+strict_optional = True

dev_config/python/ruff.toml

@@ -0,0 +1,26 @@
+[lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "Q",
+    "B",
+]
+
+ignore = [
+    "E501",
+    "B003",
+    "B007",
+    "B009",
+    "B010",
+    "B904",
+    "B018",
+]
+
+[lint.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "single"
+
+[format]
+quote-style = "single"

docker-compose.yml

@@ -0,0 +1,23 @@
+
+services:
+  openhands:
+    build:
+      context: ./
+      dockerfile: ./containers/app/Dockerfile
+    image: openhands:latest
+    container_name: openhands-app-${DATE:-}
+    environment:
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik}
+      #- SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234} # enable this only if you want a specific non-root sandbox user but you will have to manually adjust permissions of openhands-state for this user
+      - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
+    ports:
+      - "3000:3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ~/.openhands-state:/.openhands-state
+      - ${WORKSPACE_BASE:-$PWD/workspace}:/opt/workspace_base
+    pull_policy: build
+    stdin_open: true
+    tty: true

entrypoint.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -eo pipefail
+
+echo "Starting OpenHands..."
+if [[ $NO_SETUP == "true" ]]; then
+  echo "Skipping setup, running as $(whoami)"
+  "$@"
+  exit 0
+fi
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "The OpenHands entrypoint.sh must run as root"
+  exit 1
+fi
+
+if [ -z "$SANDBOX_USER_ID" ]; then
+  echo "SANDBOX_USER_ID is not set"
+  exit 1
+fi
+
+if [ -z "$WORKSPACE_MOUNT_PATH" ]; then
+  # This is set to /opt/workspace in the Dockerfile. But if the user isn't mounting, we want to unset it so that OpenHands doesn't mount at all
+  unset WORKSPACE_BASE
+fi
+
+if [[ "$SANDBOX_USER_ID" -eq 0 ]]; then
+  echo "Running OpenHands as root"
+  export RUN_AS_OPENHANDS=false
+  mkdir -p /root/.cache/ms-playwright/
+  if [ -d "/home/openhands/.cache/ms-playwright/" ]; then
+    mv /home/openhands/.cache/ms-playwright/ /root/.cache/
+  fi
+  "$@"
+else
+  echo "Setting up enduser with id $SANDBOX_USER_ID"
+  if id "enduser" &>/dev/null; then
+    echo "User enduser already exists. Skipping creation."
+  else
+    if ! useradd -l -m -u $SANDBOX_USER_ID -s /bin/bash enduser; then
+      echo "Failed to create user enduser with id $SANDBOX_USER_ID. Moving openhands user."
+      incremented_id=$(($SANDBOX_USER_ID + 1))
+      usermod -u $incremented_id openhands
+      if ! useradd -l -m -u $SANDBOX_USER_ID -s /bin/bash enduser; then
+        echo "Failed to create user enduser with id $SANDBOX_USER_ID for a second time. Exiting."
+        exit 1
+      fi
+    fi
+  fi
+  usermod -aG app enduser
+  # get the user group of /var/run/docker.sock and set openhands to that group
+  DOCKER_SOCKET_GID=$(stat -c '%g' /var/run/docker.sock)
+  echo "Docker socket group id: $DOCKER_SOCKET_GID"
+  if getent group $DOCKER_SOCKET_GID; then
+    echo "Group with id $DOCKER_SOCKET_GID already exists"
+  else
+    echo "Creating group with id $DOCKER_SOCKET_GID"
+    groupadd -g $DOCKER_SOCKET_GID docker
+  fi
+
+  mkdir -p /home/enduser/.cache/huggingface/hub/
+  mkdir -p /home/enduser/.cache/ms-playwright/
+  if [ -d "/home/openhands/.cache/ms-playwright/" ]; then
+    mv /home/openhands/.cache/ms-playwright/ /home/enduser/.cache/
+  fi
+
+  usermod -aG $DOCKER_SOCKET_GID enduser
+  echo "Running as enduser"
+  su enduser /bin/bash -c "${*@Q}" # This magically runs any arguments passed to the script as a command
+fi

evaluation/README.md

@@ -0,0 +1,94 @@
+# Evaluation
+
+This folder contains code and resources to run experiments and evaluations.
+
+## For Benchmark Users
+
+### Setup
+
+Before starting evaluation, follow the instructions [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
+
+Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the [evaluation directory](#supported-benchmarks).
+Generally these will involve running `run_infer.py` to perform inference with the agents.
+
+### Implementing and Evaluating an Agent
+
+To add an agent to OpenHands, you will need to implement it in the [agenthub directory](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/agenthub). There is a README there with more information.
+
+To evaluate an agent, you can provide the agent's name to the `run_infer.py` program.
+
+### Evaluating Different LLMs
+
+OpenHands in development mode uses `config.toml` to keep track of most configuration.
+Here's an example configuration file you can use to define and use multiple LLMs:
+
+```toml
+[llm]
+# IMPORTANT: add your API key here, and set the model to the one you want to evaluate
+model = "gpt-4o-2024-05-13"
+api_key = "sk-XXX"
+
+[llm.eval_gpt4_1106_preview_llm]
+model = "gpt-4-1106-preview"
+api_key = "XXX"
+temperature = 0.0
+
+[llm.eval_some_openai_compatible_model_llm]
+model = "openai/MODEL_NAME"
+base_url = "https://OPENAI_COMPATIBLE_URL/v1"
+api_key = "XXX"
+temperature = 0.0
+```
+
+## Supported Benchmarks
+
+The OpenHands evaluation harness supports a wide variety of benchmarks across [software engineering](#software-engineering), [web browsing](#web-browsing), [miscellaneous assistance](#misc-assistance), and [real-world](#real-world) tasks.
+
+### Software Engineering
+
+- SWE-Bench: [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench)
+- HumanEvalFix: [`evaluation/benchmarks/humanevalfix`](./benchmarks/humanevalfix)
+- BIRD: [`evaluation/benchmarks/bird`](./benchmarks/bird)
+- BioCoder: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
+- ML-Bench: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
+- APIBench: [`evaluation/benchmarks/gorilla`](./benchmarks/gorilla/)
+- ToolQA: [`evaluation/benchmarks/toolqa`](./benchmarks/toolqa/)
+- AiderBench: [`evaluation/benchmarks/aider_bench`](./benchmarks/aider_bench/)
+- Commit0: [`evaluation/benchmarks/commit0_bench`](./benchmarks/commit0_bench/)
+- DiscoveryBench: [`evaluation/benchmarks/discoverybench`](./benchmarks/discoverybench/)
+
+### Web Browsing
+
+- WebArena: [`evaluation/benchmarks/webarena`](./benchmarks/webarena/)
+- MiniWob++: [`evaluation/benchmarks/miniwob`](./benchmarks/miniwob/)
+- Browsing Delegation: [`evaluation/benchmarks/browsing_delegation`](./benchmarks/browsing_delegation/)
+
+### Misc. Assistance
+
+- GAIA: [`evaluation/benchmarks/gaia`](./benchmarks/gaia)
+- GPQA: [`evaluation/benchmarks/gpqa`](./benchmarks/gpqa)
+- AgentBench: [`evaluation/benchmarks/agent_bench`](./benchmarks/agent_bench)
+- MINT: [`evaluation/benchmarks/mint`](./benchmarks/mint)
+- Entity deduction Arena (EDA): [`evaluation/benchmarks/EDA`](./benchmarks/EDA)
+- ProofWriter: [`evaluation/benchmarks/logic_reasoning`](./benchmarks/logic_reasoning)
+- ScienceAgentBench: [`evaluation/benchmarks/scienceagentbench`](./benchmarks/scienceagentbench)
+
+### Real World
+
+- TheAgentCompany: [`evaluation/benchmarks/the_agent_company`](./benchmarks/the_agent_company)
+
+## Result Visualization
+
+Check [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization of existing experimental results.
+
+You can start your own fork of [our huggingface evaluation outputs](https://huggingface.co/spaces/OpenHands/evaluation) and submit a PR of your evaluation results to our hosted huggingface repo via PR following the guide [here](https://huggingface.co/docs/hub/en/repositories-pull-requests-discussions#pull-requests-and-discussions).
+
+## For Benchmark Developers
+
+To learn more about how to integrate your benchmark into OpenHands, check out [tutorial here](https://docs.all-hands.dev/modules/usage/how-to/evaluation-harness). Briefly,
+
+- Each subfolder contains a specific benchmark or experiment. For example, [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench) should contain
+all the preprocessing/evaluation/analysis scripts.
+- Raw data and experimental records should not be stored within this repo.
+- For model outputs, they should be stored at [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization.
+- Important data files of manageable size and analysis scripts (e.g., jupyter notebooks) can be directly uploaded to this repo.

evaluation/benchmarks/EDA/README.md

@@ -0,0 +1,46 @@
+# EDA Evaluation
+
+This folder contains evaluation harness for evaluating agents on the Entity-deduction-Arena Benchmark, from the paper [Probing the Multi-turn Planning Capabilities of LLMs via 20 Question Games](https://arxiv.org/abs/2310.01468), presented in ACL 2024 main conference.
+
+## Setup Environment and LLM Configuration
+
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
+
+## Start the evaluation
+
+```bash
+export OPENAI_API_KEY="sk-XXX"; # This is required for evaluation (to simulate another party of conversation)
+./evaluation/benchmarks/EDA/scripts/run_infer.sh [model_config] [git-version] [agent] [dataset] [eval_limit]
+```
+
+where `model_config` is mandatory, while `git-version`, `agent`, `dataset` and `eval_limit` are optional.
+
+- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
+LLM settings, as defined in your `config.toml`.
+
+- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
+like to evaluate. It could also be a release tag like `0.6.2`.
+
+- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
+to `CodeActAgent`.
+
+- `dataset`: There are two tasks in this evaluation. Specify `dataset` to test on either `things` or `celebs` task.
+
+- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By default it infers all instances.
+
+For example,
+
+```bash
+./evaluation/benchmarks/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 0.6.2 CodeActAgent things
+```
+
+## Reference
+
+```bibtex
+@inproceedings{zhang2023entity,
+  title={Probing the Multi-turn Planning Capabilities of LLMs via 20 Question Games},
+  author={Zhang, Yizhe and Lu, Jiarui and Jaitly, Navdeep},
+  journal={ACL},
+  year={2024}
+}
+```

evaluation/benchmarks/EDA/game.py

@@ -0,0 +1,203 @@
+import logging
+import re
+
+import openai
+import requests.exceptions
+from openai import OpenAI
+from retry import retry
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Q20Game:
+    def __init__(
+        self,
+        item: str,
+        answerer_model: str = 'gpt-3.5-turbo-0613',
+        guesser_model: str = 'gpt-3.5-turbo-0613',
+        num_turns: int = 20,
+        temperature: float = 0.8,
+        openai_api: bool = True,
+        openai_api_key: str | None = None,
+        guesser_kargs=None,
+    ) -> None:
+        if guesser_kargs is None:
+            guesser_kargs = {}
+        self.item = item
+        self.answerer_model = answerer_model
+        self.guesser_model = guesser_model
+        self.num_turns = num_turns
+        self.temperature = temperature
+        self.openai_api = openai_api
+        self.guesser_kargs = guesser_kargs
+        self.vicuna_prompt = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
+        self.first_user_utterance = (
+            'Your task is to ask a series of questions to deduce the entity '
+            "that I'm thinking of with as few queries as possible. "
+            "Only ask questions that can be answered by 'yes', 'no' or 'maybe'. "
+            'Do not ask for hint. Make your question brief with no linebreaker. '
+            'Now start asking a question.'
+        )
+        self.guesser_win = False
+        self.curr_turn = 0
+        if openai_api_key is not None:
+            openai.api_key = openai_api_key
+
+        if isinstance(answerer_model, str) and not answerer_model.startswith('gpt'):
+            self.user_api_base = 'http://0.0.0.0:8000/v1'
+        else:
+            self.user_api_base = 'https://api.openai.com/v1'
+
+        if isinstance(guesser_model, str) and not guesser_model.startswith('gpt'):
+            self.guesser_api_base = 'http://0.0.0.0:8000/v1'
+        else:
+            self.guesser_api_base = 'https://api.openai.com/v1'
+
+        self.guesser_messages = []
+
+    def preprocess_response(self, response):
+        response = re.sub(r'the entity you are thinking of', 'it', response)
+        response = re.sub(r"the entity you're thinking of", 'it', response)
+        response = re.sub(r" you're thinking of", '', response)
+        response = re.sub(r' you are thinking of', '', response)
+        return response
+
+    def judge_winner(self, response):
+        guesser_question = response.strip()
+
+        if self.curr_turn == self.num_turns - 1:
+            guesser_question += ' Is it right?'
+
+        self.guesser_messages.append({'role': 'assistant', 'content': guesser_question})
+        # ask for answer
+        usr_msg = self.answerer(guesser_question)
+
+        self.guesser_messages.append(
+            {'role': 'user', 'content': f"{usr_msg['content'].strip()}"}
+        )
+
+        if 'bingo' in usr_msg['content'].lower():
+            self.guesser_win = True
+            return True, ''
+
+        return False, usr_msg['content'].strip()
+
+    def generate_user_response(self, response):
+        response = self.preprocess_response(response)
+        # others
+        bingo, anwser_reply = self.judge_winner(response)
+        if bingo:
+            return 'You are bingo! Use the "finish" tool to finish the interaction.\n'
+        if self.curr_turn == self.num_turns - 2:
+            anwser_reply += " You must guess now, what's it?"
+        return anwser_reply
+
+    def reward(self):
+        if self.guesser_win:
+            n_turns = (len(self.guesser_messages) + 1) // 2
+            return 1 - max(n_turns - 5, 0) * 0.02
+        return 0
+
+    @retry(
+        (
+            openai.Timeout,
+            requests.exceptions.ReadTimeout,
+            openai.RateLimitError,
+            openai.APIError,
+            openai.APIConnectionError,
+        ),
+        tries=5,
+        delay=0.5,
+        backoff=0.5,
+        max_delay=2,
+        logger=LOGGER,
+    )
+    def answerer(self, question):
+        openai.api_base = self.user_api_base
+        client = OpenAI(api_key=openai.api_key)
+        user_messages = [
+            {
+                'role': 'user',
+                'content': f'Based on your knowledge about {self.item}, '
+                f'respond to the following question or guess. '
+                f"Limit your respond to only 'Yes.', 'No.' or 'Maybe.', with no explanation or other words. "
+                f'Never say the answer {self.item} in your response. '
+                f"If the question is to solicit the answer, respond 'No.'.",
+            },
+            {
+                'role': 'user',
+                'content': f'For the entity {self.item}, {question} (Yes/No/Maybe)',
+            },
+        ]
+
+        response = client.chat.completions.create(
+            model=self.answerer_model,
+            messages=user_messages,
+            max_tokens=6,
+            n=1,
+            stop=None,
+            temperature=0.2,
+        )
+        if any(
+            [
+                re.search(rf'(?:^|\W){i.strip().lower()}(?:$|\W)', question.lower())
+                for i in self.item.lower().split('|')
+            ]
+        ):
+            response.choices[0].message.content = 'Bingo!'
+        return response.choices[0].message.to_dict()
+
+
+class Q20GameCelebrity(Q20Game):
+    def __init__(self, item: str, **kwargs) -> None:
+        super().__init__(item, **kwargs)
+        self.first_user_utterance = (
+            'Your task is to ask a series of questions to deduce the celebrity '
+            "that I'm thinking of with as few queries as possible. "
+            "Only ask factual questions that can be answered by 'Yes.', 'No.' or 'Dunno.'. Do not ask for hint. Make your question brief with no linebreaker. "
+            'Now start asking a question.'
+        )
+
+    @retry(
+        (
+            openai.Timeout,
+            requests.exceptions.ReadTimeout,
+            openai.RateLimitError,
+            openai.APIError,
+            openai.APIConnectionError,
+        ),
+        tries=5,
+        delay=0.5,
+        backoff=0.5,
+        max_delay=2,
+        logger=LOGGER,
+    )
+    def answerer(self, question):
+        openai.api_base = self.user_api_base
+        client = OpenAI(api_key=openai.api_key)
+        user_messages = [
+            {
+                'role': 'system',
+                'content': f'Based on your knowledge about the celebrity: {self.item}, '
+                f'respond to the following question or guess. '
+                f"Limit your respond to only 'Yes.', 'No.' or 'Dunno.', with no explanation or other words. "
+                f"Never say the name {self.item} in your response. Do not say 'Dunno.' if it can be answered by 'Yes.' or 'No.' "
+                f"If the question is to solicit the answer, respond 'No.'.",
+            },
+            {
+                'role': 'user',
+                'content': f'For the celebrity {self.item}, {question}(Yes/No/Dunno)',
+            },
+        ]
+
+        response = client.chat.completions.create(
+            model=self.answerer_model,
+            messages=user_messages,
+            max_tokens=6,
+            n=1,
+            stop=None,
+            temperature=0.2,
+        )
+        if re.search(rf'(?:^|\W){self.item.lower()}(?:$|\W)', question.lower()):
+            response.choices[0].message.content = 'Bingo!'
+        return response.choices[0].message.to_dict()

evaluation/benchmarks/EDA/run_infer.py

@@ -0,0 +1,238 @@
+import asyncio
+import os
+
+import pandas as pd
+from datasets import load_dataset
+
+from evaluation.benchmarks.EDA.game import Q20Game, Q20GameCelebrity
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    compatibility_for_eval_history_pairs,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    get_parser,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import MessageAction
+from openhands.utils.async_utils import call_async_from_sync
+
+game = None
+
+
+def codeact_user_response_eda(state: State) -> str:
+    global game
+    model_guess = ''
+
+    # retrieve the latest model message from history
+    if state.history:
+        last_agent_message = state.get_last_agent_message()
+        model_guess = last_agent_message.content if last_agent_message else ''
+
+    assert game is not None, 'Game is not initialized.'
+    msg = game.generate_user_response(model_guess)
+    game.curr_turn += 1
+    logger.info(f'Model guess: {model_guess}')
+    logger.info(f'Answer response: {msg}')
+    if 'bingo!' in msg.lower():
+        return '/exit'
+    return msg
+
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response_eda,
+}
+
+AGENT_CLS_TO_INST_SUFFIX = {
+    'CodeActAgent': 'When you think you have solved the question, please first send your answer to user through message and then exit.\n'
+}
+
+
+def get_config(
+    metadata: EvalMetadata,
+) -> AppConfig:
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime='docker',
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='python:3.12-bookworm',
+            enable_auto_lint=False,
+            use_host_network=False,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(metadata.llm_config)
+    agent_config = config.get_agent_config(metadata.agent_class)
+    agent_config.enable_prompt_extensions = False
+    return config
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+) -> EvalOutput:
+    config = get_config(metadata)
+    instance_id = instance['text'].strip()
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance_id}.')
+
+    # Prepare instruction
+    _game_class = {'eda-things': Q20Game, 'eda-celebs': Q20GameCelebrity}
+
+    guesser_kargs = {
+        'max_new_tokens': 64,
+        'temperature': 0.8,
+        'repetition_penalty': 1.0,
+        'do_sample': True,
+    }  # no penalty
+
+    # Use codeactagent as guesser_model
+    global game
+    assert metadata.dataset is not None
+    assert metadata.details is not None
+    game = _game_class[metadata.dataset](
+        item=instance['text'].strip(),
+        answerer_model=metadata.details['answerer_model'],
+        guesser_model=None,
+        num_turns=metadata.max_iterations,
+        openai_api_key=metadata.details['openai_api_key'],
+        guesser_kargs=guesser_kargs,
+    )
+
+    instruction = f'{game.first_user_utterance}'
+    logger.info(f'Instruction: {instruction}')
+    instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
+
+    # Here's how you can run the agent (similar to the `main` function) and get the final task state
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            initial_user_action=MessageAction(content=instruction),
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                metadata.agent_class
+            ],
+        )
+    )
+    # ======= Attempt to evaluate the agent's edits =======
+    # If you are working on simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    last_agent_message = state.get_last_agent_message()
+    final_message = last_agent_message.content if last_agent_message else ''
+
+    logger.info(f'Final message: {final_message} | Ground truth: {instance["text"]}')
+    test_result = game.reward()
+    metrics = state.metrics.get() if state.metrics else None
+
+    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+    # for compatibility with the existing output format, we can remake the pairs here
+    # remove when it becomes unnecessary
+    histories = compatibility_for_eval_history_pairs(state.history)
+
+    # Save the output
+    output = EvalOutput(
+        instance_id=instance_id,
+        instance=instance.to_dict(),
+        instruction=instruction,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+        test_result={
+            'success': test_result,
+            'final_message': final_message,
+            'ground_truth': instance['text'],
+        },
+    )
+    return output
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    parser.add_argument(
+        '--answerer_model', '-a', default='gpt-3.5-turbo', help='answerer model'
+    )
+    parser.add_argument(
+        '--dataset',
+        default='things',
+        choices=['things', 'celebs'],
+        type=str,
+        help='dataset to be used',
+    )
+    parser.add_argument(
+        '--OPENAI_API_KEY', type=str, required=True, help='Your OpenAI API key'
+    )
+    parser.add_argument(
+        '--data-split',
+        default='test',
+        type=str,
+        help='data split, eg, test',
+    )
+    args, _ = parser.parse_known_args()
+
+    eda_dataset = load_dataset(
+        'yizheapple/entity-deduction-arena', name=args.dataset, split=args.data_split
+    )
+    eda_dataset.rename(columns={'text': 'instance_id'}, inplace=True)
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    metadata = make_metadata(
+        llm_config,
+        f'eda-{args.dataset}',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+        data_split=args.data_split,
+        details={
+            'answerer_model': str(args.answerer_model),
+            'openai_api_key': str(args.OPENAI_API_KEY),
+        },
+    )
+
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    prepared_dataset = prepare_dataset(
+        eda_dataset.to_pandas(), output_file, args.eval_n_limit
+    )
+
+    run_evaluation(
+        prepared_dataset,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+    )

evaluation/benchmarks/EDA/scripts/run_infer.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+DATASET=$4
+EVAL_LIMIT=$5
+NUM_WORKERS=$6
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+get_openhands_version
+
+if [ -z "$DATASET" ]; then
+  echo "Dataset not specified, use default 'things'"
+  DATASET="things"
+fi
+
+# check if OPENAI_API_KEY is set
+if [ -z "$OPENAI_API_KEY" ]; then
+  echo "OPENAI_API_KEY is not set, please set it to run the script"
+  exit 1
+fi
+
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+echo "DATASET: $DATASET"
+
+COMMAND="poetry run python evaluation/benchmarks/EDA/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --dataset $DATASET \
+  --data-split test \
+  --max-iterations 20 \
+  --OPENAI_API_KEY $OPENAI_API_KEY \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note ${OPENHANDS_VERSION}_${DATASET}"
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+echo $COMMAND
+eval $COMMAND

evaluation/benchmarks/agent_bench/README.md

@@ -0,0 +1,56 @@
+# AgentBench Evaluation
+
+This folder contains evaluation harness for evaluating agents on the [AgentBench: Evaluating LLMs as Agents](https://arxiv.org/abs/2308.03688). We currently only support running on the `osbench` subset.
+
+## Setup Environment and LLM Configuration
+
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
+
+## Start the evaluation
+
+```bash
+./evaluation/benchmarks/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
+```
+
+- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
+LLM settings, as defined in your `config.toml`.
+- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
+like to evaluate. It could also be a release tag like `0.6.2`.
+- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
+to `CodeActAgent`.
+- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
+default, the script evaluates the entire SWE-bench_Lite test set (300 issues). Note:
+in order to use `eval_limit`, you must also set `agent`.
+
+
+Following is the basic command to start the evaluation.
+
+You can update the arguments in the script `evaluation/benchmarks/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
+
+- `--agent-cls`, the agent to use. For example, `CodeActAgent`.
+- `--llm-config`: the LLM configuration to use. For example, `eval_gpt4_1106_preview`.
+- `--max-iterations`: the number of iterations to run the evaluation. For example, `30`.
+- `--eval-num-workers`: the number of workers to use for evaluation. For example, `5`.
+- `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
+
+```bash
+./evaluation/benchmarks/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
+```
+
+## Run with Remote Runtime (experimental)
+
+You can run the evaluation using a remote runtime instead of a local Docker container. This is useful when you want to run the evaluation in a cloud environment or when you don't have Docker installed locally.
+
+To use the remote runtime, set the following environment variables:
+
+```bash
+# Required environment variables
+export ALLHANDS_API_KEY="your-api-key"  # Contact the team to get an API key
+export RUNTIME=remote
+export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
+
+# Run the evaluation
+./evaluation/benchmarks/agent_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview HEAD CodeActAgent 1
+```
+
+The remote runtime will build a container image and run the evaluation in a cloud environment. The results will be saved locally in the same way as when running with a local runtime.

evaluation/benchmarks/agent_bench/helper.py

@@ -0,0 +1,77 @@
+import os
+import re
+from functools import partial
+
+from evaluation.utils.shared import codeact_user_response
+from openhands.events.action import CmdRunAction, MessageAction
+
+
+def try_parse_answer(act) -> str | None:
+    raw_ans = ''
+    if isinstance(act, MessageAction) and act.source == 'agent':
+        raw_ans = act.content
+    elif isinstance(act, CmdRunAction) and act.source == 'agent':
+        raw_ans = act.thought
+    else:
+        return None
+    agent_answer = re.findall(r'<solution>(.*?)</solution>', raw_ans, re.DOTALL)
+    if not agent_answer:
+        return None
+    return agent_answer[0].strip()
+
+
+FAKE_RESPONSES = {
+    'CodeActAgent': partial(
+        codeact_user_response, encapsulate_solution=True, try_parse=try_parse_answer
+    ),
+}
+
+INST_SUFFIXES: dict[str, str] = {
+    'CodeActAgent': (
+        'When you think you have solved the question, '
+        'please first send your answer to user through message and then exit.\n'
+    )
+}
+
+
+def analysis_size(size_str):
+    size_str = size_str.strip()
+    avails = {
+        'B': 1,
+        'Byte': 1,
+        'K': 1024,
+        'KB': 1024,
+        'M': 1024 * 1024,
+        'MB': 1024 * 1024,
+        'G': 1024 * 1024 * 1024,
+        'GB': 1024 * 1024 * 1024,
+        'T': 1024 * 1024 * 1024 * 1024,
+        'TB': 1024 * 1024 * 1024 * 1024,
+        'P': 1024 * 1024 * 1024 * 1024 * 1024,
+        'PB': 1024 * 1024 * 1024 * 1024 * 1024,
+    }
+    for size_unit in avails:
+        if size_str.endswith(size_unit):
+            return int(size_str[: -len(size_unit)]) * avails[size_unit]
+    return int(size_str)
+
+
+def compare_results(check_method: str, model_answer: str, final_ans: str) -> bool:
+    try:
+        match check_method:
+            case 'check/integer-match.py':
+                return int(model_answer) == int(final_ans)
+            case 'check/size-match.py':
+                return analysis_size(model_answer) == analysis_size(final_ans)
+        return (
+            model_answer.replace('\r\n', '\n').replace('\r', '\n').strip()
+            == final_ans.replace('\r\n', '\n').replace('\r', '\n').strip()
+        )
+    except Exception:
+        return False
+
+
+def create_sh_file(filename: str, cmds: str) -> None:
+    with open(filename, 'w', encoding='utf-8') as file:
+        file.write(cmds.replace('\r\n', '\n'))
+    os.chmod(filename, 0o755)

evaluation/benchmarks/agent_bench/run_infer.py

@@ -0,0 +1,329 @@
+import asyncio
+import os
+import re
+import tempfile
+from typing import Any
+
+import pandas as pd
+from datasets import load_dataset
+
+from evaluation.benchmarks.agent_bench.helper import (
+    FAKE_RESPONSES,
+    INST_SUFFIXES,
+    compare_results,
+    create_sh_file,
+)
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    compatibility_for_eval_history_pairs,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    parse_arguments,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+
+
+def get_config(
+    metadata: EvalMetadata,
+) -> AppConfig:
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime=os.environ.get('RUNTIME', 'docker'),
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='python:3.12-slim',
+            enable_auto_lint=True,
+            use_host_network=False,
+            api_key=os.environ.get('ALLHANDS_API_KEY', None),
+            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
+            keep_runtime_alive=False,
+            remote_runtime_init_timeout=3600,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(metadata.llm_config)
+    agent_config = config.get_agent_config(metadata.agent_class)
+    agent_config.enable_prompt_extensions = False
+    return config
+
+
+def initialize_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required
+):
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    # Set instance id
+    action = CmdRunAction(command='mkdir -p /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    action = CmdRunAction(command='cd /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    init_cmd = instance.init
+    if init_cmd is not None:
+        script_name = f'{instance.instance_id}_init.sh'
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            host_script_path = os.path.join(tmpdir, script_name)
+            create_sh_file(host_script_path, init_cmd)
+            runtime.copy_to(
+                host_script_path,
+                '/workspace',
+            )
+
+        logger.info(f'Running init script: {script_name}')
+        action = CmdRunAction(command=f'chmod +x ./{script_name} && ./{script_name}')
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        assert obs.exit_code == 0
+
+    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+
+
+def complete_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    agent_answer = None
+    get_agent_result_cmd = instance.get_agent_result
+    if get_agent_result_cmd is not None:
+        script_name = 'get_agent_result.sh'
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            host_script_path = os.path.join(tmpdir, script_name)
+            create_sh_file(host_script_path, get_agent_result_cmd)
+            runtime.copy_to(
+                host_script_path,
+                '/workspace',
+            )
+            logger.info(f'Running get agent result cmd: {script_name}')
+
+        action = CmdRunAction(
+            command=f'chmod +x ./{script_name} && ./{script_name}',
+        )
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        assert obs.exit_code == 0
+        agent_answer = obs.content
+    # IF the agent answer is not found, retrieve it from the history
+    # We wait until the controller finishes
+
+    final_ans = None
+    if instance.ground_truth is not None:
+        final_ans = instance.ground_truth
+    else:
+        get_ground_truth_cmd = instance.get_ground_truth
+        if get_ground_truth_cmd is not None:
+            script_name = 'get_ground_truth.sh'
+            with tempfile.TemporaryDirectory() as tmpdir:
+                host_script_path = os.path.join(tmpdir, script_name)
+                create_sh_file(host_script_path, get_ground_truth_cmd)
+                runtime.copy_to(
+                    host_script_path,
+                    '/workspace',
+                )
+            logger.info(f'Running get ground truth cmd: {script_name}')
+
+            action = CmdRunAction(
+                command=f'chmod +x ./{script_name} && ./{script_name}'
+            )
+            logger.info(action, extra={'msg_type': 'ACTION'})
+            obs = runtime.run_action(action)
+            logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+            final_ans = obs.content
+
+    logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
+    return {
+        'final_ans': final_ans,
+        'agent_answer': agent_answer,
+    }
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+) -> EvalOutput:
+    config = get_config(metadata)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    # =============================================
+    # build instruction
+    # =============================================
+
+    # Prepare instruction
+    instruction = (
+        f'Please fix the following issue.\n'
+        'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
+        'Please encapsulate your final answer (answer ONLY) within <solution> and </solution>.\n'
+        'For example: The answer to the question is <solution> 42 </solution>.\n'
+        '# Problem \n'
+        f'{instance.description}\n\n'
+    )
+    instruction += (
+        'IMPORTANT: You should ONLY interact with the environment provided '
+        'to you AND NEVER ASK FOR HUMAN HELP.\n'
+    )
+    # NOTE: You can actually set slightly different instruction for different agents
+    instruction += INST_SUFFIXES[metadata.agent_class]
+
+    # =============================================
+    # create sandbox and run the agent
+    # =============================================
+
+    runtime: Runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+
+    initialize_runtime(runtime, instance=instance)
+
+    # Here's how you can run the agent (similar to the `main` function) and get the final task state
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            initial_user_action=MessageAction(content=instruction),
+            runtime=runtime,
+            fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class],
+        )
+    )
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    # =============================================
+    # result evaluation
+    # =============================================
+
+    return_val = complete_runtime(runtime, instance)
+    agent_answer = return_val['agent_answer']
+    final_ans = return_val['final_ans']
+
+    # If the agent answer is not found, retrieve it from the history
+    if agent_answer is None:
+        agent_answer = ''
+        logger.info('Retrieving agent answer from history.')
+        raw_ans = ''
+
+        # retrieve the last agent message or thought
+        for event in reversed(state.history):
+            if event.source == 'agent':
+                if isinstance(event, AgentFinishAction):
+                    raw_ans = event.thought
+                    break
+                elif isinstance(event, MessageAction):
+                    raw_ans = event.content
+                    break
+                elif isinstance(event, CmdRunAction):
+                    raw_ans = event.thought
+                    break
+
+        # parse the answer for a solution tag
+        agent_answer = re.findall(r'<solution>(.*?)</solution>', raw_ans, re.DOTALL)
+        if len(agent_answer) == 0:
+            logger.warning(f'Failed to parse model answer: {raw_ans}')
+            agent_answer = raw_ans
+        else:
+            agent_answer = agent_answer[0]
+
+    comparison_method = instance.comparison_method
+    logger.info(
+        f'Final message: {agent_answer} | Ground truth: {final_ans} | Comparison method: {comparison_method}'
+    )
+    test_result = compare_results(comparison_method, agent_answer, final_ans)
+
+    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+    # for compatibility with the existing output format, we can remake the pairs here
+    # remove when it becomes unnecessary
+    histories = compatibility_for_eval_history_pairs(state.history)
+
+    metrics = state.metrics.get() if state.metrics else None
+
+    # Save the output
+    output = EvalOutput(
+        instance_id=instance.instance_id,
+        instance=instance.to_dict(),
+        instruction=instruction,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+        test_result={
+            'agent_answer': agent_answer,
+            'final_answer': final_ans,
+            'check_method': comparison_method,
+            'result': test_result,
+        },
+    )
+    return output
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+    dataset = load_dataset('iFurySt/AgentBench')
+    agent_bench_tests = dataset['osbench'].to_pandas()
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    metadata = make_metadata(
+        llm_config,
+        'AgentBench-OS',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+    )
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    instances = prepare_dataset(agent_bench_tests, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
+    )

evaluation/benchmarks/agent_bench/scripts/run_infer.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+NUM_WORKERS=$5
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+COMMAND="export PYTHONPATH=evaluation/benchmarks/agent_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/agent_bench/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --max-iterations 30 \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note $OPENHANDS_VERSION"
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+eval $COMMAND

evaluation/benchmarks/agent_bench/scripts/summarise_results.py

@@ -0,0 +1,37 @@
+import json
+import sys
+
+
+def extract_test_results(res_file_path: str) -> tuple[list[str], list[str]]:
+    passed = []
+    failed = []
+    with open(res_file_path, 'r') as file:
+        for line in file:
+            data = json.loads(line.strip())
+            instance_id = data['instance_id']
+            resolved = False
+            if 'test_result' in data and 'result' in data['test_result']:
+                resolved = data['test_result']['result']
+            if resolved:
+                passed.append(instance_id)
+            else:
+                failed.append(instance_id)
+    return passed, failed
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print(
+            'Usage: poetry run python summarise_results.py <path_to_output_jsonl_file>'
+        )
+        sys.exit(1)
+    json_file_path = sys.argv[1]
+    passed_tests, failed_tests = extract_test_results(json_file_path)
+    succ_rate = len(passed_tests) / (len(passed_tests) + len(failed_tests))
+    print(
+        f'\nPassed {len(passed_tests)} tests, failed {len(failed_tests)} tests, resolve rate = {succ_rate}'
+    )
+    print('PASSED TESTS:')
+    print(passed_tests)
+    print('FAILED TESTS:')
+    print(failed_tests)

evaluation/benchmarks/aider_bench/README.md

@@ -0,0 +1,95 @@
+# AiderBench Evaluation
+
+This folder contains evaluation harness for evaluating agents on the
+[Aider Editing Benchmark](https://github.com/paul-gauthier/aider/blob/main/benchmark/README.md).
+This will allow us to develop better editing approach without running the full
+SWE-bench. The benchmark uses the
+[RajMaheshwari/Exercism-Python](https://huggingface.co/datasets/RajMaheshwari/Exercism-Python)
+Hugging Face dataset based on the
+[Exercism python coding exercises](https://github.com/exercism/python).
+
+## Setup Environment and LLM Configuration
+
+Please follow instruction [here](../../README.md#setup) to setup your local
+development environment and LLM.
+
+## Start the evaluation
+
+```bash
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
+```
+
+- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for
+    your LLM settings, as defined in your `config.toml`.
+- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version
+    you would like to evaluate. It could also be a release tag like `0.9.0`.
+- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks,
+    defaulting to `CodeActAgent`.
+- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit`
+    instances. By default, the script evaluates the entire Exercism test set
+    (133 issues). Note: in order to use `eval_limit`, you must also set `agent`.
+- `eval-num-workers`: the number of workers to use for evaluation. Default: `1`.
+- `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the
+    given IDs (comma separated).
+
+There are also following optional environment variables you can set:
+
+```bash
+export USE_UNIT_TESTS=true # if you want to allow the Agent to verify correctness using unittests. Default to false.
+export SKIP_NUM=12 # skip the first 12 instances from the dataset
+```
+
+Following is the basic command to start the evaluation.
+
+You can update the arguments in the script
+`evaluation/benchmarks/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
+`--eval-num-workers` and so on:
+
+- `--agent-cls`, the agent to use. For example, `CodeActAgent`.
+- `--llm-config`: the LLM configuration to use. For example, `eval_gpt4_1106_preview`.
+- `--max-iterations`: the max allowed number of iterations to run the evaluation. Default: `30`.
+- `--eval-num-workers`: the number of workers to use for evaluation. Default: `1`.
+- `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
+- `--eval-ids`: the IDs of the examples to evaluate (comma separated). For example, `"1,3,10"`.
+
+```bash
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
+```
+
+### Run Inference on `RemoteRuntime` (experimental)
+
+This is in limited beta. Contact Xingyao over slack if you want to try this out!
+
+```bash
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
+
+# Example - This runs evaluation on CodeActAgent for 133 instances on aider_bench test set, with 2 workers running in parallel
+export ALLHANDS_API_KEY="YOUR-API-KEY"
+export RUNTIME=remote
+export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
+```
+
+## Summarize Results
+
+```bash
+poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file]
+```
+
+Full example:
+
+```bash
+poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl
+```
+
+This will list the instances that passed and the instances that failed. For each
+instance, the corresponding set of test cases (which can vary for each instance)
+are run on the file edited by the agent. We consider an instance to be passed
+only if ALL test cases are passed. Sometimes even a single failed test case will
+cause the entire instance to be marked as failed.
+
+You can inspect the `test_results` field in the `output.jsonl` file to find the exact
+outcome of the tests. If there are no syntax or indentation errors, you can
+expect to see something like "`..F...EF..`", where "`.`" means the test case
+passed, "`E`" means there was an error while executing the test case and "`F`"
+means some assertion failed and some returned output was not as expected.

evaluation/benchmarks/aider_bench/create_dataset.py

@@ -0,0 +1,47 @@
+# This file was used to create the hugging face dataset from the exercism/python
+# github repo.
+# Refer to: https://github.com/exercism/python/tree/main/exercises/practice
+
+import os
+from pathlib import Path
+
+from datasets import Dataset
+
+tests = sorted(os.listdir('practice/'))
+dataset = {
+    'instance_id': [],
+    'instance_name': [],
+    'instruction': [],
+    'signature': [],
+    'test': [],
+}
+
+for i, test in enumerate(tests):
+    testdir = Path(f'practice/{test}/')
+
+    dataset['instance_id'].append(i)
+    dataset['instance_name'].append(testdir.name.replace('-', '_'))
+
+    # if len(glob.glob(f'practice/{testdir.name}/*.py')) != 2:
+    #     print(testdir.name)
+
+    instructions = ''
+    introduction = testdir / '.docs/introduction.md'
+    if introduction.exists():
+        instructions += introduction.read_text()
+    instructions += (testdir / '.docs/instructions.md').read_text()
+    instructions_append = testdir / '.docs/instructions.append.md'
+    if instructions_append.exists():
+        instructions += instructions_append.read_text()
+
+    dataset['instruction'].append(instructions)
+
+    signature_file = testdir / (testdir.name + '.py').replace('-', '_')
+    dataset['signature'].append(signature_file.read_text())
+
+    test_file = testdir / (testdir.name + '_test.py').replace('-', '_')
+    dataset['test'].append(test_file.read_text())
+
+ds = Dataset.from_dict(dataset)
+
+ds.push_to_hub('RajMaheshwari/Exercism-Python')