From dc42bd525a4004f6166bf881baff52497688616a Mon Sep 17 00:00:00 2001
From: liangxinbing <1580466765@qq.com>
Date: Tue, 18 Mar 2025 22:51:27 +0800
Subject: [PATCH] add BrowserAgent and update Manus
---
app/agent/__init__.py | 2 +
app/agent/browser.py | 129 ++++++++++++++++++++++++++++++++++++++++++
app/agent/manus.py | 63 ++++++---------------
app/prompt/browser.py | 92 ++++++++++++++++++++++++++++++
app/tool/__init__.py | 2 +
5 files changed, 243 insertions(+), 45 deletions(-)
create mode 100644 app/agent/browser.py
create mode 100644 app/prompt/browser.py
diff --git a/app/agent/__init__.py b/app/agent/__init__.py
index a7b69c7..4b8fb9d 100644
--- a/app/agent/__init__.py
+++ b/app/agent/__init__.py
@@ -1,4 +1,5 @@
from app.agent.base import BaseAgent
+from app.agent.browser import BrowserAgent
from app.agent.planning import PlanningAgent
from app.agent.react import ReActAgent
from app.agent.swe import SWEAgent
@@ -7,6 +8,7 @@ from app.agent.toolcall import ToolCallAgent
__all__ = [
"BaseAgent",
+ "BrowserAgent",
"PlanningAgent",
"ReActAgent",
"SWEAgent",
diff --git a/app/agent/browser.py b/app/agent/browser.py
new file mode 100644
index 0000000..ae0ce2f
--- /dev/null
+++ b/app/agent/browser.py
@@ -0,0 +1,129 @@
+import json
+from typing import Any, Optional
+
+from pydantic import Field
+
+from app.agent.toolcall import ToolCallAgent
+from app.logger import logger
+from app.prompt.browser import NEXT_STEP_PROMPT, SYSTEM_PROMPT
+from app.schema import Message, ToolChoice
+from app.tool import BrowserUseTool, Terminate, ToolCollection
+
+
+class BrowserAgent(ToolCallAgent):
+ """
+ A browser agent that uses the browser_use library to control a browser.
+
+ This agent can navigate web pages, interact with elements, fill forms,
+ extract content, and perform other browser-based actions to accomplish tasks.
+ """
+
+ name: str = "browser"
+ description: str = "A browser agent that can control a browser to accomplish tasks"
+
+ system_prompt: str = SYSTEM_PROMPT
+ next_step_prompt: str = NEXT_STEP_PROMPT
+
+ max_observe: int = 10000
+ max_steps: int = 20
+
+ # Configure the available tools
+ available_tools: ToolCollection = Field(
+ default_factory=lambda: ToolCollection(BrowserUseTool(), Terminate())
+ )
+
+ # Use Auto for tool choice to allow both tool usage and free-form responses
+ tool_choices: ToolChoice = ToolChoice.AUTO
+ special_tool_names: list[str] = Field(default_factory=lambda: [Terminate().name])
+
+ _current_base64_image: Optional[str] = None
+
+ async def _handle_special_tool(self, name: str, result: Any, **kwargs):
+ if not self._is_special_tool(name):
+ return
+ else:
+ await self.available_tools.get_tool(BrowserUseTool().name).cleanup()
+ await super()._handle_special_tool(name, result, **kwargs)
+
+ async def get_browser_state(self) -> Optional[dict]:
+ """Get the current browser state for context in next steps."""
+ browser_tool = self.available_tools.get_tool(BrowserUseTool().name)
+ if not browser_tool:
+ return None
+
+ try:
+ # Get browser state directly from the tool
+ result = await browser_tool.get_current_state()
+
+ if result.error:
+ logger.debug(f"Browser state error: {result.error}")
+ return None
+
+ # Store screenshot if available
+ if hasattr(result, "base64_image") and result.base64_image:
+ self._current_base64_image = result.base64_image
+
+ # Parse the state info
+ return json.loads(result.output)
+
+ except Exception as e:
+ logger.debug(f"Failed to get browser state: {str(e)}")
+ return None
+
+ async def think(self) -> bool:
+ """Process current state and decide next actions using tools, with browser state info added"""
+ # Add browser state to the context
+ browser_state = await self.get_browser_state()
+
+ # Initialize placeholder values
+ url_info = ""
+ tabs_info = ""
+ content_above_info = ""
+ content_below_info = ""
+ results_info = ""
+
+ if browser_state and not browser_state.get("error"):
+ # URL and title info
+ url_info = f"\n URL: {browser_state.get('url', 'N/A')}\n Title: {browser_state.get('title', 'N/A')}"
+
+ # Tab information
+ if "tabs" in browser_state:
+ tabs = browser_state.get("tabs", [])
+ if tabs:
+ tabs_info = f"\n {len(tabs)} tab(s) available"
+
+ # Content above/below viewport
+ pixels_above = browser_state.get("pixels_above", 0)
+ pixels_below = browser_state.get("pixels_below", 0)
+
+ if pixels_above > 0:
+ content_above_info = f" ({pixels_above} pixels)"
+
+ if pixels_below > 0:
+ content_below_info = f" ({pixels_below} pixels)"
+
+ # Add screenshot as base64 if available
+ if self._current_base64_image:
+ # Create a message with image attachment
+ image_message = Message.user_message(
+ content="Current browser screenshot:",
+ base64_image=self._current_base64_image,
+ )
+ self.memory.add_message(image_message)
+
+ # Replace placeholders with actual browser state info
+ self.next_step_prompt = NEXT_STEP_PROMPT.format(
+ url_placeholder=url_info,
+ tabs_placeholder=tabs_info,
+ content_above_placeholder=content_above_info,
+ content_below_placeholder=content_below_info,
+ results_placeholder=results_info,
+ )
+
+ # Call parent implementation
+ result = await super().think()
+
+ # Reset the next_step_prompt to its original state
+ self.next_step_prompt = NEXT_STEP_PROMPT
+
+ return result
diff --git a/app/agent/manus.py b/app/agent/manus.py
index fd3e843..8cba71a 100644
--- a/app/agent/manus.py
+++ b/app/agent/manus.py
@@ -1,12 +1,9 @@
-import json
import os
from pathlib import Path
-from typing import Any, Optional
from pydantic import Field
-from app.agent.toolcall import ToolCallAgent
-from app.logger import logger
+from app.agent.browser import BrowserAgent
from app.prompt.manus import NEXT_STEP_PROMPT, SYSTEM_PROMPT
from app.tool import Terminate, ToolCollection
from app.tool.browser_use_tool import BrowserUseTool
@@ -17,11 +14,11 @@ from app.tool.str_replace_editor import StrReplaceEditor
initial_working_directory = Path(os.getcwd()) / "workspace"
-class Manus(ToolCallAgent):
+class Manus(BrowserAgent):
"""
A versatile general-purpose agent that uses planning to solve various tasks.
- This agent extends PlanningAgent with a comprehensive set of tools and capabilities,
+ This agent extends BrowserAgent with a comprehensive set of tools and capabilities,
including Python execution, web browsing, file operations, and information retrieval
to handle a wide range of user requests.
"""
@@ -44,48 +41,24 @@ class Manus(ToolCallAgent):
)
)
- async def _handle_special_tool(self, name: str, result: Any, **kwargs):
- if not self._is_special_tool(name):
- return
- else:
- await self.available_tools.get_tool(BrowserUseTool().name).cleanup()
- await super()._handle_special_tool(name, result, **kwargs)
-
- async def get_browser_state(self) -> Optional[dict]:
- """Get the current browser state for context in next steps."""
- browser_tool = self.available_tools.get_tool(BrowserUseTool().name)
- if not browser_tool:
- return None
-
- try:
- # Get browser state directly from the tool with no context parameter
- result = await browser_tool.get_current_state()
-
- if result.error:
- logger.debug(f"Browser state error: {result.error}")
- return None
-
- # Store screenshot if available
- if hasattr(result, "base64_image") and result.base64_image:
- self._current_base64_image = result.base64_image
-
- # Parse the state info
- return json.loads(result.output)
-
- except Exception as e:
- logger.debug(f"Failed to get browser state: {str(e)}")
- return None
-
async def think(self) -> bool:
- # Add your custom pre-processing here
- browser_state = await self.get_browser_state()
-
- # Modify the next_step_prompt temporarily
+ """Process current state and decide next actions with appropriate context."""
+ # Store original prompt
original_prompt = self.next_step_prompt
- if browser_state and not browser_state.get("error"):
- self.next_step_prompt += f"\nCurrent browser state:\nURL: {browser_state.get('url', 'N/A')}\nTitle: {browser_state.get('title', 'N/A')}\n"
- # Call parent implementation
+ # Only check recent messages (last 3) for browser activity
+ recent_messages = self.memory.messages[-3:] if self.memory.messages else []
+ browser_in_use = any(
+ "browser_use" in msg.content.lower()
+ for msg in recent_messages
+ if hasattr(msg, "content") and isinstance(msg.content, str)
+ )
+
+ if browser_in_use:
+ # Override with parent class's prompt temporarily to get browser context
+ self.next_step_prompt = BrowserAgent.next_step_prompt
+
+ # Call parent's think method
result = await super().think()
# Restore original prompt
diff --git a/app/prompt/browser.py b/app/prompt/browser.py
new file mode 100644
index 0000000..70fed30
--- /dev/null
+++ b/app/prompt/browser.py
@@ -0,0 +1,92 @@
+SYSTEM_PROMPT = """\
+You are an AI agent designed to automate browser tasks. Your goal is to accomplish the ultimate task following the rules.
+
+# Input Format
+Task
+Previous steps
+Current URL
+Open Tabs
+Interactive Elements
+[index]text
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, etc.)
+- text: Element description
+Example:
+[33]
+
+- Only elements with numeric indexes in [] are interactive
+- elements without [] provide only context
+
+# Response Rules
+1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format:
+{{"current_state": {{"evaluation_previous_goal": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Mention if something unexpected happened. Shortly state why/why not",
+"memory": "Description of what has been done and what you need to remember. Be very specific. Count here ALWAYS how many times you have done something and how many remain. E.g. 0 out of 10 websites analyzed. Continue with abc and xyz",
+"next_goal": "What needs to be done with the next immediate action"}},
+"action":[{{"one_action_name": {{// action-specific parameter}}}}, // ... more actions in sequence]}}
+
+2. ACTIONS: You can specify multiple actions in the list to be executed in sequence. But always specify only one action name per item. Use maximum {{max_actions}} actions per sequence.
+Common action sequences:
+- Form filling: [{{"input_text": {{"index": 1, "text": "username"}}}}, {{"input_text": {{"index": 2, "text": "password"}}}}, {{"click_element": {{"index": 3}}}}]
+- Navigation and extraction: [{{"go_to_url": {{"url": "https://example.com"}}}}, {{"extract_content": {{"goal": "extract the names"}}}}]
+- Actions are executed in the given order
+- If the page changes after an action, the sequence is interrupted and you get the new state.
+- Only provide the action sequence until an action which changes the page state significantly.
+- Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page
+- only use multiple actions if it makes sense.
+
+3. ELEMENT INTERACTION:
+- Only use indexes of the interactive elements
+- Elements marked with "[]Non-interactive text" are non-interactive
+
+4. NAVIGATION & ERROR HANDLING:
+- If no suitable elements exist, use other functions to complete the task
+- If stuck, try alternative approaches - like going back to a previous page, new search, new tab etc.
+- Handle popups/cookies by accepting or closing them
+- Use scroll to find elements you are looking for
+- If you want to research something, open a new tab instead of using the current tab
+- If captcha pops up, try to solve it - else try a different approach
+- If the page is not fully loaded, use wait action
+
+5. TASK COMPLETION:
+- Use the done action as the last action as soon as the ultimate task is complete
+- Dont use "done" before you are done with everything the user asked you, except you reach the last step of max_steps.
+- If you reach your last step, use the done action even if the task is not fully finished. Provide all the information you have gathered so far. If the ultimate task is completly finished set success to true. If not everything the user asked for is completed set success in done to false!
+- If you have to do something repeatedly for example the task says for "each", or "for all", or "x times", count always inside "memory" how many times you have done it and how many remain. Don't stop until you have completed like the task asked you. Only call done after the last step.
+- Don't hallucinate actions
+- Make sure you include everything you found out for the ultimate task in the done text parameter. Do not just say you are done, but include the requested information of the task.
+
+6. VISUAL CONTEXT:
+- When an image is provided, use it to understand the page layout
+- Bounding boxes with labels on their top right corner correspond to element indexes
+
+7. Form filling:
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+
+8. Long tasks:
+- Keep track of the status and subresults in the memory.
+
+9. Extraction:
+- If your task is to find information - call extract_content on the specific pages to get and store the information.
+Your responses must be always JSON with the specified format.
+"""
+
+NEXT_STEP_PROMPT = """
+What should I do next to achieve my goal?
+
+When you see [Current state starts here], focus on the following:
+- Current URL and page title{url_placeholder}
+- Available tabs{tabs_placeholder}
+- Interactive elements and their indices
+- Content above{content_above_placeholder} or below{content_below_placeholder} the viewport (if indicated)
+- Any action results or errors{results_placeholder}
+
+For browser interactions:
+- To navigate: browser_use with action="go_to_url", url="..."
+- To click: browser_use with action="click_element", index=N
+- To type: browser_use with action="input_text", index=N, text="..."
+- To extract: browser_use with action="extract_content", goal="..."
+- To scroll: browser_use with action="scroll_down" or "scroll_up"
+
+Consider both what's visible and what might be beyond the current viewport.
+Be methodical - remember your progress and what you've learned so far.
+"""
diff --git a/app/tool/__init__.py b/app/tool/__init__.py
index 9ab569a..6fbd1bc 100644
--- a/app/tool/__init__.py
+++ b/app/tool/__init__.py
@@ -1,5 +1,6 @@
from app.tool.base import BaseTool
from app.tool.bash import Bash
+from app.tool.browser_use_tool import BrowserUseTool
from app.tool.create_chat_completion import CreateChatCompletion
from app.tool.planning import PlanningTool
from app.tool.str_replace_editor import StrReplaceEditor
@@ -10,6 +11,7 @@ from app.tool.tool_collection import ToolCollection
__all__ = [
"BaseTool",
"Bash",
+ "BrowserUseTool",
"Terminate",
"StrReplaceEditor",
"ToolCollection",