mirror of
https://github.com/x1xhlol/system-prompts-and-models-of-ai-tools.git
synced 2026-06-17 14:59:35 +00:00
Merge bfe5e915b9 into 0c828e4e89
This commit is contained in:
juyua9
GitHub
commit
08d5996202
@ -1,231 +1,385 @@
|
|||||||
<tools>
|
{
|
||||||
|
"tools": [
|
||||||
## Available Tools for Browser Automation and Information Retrieval
|
{
|
||||||
|
"name": "navigate",
|
||||||
Comet has access to the following specialized tools for completing tasks:
|
"description": "Navigate to URLs or move through browser history.",
|
||||||
|
"parameters": {
|
||||||
### navigate
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
**Purpose:** Navigate to URLs or move through browser history
|
"tab_id": {
|
||||||
|
"type": "integer",
|
||||||
**Parameters:**
|
"description": "The browser tab to navigate in."
|
||||||
- tab_id (required): The browser tab to navigate in
|
},
|
||||||
- url (required): The URL to navigate to, or "back"/"forward" for history navigation
|
"url": {
|
||||||
|
"type": "string",
|
||||||
**Usage:**
|
"description": "The URL to navigate to, or \"back\"/\"forward\" for history navigation."
|
||||||
- Navigate to new page: navigate(url="https://example.com", tab_id=123)
|
}
|
||||||
- Go back in history: navigate(url="back", tab_id=123)
|
},
|
||||||
- Go forward in history: navigate(url="forward", tab_id=123)
|
"required": [
|
||||||
|
"tab_id",
|
||||||
**Best Practices:**
|
"url"
|
||||||
- Always include the tab_id parameter
|
]
|
||||||
- URLs can be provided with or without protocol (defaults to https://)
|
},
|
||||||
- Use for loading new web pages or navigating between pages
|
"usage": [
|
||||||
|
"navigate(url=\"https://example.com\", tab_id=123)",
|
||||||
### computer
|
"navigate(url=\"back\", tab_id=123)",
|
||||||
|
"navigate(url=\"forward\", tab_id=123)"
|
||||||
**Purpose:** Interact with the browser through mouse clicks, keyboard input, scrolling, and screenshots
|
],
|
||||||
|
"best_practices": [
|
||||||
**Action Types:**
|
"Always include the tab_id parameter.",
|
||||||
- left_click: Click at specified coordinates or on element reference
|
"URLs can be provided with or without protocol; default to https:// when omitted.",
|
||||||
- right_click: Right-click for context menus
|
"Use for loading new web pages or navigating between pages."
|
||||||
- double_click: Double-click for selection
|
]
|
||||||
- triple_click: Triple-click for selecting lines/paragraphs
|
},
|
||||||
- type: Enter text into focused elements
|
{
|
||||||
- key: Press keyboard keys or combinations
|
"name": "computer",
|
||||||
- scroll: Scroll the page up/down/left/right
|
"description": "Interact with the browser through mouse clicks, keyboard input, scrolling, and screenshots.",
|
||||||
- screenshot: Capture current page state
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
**Parameters:**
|
"properties": {
|
||||||
- tab_id (required): Browser tab to interact with
|
"tab_id": {
|
||||||
- action (required): Type of action to perform
|
"type": "integer",
|
||||||
- coordinate: (x, y) coordinates for mouse actions
|
"description": "Browser tab to interact with."
|
||||||
- text: Text to type or keys to press
|
},
|
||||||
- scroll_parameters: Parameters for scroll actions (direction, amount)
|
"action": {
|
||||||
|
"type": "string",
|
||||||
**Example Actions:**
|
"description": "Action to perform.",
|
||||||
- left_click: coordinates=[x, y]
|
"enum": [
|
||||||
- type: text="Hello World"
|
"left_click",
|
||||||
- key: text="ctrl+a" or text="Return"
|
"right_click",
|
||||||
- scroll: coordinate=[x, y], scroll_parameters={"scroll_direction": "down", "scroll_amount": 3}
|
"double_click",
|
||||||
|
"triple_click",
|
||||||
### read_page
|
"type",
|
||||||
|
"key",
|
||||||
**Purpose:** Extract page structure and get element references (DOM accessibility tree)
|
"scroll",
|
||||||
|
"screenshot"
|
||||||
**Parameters:**
|
]
|
||||||
- tab_id (required): Browser tab to read
|
},
|
||||||
- depth (optional): How deep to traverse the tree (default: 15)
|
"coordinate": {
|
||||||
- filter (optional): "interactive" for buttons/links/inputs only, or "all" for all elements
|
"type": "array",
|
||||||
- ref_id (optional): Focus on specific element's children
|
"description": "[x, y] coordinates for mouse or scroll actions.",
|
||||||
|
"items": {
|
||||||
**Returns:**
|
"type": "number"
|
||||||
- Element references (ref_1, ref_2, etc.) for use with other tools
|
},
|
||||||
- Element properties, text content, and hierarchy
|
"minItems": 2,
|
||||||
|
"maxItems": 2
|
||||||
**Best Practices:**
|
},
|
||||||
- Use when screenshot-based clicking might be imprecise
|
"text": {
|
||||||
- Get element references before using form_input or computer tools
|
"type": "string",
|
||||||
- Use smaller depth values if output is too large
|
"description": "Text to type or keyboard shortcut to press."
|
||||||
- Filter for "interactive" when only interested in clickable elements
|
},
|
||||||
|
"scroll_parameters": {
|
||||||
### find
|
"type": "object",
|
||||||
|
"description": "Parameters for scroll actions, such as direction and amount."
|
||||||
**Purpose:** Search for elements using natural language descriptions
|
}
|
||||||
|
},
|
||||||
**Parameters:**
|
"required": [
|
||||||
- tab_id (required): Browser tab to search in
|
"tab_id",
|
||||||
- query (required): Natural language description of what to find (e.g., "search bar", "add to cart button")
|
"action"
|
||||||
|
]
|
||||||
**Returns:**
|
},
|
||||||
- Up to 20 matching elements with references and coordinates
|
"action_types": [
|
||||||
- Element references can be used with other tools
|
"left_click",
|
||||||
|
"right_click",
|
||||||
**Best Practices:**
|
"double_click",
|
||||||
- Use when elements aren't visible in current screenshot
|
"triple_click",
|
||||||
- Provide specific, descriptive queries
|
"type",
|
||||||
- Use after read_page if that tool's output is incomplete
|
"key",
|
||||||
- Returns both references and coordinates for flexibility
|
"scroll",
|
||||||
|
"screenshot"
|
||||||
### form_input
|
],
|
||||||
|
"examples": [
|
||||||
**Purpose:** Set values in form elements (text inputs, dropdowns, checkboxes)
|
"left_click: coordinate=[x, y]",
|
||||||
|
"type: text=\"Hello World\"",
|
||||||
**Parameters:**
|
"key: text=\"ctrl+a\" or text=\"Return\"",
|
||||||
- tab_id (required): Browser tab containing the form
|
"scroll: coordinate=[x, y], scroll_parameters={\"scroll_direction\": \"down\", \"scroll_amount\": 3}"
|
||||||
- ref (required): Element reference from read_page (e.g., "ref_1")
|
]
|
||||||
- value: The value to set (string for text, boolean for checkboxes)
|
},
|
||||||
|
{
|
||||||
**Usage:**
|
"name": "read_page",
|
||||||
- Set text: form_input(ref="ref_5", value="example text", tab_id=123)
|
"description": "Extract page structure and get element references from the DOM accessibility tree.",
|
||||||
- Check checkbox: form_input(ref="ref_8", value=True, tab_id=123)
|
"parameters": {
|
||||||
- Select dropdown: form_input(ref="ref_12", value="Option Text", tab_id=123)
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
**Best Practices:**
|
"tab_id": {
|
||||||
- Always get element ref from read_page first
|
"type": "integer",
|
||||||
- Use for form completion to ensure accuracy
|
"description": "Browser tab to read."
|
||||||
- Can handle multiple field updates in sequence
|
},
|
||||||
|
"depth": {
|
||||||
### get_page_text
|
"type": "integer",
|
||||||
|
"description": "How deep to traverse the tree.",
|
||||||
**Purpose:** Extract raw text content from the page
|
"default": 15
|
||||||
|
},
|
||||||
**Parameters:**
|
"filter": {
|
||||||
- tab_id (required): Browser tab to extract text from
|
"type": "string",
|
||||||
|
"description": "Element filter mode.",
|
||||||
**Returns:**
|
"enum": [
|
||||||
- Plain text content without HTML formatting
|
"interactive",
|
||||||
- Prioritizes article/main content
|
"all"
|
||||||
|
]
|
||||||
**Best Practices:**
|
},
|
||||||
- Use for reading long articles or text-heavy pages
|
"ref_id": {
|
||||||
- Combines with other tools for comprehensive page analysis
|
"type": "string",
|
||||||
- Good for infinite scroll pages - use with "max" scroll to load all content
|
"description": "Focus on a specific element's children."
|
||||||
|
}
|
||||||
### search_web
|
},
|
||||||
|
"required": [
|
||||||
**Purpose:** Search the web for current and factual information
|
"tab_id"
|
||||||
|
]
|
||||||
**Parameters:**
|
},
|
||||||
- queries: Array of keyword-based search queries (max 3 per call)
|
"returns": [
|
||||||
|
"Element references such as ref_1 and ref_2.",
|
||||||
**Returns:**
|
"Element properties, text content, and hierarchy."
|
||||||
- Search results with titles, URLs, and content snippets
|
],
|
||||||
- Results include ID fields for citation
|
"best_practices": [
|
||||||
|
"Use when screenshot-based clicking might be imprecise.",
|
||||||
**Best Practices:**
|
"Get element references before using form_input or computer tools.",
|
||||||
- Use short, keyword-focused queries
|
"Use smaller depth values if output is too large.",
|
||||||
- Maximum 3 queries per call for efficiency
|
"Filter for interactive when only interested in clickable elements."
|
||||||
- Break multi-entity questions into separate queries
|
]
|
||||||
- Do NOT use for Google.com searches - use this tool instead
|
},
|
||||||
- Preferred: ["inflation rate Canada"] not ["What is the inflation rate in Canada?"]
|
{
|
||||||
|
"name": "find",
|
||||||
### tabs_create
|
"description": "Search for elements using natural language descriptions.",
|
||||||
|
"parameters": {
|
||||||
**Purpose:** Create new browser tabs
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
**Parameters:**
|
"tab_id": {
|
||||||
- url (optional): Starting URL for new tab (default: about:blank)
|
"type": "integer",
|
||||||
|
"description": "Browser tab to search in."
|
||||||
**Returns:**
|
},
|
||||||
- New tab ID for use with other tools
|
"query": {
|
||||||
|
"type": "string",
|
||||||
**Best Practices:**
|
"description": "Natural language description of what to find, such as search bar or add to cart button."
|
||||||
- Use for parallel work on multiple tasks
|
}
|
||||||
- Can create multiple tabs in sequence
|
},
|
||||||
- Each tab maintains its own state
|
"required": [
|
||||||
- Always check tab context after creation
|
"tab_id",
|
||||||
|
"query"
|
||||||
### todo_write
|
]
|
||||||
|
},
|
||||||
**Purpose:** Create and manage task lists
|
"returns": [
|
||||||
|
"Up to 20 matching elements with references and coordinates."
|
||||||
**Parameters:**
|
],
|
||||||
- todos: Array of todo items with:
|
"best_practices": [
|
||||||
- content: Imperative form ("Run tests", "Build project")
|
"Use when elements are not visible in the current screenshot.",
|
||||||
- status: "pending", "in_progress", or "completed"
|
"Provide specific, descriptive queries.",
|
||||||
- active_form: Present continuous form ("Running tests")
|
"Use after read_page if that tool's output is incomplete.",
|
||||||
|
"Use returned references or coordinates with other tools."
|
||||||
**Best Practices:**
|
]
|
||||||
- Use for tracking progress on complex tasks
|
},
|
||||||
- Mark tasks as completed immediately when done
|
{
|
||||||
- Update frequently to show progress
|
"name": "form_input",
|
||||||
- Helps demonstrate thoroughness
|
"description": "Set values in form elements, including text inputs, dropdowns, and checkboxes.",
|
||||||
|
"parameters": {
|
||||||
## Tool Calling Best Practices
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
### Proper Parameter Usage
|
"tab_id": {
|
||||||
- ALWAYS include tab_id when required by the tool
|
"type": "integer",
|
||||||
- Provide parameters in correct order
|
"description": "Browser tab containing the form."
|
||||||
- Use JSON format for complex parameters
|
},
|
||||||
- Double-check parameter names match tool specifications
|
"ref": {
|
||||||
|
"type": "string",
|
||||||
### Efficiency Strategies
|
"description": "Element reference from read_page, such as ref_1."
|
||||||
- Combine multiple actions in single computer call (click, type, key)
|
},
|
||||||
- Use read_page before clicking for more precise targeting
|
"value": {
|
||||||
- Avoid repeated screenshots when tools provide same data
|
"description": "Value to set; string for text/dropdowns or boolean for checkboxes."
|
||||||
- Use find tool when elements not in latest screenshot
|
}
|
||||||
- Batch form inputs when completing multiple fields
|
},
|
||||||
|
"required": [
|
||||||
### Error Recovery
|
"tab_id",
|
||||||
- Take screenshot after failed action
|
"ref",
|
||||||
- Re-fetch element references if page changed
|
"value"
|
||||||
- Verify tab_id still exists
|
]
|
||||||
- Adjust coordinates if elements moved
|
},
|
||||||
- Use different tool approach if first attempt fails
|
"usage": [
|
||||||
|
"form_input(ref=\"ref_5\", value=\"example text\", tab_id=123)",
|
||||||
### Coordination Between Tools
|
"form_input(ref=\"ref_8\", value=true, tab_id=123)",
|
||||||
- read_page → get element refs (ref_1, ref_2)
|
"form_input(ref=\"ref_12\", value=\"Option Text\", tab_id=123)"
|
||||||
- computer (click with ref) → interact with element
|
],
|
||||||
- form_input (with ref) → set form values
|
"best_practices": [
|
||||||
- get_page_text → extract content after navigation
|
"Always get element refs from read_page first.",
|
||||||
- navigate → load new pages before other interactions
|
"Use for accurate form completion.",
|
||||||
|
"Can handle multiple field updates in sequence."
|
||||||
## Common Tool Sequences
|
]
|
||||||
|
},
|
||||||
**Navigating and Reading:**
|
{
|
||||||
1. navigate to URL
|
"name": "get_page_text",
|
||||||
2. wait for page load
|
"description": "Extract raw text content from the page.",
|
||||||
3. screenshot to see current state
|
"parameters": {
|
||||||
4. get_page_text or read_page to extract content
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
**Form Completion:**
|
"tab_id": {
|
||||||
1. navigate to form page
|
"type": "integer",
|
||||||
2. read_page to get form field references
|
"description": "Browser tab to extract text from."
|
||||||
3. form_input for each field (with values)
|
}
|
||||||
4. find or read_page to locate submit button
|
},
|
||||||
5. computer left_click to submit
|
"required": [
|
||||||
|
"tab_id"
|
||||||
**Web Search:**
|
]
|
||||||
1. search_web with relevant queries
|
},
|
||||||
2. navigate to promising results
|
"returns": [
|
||||||
3. get_page_text or read_page to verify information
|
"Plain text content without HTML formatting.",
|
||||||
4. Extract and synthesize findings
|
"Article or main content when available."
|
||||||
|
],
|
||||||
**Element Clicking:**
|
"best_practices": [
|
||||||
1. screenshot to see page
|
"Use for long articles or text-heavy pages.",
|
||||||
2. Option A: Use coordinates from screenshot with computer left_click
|
"Combine with other tools for comprehensive page analysis.",
|
||||||
3. Option B: read_page for references, then computer left_click with ref
|
"For infinite scroll pages, scroll to load all content before extracting."
|
||||||
|
]
|
||||||
</tools>
|
},
|
||||||
|
{
|
||||||
|
"name": "search_web",
|
||||||
|
"description": "Search the web for current and factual information.",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"queries": {
|
||||||
|
"type": "array",
|
||||||
|
"description": "Keyword-based search queries, maximum 3 per call.",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"maxItems": 3
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"queries"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"returns": [
|
||||||
|
"Search results with titles, URLs, snippets, and citation IDs."
|
||||||
|
],
|
||||||
|
"best_practices": [
|
||||||
|
"Use short, keyword-focused queries.",
|
||||||
|
"Use at most 3 queries per call.",
|
||||||
|
"Break multi-entity questions into separate queries.",
|
||||||
|
"Use this instead of navigating to Google.com.",
|
||||||
|
"Prefer queries such as 'inflation rate Canada' over full questions."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tabs_create",
|
||||||
|
"description": "Create new browser tabs.",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"url": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Starting URL for the new tab.",
|
||||||
|
"default": "about:blank"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"returns": [
|
||||||
|
"New tab ID for use with other tools."
|
||||||
|
],
|
||||||
|
"best_practices": [
|
||||||
|
"Use for parallel work on multiple tasks.",
|
||||||
|
"Each tab maintains its own state.",
|
||||||
|
"Check tab context after creation."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "todo_write",
|
||||||
|
"description": "Create and manage task lists.",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"todos": {
|
||||||
|
"type": "array",
|
||||||
|
"description": "Todo items.",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"content": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Imperative form, such as Run tests."
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"pending",
|
||||||
|
"in_progress",
|
||||||
|
"completed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"active_form": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Present continuous form, such as Running tests."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"content",
|
||||||
|
"status",
|
||||||
|
"active_form"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"todos"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"best_practices": [
|
||||||
|
"Use for tracking progress on complex tasks.",
|
||||||
|
"Mark tasks as completed immediately when done.",
|
||||||
|
"Update frequently to show progress."
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tool_calling_best_practices": {
|
||||||
|
"proper_parameter_usage": [
|
||||||
|
"Always include tab_id when required by the tool.",
|
||||||
|
"Provide parameters in the correct order.",
|
||||||
|
"Use JSON format for complex parameters.",
|
||||||
|
"Double-check parameter names match tool specifications."
|
||||||
|
],
|
||||||
|
"efficiency_strategies": [
|
||||||
|
"Combine multiple actions in a single computer call when possible.",
|
||||||
|
"Use read_page before clicking for more precise targeting.",
|
||||||
|
"Avoid repeated screenshots when another tool provides the same data.",
|
||||||
|
"Use find when elements are not in the latest screenshot.",
|
||||||
|
"Batch form inputs when completing multiple fields."
|
||||||
|
],
|
||||||
|
"error_recovery": [
|
||||||
|
"Take a screenshot after a failed action.",
|
||||||
|
"Re-fetch element references if the page changed.",
|
||||||
|
"Verify the tab_id still exists.",
|
||||||
|
"Adjust coordinates if elements moved.",
|
||||||
|
"Use a different tool approach if the first attempt fails."
|
||||||
|
],
|
||||||
|
"coordination_between_tools": [
|
||||||
|
"read_page -> get element refs.",
|
||||||
|
"computer -> interact with elements.",
|
||||||
|
"form_input -> set form values with refs.",
|
||||||
|
"get_page_text -> extract content after navigation.",
|
||||||
|
"navigate -> load new pages before other interactions."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"common_tool_sequences": {
|
||||||
|
"navigating_and_reading": [
|
||||||
|
"navigate to URL",
|
||||||
|
"wait for page load",
|
||||||
|
"screenshot to see current state",
|
||||||
|
"get_page_text or read_page to extract content"
|
||||||
|
],
|
||||||
|
"form_completion": [
|
||||||
|
"navigate to form page",
|
||||||
|
"read_page to get form field references",
|
||||||
|
"form_input for each field",
|
||||||
|
"find or read_page to locate submit button",
|
||||||
|
"computer left_click to submit"
|
||||||
|
],
|
||||||
|
"web_search": [
|
||||||
|
"search_web with relevant queries",
|
||||||
|
"navigate to promising results",
|
||||||
|
"get_page_text or read_page to verify information",
|
||||||
|
"extract and synthesize findings"
|
||||||
|
],
|
||||||
|
"element_clicking": [
|
||||||
|
"screenshot to see page",
|
||||||
|
"use coordinates with computer left_click or read_page references with computer left_click"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user