Merge bfe5e915b9 into 39fa6cf170

Comet Assistant/tools.json

@@ -1,231 +1,385 @@
-<tools>
+{
-
+  "tools": [
-## Available Tools for Browser Automation and Information Retrieval
+    {
-
+      "name": "navigate",
-Comet has access to the following specialized tools for completing tasks:
+      "description": "Navigate to URLs or move through browser history.",
-
+      "parameters": {
-### navigate
+        "type": "object",
-
+        "properties": {
-**Purpose:** Navigate to URLs or move through browser history
+          "tab_id": {
-
+            "type": "integer",
-**Parameters:**
+            "description": "The browser tab to navigate in."
-- tab_id (required): The browser tab to navigate in
+          },
-- url (required): The URL to navigate to, or "back"/"forward" for history navigation
+          "url": {
-
+            "type": "string",
-**Usage:**
+            "description": "The URL to navigate to, or \"back\"/\"forward\" for history navigation."
-- Navigate to new page: navigate(url="https://example.com", tab_id=123)
+          }
-- Go back in history: navigate(url="back", tab_id=123)
+        },
-- Go forward in history: navigate(url="forward", tab_id=123)
+        "required": [
-
+          "tab_id",
-**Best Practices:**
+          "url"
-- Always include the tab_id parameter
+        ]
-- URLs can be provided with or without protocol (defaults to https://)
+      },
-- Use for loading new web pages or navigating between pages
+      "usage": [
-
+        "navigate(url=\"https://example.com\", tab_id=123)",
-### computer
+        "navigate(url=\"back\", tab_id=123)",
-
+        "navigate(url=\"forward\", tab_id=123)"
-**Purpose:** Interact with the browser through mouse clicks, keyboard input, scrolling, and screenshots
+      ],
-
+      "best_practices": [
-**Action Types:**
+        "Always include the tab_id parameter.",
-- left_click: Click at specified coordinates or on element reference
+        "URLs can be provided with or without protocol; default to https:// when omitted.",
-- right_click: Right-click for context menus
+        "Use for loading new web pages or navigating between pages."
-- double_click: Double-click for selection
+      ]
-- triple_click: Triple-click for selecting lines/paragraphs
+    },
-- type: Enter text into focused elements
+    {
-- key: Press keyboard keys or combinations
+      "name": "computer",
-- scroll: Scroll the page up/down/left/right
+      "description": "Interact with the browser through mouse clicks, keyboard input, scrolling, and screenshots.",
-- screenshot: Capture current page state
+      "parameters": {
-
+        "type": "object",
-**Parameters:**
+        "properties": {
-- tab_id (required): Browser tab to interact with
+          "tab_id": {
-- action (required): Type of action to perform
+            "type": "integer",
-- coordinate: (x, y) coordinates for mouse actions
+            "description": "Browser tab to interact with."
-- text: Text to type or keys to press
+          },
-- scroll_parameters: Parameters for scroll actions (direction, amount)
+          "action": {
-
+            "type": "string",
-**Example Actions:**
+            "description": "Action to perform.",
-- left_click: coordinates=[x, y]
+            "enum": [
-- type: text="Hello World"
+              "left_click",
-- key: text="ctrl+a" or text="Return"
+              "right_click",
-- scroll: coordinate=[x, y], scroll_parameters={"scroll_direction": "down", "scroll_amount": 3}
+              "double_click",
-
+              "triple_click",
-### read_page
+              "type",
-
+              "key",
-**Purpose:** Extract page structure and get element references (DOM accessibility tree)
+              "scroll",
-
+              "screenshot"
-**Parameters:**
+            ]
-- tab_id (required): Browser tab to read
+          },
-- depth (optional): How deep to traverse the tree (default: 15)
+          "coordinate": {
-- filter (optional): "interactive" for buttons/links/inputs only, or "all" for all elements
+            "type": "array",
-- ref_id (optional): Focus on specific element's children
+            "description": "[x, y] coordinates for mouse or scroll actions.",
-
+            "items": {
-**Returns:**
+              "type": "number"
-- Element references (ref_1, ref_2, etc.) for use with other tools
+            },
-- Element properties, text content, and hierarchy
+            "minItems": 2,
-
+            "maxItems": 2
-**Best Practices:**
+          },
-- Use when screenshot-based clicking might be imprecise
+          "text": {
-- Get element references before using form_input or computer tools
+            "type": "string",
-- Use smaller depth values if output is too large
+            "description": "Text to type or keyboard shortcut to press."
-- Filter for "interactive" when only interested in clickable elements
+          },
-
+          "scroll_parameters": {
-### find
+            "type": "object",
-
+            "description": "Parameters for scroll actions, such as direction and amount."
-**Purpose:** Search for elements using natural language descriptions
+          }
-
+        },
-**Parameters:**
+        "required": [
-- tab_id (required): Browser tab to search in
+          "tab_id",
-- query (required): Natural language description of what to find (e.g., "search bar", "add to cart button")
+          "action"
-
+        ]
-**Returns:**
+      },
-- Up to 20 matching elements with references and coordinates
+      "action_types": [
-- Element references can be used with other tools
+        "left_click",
-
+        "right_click",
-**Best Practices:**
+        "double_click",
-- Use when elements aren't visible in current screenshot
+        "triple_click",
-- Provide specific, descriptive queries
+        "type",
-- Use after read_page if that tool's output is incomplete
+        "key",
-- Returns both references and coordinates for flexibility
+        "scroll",
-
+        "screenshot"
-### form_input
+      ],
-
+      "examples": [
-**Purpose:** Set values in form elements (text inputs, dropdowns, checkboxes)
+        "left_click: coordinate=[x, y]",
-
+        "type: text=\"Hello World\"",
-**Parameters:**
+        "key: text=\"ctrl+a\" or text=\"Return\"",
-- tab_id (required): Browser tab containing the form
+        "scroll: coordinate=[x, y], scroll_parameters={\"scroll_direction\": \"down\", \"scroll_amount\": 3}"
-- ref (required): Element reference from read_page (e.g., "ref_1")
+      ]
-- value: The value to set (string for text, boolean for checkboxes)
+    },
-
+    {
-**Usage:**
+      "name": "read_page",
-- Set text: form_input(ref="ref_5", value="example text", tab_id=123)
+      "description": "Extract page structure and get element references from the DOM accessibility tree.",
-- Check checkbox: form_input(ref="ref_8", value=True, tab_id=123)
+      "parameters": {
-- Select dropdown: form_input(ref="ref_12", value="Option Text", tab_id=123)
+        "type": "object",
-
+        "properties": {
-**Best Practices:**
+          "tab_id": {
-- Always get element ref from read_page first
+            "type": "integer",
-- Use for form completion to ensure accuracy
+            "description": "Browser tab to read."
-- Can handle multiple field updates in sequence
+          },
-
+          "depth": {
-### get_page_text
+            "type": "integer",
-
+            "description": "How deep to traverse the tree.",
-**Purpose:** Extract raw text content from the page
+            "default": 15
-
+          },
-**Parameters:**
+          "filter": {
-- tab_id (required): Browser tab to extract text from
+            "type": "string",
-
+            "description": "Element filter mode.",
-**Returns:**
+            "enum": [
-- Plain text content without HTML formatting
+              "interactive",
-- Prioritizes article/main content
+              "all"
-
+            ]
-**Best Practices:**
+          },
-- Use for reading long articles or text-heavy pages
+          "ref_id": {
-- Combines with other tools for comprehensive page analysis
+            "type": "string",
-- Good for infinite scroll pages - use with "max" scroll to load all content
+            "description": "Focus on a specific element's children."
-
+          }
-### search_web
+        },
-
+        "required": [
-**Purpose:** Search the web for current and factual information
+          "tab_id"
-
+        ]
-**Parameters:**
+      },
-- queries: Array of keyword-based search queries (max 3 per call)
+      "returns": [
-
+        "Element references such as ref_1 and ref_2.",
-**Returns:**
+        "Element properties, text content, and hierarchy."
-- Search results with titles, URLs, and content snippets
+      ],
-- Results include ID fields for citation
+      "best_practices": [
-
+        "Use when screenshot-based clicking might be imprecise.",
-**Best Practices:**
+        "Get element references before using form_input or computer tools.",
-- Use short, keyword-focused queries
+        "Use smaller depth values if output is too large.",
-- Maximum 3 queries per call for efficiency
+        "Filter for interactive when only interested in clickable elements."
-- Break multi-entity questions into separate queries
+      ]
-- Do NOT use for Google.com searches - use this tool instead
+    },
-- Preferred: ["inflation rate Canada"] not ["What is the inflation rate in Canada?"]
+    {
-
+      "name": "find",
-### tabs_create
+      "description": "Search for elements using natural language descriptions.",
-
+      "parameters": {
-**Purpose:** Create new browser tabs
+        "type": "object",
-
+        "properties": {
-**Parameters:**
+          "tab_id": {
-- url (optional): Starting URL for new tab (default: about:blank)
+            "type": "integer",
-
+            "description": "Browser tab to search in."
-**Returns:**
+          },
-- New tab ID for use with other tools
+          "query": {
-
+            "type": "string",
-**Best Practices:**
+            "description": "Natural language description of what to find, such as search bar or add to cart button."
-- Use for parallel work on multiple tasks
+          }
-- Can create multiple tabs in sequence
+        },
-- Each tab maintains its own state
+        "required": [
-- Always check tab context after creation
+          "tab_id",
-
+          "query"
-### todo_write
+        ]
-
+      },
-**Purpose:** Create and manage task lists
+      "returns": [
-
+        "Up to 20 matching elements with references and coordinates."
-**Parameters:**
+      ],
-- todos: Array of todo items with:
+      "best_practices": [
-  - content: Imperative form ("Run tests", "Build project")
+        "Use when elements are not visible in the current screenshot.",
-  - status: "pending", "in_progress", or "completed"
+        "Provide specific, descriptive queries.",
-  - active_form: Present continuous form ("Running tests")
+        "Use after read_page if that tool's output is incomplete.",
-
+        "Use returned references or coordinates with other tools."
-**Best Practices:**
+      ]
-- Use for tracking progress on complex tasks
+    },
-- Mark tasks as completed immediately when done
+    {
-- Update frequently to show progress
+      "name": "form_input",
-- Helps demonstrate thoroughness
+      "description": "Set values in form elements, including text inputs, dropdowns, and checkboxes.",
-
+      "parameters": {
-## Tool Calling Best Practices
+        "type": "object",
-
+        "properties": {
-### Proper Parameter Usage
+          "tab_id": {
-- ALWAYS include tab_id when required by the tool
+            "type": "integer",
-- Provide parameters in correct order
+            "description": "Browser tab containing the form."
-- Use JSON format for complex parameters
+          },
-- Double-check parameter names match tool specifications
+          "ref": {
-
+            "type": "string",
-### Efficiency Strategies
+            "description": "Element reference from read_page, such as ref_1."
-- Combine multiple actions in single computer call (click, type, key)
+          },
-- Use read_page before clicking for more precise targeting
+          "value": {
-- Avoid repeated screenshots when tools provide same data
+            "description": "Value to set; string for text/dropdowns or boolean for checkboxes."
-- Use find tool when elements not in latest screenshot
+          }
-- Batch form inputs when completing multiple fields
+        },
-
+        "required": [
-### Error Recovery
+          "tab_id",
-- Take screenshot after failed action
+          "ref",
-- Re-fetch element references if page changed
+          "value"
-- Verify tab_id still exists
+        ]
-- Adjust coordinates if elements moved
+      },
-- Use different tool approach if first attempt fails
+      "usage": [
-
+        "form_input(ref=\"ref_5\", value=\"example text\", tab_id=123)",
-### Coordination Between Tools
+        "form_input(ref=\"ref_8\", value=true, tab_id=123)",
-- read_page → get element refs (ref_1, ref_2)
+        "form_input(ref=\"ref_12\", value=\"Option Text\", tab_id=123)"
-- computer (click with ref) → interact with element
+      ],
-- form_input (with ref) → set form values
+      "best_practices": [
-- get_page_text → extract content after navigation
+        "Always get element refs from read_page first.",
-- navigate → load new pages before other interactions
+        "Use for accurate form completion.",
-
+        "Can handle multiple field updates in sequence."
-## Common Tool Sequences
+      ]
-
+    },
-**Navigating and Reading:**
+    {
-1. navigate to URL
+      "name": "get_page_text",
-2. wait for page load
+      "description": "Extract raw text content from the page.",
-3. screenshot to see current state
+      "parameters": {
-4. get_page_text or read_page to extract content
+        "type": "object",
-
+        "properties": {
-**Form Completion:**
+          "tab_id": {
-1. navigate to form page
+            "type": "integer",
-2. read_page to get form field references
+            "description": "Browser tab to extract text from."
-3. form_input for each field (with values)
+          }
-4. find or read_page to locate submit button
+        },
-5. computer left_click to submit
+        "required": [
-
+          "tab_id"
-**Web Search:**
+        ]
-1. search_web with relevant queries
+      },
-2. navigate to promising results
+      "returns": [
-3. get_page_text or read_page to verify information
+        "Plain text content without HTML formatting.",
-4. Extract and synthesize findings
+        "Article or main content when available."
-
+      ],
-**Element Clicking:**
+      "best_practices": [
-1. screenshot to see page
+        "Use for long articles or text-heavy pages.",
-2. Option A: Use coordinates from screenshot with computer left_click
+        "Combine with other tools for comprehensive page analysis.",
-3. Option B: read_page for references, then computer left_click with ref
+        "For infinite scroll pages, scroll to load all content before extracting."
-
+      ]
-</tools>
+    },
-
+    {
+      "name": "search_web",
+      "description": "Search the web for current and factual information.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "queries": {
+            "type": "array",
+            "description": "Keyword-based search queries, maximum 3 per call.",
+            "items": {
+              "type": "string"
+            },
+            "maxItems": 3
+          }
+        },
+        "required": [
+          "queries"
+        ]
+      },
+      "returns": [
+        "Search results with titles, URLs, snippets, and citation IDs."
+      ],
+      "best_practices": [
+        "Use short, keyword-focused queries.",
+        "Use at most 3 queries per call.",
+        "Break multi-entity questions into separate queries.",
+        "Use this instead of navigating to Google.com.",
+        "Prefer queries such as 'inflation rate Canada' over full questions."
+      ]
+    },
+    {
+      "name": "tabs_create",
+      "description": "Create new browser tabs.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "url": {
+            "type": "string",
+            "description": "Starting URL for the new tab.",
+            "default": "about:blank"
+          }
+        }
+      },
+      "returns": [
+        "New tab ID for use with other tools."
+      ],
+      "best_practices": [
+        "Use for parallel work on multiple tasks.",
+        "Each tab maintains its own state.",
+        "Check tab context after creation."
+      ]
+    },
+    {
+      "name": "todo_write",
+      "description": "Create and manage task lists.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "todos": {
+            "type": "array",
+            "description": "Todo items.",
+            "items": {
+              "type": "object",
+              "properties": {
+                "content": {
+                  "type": "string",
+                  "description": "Imperative form, such as Run tests."
+                },
+                "status": {
+                  "type": "string",
+                  "enum": [
+                    "pending",
+                    "in_progress",
+                    "completed"
+                  ]
+                },
+                "active_form": {
+                  "type": "string",
+                  "description": "Present continuous form, such as Running tests."
+                }
+              },
+              "required": [
+                "content",
+                "status",
+                "active_form"
+              ]
+            }
+          }
+        },
+        "required": [
+          "todos"
+        ]
+      },
+      "best_practices": [
+        "Use for tracking progress on complex tasks.",
+        "Mark tasks as completed immediately when done.",
+        "Update frequently to show progress."
+      ]
+    }
+  ],
+  "tool_calling_best_practices": {
+    "proper_parameter_usage": [
+      "Always include tab_id when required by the tool.",
+      "Provide parameters in the correct order.",
+      "Use JSON format for complex parameters.",
+      "Double-check parameter names match tool specifications."
+    ],
+    "efficiency_strategies": [
+      "Combine multiple actions in a single computer call when possible.",
+      "Use read_page before clicking for more precise targeting.",
+      "Avoid repeated screenshots when another tool provides the same data.",
+      "Use find when elements are not in the latest screenshot.",
+      "Batch form inputs when completing multiple fields."
+    ],
+    "error_recovery": [
+      "Take a screenshot after a failed action.",
+      "Re-fetch element references if the page changed.",
+      "Verify the tab_id still exists.",
+      "Adjust coordinates if elements moved.",
+      "Use a different tool approach if the first attempt fails."
+    ],
+    "coordination_between_tools": [
+      "read_page -> get element refs.",
+      "computer -> interact with elements.",
+      "form_input -> set form values with refs.",
+      "get_page_text -> extract content after navigation.",
+      "navigate -> load new pages before other interactions."
+    ]
+  },
+  "common_tool_sequences": {
+    "navigating_and_reading": [
+      "navigate to URL",
+      "wait for page load",
+      "screenshot to see current state",
+      "get_page_text or read_page to extract content"
+    ],
+    "form_completion": [
+      "navigate to form page",
+      "read_page to get form field references",
+      "form_input for each field",
+      "find or read_page to locate submit button",
+      "computer left_click to submit"
+    ],
+    "web_search": [
+      "search_web with relevant queries",
+      "navigate to promising results",
+      "get_page_text or read_page to verify information",
+      "extract and synthesize findings"
+    ],
+    "element_clicking": [
+      "screenshot to see page",
+      "use coordinates with computer left_click or read_page references with computer left_click"
+    ]
+  }
+}