use observe in act (#518)

* use observe in act

* changeset

* fix act from observe (#531)

* fix act from observe

* remove arbitrary log

* refactor into actHandler and solved recursing issue

* adding the missing piece to observe prompt

---------

Co-authored-by: miguel <[email protected]>
Co-authored-by: Anirudh Kamath <[email protected]>

Author: 3 people

Diff for: .changeset/wild-hats-turn.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": minor
+---
+
+`act()` now uses `observe()` under the hood. This results in significant performance improvements.

Diff for: evals/tasks/nonsense_action.ts

@@ -13,22 +13,11 @@ export const nonsense_action: EvalFunction = async ({ modelName, logger }) => {
     await stagehand.page.goto("https://www.homedepot.com/");
 
     const result = await stagehand.page.act({
-      action: "click on the first banana",
+      action: "what is the capital of the moon?",
     });
-    console.log("result", result);
-
-    // Assert the output
-    const expectedResult = {
-      success: false,
-      message: "Action was not able to be completed.",
-      action: "click on the first banana",
-    };
-
-    const isResultCorrect =
-      JSON.stringify(result) === JSON.stringify(expectedResult);
 
     return {
-      _success: isResultCorrect,
+      _success: !result.success, // We expect this to fail
       debugUrl,
       sessionUrl,
       logs: logger.getLogs(),

Diff for: evals/tasks/simple_google_search.ts

@@ -15,9 +15,11 @@ export const simple_google_search: EvalFunction = async ({
   await stagehand.page.goto("https://www.google.com");
 
   await stagehand.page.act({
-    action: 'Search for "OpenAI"',
+    action: 'type "OpenAI" into the search bar',
   });
 
+  await stagehand.page.act("click the search button");
+
   const expectedUrl = "https://www.google.com/search?q=OpenAI";
   const currentUrl = stagehand.page.url();
 

Diff for: lib/StagehandPage.ts

@@ -465,6 +465,7 @@ export class StagehandPage {
       useVision, // still destructure this but will not pass it on
       variables = {},
       domSettleTimeoutMs,
+      slowDomBasedAct = false,
     } = actionOrOptions;
 
     if (typeof useVision !== "undefined") {
@@ -487,6 +488,10 @@ export class StagehandPage {
       ? this.stagehand.llmProvider.getClient(modelName, modelClientOptions)
       : this.llmClient;
 
+    if (!slowDomBasedAct) {
+      return this.actHandler.observeAct(action);
+    }
+
     this.stagehand.log({
       category: "act",
       message: "running act",

Diff for: lib/handlers/actHandler.ts

@@ -11,7 +11,8 @@ import { LLMProvider } from "../llm/LLMProvider";
 import { StagehandContext } from "../StagehandContext";
 import { StagehandPage } from "../StagehandPage";
 import { generateId } from "../utils";
-import { ObserveResult } from "@/types/stagehand";
+import { ActResult, ObserveResult } from "@/types/stagehand";
+import { SupportedPlaywrightAction } from "@/types/act";
 
 /**
  * NOTE: Vision support has been removed from this version of Stagehand.
@@ -71,7 +72,7 @@ export class StagehandActHandler {
    */
   public async actFromObserveResult(
     observe: ObserveResult,
-  ): Promise<{ success: boolean; message: string; action: string }> {
+  ): Promise<ActResult> {
     this.logger({
       category: "action",
       message: "Performing act from an ObserveResult",
@@ -98,7 +99,10 @@ export class StagehandActHandler {
         action: observe.description || `ObserveResult action (${method})`,
       };
     } catch (err) {
-      if (!this.selfHeal) {
+      if (
+        !this.selfHeal ||
+        err instanceof PlaywrightCommandMethodNotSupportedException
+      ) {
         this.logger({
           category: "action",
           message: "Error performing act from an ObserveResult",
@@ -136,11 +140,14 @@ export class StagehandActHandler {
             ? `${method} ${observe.description}`
             : observe.description;
         // Call act with the ObserveResult description
-        await this.stagehandPage.act(actCommand);
+        await this.stagehandPage.act({
+          action: actCommand,
+          slowDomBasedAct: true,
+        });
       } catch (err) {
         this.logger({
           category: "action",
-          message: "Error performing act from an ObserveResult",
+          message: "Error performing act from an ObserveResult on fallback",
           level: 1,
           auxiliary: {
             error: { value: err.message, type: "string" },
@@ -156,6 +163,29 @@ export class StagehandActHandler {
     }
   }
 
+  /**
+   * Perform an act based on an instruction.
+   * This method will observe the page and then perform the act on the first element returned.
+   */
+  public async observeAct(instruction: string): Promise<ActResult> {
+    const observeResults = await this.stagehandPage.observe(
+      `Find the most relevant element to perform an action on given the following action: ${instruction}. 
+      Provide an action for this element such as ${Object.values(SupportedPlaywrightAction).join(", ")}, or any other playwright locator method. Remember that to users, buttons and links look the same in most cases.
+      If the action is completely unrelated to a potential action to be taken on the page, return an empty array. 
+      ONLY return one action. If multiple actions are relevant, return the most relevant one.`,
+    );
+    if (observeResults.length === 0) {
+      return {
+        success: false,
+        message: `Failed to perform act: No observe results found for action"`,
+        action: instruction,
+      };
+    }
+    // Picking the first element observe returns
+    const element = observeResults[0];
+    return this.actFromObserveResult(element);
+  }
+
   private async _recordAction(action: string, result: string): Promise<string> {
     const id = generateId(action);
 

Diff for: package.json

@@ -6,24 +6,24 @@
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "scripts": {
-    "2048": "npm run build-dom-scripts && tsx examples/2048.ts",
-    "popup": "npm run build-dom-scripts && tsx examples/popup.ts",
-    "example": "npm run build-dom-scripts && tsx examples/example.ts",
-    "langchain": "npm run build-dom-scripts && tsx examples/langchain.ts",
-    "debug-url": "npm run build-dom-scripts && tsx examples/debugUrl.ts",
-    "external-client": "npm run build-dom-scripts && tsx examples/external_client.ts",
-    "instructions": "npm run build-dom-scripts && tsx examples/instructions.ts",
-    "ai-sdk-client": "npm run build-dom-scripts && tsx examples/ai_sdk_example.ts",
-    "actionable_observe_example": "npm run build-dom-scripts && tsx examples/actionable_observe_example.ts",
+    "2048": "npm run build && tsx examples/2048.ts",
+    "popup": "npm run build && tsx examples/popup.ts",
+    "example": "npm run build && tsx examples/example.ts",
+    "langchain": "npm run build && tsx examples/langchain.ts",
+    "debug-url": "npm run build && tsx examples/debugUrl.ts",
+    "external-client": "npm run build && tsx examples/external_client.ts",
+    "instructions": "npm run build && tsx examples/instructions.ts",
+    "ai-sdk-client": "npm run build && tsx examples/ai_sdk_example.ts",
+    "actionable_observe_example": "npm run build && tsx examples/actionable_observe_example.ts",
     "format": "prettier --write .",
     "prettier": "prettier --check .",
     "prettier:fix": "prettier --write .",
     "eslint": "eslint .",
     "cache:clear": "rm -rf .cache",
-    "evals": "npm run build-dom-scripts && tsx evals/index.eval.ts",
-    "e2e": "npm run build-dom-scripts && cd evals/deterministic && npx playwright test --config=e2e.playwright.config.ts",
-    "e2e:bb": "npm run build-dom-scripts && cd evals/deterministic && npx playwright test --config=bb.playwright.config.ts",
-    "e2e:local": "npm run build-dom-scripts && cd evals/deterministic && npx playwright test --config=local.playwright.config.ts",
+    "evals": "npm run build && tsx evals/index.eval.ts",
+    "e2e": "npm run build && cd evals/deterministic && npx playwright test --config=e2e.playwright.config.ts",
+    "e2e:bb": "npm run build && cd evals/deterministic && npx playwright test --config=bb.playwright.config.ts",
+    "e2e:local": "npm run build && cd evals/deterministic && npx playwright test --config=local.playwright.config.ts",
     "build-dom-scripts": "tsx lib/dom/genDomScripts.ts",
     "build-types": "tsc --emitDeclarationOnly --outDir dist",
     "build-js": "tsup lib/index.ts --dts",

Diff for: types/act.ts

@@ -26,3 +26,10 @@ export interface ActCommandResult {
   step: string;
   why?: string;
 }
+
+// We can use this enum to list the actions supported in performPlaywrightMethod
+export enum SupportedPlaywrightAction {
+  CLICK = "click",
+  FILL = "fill",
+  TYPE = "type",
+}

Diff for: types/stagehand.ts

@@ -77,6 +77,13 @@ export interface ActOptions {
   useVision?: boolean;
   variables?: Record<string, string>;
   domSettleTimeoutMs?: number;
+  /**
+   * If true, the action will be performed in a slow manner that allows the DOM to settle.
+   * This is useful for debugging.
+   *
+   * @default false
+   */
+  slowDomBasedAct?: boolean;
 }
 
 export interface ActResult {