Allow act to accept an ObserveResult (#441)

* flagged return action v0.1

* isVisible flag added to observe

* prettier and cleanup of fs functions

* simplifying observe flagging

* two flags: useAccessibilityTree and returnAction. No more processAlDOM on a11y context

* github eval

* add back in a11y param (whoops)

* google search observe eval (returnAction)

* fix my terrible grammar in the instruction

* amazon actionable eval

* add gh eval to config

* vtj eval

* added evals to config.json

* fixing lint/build issues

* compare element handles

* yc eval

* changed useAccessibilityTree to onlyVisible

* removing useAccessibilityTree from evals

* mostly removing comments

* accept multiple selectors

* added changeset

* allow act to accept ObserveResult

* actionable observe example

* changeset

---------

Co-authored-by: Miguel <[email protected]>

Author: seanmcguire12

Diff for: .changeset/ninety-drinks-warn.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": minor
+---
+
+allow act to accept observe output

Diff for: examples/actionable_observe_example.ts

@@ -0,0 +1,71 @@
+/**
+ * This file is meant to be used as a scratchpad for trying out actionable observe.
+ * To create a Stagehand project with best practices and configuration, run:
+ *
+ * npx create-browser-app@latest my-browser-app
+ */
+
+import { Stagehand } from "@/dist";
+import stagehandConfig from "@/stagehand.config";
+
+async function example() {
+  const stagehand = new Stagehand(stagehandConfig);
+  await stagehand.init();
+  await stagehand.page.goto("https://www.apartments.com/san-francisco-ca/");
+
+  await new Promise((resolve) => setTimeout(resolve, 3000));
+  const observations1 = await stagehand.page.observe({
+    instruction: "find the 'all filters' button",
+  });
+  await stagehand.page.act(observations1[0]);
+
+  await new Promise((resolve) => setTimeout(resolve, 3000));
+  const observations2 = await stagehand.page.observe({
+    instruction: "find the '1+' button in the 'beds' section",
+  });
+  await stagehand.page.act(observations2[0]);
+
+  await new Promise((resolve) => setTimeout(resolve, 3000));
+  const observations3 = await stagehand.page.observe({
+    instruction: "find the 'apartments' button in the 'home type' section",
+  });
+  await stagehand.page.act(observations3[0]);
+
+  await new Promise((resolve) => setTimeout(resolve, 3000));
+  const observations4 = await stagehand.page.observe({
+    instruction: "find the pet policy dropdown to click on.",
+  });
+  await stagehand.page.act(observations4[0]);
+
+  await new Promise((resolve) => setTimeout(resolve, 3000));
+  const observations5 = await stagehand.page.observe({
+    instruction: "find the 'Dog Friendly' option to click on",
+  });
+  await stagehand.page.act(observations5[0]);
+
+  await new Promise((resolve) => setTimeout(resolve, 3000));
+  const observations6 = await stagehand.page.observe({
+    instruction: "find the 'see results' section",
+  });
+  await stagehand.page.act(observations6[0]);
+
+  const currentUrl = await stagehand.page.url();
+  await stagehand.close();
+  if (
+    currentUrl.includes(
+      "https://www.apartments.com/apartments/san-francisco-ca/min-1-bedrooms-pet-friendly-dog/",
+    )
+  ) {
+    console.log("✅ Success! we made it to the correct page");
+  } else {
+    console.log(
+      "❌ Whoops, looks like we didnt make it to the correct page. " +
+        "\nThanks for testing out this new Stagehand feature!" +
+        "\nReach us on Slack if you have any feedback/questions/suggestions!",
+    );
+  }
+}
+
+(async () => {
+  await example();
+})();

Diff for: lib/StagehandPage.ts

@@ -285,15 +285,41 @@ export class StagehandPage {
     }
   }
 
-  async act(actionOrOptions: string | ActOptions): Promise<ActResult> {
+  async act(
+    actionOrOptions: string | ActOptions | ObserveResult,
+  ): Promise<ActResult> {
     if (!this.actHandler) {
       throw new Error("Act handler not initialized");
     }
 
-    const options: ActOptions =
-      typeof actionOrOptions === "string"
-        ? { action: actionOrOptions }
-        : actionOrOptions;
+    // If actionOrOptions is an ObserveResult, we call actFromObserveResult.
+    // We need to ensure there is both a selector and a method in the ObserveResult.
+    if (typeof actionOrOptions === "object" && actionOrOptions !== null) {
+      // If it has selector AND method => treat as ObserveResult
+      if ("selector" in actionOrOptions && "method" in actionOrOptions) {
+        const observeResult = actionOrOptions as ObserveResult;
+        // validate observeResult.method, etc.
+        return this.actHandler.actFromObserveResult(observeResult);
+      } else {
+        // If it's an object but no selector/method,
+        // check that it’s truly ActOptions (i.e., has an `action` field).
+        if (!("action" in actionOrOptions)) {
+          throw new Error(
+            "Invalid argument. Valid arguments are: a string, an ActOptions object, " +
+              "or an ObserveResult WITH 'selector' and 'method' fields.",
+          );
+        }
+      }
+    } else if (typeof actionOrOptions === "string") {
+      // Convert string to ActOptions
+      actionOrOptions = { action: actionOrOptions };
+    } else {
+      throw new Error(
+        "Invalid argument: you may have called act with an empty ObserveResult.\n" +
+          "Valid arguments are: a string, an ActOptions object, or an ObserveResult " +
+          "WITH 'selector' and 'method' fields.",
+      );
+    }
 
     const {
       action,
@@ -302,7 +328,7 @@ export class StagehandPage {
       useVision, // still destructure this but will not pass it on
       variables = {},
       domSettleTimeoutMs,
-    } = options;
+    } = actionOrOptions;
 
     if (typeof useVision !== "undefined") {
       this.stagehand.log({

Diff for: lib/handlers/actHandler.ts

@@ -11,6 +11,7 @@ import { LLMProvider } from "../llm/LLMProvider";
 import { StagehandContext } from "../StagehandContext";
 import { StagehandPage } from "../StagehandPage";
 import { generateId } from "../utils";
+import { ObserveResult } from "@/types/stagehand";
 
 /**
  * NOTE: Vision support has been removed from this version of Stagehand.
@@ -56,6 +57,57 @@ export class StagehandActHandler {
     this.userProvidedInstructions = userProvidedInstructions;
   }
 
+  /**
+   * Perform an immediate Playwright action based on an ObserveResult object
+   * that was returned from `page.observe(...)`.
+   */
+  public async actFromObserveResult(
+    observe: ObserveResult,
+  ): Promise<{ success: boolean; message: string; action: string }> {
+    this.logger({
+      category: "action",
+      message: "Performing act from an ObserveResult",
+      level: 1,
+      auxiliary: {
+        observeResult: {
+          value: JSON.stringify(observe),
+          type: "object",
+        },
+      },
+    });
+
+    const method = observe.method;
+    const args = observe.arguments ?? [];
+    // remove the xpath prefix on the selector
+    const selector = observe.selector.replace("xpath=", "");
+
+    try {
+      await this._performPlaywrightMethod(method, args, selector);
+
+      return {
+        success: true,
+        message: `Action [${method}] performed successfully on selector: ${selector}`,
+        action: observe.description || `ObserveResult action (${method})`,
+      };
+    } catch (err) {
+      this.logger({
+        category: "action",
+        message: "Error performing act from an ObserveResult",
+        level: 1,
+        auxiliary: {
+          error: { value: err.message, type: "string" },
+          trace: { value: err.stack, type: "string" },
+          observeResult: { value: JSON.stringify(observe), type: "object" },
+        },
+      });
+      return {
+        success: false,
+        message: `Failed to perform act: ${err.message}`,
+        action: observe.description || `ObserveResult action (${method})`,
+      };
+    }
+  }
+
   private async _recordAction(action: string, result: string): Promise<string> {
     const id = generateId(action);
 
@@ -361,6 +413,13 @@ export class StagehandActHandler {
             },
           },
         });
+        // try {
+        //   // Force-click here
+        //   await locator.click({ force: true });
+        // } catch (e) {
+        //   // handle/log exception
+        //   throw new PlaywrightCommandException(e.message);
+        // }
 
         // NAVIDNOTE: Should this happen before we wait for locator[method]?
         const newOpenedTab = await Promise.race([

Diff for: package.json

@@ -13,6 +13,7 @@
     "external-client": "npm run build-dom-scripts && tsx examples/external_client.ts",
     "instructions": "npm run build-dom-scripts && tsx examples/instructions.ts",
     "ai-sdk-client": "npm run build-dom-scripts && tsx examples/ai_sdk_example.ts",
+    "actionable_observe_example": "npm run build-dom-scripts && tsx examples/actionable_observe_example.ts",
     "format": "prettier --write .",
     "prettier": "prettier --check .",
     "prettier:fix": "prettier --write .",

Diff for: types/page.ts

@@ -20,6 +20,7 @@ export const defaultExtractSchema = z.object({
 export interface Page extends Omit<PlaywrightPage, "on"> {
   act(action: string): Promise<ActResult>;
   act(options: ActOptions): Promise<ActResult>;
+  act(observation: ObserveResult): Promise<ActResult>;
 
   extract(
     instruction: string,