added fallback try to actFromObserveResult (#480)

* added fallback try to actFromObserveResult

* linting fix

* defaulting flags, adding comments, added selfHeal flag

* adding proxy param to page

* moved selfHeal flag to stagehand config from act param

Author: miguelg719

Diff for: .changeset/gentle-grapes-join.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Adding a fallback try on actFromObserveResult to use the description from observe and call regular act.

Diff for: lib/StagehandPage.ts

@@ -71,6 +71,7 @@ export class StagehandPage {
         stagehandContext: this.intContext,
         llmClient: llmClient,
         userProvidedInstructions,
+        selfHeal: this.stagehand.selfHeal,
       });
       this.extractHandler = new StagehandExtractHandler({
         stagehand: this.stagehand,
@@ -305,7 +306,7 @@ export class StagehandPage {
         return this.actHandler.actFromObserveResult(observeResult);
       } else {
         // If it's an object but no selector/method,
-        // check that it’s truly ActOptions (i.e., has an `action` field).
+        // check that it's truly ActOptions (i.e., has an `action` field).
         if (!("action" in actionOrOptions)) {
           throw new Error(
             "Invalid argument. Valid arguments are: a string, an ActOptions object, " +
@@ -509,7 +510,7 @@ export class StagehandPage {
       modelClientOptions,
       useVision, // still destructure but will not pass it on
       domSettleTimeoutMs,
-      returnAction = false,
+      returnAction = true,
       onlyVisible = false,
       useAccessibilityTree,
       drawOverlay,

Diff for: lib/handlers/actHandler.ts

@@ -29,6 +29,7 @@ export class StagehandActHandler {
     [key: string]: { result: string; action: string };
   };
   private readonly userProvidedInstructions?: string;
+  private readonly selfHeal: boolean;
 
   constructor({
     verbose,
@@ -37,6 +38,7 @@ export class StagehandActHandler {
     logger,
     stagehandPage,
     userProvidedInstructions,
+    selfHeal,
   }: {
     verbose: 0 | 1 | 2;
     llmProvider: LLMProvider;
@@ -46,6 +48,7 @@ export class StagehandActHandler {
     stagehandPage: StagehandPage;
     stagehandContext: StagehandContext;
     userProvidedInstructions?: string;
+    selfHeal: boolean;
   }) {
     this.verbose = verbose;
     this.llmProvider = llmProvider;
@@ -55,6 +58,7 @@ export class StagehandActHandler {
     this.actions = {};
     this.stagehandPage = stagehandPage;
     this.userProvidedInstructions = userProvidedInstructions;
+    this.selfHeal = selfHeal;
   }
 
   /**
@@ -90,21 +94,61 @@ export class StagehandActHandler {
         action: observe.description || `ObserveResult action (${method})`,
       };
     } catch (err) {
+      if (!this.selfHeal) {
+        this.logger({
+          category: "action",
+          message: "Error performing act from an ObserveResult",
+          level: 1,
+          auxiliary: {
+            error: { value: err.message, type: "string" },
+            trace: { value: err.stack, type: "string" },
+          },
+        });
+        return {
+          success: false,
+          message: `Failed to perform act: ${err.message}`,
+          action: observe.description || `ObserveResult action (${method})`,
+        };
+      }
+      // We will try to use regular act on a failed ObserveResult-act if selfHeal is true
       this.logger({
         category: "action",
-        message: "Error performing act from an ObserveResult",
+        message:
+          "Error performing act from an ObserveResult. Trying again with regular act method",
         level: 1,
         auxiliary: {
           error: { value: err.message, type: "string" },
           trace: { value: err.stack, type: "string" },
           observeResult: { value: JSON.stringify(observe), type: "object" },
         },
       });
-      return {
-        success: false,
-        message: `Failed to perform act: ${err.message}`,
-        action: observe.description || `ObserveResult action (${method})`,
-      };
+      try {
+        // Remove redundancy from method-description
+        const actCommand = observe.description
+          .toLowerCase()
+          .startsWith(method.toLowerCase())
+          ? observe.description
+          : method
+            ? `${method} ${observe.description}`
+            : observe.description;
+        // Call act with the ObserveResult description
+        await this.stagehandPage.act(actCommand);
+      } catch (err) {
+        this.logger({
+          category: "action",
+          message: "Error performing act from an ObserveResult",
+          level: 1,
+          auxiliary: {
+            error: { value: err.message, type: "string" },
+            trace: { value: err.stack, type: "string" },
+          },
+        });
+        return {
+          success: false,
+          message: `Failed to perform act: ${err.message}`,
+          action: observe.description || `ObserveResult action (${method})`,
+        };
+      }
     }
   }
 
@@ -369,7 +413,7 @@ export class StagehandActHandler {
         const clickArg = args.length ? args[0] : undefined;
 
         if (isRadio) {
-          // if it’s a radio button, try to find a label to click
+          // if it's a radio button, try to find a label to click
           const inputId = await locator.evaluate((el) => el.id);
           let labelLocator;
 
@@ -380,7 +424,7 @@ export class StagehandActHandler {
             );
           }
           if (!labelLocator || (await labelLocator.count()) < 1) {
-            // if no label was found or the label doesn’t exist, check if
+            // if no label was found or the label doesn't exist, check if
             // there is an ancestor <label>
             labelLocator = this.stagehandPage.page
               .locator(`xpath=${xpath}/ancestor::label`)

Diff for: lib/index.ts

@@ -327,6 +327,7 @@ export class Stagehand {
   private contextPath?: string;
   private llmClient: LLMClient;
   private userProvidedInstructions?: string;
+  public readonly selfHeal: boolean;
 
   constructor(
     {
@@ -346,6 +347,7 @@ export class Stagehand {
       modelName,
       modelClientOptions,
       systemPrompt,
+      selfHeal = true,
     }: ConstructorParams = {
       env: "BROWSERBASE",
     },
@@ -380,6 +382,7 @@ export class Stagehand {
     this.browserbaseSessionCreateParams = browserbaseSessionCreateParams;
     this.browserbaseSessionID = browserbaseSessionID;
     this.userProvidedInstructions = systemPrompt;
+    this.selfHeal = selfHeal;
   }
 
   public get logger(): (logLine: LogLine) => void {

Diff for: types/stagehand.ts

@@ -26,6 +26,7 @@ export interface ConstructorParams {
    * Instructions for stagehand.
    */
   systemPrompt?: string;
+  selfHeal?: boolean;
 }
 
 export interface InitOptions {