added iframes as the last observeResult on pages with 1+ iframes (#500)

* added iframes as the last observeResult on pages with 1+ iframes

* added changeset

Author: miguelg719

Diff for: .changeset/odd-eggs-bake.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": minor
+---
+
+Including Iframes in ObserveResults. This appends any iframe(s) found in the page to the end of observe results on any observe call.

Diff for: evals/evals.config.json

@@ -239,6 +239,14 @@
     {
       "name": "observe_taxes",
       "categories": ["observe"]
+    },
+    {
+      "name": "observe_iframes1",
+      "categories": ["observe"]
+    },
+    {
+      "name": "observe_iframes2",
+      "categories": ["observe"]
     }
   ]
 }

Diff for: evals/tasks/observe_iframes1.ts

@@ -0,0 +1,90 @@
+import { initStagehand } from "@/evals/initStagehand";
+import { EvalFunction } from "@/types/evals";
+
+export const observe_iframes1: EvalFunction = async ({ modelName, logger }) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto("https://tucowsdomains.com/abuse-form/phishing/");
+
+  const observations = await stagehand.page.observe({
+    instruction: "find the main header of the page",
+  });
+
+  if (observations.length === 0) {
+    await stagehand.close();
+    return {
+      _success: false,
+      observations,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  }
+
+  const possibleLocators = [
+    `#primary > div.singlePage > section > div > div > article > div > iframe`,
+    `#primary > div.heroBanner > section > div > h1`,
+  ];
+
+  const possibleHandles = [];
+  for (const locatorStr of possibleLocators) {
+    const locator = stagehand.page.locator(locatorStr);
+    const handle = await locator.elementHandle();
+    if (handle) {
+      possibleHandles.push({ locatorStr, handle });
+    }
+  }
+
+  let foundMatch = false;
+  let matchedLocator: string | null = null;
+
+  for (const observation of observations) {
+    try {
+      const observationLocator = stagehand.page
+        .locator(observation.selector)
+        .first();
+      const observationHandle = await observationLocator.elementHandle();
+      if (!observationHandle) {
+        continue;
+      }
+
+      for (const { locatorStr, handle: candidateHandle } of possibleHandles) {
+        const isSameNode = await observationHandle.evaluate(
+          (node, otherNode) => node === otherNode,
+          candidateHandle,
+        );
+        if (isSameNode) {
+          foundMatch = true;
+          matchedLocator = locatorStr;
+          break;
+        }
+      }
+
+      if (foundMatch) {
+        break;
+      }
+    } catch (error) {
+      console.warn(
+        `Failed to check observation with selector ${observation.selector}:`,
+        error.message,
+      );
+      continue;
+    }
+  }
+
+  await stagehand.close();
+
+  return {
+    _success: foundMatch,
+    matchedLocator,
+    observations,
+    debugUrl,
+    sessionUrl,
+    logs: logger.getLogs(),
+  };
+};

Diff for: evals/tasks/observe_iframes2.ts

@@ -0,0 +1,90 @@
+import { initStagehand } from "@/evals/initStagehand";
+import { EvalFunction } from "@/types/evals";
+
+export const observe_iframes2: EvalFunction = async ({ modelName, logger }) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto(
+    "https://iframetester.com/?url=https://shopify.com",
+  );
+  await new Promise((resolve) => setTimeout(resolve, 5000));
+
+  const observations = await stagehand.page.observe({
+    instruction: "find the main header of the page",
+  });
+
+  if (observations.length === 0) {
+    await stagehand.close();
+    return {
+      _success: false,
+      observations,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  }
+
+  const possibleLocators = [`#iframe-window`, `body > header > h1`];
+
+  const possibleHandles = [];
+  for (const locatorStr of possibleLocators) {
+    const locator = stagehand.page.locator(locatorStr);
+    const handle = await locator.elementHandle();
+    if (handle) {
+      possibleHandles.push({ locatorStr, handle });
+    }
+  }
+
+  let foundMatch = false;
+  let matchedLocator: string | null = null;
+
+  for (const observation of observations) {
+    try {
+      const observationLocator = stagehand.page
+        .locator(observation.selector)
+        .first();
+      const observationHandle = await observationLocator.elementHandle();
+      if (!observationHandle) {
+        continue;
+      }
+
+      for (const { locatorStr, handle: candidateHandle } of possibleHandles) {
+        const isSameNode = await observationHandle.evaluate(
+          (node, otherNode) => node === otherNode,
+          candidateHandle,
+        );
+        if (isSameNode) {
+          foundMatch = true;
+          matchedLocator = locatorStr;
+          break;
+        }
+      }
+
+      if (foundMatch) {
+        break;
+      }
+    } catch (error) {
+      console.warn(
+        `Failed to check observation with selector ${observation.selector}:`,
+        error.message,
+      );
+      continue;
+    }
+  }
+
+  await stagehand.close();
+
+  return {
+    _success: foundMatch,
+    matchedLocator,
+    observations,
+    debugUrl,
+    sessionUrl,
+    logs: logger.getLogs(),
+  };
+};

Diff for: lib/a11y/utils.ts

@@ -156,6 +156,7 @@ export async function buildHierarchicalTree(
 ): Promise<TreeResult> {
   // Map to store processed nodes for quick lookup
   const nodeMap = new Map<string, AccessibilityNode>();
+  const iframe_list: AccessibilityNode[] = [];
 
   // First pass: Create nodes that are meaningful
   // We only keep nodes that either have a name or children to avoid cluttering the tree
@@ -194,6 +195,15 @@ export async function buildHierarchicalTree(
   // Second pass: Establish parent-child relationships
   // This creates the actual tree structure by connecting nodes based on parentId
   nodes.forEach((node) => {
+    // Add iframes to a list and include in the return object
+    const isIframe = node.role === "Iframe";
+    if (isIframe) {
+      const iframeNode = {
+        role: node.role,
+        nodeId: node.nodeId,
+      };
+      iframe_list.push(iframeNode);
+    }
     if (node.parentId && nodeMap.has(node.nodeId)) {
       const parentNode = nodeMap.get(node.parentId);
       const currentNode = nodeMap.get(node.nodeId);
@@ -228,6 +238,7 @@ export async function buildHierarchicalTree(
   return {
     tree: finalTree,
     simplified: simplifiedFormat,
+    iframes: iframe_list,
   };
 }
 
@@ -283,7 +294,6 @@ export async function getAccessibilityTree(
       message: `got accessibility tree in ${Date.now() - startTime}ms`,
       level: 1,
     });
-
     return hierarchicalTree;
   } catch (error) {
     logger({

Diff for: lib/handlers/observeHandler.ts

@@ -8,6 +8,7 @@ import {
   getAccessibilityTree,
   getXPathByResolvedObjectId,
 } from "../a11y/utils";
+import { AccessibilityNode } from "../../types/context";
 
 export class StagehandObserveHandler {
   private readonly stagehand: Stagehand;
@@ -83,6 +84,7 @@ export class StagehandObserveHandler {
 
     let selectorMap: Record<string, string[]> = {};
     let outputString: string;
+    let iframes: AccessibilityNode[] = [];
     const useAccessibilityTree = !onlyVisible;
     if (useAccessibilityTree) {
       await this.stagehandPage._waitForSettledDom();
@@ -93,6 +95,7 @@ export class StagehandObserveHandler {
         level: 1,
       });
       outputString = tree.simplified;
+      iframes = tree.iframes;
     } else {
       const evalResult = await this.stagehand.page.evaluate(() => {
         return window.processAllOfDom().then((result) => result);
@@ -111,6 +114,18 @@ export class StagehandObserveHandler {
       isUsingAccessibilityTree: useAccessibilityTree,
       returnAction,
     });
+
+    //Add iframes to the observation response if there are any on the page
+    if (iframes.length > 0) {
+      iframes.forEach((iframe) => {
+        observationResponse.elements.push({
+          elementId: Number(iframe.nodeId),
+          description: "an iframe",
+          method: "not-supported",
+          arguments: [],
+        });
+      });
+    }
     const elementsWithSelectors = await Promise.all(
       observationResponse.elements.map(async (element) => {
         const { elementId, ...rest } = element;

Diff for: types/context.ts

@@ -24,4 +24,5 @@ export type AccessibilityNode = {
 export interface TreeResult {
   tree: AccessibilityNode[];
   simplified: string;
+  iframes?: AccessibilityNode[];
 }