Scroll on 'main scrollable element' when possible (#405)

Author: seanmcguire12

Diff for: .changeset/stupid-starfishes-type.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": minor
+---
+
+in ProcessAllOfDom, scroll on large scrollable elements instead of just the root DOM

Diff for: evals/evals.config.json

@@ -207,6 +207,14 @@
     {
       "name": "extract_jstor_news",
       "categories": ["text_extract"]
+    },
+    {
+      "name": "extract_apartments",
+      "categories": ["text_extract"]
+    },
+    {
+      "name": "extract_zillow",
+      "categories": ["text_extract"]
     }
   ]
 }

Diff for: evals/tasks/extract_apartments.ts

@@ -0,0 +1,70 @@
+import { z } from "zod";
+import { initStagehand } from "../initStagehand";
+import { EvalFunction } from "../../types/evals";
+
+export const extract_apartments: EvalFunction = async ({
+  modelName,
+  logger,
+  useTextExtract,
+}) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+    domSettleTimeoutMs: 3000,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto(
+    "https://www.apartments.com/san-francisco-ca/2-bedrooms/",
+  );
+  const apartment_listings = await stagehand.page.extract({
+    instruction:
+      "Extract all the apartment listings with their prices and their addresses.",
+    schema: z.object({
+      listings: z.array(
+        z.object({
+          price: z.string().describe("The price of the listing"),
+          trails: z.string().describe("The address of the listing"),
+        }),
+      ),
+    }),
+    modelName,
+    useTextExtract,
+  });
+
+  await stagehand.close();
+  const listings = apartment_listings.listings;
+  const expectedLength = 40;
+
+  if (listings.length < expectedLength) {
+    logger.error({
+      message: "Incorrect number of listings extracted",
+      level: 0,
+      auxiliary: {
+        expected: {
+          value: expectedLength.toString(),
+          type: "integer",
+        },
+        actual: {
+          value: listings.length.toString(),
+          type: "integer",
+        },
+      },
+    });
+    return {
+      _success: false,
+      error: "Incorrect number of listings extracted",
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+    };
+  }
+
+  return {
+    _success: true,
+    logs: logger.getLogs(),
+    debugUrl,
+    sessionUrl,
+  };
+};

Diff for: evals/tasks/extract_zillow.ts

@@ -0,0 +1,73 @@
+import { z } from "zod";
+import { initStagehand } from "../initStagehand";
+import { EvalFunction } from "../../types/evals";
+
+export const extract_zillow: EvalFunction = async ({
+  modelName,
+  logger,
+  useTextExtract,
+}) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+    domSettleTimeoutMs: 3000,
+    configOverrides: {
+      debugDom: false,
+    },
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto("https://zillow-eval.surge.sh/");
+  // timeout for 5 seconds
+  await stagehand.page.waitForTimeout(5000);
+  const real_estate_listings = await stagehand.page.extract({
+    instruction:
+      "Extract EACH AND EVERY HOME PRICE AND ADDRESS ON THE PAGE. DO NOT MISS ANY OF THEM.",
+    schema: z.object({
+      listings: z.array(
+        z.object({
+          price: z.string().describe("The price of the home"),
+          trails: z.string().describe("The address of the home"),
+        }),
+      ),
+    }),
+    modelName,
+    useTextExtract,
+  });
+
+  await stagehand.close();
+  const listings = real_estate_listings.listings;
+  const expectedLength = 38;
+
+  if (listings.length < expectedLength) {
+    logger.error({
+      message: "Incorrect number of listings extracted",
+      level: 0,
+      auxiliary: {
+        expected: {
+          value: expectedLength.toString(),
+          type: "integer",
+        },
+        actual: {
+          value: listings.length.toString(),
+          type: "integer",
+        },
+      },
+    });
+    return {
+      _success: false,
+      error: "Incorrect number of listings extracted",
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+    };
+  }
+
+  return {
+    _success: true,
+    logs: logger.getLogs(),
+    debugUrl,
+    sessionUrl,
+  };
+};

Diff for: lib/dom/ElementContainer.ts

@@ -0,0 +1,34 @@
+import { StagehandContainer } from "./StagehandContainer";
+
+export class ElementContainer implements StagehandContainer {
+  constructor(private el: HTMLElement) {}
+
+  public getViewportHeight(): number {
+    return this.el.clientHeight;
+  }
+
+  public getScrollHeight(): number {
+    return this.el.scrollHeight;
+  }
+
+  public async scrollTo(offset: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, 1500));
+    this.el.scrollTo({ top: offset, left: 0, behavior: "smooth" });
+    await this.waitForScrollEnd();
+  }
+
+  private async waitForScrollEnd(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      let scrollEndTimer: number;
+      const handleScroll = () => {
+        clearTimeout(scrollEndTimer);
+        scrollEndTimer = window.setTimeout(() => {
+          this.el.removeEventListener("scroll", handleScroll);
+          resolve();
+        }, 100);
+      };
+      this.el.addEventListener("scroll", handleScroll, { passive: true });
+      handleScroll();
+    });
+  }
+}

Diff for: lib/dom/GlobalPageContainer.ts

@@ -0,0 +1,33 @@
+import { StagehandContainer } from "./StagehandContainer";
+import { calculateViewportHeight } from "./utils";
+
+export class GlobalPageContainer implements StagehandContainer {
+  public getViewportHeight(): number {
+    return calculateViewportHeight();
+  }
+
+  public getScrollHeight(): number {
+    return document.documentElement.scrollHeight;
+  }
+
+  public async scrollTo(offset: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, 1500));
+    window.scrollTo({ top: offset, left: 0, behavior: "smooth" });
+    await this.waitForScrollEnd();
+  }
+
+  private async waitForScrollEnd(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      let scrollEndTimer: number;
+      const handleScroll = () => {
+        clearTimeout(scrollEndTimer);
+        scrollEndTimer = window.setTimeout(() => {
+          window.removeEventListener("scroll", handleScroll);
+          resolve();
+        }, 100);
+      };
+      window.addEventListener("scroll", handleScroll, { passive: true });
+      handleScroll();
+    });
+  }
+}

Diff for: lib/dom/StagehandContainer.ts

@@ -0,0 +1,7 @@
+export interface StagehandContainer {
+  getViewportHeight(): number;
+
+  getScrollHeight(): number;
+
+  scrollTo(offset: number): Promise<void>;
+}

Diff for: lib/dom/containerFactory.ts

@@ -0,0 +1,16 @@
+import { StagehandContainer } from "./StagehandContainer";
+import { GlobalPageContainer } from "./GlobalPageContainer";
+import { ElementContainer } from "./ElementContainer";
+
+/**
+ * Decide which container to create.
+ */
+export function createStagehandContainer(
+  obj: Window | HTMLElement,
+): StagehandContainer {
+  if (obj instanceof Window) {
+    return new GlobalPageContainer();
+  } else {
+    return new ElementContainer(obj);
+  }
+}

Diff for: lib/dom/global.d.ts

@@ -1,3 +1,5 @@
+import { StagehandContainer } from "./StagehandContainer";
+
 export {};
 declare global {
   interface Window {
@@ -19,7 +21,7 @@ declare global {
     }>;
     debugDom: () => Promise<void>;
     cleanupDebug: () => void;
-    scrollToHeight: (height: number) => Promise<void>;
+    createStagehandContainer: (obj: Window | HTMLElement) => StagehandContainer;
     waitForDomSettle: () => Promise<void>;
     __playwright?: unknown;
     __pw_manual?: unknown;