support scrolling (#563)

* support scrolling

* changeset

* move window declaration to process.ts

* use existing getNodeFromXpath

* log warning if xpath not found

* handle mouse.wheel

* evals

* add range tolerance of +- 200px

Author: seanmcguire12

Diff for: .changeset/plenty-ties-float.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+support scrolling in `act`

Diff for: evals/evals.config.json

@@ -279,6 +279,14 @@
     {
       "name": "extract_geniusee_2",
       "categories": ["targeted_extract"]
+    },
+    {
+      "name": "scroll_50",
+      "categories": ["act"]
+    },
+    {
+      "name": "scroll_75",
+      "categories": ["act"]
     }
   ]
 }

Diff for: evals/tasks/scroll_50.ts

@@ -0,0 +1,48 @@
+import { initStagehand } from "@/evals/initStagehand";
+import { EvalFunction } from "@/types/evals";
+
+export const scroll_50: EvalFunction = async ({ modelName, logger }) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+    domSettleTimeoutMs: 3000,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+  await stagehand.page.goto("https://aigrant.com/");
+  await stagehand.page.act({
+    action: "Scroll 50% down the page",
+    slowDomBasedAct: false,
+  });
+
+  await new Promise((resolve) => setTimeout(resolve, 5000));
+
+  // Get the current scroll position and total scroll height
+  const scrollInfo = await stagehand.page.evaluate(() => {
+    return {
+      scrollTop: window.scrollY + window.innerHeight / 2,
+      scrollHeight: document.documentElement.scrollHeight,
+    };
+  });
+
+  await stagehand.close();
+
+  const halfwayScroll = scrollInfo.scrollHeight / 2;
+  const halfwayReached = Math.abs(scrollInfo.scrollTop - halfwayScroll) <= 200;
+  const evaluationResult = halfwayReached
+    ? {
+        _success: true,
+        logs: logger.getLogs(),
+        debugUrl,
+        sessionUrl,
+      }
+    : {
+        _success: false,
+        logs: logger.getLogs(),
+        debugUrl,
+        sessionUrl,
+        message: `Scroll position (${scrollInfo.scrollTop}px) is not halfway down the page (${halfwayScroll}px).`,
+      };
+
+  return evaluationResult;
+};

Diff for: evals/tasks/scroll_75.ts

@@ -0,0 +1,49 @@
+import { initStagehand } from "@/evals/initStagehand";
+import { EvalFunction } from "@/types/evals";
+
+export const scroll_75: EvalFunction = async ({ modelName, logger }) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+    domSettleTimeoutMs: 3000,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+  await stagehand.page.goto("https://aigrant.com/");
+  await stagehand.page.act({
+    action: "Scroll 75% down the page",
+    slowDomBasedAct: false,
+  });
+
+  await new Promise((resolve) => setTimeout(resolve, 5000));
+
+  // Get the current scroll position and total scroll height
+  const scrollInfo = await stagehand.page.evaluate(() => {
+    return {
+      scrollTop: window.scrollY + window.innerHeight * 0.75,
+      scrollHeight: document.documentElement.scrollHeight,
+    };
+  });
+
+  await stagehand.close();
+
+  const threeQuartersScroll = scrollInfo.scrollHeight * 0.75;
+  const threeQuartersReached =
+    Math.abs(scrollInfo.scrollTop - threeQuartersScroll) <= 200;
+  const evaluationResult = threeQuartersReached
+    ? {
+        _success: true,
+        logs: logger.getLogs(),
+        debugUrl,
+        sessionUrl,
+      }
+    : {
+        _success: false,
+        logs: logger.getLogs(),
+        debugUrl,
+        sessionUrl,
+        message: `Scroll position (${scrollInfo.scrollTop}px) is not three quarters down the page (${threeQuartersScroll}px).`,
+      };
+
+  return evaluationResult;
+};

Diff for: lib/dom/process.ts

@@ -3,6 +3,7 @@ import {
   calculateViewportHeight,
   canElementScroll,
   getNodeFromXpath,
+  waitForDomSettle,
 } from "./utils";
 import { createStagehandContainer } from "./containerFactory";
 import { StagehandContainer } from "./StagehandContainer";
@@ -518,6 +519,7 @@ export function getElementBoundingBoxes(xpath: string): Array<{
   return boundingBoxes;
 }
 
+window.waitForDomSettle = waitForDomSettle;
 window.processDom = processDom;
 window.processAllOfDom = processAllOfDom;
 window.storeDOM = storeDOM;

Diff for: lib/dom/utils.ts

@@ -14,8 +14,6 @@ export async function waitForDomSettle() {
   });
 }
 
-window.waitForDomSettle = waitForDomSettle;
-
 export function calculateViewportHeight() {
   return Math.ceil(window.innerHeight * 0.75);
 }

Diff for: lib/handlers/actHandler.ts

@@ -19,6 +19,7 @@ import {
 } from "@/types/stagehand";
 import { SupportedPlaywrightAction } from "@/types/act";
 import { buildActObservePrompt } from "../prompt";
+import { getNodeFromXpath } from "@/lib/dom/utils";
 /**
  * NOTE: Vision support has been removed from this version of Stagehand.
  * If useVision or verifierUseVision is set to true, a warning is logged and
@@ -448,6 +449,82 @@ export class StagehandActHandler {
 
         throw new PlaywrightCommandException(e.message);
       }
+    } else if (
+      method === "scrollTo" ||
+      method === "scroll" ||
+      method === "mouse.wheel"
+    ) {
+      this.logger({
+        category: "action",
+        message: "scrolling element vertically to specified percentage",
+        level: 2,
+        auxiliary: {
+          xpath: { value: xpath, type: "string" },
+          coordinate: { value: JSON.stringify(args), type: "string" },
+        },
+      });
+
+      try {
+        const [yArg = "0%"] = args as string[];
+
+        await this.stagehandPage.page.evaluate(
+          ({ xpath, yArg }) => {
+            function parsePercent(val: string): number {
+              const cleaned = val.trim().replace("%", "");
+              const num = parseFloat(cleaned);
+              return Number.isNaN(num) ? 0 : Math.max(0, Math.min(num, 100));
+            }
+
+            const elementNode = getNodeFromXpath(xpath);
+            if (!elementNode || elementNode.nodeType !== Node.ELEMENT_NODE) {
+              console.warn(`Could not locate element to scroll on.`);
+              return;
+            }
+
+            const element = elementNode as HTMLElement;
+            const yPct = parsePercent(yArg);
+
+            // Determine if <html> is actually the scrolled container
+            if (element.tagName.toLowerCase() === "html") {
+              // Scroll the entire page (window)
+              const scrollHeight = document.body.scrollHeight;
+              const viewportHeight = window.innerHeight;
+              const scrollTop = (scrollHeight - viewportHeight) * (yPct / 100);
+
+              window.scrollTo({
+                top: scrollTop,
+                left: window.scrollX,
+                behavior: "smooth",
+              });
+            } else {
+              // Otherwise, scroll the element itself
+              const scrollHeight = element.scrollHeight;
+              const clientHeight = element.clientHeight;
+              const scrollTop = (scrollHeight - clientHeight) * (yPct / 100);
+
+              element.scrollTo({
+                top: scrollTop,
+                left: element.scrollLeft,
+                behavior: "smooth",
+              });
+            }
+          },
+          { xpath, yArg },
+        );
+      } catch (e) {
+        this.logger({
+          category: "action",
+          message: "error scrolling element vertically to percentage",
+          level: 1,
+          auxiliary: {
+            error: { value: (e as Error).message, type: "string" },
+            trace: { value: (e as Error).stack, type: "string" },
+            xpath: { value: xpath, type: "string" },
+            args: { value: JSON.stringify(args), type: "object" },
+          },
+        });
+        throw new PlaywrightCommandException((e as Error).message);
+      }
     } else if (method === "fill" || method === "type") {
       try {
         await locator.fill("");

Diff for: lib/prompt.ts

@@ -401,7 +401,7 @@ export function buildActObservePrompt(
   let instruction = `Find the most relevant element to perform an action on given the following action: ${action}. 
   Provide an action for this element such as ${supportedActions.join(", ")}, or any other playwright locator method. Remember that to users, buttons and links look the same in most cases.
   If the action is completely unrelated to a potential action to be taken on the page, return an empty array. 
-  ONLY return one action. If multiple actions are relevant, return the most relevant one.`;
+  ONLY return one action. If multiple actions are relevant, return the most relevant one. If the user is asking to scroll to a position on the page, e.g., 'halfway' or 0.75, etc, you must return the argument formatted as the correct percentage, e.g., '50%' or '75%', etc.`;
 
   // Add variable names (not values) to the instruction if any
   if (variables && Object.keys(variables).length > 0) {

Diff for: types/act.ts

@@ -32,4 +32,5 @@ export enum SupportedPlaywrightAction {
   CLICK = "click",
   FILL = "fill",
   TYPE = "type",
+  SCROLL = "scrollTo",
 }