extract() -> full page text (#544)

* extract page text

* changeset

Author: seanmcguire12

Diff for: .changeset/seven-glasses-drop.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": minor
+---
+
+deterministically get the full text representation of a webpage by calling `extract()` (with no arguments)

Diff for: lib/StagehandPage.ts

@@ -574,14 +574,19 @@ export class StagehandPage {
   }
 
   async extract<T extends z.AnyZodObject = typeof defaultExtractSchema>(
-    instructionOrOptions: string | ExtractOptions<T>,
+    instructionOrOptions?: string | ExtractOptions<T>,
   ): Promise<ExtractResult<T>> {
     if (!this.extractHandler) {
       throw new Error("Extract handler not initialized");
     }
 
     await clearOverlays(this.page);
 
+    // check if user called extract() with no arguments
+    if (!instructionOrOptions) {
+      return this.extractHandler.extract();
+    }
+
     const options: ExtractOptions<T> =
       typeof instructionOrOptions === "string"
         ? {

Diff for: lib/handlers/extractHandler.ts

@@ -6,6 +6,7 @@ import { LLMClient } from "../llm/LLMClient";
 import { formatText } from "../utils";
 import { StagehandPage } from "../StagehandPage";
 import { Stagehand } from "../index";
+import { pageTextSchema } from "../../types/page";
 
 const PROXIMITY_THRESHOLD = 15;
 
@@ -118,16 +119,26 @@ export class StagehandExtractHandler {
     useTextExtract = false,
     selector,
   }: {
-    instruction: string;
-    schema: T;
+    instruction?: string;
+    schema?: T;
     content?: z.infer<T>;
     chunksSeen?: Array<number>;
-    llmClient: LLMClient;
+    llmClient?: LLMClient;
     requestId?: string;
     domSettleTimeoutMs?: number;
     useTextExtract?: boolean;
     selector?: string;
-  }): Promise<z.infer<T>> {
+  } = {}): Promise<z.infer<T>> {
+    const noArgsCalled = !instruction && !schema && !llmClient && !selector;
+    if (noArgsCalled) {
+      this.logger({
+        category: "extraction",
+        message: "Extracting the entire page text.",
+        level: 1,
+      });
+      return this.extractPageText();
+    }
+
     if (useTextExtract) {
       return this.textExtract({
         instruction,
@@ -151,6 +162,52 @@ export class StagehandExtractHandler {
     }
   }
 
+  private async extractPageText(): Promise<{ page_text?: string }> {
+    await this.stagehandPage._waitForSettledDom();
+    await this.stagehandPage.startDomDebug();
+
+    const originalDOM = await this.stagehandPage.page.evaluate(() =>
+      window.storeDOM(undefined),
+    );
+
+    const { selectorMap }: { selectorMap: Record<number, string[]> } =
+      await this.stagehand.page.evaluate(() =>
+        window.processAllOfDom(undefined),
+      );
+
+    await this.stagehand.page.evaluate(() =>
+      window.createTextBoundingBoxes(undefined),
+    );
+
+    const containerDims = await this.getTargetDimensions();
+
+    const allAnnotations = await this.collectAllAnnotations(
+      selectorMap,
+      containerDims.width,
+      containerDims.height,
+      containerDims.offsetLeft,
+      containerDims.offsetTop,
+    );
+
+    const deduplicatedTextAnnotations =
+      this.deduplicateAnnotations(allAnnotations);
+
+    await this.stagehandPage.page.evaluate(
+      (dom) => window.restoreDOM(dom, undefined),
+      originalDOM,
+    );
+
+    const formattedText = formatText(
+      deduplicatedTextAnnotations,
+      containerDims.width,
+    );
+
+    await this.stagehandPage.cleanupDomDebug();
+
+    const result = { page_text: formattedText };
+    return pageTextSchema.parse(result);
+  }
+
   private async textExtract<T extends z.AnyZodObject>({
     instruction,
     schema,
@@ -160,10 +217,10 @@ export class StagehandExtractHandler {
     domSettleTimeoutMs,
     selector,
   }: {
-    instruction: string;
-    schema: T;
+    instruction?: string;
+    schema?: T;
     content?: z.infer<T>;
-    llmClient: LLMClient;
+    llmClient?: LLMClient;
     requestId?: string;
     domSettleTimeoutMs?: number;
     selector?: string;
@@ -611,4 +668,41 @@ export class StagehandExtractHandler {
 
     return allAnnotations;
   }
+
+  /**
+   * Deduplicate text annotations by grouping them by text, then removing duplicates
+   * within a certain proximity threshold.
+   */
+  private deduplicateAnnotations(
+    annotations: TextAnnotation[],
+  ): TextAnnotation[] {
+    const annotationsGroupedByText = new Map<string, TextAnnotation[]>();
+    const deduplicated: TextAnnotation[] = [];
+
+    for (const annotation of annotations) {
+      if (!annotationsGroupedByText.has(annotation.text)) {
+        annotationsGroupedByText.set(annotation.text, []);
+      }
+      annotationsGroupedByText.get(annotation.text)!.push(annotation);
+    }
+
+    for (const [text, group] of annotationsGroupedByText.entries()) {
+      for (const annotation of group) {
+        const isDuplicate = deduplicated.some((existing) => {
+          if (existing.text !== text) return false;
+
+          const dx = existing.bottom_left.x - annotation.bottom_left.x;
+          const dy = existing.bottom_left.y - annotation.bottom_left.y;
+          const distance = Math.hypot(dx, dy);
+          return distance < PROXIMITY_THRESHOLD;
+        });
+
+        if (!isDuplicate) {
+          deduplicated.push(annotation);
+        }
+      }
+    }
+
+    return deduplicated;
+  }
 }

Diff for: types/page.ts

@@ -17,6 +17,10 @@ export const defaultExtractSchema = z.object({
   extraction: z.string(),
 });
 
+export const pageTextSchema = z.object({
+  page_text: z.string(),
+});
+
 export interface Page extends Omit<PlaywrightPage, "on"> {
   act(action: string): Promise<ActResult>;
   act(options: ActOptions): Promise<ActResult>;
@@ -28,6 +32,7 @@ export interface Page extends Omit<PlaywrightPage, "on"> {
   extract<T extends z.AnyZodObject>(
     options: ExtractOptions<T>,
   ): Promise<ExtractResult<T>>;
+  extract(): Promise<ExtractResult<typeof pageTextSchema>>;
 
   observe(): Promise<ObserveResult[]>;
   observe(instruction: string): Promise<ObserveResult[]>;