telemetry + debug logging (#569)

* telemetry

* empty catch block for now

* prettier

* stagehand.metrics instead of stagehand.getMetrics()

* add inferenceLogUtils file

* collect tokens from anthropic

* changeset

* single updateMetrics function

* rm import

Author: seanmcguire12

Diff for: .changeset/fifty-crabs-arrive.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+you can now call stagehand.metrics to get token usage metrics. you can also set logInferenceToFile in stagehand config to log the entire call/response history from stagehand & the LLM.

Diff for: lib/StagehandPage.ts

@@ -89,6 +89,7 @@ export class StagehandPage {
 
     if (this.llmClient) {
       this.actHandler = new StagehandActHandler({
+        stagehand: this.stagehand,
         verbose: this.stagehand.verbose,
         llmProvider: this.stagehand.llmProvider,
         enableCaching: this.stagehand.enableCaching,

Diff for: lib/handlers/actHandler.ts

@@ -16,20 +16,22 @@ import {
   ObserveResult,
   ActOptions,
   ObserveOptions,
+  StagehandFunctionName,
 } from "@/types/stagehand";
 import { MethodHandlerContext, SupportedPlaywrightAction } from "@/types/act";
 import { buildActObservePrompt } from "../prompt";
 import {
   methodHandlerMap,
   fallbackLocatorMethod,
 } from "./handlerUtils/actHandlerUtils";
-
+import { Stagehand } from "@/lib";
 /**
  * NOTE: Vision support has been removed from this version of Stagehand.
  * If useVision or verifierUseVision is set to true, a warning is logged and
  * the flow continues as if vision = false.
  */
 export class StagehandActHandler {
+  private readonly stagehand: Stagehand;
   private readonly stagehandPage: StagehandPage;
   private readonly verbose: 0 | 1 | 2;
   private readonly llmProvider: LLMProvider;
@@ -44,6 +46,7 @@ export class StagehandActHandler {
   private readonly waitForCaptchaSolves: boolean;
 
   constructor({
+    stagehand,
     verbose,
     llmProvider,
     enableCaching,
@@ -53,6 +56,7 @@ export class StagehandActHandler {
     selfHeal,
     waitForCaptchaSolves,
   }: {
+    stagehand: Stagehand;
     verbose: 0 | 1 | 2;
     llmProvider: LLMProvider;
     enableCaching: boolean;
@@ -64,6 +68,7 @@ export class StagehandActHandler {
     selfHeal: boolean;
     waitForCaptchaSolves: boolean;
   }) {
+    this.stagehand = stagehand;
     this.verbose = verbose;
     this.llmProvider = llmProvider;
     this.enableCaching = enableCaching;
@@ -337,15 +342,17 @@ export class StagehandActHandler {
       });
 
       // Always use text-based DOM verification (no vision).
-      actionCompleted = await verifyActCompletion({
+      const verifyResult = await verifyActCompletion({
         goal: action,
         steps,
         llmProvider: this.llmProvider,
         llmClient: verifyLLmClient,
         domElements,
         logger: this.logger,
         requestId,
+        logInferenceToFile: this.stagehand.logInferenceToFile,
       });
+      actionCompleted = verifyResult.completed;
 
       this.logger({
         category: "action",
@@ -362,6 +369,12 @@ export class StagehandActHandler {
           },
         },
       });
+      this.stagehand.updateMetrics(
+        StagehandFunctionName.ACT,
+        verifyResult.prompt_tokens,
+        verifyResult.completion_tokens,
+        verifyResult.inference_time_ms,
+      );
     }
 
     return actionCompleted;
@@ -681,6 +694,15 @@ export class StagehandActHandler {
         requestId,
         variables,
         userProvidedInstructions: this.userProvidedInstructions,
+        onActMetrics: (promptTokens, completionTokens, inferenceTimeMs) => {
+          this.stagehand.updateMetrics(
+            StagehandFunctionName.ACT,
+            promptTokens,
+            completionTokens,
+            inferenceTimeMs,
+          );
+        },
+        logInferenceToFile: this.stagehand.logInferenceToFile,
       });
 
       this.logger({

Diff for: lib/handlers/extractHandler.ts

@@ -5,7 +5,7 @@ import { extract } from "../inference";
 import { LLMClient } from "../llm/LLMClient";
 import { formatText } from "../utils";
 import { StagehandPage } from "../StagehandPage";
-import { Stagehand } from "../index";
+import { Stagehand, StagehandFunctionName } from "../index";
 import { pageTextSchema } from "../../types/page";
 
 const PROXIMITY_THRESHOLD = 15;
@@ -353,13 +353,24 @@ export class StagehandExtractHandler {
       requestId,
       userProvidedInstructions: this.userProvidedInstructions,
       logger: this.logger,
+      logInferenceToFile: this.stagehand.logInferenceToFile,
     });
 
     const {
       metadata: { completed },
+      prompt_tokens: promptTokens,
+      completion_tokens: completionTokens,
+      inference_time_ms: inferenceTimeMs,
       ...output
     } = extractionResponse;
 
+    this.stagehand.updateMetrics(
+      StagehandFunctionName.EXTRACT,
+      promptTokens,
+      completionTokens,
+      inferenceTimeMs,
+    );
+
     // **11:** Handle the extraction response and log the results
     this.logger({
       category: "extraction",
@@ -481,13 +492,24 @@ export class StagehandExtractHandler {
       isUsingTextExtract: false,
       userProvidedInstructions: this.userProvidedInstructions,
       logger: this.logger,
+      logInferenceToFile: this.stagehand.logInferenceToFile,
     });
 
     const {
       metadata: { completed },
+      prompt_tokens: promptTokens,
+      completion_tokens: completionTokens,
+      inference_time_ms: inferenceTimeMs,
       ...output
     } = extractionResponse;
 
+    this.stagehand.updateMetrics(
+      StagehandFunctionName.EXTRACT,
+      promptTokens,
+      completionTokens,
+      inferenceTimeMs,
+    );
+
     this.logger({
       category: "extraction",
       message: "received extraction response",

Diff for: lib/handlers/observeHandler.ts

@@ -1,5 +1,5 @@
 import { LogLine } from "../../types/log";
-import { Stagehand } from "../index";
+import { Stagehand, StagehandFunctionName } from "../index";
 import { observe } from "../inference";
 import { LLMClient } from "../llm/LLMClient";
 import { StagehandPage } from "../StagehandPage";
@@ -113,8 +113,22 @@ export class StagehandObserveHandler {
       logger: this.logger,
       isUsingAccessibilityTree: useAccessibilityTree,
       returnAction,
+      logInferenceToFile: this.stagehand.logInferenceToFile,
     });
 
+    const {
+      prompt_tokens = 0,
+      completion_tokens = 0,
+      inference_time_ms = 0,
+    } = observationResponse;
+
+    this.stagehand.updateMetrics(
+      StagehandFunctionName.OBSERVE,
+      prompt_tokens,
+      completion_tokens,
+      inference_time_ms,
+    );
+
     //Add iframes to the observation response if there are any on the page
     if (iframes.length > 0) {
       iframes.forEach((iframe) => {

Diff for: lib/index.ts

@@ -26,6 +26,8 @@ import {
   ObserveOptions,
   ObserveResult,
   AgentConfig,
+  StagehandMetrics,
+  StagehandFunctionName,
 } from "../types/stagehand";
 import { StagehandContext } from "./StagehandContext";
 import { StagehandPage } from "./StagehandPage";
@@ -382,7 +384,7 @@ export class Stagehand {
   public readonly selfHeal: boolean;
   private cleanupCalled = false;
   public readonly actTimeoutMs: number;
-
+  public readonly logInferenceToFile?: boolean;
   protected setActivePage(page: StagehandPage): void {
     this.stagehandPage = page;
   }
@@ -396,6 +398,63 @@ export class Stagehand {
     return this.stagehandPage.page;
   }
 
+  public stagehandMetrics: StagehandMetrics = {
+    actPromptTokens: 0,
+    actCompletionTokens: 0,
+    actInferenceTimeMs: 0,
+    extractPromptTokens: 0,
+    extractCompletionTokens: 0,
+    extractInferenceTimeMs: 0,
+    observePromptTokens: 0,
+    observeCompletionTokens: 0,
+    observeInferenceTimeMs: 0,
+    totalPromptTokens: 0,
+    totalCompletionTokens: 0,
+    totalInferenceTimeMs: 0,
+  };
+
+  public get metrics(): StagehandMetrics {
+    return this.stagehandMetrics;
+  }
+
+  public updateMetrics(
+    functionName: StagehandFunctionName,
+    promptTokens: number,
+    completionTokens: number,
+    inferenceTimeMs: number,
+  ): void {
+    switch (functionName) {
+      case StagehandFunctionName.ACT:
+        this.stagehandMetrics.actPromptTokens += promptTokens;
+        this.stagehandMetrics.actCompletionTokens += completionTokens;
+        this.stagehandMetrics.actInferenceTimeMs += inferenceTimeMs;
+        break;
+
+      case StagehandFunctionName.EXTRACT:
+        this.stagehandMetrics.extractPromptTokens += promptTokens;
+        this.stagehandMetrics.extractCompletionTokens += completionTokens;
+        this.stagehandMetrics.extractInferenceTimeMs += inferenceTimeMs;
+        break;
+
+      case StagehandFunctionName.OBSERVE:
+        this.stagehandMetrics.observePromptTokens += promptTokens;
+        this.stagehandMetrics.observeCompletionTokens += completionTokens;
+        this.stagehandMetrics.observeInferenceTimeMs += inferenceTimeMs;
+        break;
+    }
+    this.updateTotalMetrics(promptTokens, completionTokens, inferenceTimeMs);
+  }
+
+  private updateTotalMetrics(
+    promptTokens: number,
+    completionTokens: number,
+    inferenceTimeMs: number,
+  ): void {
+    this.stagehandMetrics.totalPromptTokens += promptTokens;
+    this.stagehandMetrics.totalCompletionTokens += completionTokens;
+    this.stagehandMetrics.totalInferenceTimeMs += inferenceTimeMs;
+  }
+
   constructor(
     {
       env,
@@ -419,6 +478,7 @@ export class Stagehand {
       selfHeal = true,
       waitForCaptchaSolves = false,
       actTimeoutMs = 60_000,
+      logInferenceToFile = false,
     }: ConstructorParams = {
       env: "BROWSERBASE",
     },
@@ -473,6 +533,7 @@ export class Stagehand {
     if (this.usingAPI) {
       this.registerSignalHandlers();
     }
+    this.logInferenceToFile = logInferenceToFile;
   }
 
   private registerSignalHandlers() {