diff --git a/.changeset/blue-islands-unite.md b/.changeset/blue-islands-unite.md new file mode 100644 index 000000000..5c63f7509 --- /dev/null +++ b/.changeset/blue-islands-unite.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": major +--- + +Add support for client certificates on local envs diff --git a/.changeset/icy-toes-obey.md b/.changeset/icy-toes-obey.md new file mode 100644 index 000000000..76ba1e561 --- /dev/null +++ b/.changeset/icy-toes-obey.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Add playwright arguments to agent execute response diff --git a/lib/agent/tools/act.ts b/lib/agent/tools/act.ts index b2a089192..a5b613050 100644 --- a/lib/agent/tools/act.ts +++ b/lib/agent/tools/act.ts @@ -1,7 +1,8 @@ import { tool } from "ai"; import { z } from "zod/v3"; import { StagehandPage } from "../../StagehandPage"; - +import { buildActObservePrompt } from "../../prompt"; +import { SupportedPlaywrightAction } from "@/types/act"; export const createActTool = ( stagehandPage: StagehandPage, executionModel?: string, @@ -19,37 +20,91 @@ export const createActTool = ( }), execute: async ({ action }) => { try { - let result; - if (executionModel) { - result = await stagehandPage.page.act({ - action, - modelName: executionModel, - }); - } else { - result = await stagehandPage.page.act(action); + const builtPrompt = buildActObservePrompt( + action, + Object.values(SupportedPlaywrightAction), + ); + + const observeOptions = executionModel + ? { + instruction: builtPrompt, + modelName: executionModel, + } + : { + instruction: builtPrompt, + }; + + const observeResults = await stagehandPage.page.observe(observeOptions); + + if (!observeResults || observeResults.length === 0) { + return { + success: false, + error: "No observable actions found for the given instruction", + }; } - const isIframeAction = result.action === "an iframe"; + + const observeResult = observeResults[0]; + + const isIframeAction = observeResult.description === "an iframe"; if (isIframeAction) { - const fallback = await stagehandPage.page.act( - executionModel - ? { action, modelName: executionModel, iframes: true } - : { action, iframes: true }, - ); + const iframeObserveOptions = executionModel + ? { + instruction: builtPrompt, + modelName: executionModel, + iframes: true, + } + : { + instruction: builtPrompt, + iframes: true, + }; + + const iframeObserveResults = + await stagehandPage.page.observe(iframeObserveOptions); + + if (!iframeObserveResults || iframeObserveResults.length === 0) { + return { + success: false, + error: "No observable actions found within iframe context", + isIframe: true, + }; + } + + const iframeObserveResult = iframeObserveResults[0]; + const fallback = await stagehandPage.page.act(iframeObserveResult); + return { success: fallback.success, action: fallback.action, isIframe: true, + playwrightArguments: { + description: iframeObserveResult.description, + method: iframeObserveResult.method, + arguments: iframeObserveResult.arguments, + selector: iframeObserveResult.selector, + }, }; } + const result = await stagehandPage.page.act(observeResult); + const playwrightArguments = { + description: observeResult.description, + method: observeResult.method, + arguments: observeResult.arguments, + selector: observeResult.selector, + }; + return { success: result.success, action: result.action, isIframe: false, + playwrightArguments, }; } catch (error) { - return { success: false, error: error.message }; + return { + success: false, + error: error.message, + }; } }, }); diff --git a/lib/handlers/stagehandAgentHandler.ts b/lib/handlers/stagehandAgentHandler.ts index af459f23c..18159987a 100644 --- a/lib/handlers/stagehandAgentHandler.ts +++ b/lib/handlers/stagehandAgentHandler.ts @@ -1,4 +1,9 @@ -import { AgentAction, AgentExecuteOptions, AgentResult } from "@/types/agent"; +import { + AgentAction, + AgentExecuteOptions, + AgentResult, + ActToolResult, +} from "@/types/agent"; import { LogLine } from "@/types/log"; import { StagehandPage } from "../StagehandPage"; import { LLMClient } from "../llm/LLMClient"; @@ -99,7 +104,8 @@ export class StagehandAgentHandler { }); if (event.toolCalls && event.toolCalls.length > 0) { - for (const toolCall of event.toolCalls) { + for (let i = 0; i < event.toolCalls.length; i++) { + const toolCall = event.toolCalls[i]; const args = toolCall.args as Record; if (event.text.length > 0) { @@ -122,6 +128,21 @@ export class StagehandAgentHandler { } } + // Get the tool result if available + const toolResult = event.toolResults?.[i]; + + const getPlaywrightArguments = () => { + if (toolCall.toolName !== "act" || !toolResult) { + return {}; + } + const result = toolResult.result as ActToolResult; + if (result && result.playwrightArguments) { + return { playwrightArguments: result.playwrightArguments }; + } + + return {}; + }; + const action: AgentAction = { type: toolCall.toolName, reasoning: event.text || undefined, @@ -130,6 +151,7 @@ export class StagehandAgentHandler { ? (args?.taskComplete as boolean) : false, ...args, + ...getPlaywrightArguments(), }; actions.push(action); diff --git a/lib/index.ts b/lib/index.ts index 04cd4a5bf..300e3178d 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -309,6 +309,7 @@ async function getBrowser( handleSIGINT: localBrowserLaunchOptions?.handleSIGINT ?? true, handleSIGTERM: localBrowserLaunchOptions?.handleSIGTERM ?? true, ignoreDefaultArgs: localBrowserLaunchOptions?.ignoreDefaultArgs, + clientCertificates: localBrowserLaunchOptions?.clientCertificates, }); if (localBrowserLaunchOptions?.cookies) { diff --git a/types/agent.ts b/types/agent.ts index 7bcea4992..9be344f0f 100644 --- a/types/agent.ts +++ b/types/agent.ts @@ -1,4 +1,13 @@ import { LogLine } from "./log"; +import { ObserveResult } from "./stagehand"; + +export interface ActToolResult { + success: boolean; + action?: string; + error?: string; + isIframe?: boolean; + playwrightArguments?: ObserveResult | null; +} export interface AgentAction { type: string; @@ -10,6 +19,7 @@ export interface AgentAction { pageText?: string; // ariaTree tool pageUrl?: string; // ariaTree tool instruction?: string; // various tools + playwrightArguments?: ObserveResult | null; // act tool [key: string]: unknown; } diff --git a/types/stagehand.ts b/types/stagehand.ts index 50f2ac2a5..d5fe4743a 100644 --- a/types/stagehand.ts +++ b/types/stagehand.ts @@ -174,6 +174,16 @@ export interface ObserveResult { export interface LocalBrowserLaunchOptions { args?: string[]; chromiumSandbox?: boolean; + clientCertificates?: Array<{ + origin: string; + certPath?: string; + cert?: Buffer; + keyPath?: string; + key?: Buffer; + pfxPath?: string; + pfx?: Buffer; + passphrase?: string; + }>; devtools?: boolean; env?: Record; executablePath?: string;