From 85f6e8ea084b456a8436bec32c4f7b93dc725407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 6 Nov 2026 22:23:19 +0100 Subject: [PATCH 1/4] fix(web): answer native dialogs an agent step's action opens An unhandled alert, confirm, or leave-page dialog after an agent step's action is accepted, a prompt dismissed, and the app log names it. Before, the engine dismissed it and failed the agent's next action, so leaving a dirty form or confirming a delete was out of reach for agent.act, explore, and MCP action tools. A dialog belongs to the newest action before it, never to a read, so a test's own dialog still fails its next step even while an agent judgment polls. OperationContext.agentStep marks agent step operations, replays included, so a recorded dialog step replays under --strict-cache; the web benchmark gains a deferred-dialog agent scenario with its recording. --- .changeset/agent-native-dialogs.md | 5 + .changeset/operation-agent-step.md | 5 + ...c4c71e08881c91c6b9c8deaa74a32431785e5.json | 63 ++++++ .../tests-agent/scenarios.e2e.ts | 12 +- docs/browser.mdx | 10 +- docs/reference/engine.mdx | 7 + docs/reference/mcp.mdx | 5 + docs/reference/web.mdx | 7 +- packages/e2e/src/agent/phases.ts | 11 +- packages/e2e/src/agent/step-accounting.ts | 2 +- packages/e2e/src/engine/contract.ts | 7 + packages/e2e/src/engine/protocol/client.ts | 1 + packages/e2e/src/engine/protocol/protocol.ts | 1 + packages/e2e/src/engine/protocol/server.ts | 1 + .../e2e/tests/unit/engine-protocol.test.ts | 10 + .../e2e/tests/unit/step-accounting.test.ts | 6 +- packages/web/src/browser.ts | 20 +- packages/web/src/dialogs.ts | 52 ++++- packages/web/src/surface.ts | 39 +++- .../tests/integration/agent-dialogs.test.ts | 182 ++++++++++++++++++ packages/web/tests/unit/dialogs.test.ts | 67 ++++++- plugins/e2e/skills/e2e/references/mcp.md | 5 + .../skills/e2e/references/writing-tests.md | 4 +- skills/e2e/references/mcp.md | 5 + skills/e2e/references/writing-tests.md | 4 +- 25 files changed, 490 insertions(+), 41 deletions(-) create mode 100644 .changeset/agent-native-dialogs.md create mode 100644 .changeset/operation-agent-step.md create mode 100644 apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json create mode 100644 packages/web/tests/integration/agent-dialogs.test.ts diff --git a/.changeset/agent-native-dialogs.md b/.changeset/agent-native-dialogs.md new file mode 100644 index 0000000000..0f3a2cf206 --- /dev/null +++ b/.changeset/agent-native-dialogs.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/web": patch +--- + +Answer native dialogs an agent step's own action opens instead of failing its next action. In `agent.act` and its replays, `e2e explore`, and the action tools of an `e2e mcp` session, an alert, a confirm, and a leave-page dialog with no handler are accepted, and a prompt is dismissed. The app log names each one. A dialog belongs to the newest action before it, so one a test's own step opens still fails the next step until a `browser.onDialog` handler takes it, even when an agent step reads the screen in between. diff --git a/.changeset/operation-agent-step.md b/.changeset/operation-agent-step.md new file mode 100644 index 0000000000..c3113c9b8e --- /dev/null +++ b/.changeset/operation-agent-step.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +`OperationContext.agentStep` tells an engine that an operation belongs to an agent step, including an action the replay cache repeats with the `test` origin. The web engine uses it to answer native dialogs for the agent. diff --git a/apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json b/apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json new file mode 100644 index 0000000000..e004345098 --- /dev/null +++ b/apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json @@ -0,0 +1,63 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-11-06T21:09:07.253Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Submit report\"", + "target": { + "role": "button", + "name": "Submit report", + "testId": "submit-report-button" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "tests-agent/scenarios.e2e.ts::act%20completes%20deferred-dialog", + "targetId": "web", + "instructionDigest": "0f97cef062579bf174a4272ebd0c9023c90355e5341388a94c5bf29f99c1bd37", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "recordedBy": { + "provider": "gateway", + "model": "openai/gpt-6-luna-fast" + }, + "summary": "Submitted the report and waited for the deferred confirmation; the page now shows “Report submitted after deferred confirmation.”", + "startPath": "/e/deferred-dialog", + "endPath": "/e/deferred-dialog", + "endAnchors": [ + { + "text": "Report submitted after deferred confirmation", + "testId": "success-message" + } + ], + "goneAnchors": [ + { + "text": "Submit the report; a confirmation dialog appears shortly after the click. Accept it." + }, + { + "role": "button", + "name": "Submit report", + "testId": "submit-report-button" + } + ], + "endWaitMs": 13352, + "keyedBy": { + "cacheSchema": "trace-1", + "policyVersion": "conservative/6", + "project": "6c340b4558007fd12144b6a4e0daf3abb724b92faa2d08705fa564444b68533c", + "platform": "web", + "engineName": "web", + "engineSpiVersion": 1, + "appIdentity": "dc587874e98340c0ead3b9555bef25c85bca9337c6c8e28ae9e991c982ef1202", + "agentContextDigest": "02e45116d4a18a6603260cbbc8fa198d311b6a1a01362cc9dd6aa341d16bb3cf" + } + } +} diff --git a/apps/web-benchmark/tests-agent/scenarios.e2e.ts b/apps/web-benchmark/tests-agent/scenarios.e2e.ts index a0e26f5409..2a81e40d36 100644 --- a/apps/web-benchmark/tests-agent/scenarios.e2e.ts +++ b/apps/web-benchmark/tests-agent/scenarios.e2e.ts @@ -3,9 +3,9 @@ * accessibility tree. Each step gets the catalog description as its goal and * reads the rest off the page; the deterministic check on the scenario's * success message decides the test. Scenarios that need pixels (canvas, image - * and CSS-content UIs, the aria-hidden flow, the onboarding wizard), native - * dialogs, a second tab, or a file the agent first downloads are left out: - * the grammar has no verb for them yet. + * and CSS-content UIs, the aria-hidden flow, the onboarding wizard), a typed + * answer to a native prompt, a second tab, or a file the agent first + * downloads are left out: the grammar has no verb for them yet. */ import { test } from '@e2e-dev/web'; @@ -130,6 +130,12 @@ const SCENARIOS: readonly Scenario[] = [ goal: 'buy a gift card for friend@example.com and complete the purchase', success: 'Gift card sent to friend@example.com', }, + { + // The confirm opens 900 ms after the click, and the engine accepts it for the agent, replay included. + slug: 'deferred-dialog', + goal: 'submit the report and accept the confirmation that follows', + success: 'Report submitted after deferred confirmation', + }, ]; async function complete( diff --git a/docs/browser.mdx b/docs/browser.mdx index c48395f2e4..d9e431330c 100644 --- a/docs/browser.mdx +++ b/docs/browser.mdx @@ -62,8 +62,8 @@ expect(response.status).toBe(201); ## Handle a dialog -An unhandled `alert`, `confirm`, or `prompt` fails the next step. Register a -handler first: +An unhandled `alert`, `confirm`, `prompt`, or leave-page dialog that a test's +own step opens fails the next step. Register a handler first: ```ts const off = await browser.onDialog('accept'); @@ -74,6 +74,12 @@ await off(); A function handler gets the dialog and must call `accept(text?)` or `dismiss()`. +The agent can't see native dialogs, so one its own action opens is answered +for it when no handler is registered. This covers `agent.act` and its replays, +`e2e explore`, and the action tools of an `e2e mcp` session. An alert, a +confirm, and a leave-page dialog are accepted, and a prompt is dismissed. The +app log names each one. + ## Download a file ```ts diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index f2dc8fe8e6..9728c9fb80 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -341,6 +341,13 @@ export interface OperationContext { * transition to end first. */ readonly origin: 'test' | 'agent'; + /** + * The operation is part of an agent step: the agent chose it just now, or + * the replay cache repeats a choice it made in an earlier run, whose + * `origin` is then `test`. An engine answers for the agent what the agent + * cannot see, such as a native dialog its action opens. + */ + readonly agentStep?: boolean; } ``` diff --git a/docs/reference/mcp.mdx b/docs/reference/mcp.mdx index c85fcbc9bc..e11b126c4e 100644 --- a/docs/reference/mcp.mdx +++ b/docs/reference/mcp.mdx @@ -274,6 +274,11 @@ changed on screen. `secrets` and the engine declares the `fill` action. The plaintext never reaches the agent as text; see [screenshots after a secret fill](/reference/security#screenshots-after-a-secret-fill). +- A native dialog an action opens is answered as in `agent.act`: an alert, a + confirm, and a leave-page dialog are accepted, and a prompt is dismissed. + The result doesn't say so, and the test line registers no handler, so a + test of that flow needs `browser.onDialog` first. `run_code` runs test + statements, so a dialog it opens fails the next call. #### `dismiss_keyboard` diff --git a/docs/reference/web.mdx b/docs/reference/web.mdx index f25db53014..6b544c33b1 100644 --- a/docs/reference/web.mdx +++ b/docs/reference/web.mdx @@ -857,6 +857,11 @@ await off(); `Dialog`. - A function handler must call `accept` or `dismiss`. One that does neither has the dialog dismissed, as an unhandled dialog does. +- A dialog no handler takes is dismissed and fails the next step, unless an + agent step's own action opened it, replayed from the cache or not. Then + it's answered for the agent: an alert, a confirm, and a leave-page dialog + are accepted, and a prompt is dismissed. The app log names each one. A + dialog belongs to the newest action before it, never to a read. - A handler that throws fails the next step with its error, or the attempt at its end when no step follows. One that throws before answering has its dialog dismissed. @@ -865,7 +870,7 @@ The errors surface on the next step, not on `onDialog`: | Throws | When | | --- | --- | -| `APP_NOT_OPEN` | A function handler answered neither way, or no handler took the dialog. | +| `INVALID_STATE` | A function handler answered neither way, or no handler took a dialog a test's own step opened. | | `ASSERTION_FAILED` | The handler's `expect` failed. | | `ENGINE_FAILURE` | The handler threw an unclassified error. | diff --git a/packages/e2e/src/agent/phases.ts b/packages/e2e/src/agent/phases.ts index 7fb95fd8ec..f05f5b8d5f 100644 --- a/packages/e2e/src/agent/phases.ts +++ b/packages/e2e/src/agent/phases.ts @@ -15,17 +15,18 @@ import type { StepActivity, StepEvent, StepRecorder } from '../run/steps.ts'; import { AgentError, toAgentError } from './error.ts'; /** - * One engine operation's budget: `actionTimeout`, capped by the step clock. - * Bounding each call independently is what keeps a single screen that never - * settles from consuming the whole step: the hang costs one action timeout and - * a clearly attributed failure, not the test budget. + * One engine operation of an agent step, with its budget: `actionTimeout`, + * capped by the step clock. Bounding each call independently is what keeps a + * single screen that never settles from consuming the whole step: the hang + * costs one action timeout and a clearly attributed failure, not the test + * budget. */ export function boundedOperation( engine: LocatorEngine, actionTimeout: number, deadline: Deadline, ): OperationContext { - return { ...engine.operation(Math.max(1, Math.min(actionTimeout, deadline.remaining()))), origin: 'agent' }; + return { ...engine.operation(Math.max(1, Math.min(actionTimeout, deadline.remaining()))), origin: 'agent', agentStep: true }; } /** Records one policy decision as a child event of the current step. */ diff --git a/packages/e2e/src/agent/step-accounting.ts b/packages/e2e/src/agent/step-accounting.ts index b84541a79e..e527c1671e 100644 --- a/packages/e2e/src/agent/step-accounting.ts +++ b/packages/e2e/src/agent/step-accounting.ts @@ -202,7 +202,7 @@ export class StepAccounting { ); } - /** Every operation inside a replayed span is the test's, navigation included; outside it, the agent's. */ + /** Every operation inside a replayed span acts with the test's origin, navigation included, and is still the agent step's; outside it, the agent's. */ private stamped(operation: OperationContext): OperationContext { return this.inReplay ? { ...operation, origin: 'test' } : operation; } diff --git a/packages/e2e/src/engine/contract.ts b/packages/e2e/src/engine/contract.ts index 518bece23e..a2c38b4c0d 100644 --- a/packages/e2e/src/engine/contract.ts +++ b/packages/e2e/src/engine/contract.ts @@ -50,6 +50,13 @@ export interface OperationContext { * transition to end first. */ readonly origin: 'test' | 'agent'; + /** + * The operation is part of an agent step: the agent chose it just now, or + * the replay cache repeats a choice it made in an earlier run, whose + * `origin` is then `test`. An engine answers for the agent what the agent + * cannot see, such as a native dialog its action opens. + */ + readonly agentStep?: boolean; } export type TextPattern = diff --git a/packages/e2e/src/engine/protocol/client.ts b/packages/e2e/src/engine/protocol/client.ts index 6ae9073f5e..fe99d458fd 100644 --- a/packages/e2e/src/engine/protocol/client.ts +++ b/packages/e2e/src/engine/protocol/client.ts @@ -584,6 +584,7 @@ function encodeOperation(link: Link, operation: OperationContext): WireOperation runId: operation.runId, attemptId: operation.attemptId, origin: operation.origin, + ...(operation.agentStep === true ? { agentStep: true as const } : {}), }; } diff --git a/packages/e2e/src/engine/protocol/protocol.ts b/packages/e2e/src/engine/protocol/protocol.ts index cac0358b83..37f991d243 100644 --- a/packages/e2e/src/engine/protocol/protocol.ts +++ b/packages/e2e/src/engine/protocol/protocol.ts @@ -158,6 +158,7 @@ export interface WireOperation { readonly runId: string; readonly attemptId: string; readonly origin: 'test' | 'agent'; + readonly agentStep?: true; } /** `EngineCleanupContext` on the wire. */ diff --git a/packages/e2e/src/engine/protocol/server.ts b/packages/e2e/src/engine/protocol/server.ts index de7ff4072a..c10607b07d 100644 --- a/packages/e2e/src/engine/protocol/server.ts +++ b/packages/e2e/src/engine/protocol/server.ts @@ -268,6 +268,7 @@ class EngineServer { runId: wire.runId, attemptId: wire.attemptId, origin: wire.origin, + ...(wire.agentStep === true ? { agentStep: true } : {}), }; } diff --git a/packages/e2e/tests/unit/engine-protocol.test.ts b/packages/e2e/tests/unit/engine-protocol.test.ts index 9180e8ccef..eb81b675b9 100644 --- a/packages/e2e/tests/unit/engine-protocol.test.ts +++ b/packages/e2e/tests/unit/engine-protocol.test.ts @@ -86,6 +86,16 @@ describe('engine protocol', () => { expect(Object.keys(remote.session ?? {}).toSorted()).toEqual(['open', 'reset']); }); + it("carries whether an operation is an agent step's, and leaves it out when it is not", async () => { + const received: OperationContext[] = []; + const { remote } = overLoopback(toy({ actions: ['tap'], perform: async (_ref, _action, seen) => void received.push(seen) })); + await remote.init!(initInfo()); + await remote.perform!({ id: 'n1', revision: '' }, { kind: 'tap' }, { ...operation(), agentStep: true }); + await remote.perform!({ id: 'n1', revision: '' }, { kind: 'tap' }, operation()); + expect(received.map((seen) => [seen.origin, seen.agentStep])).toEqual([['test', true], ['test', undefined]]); + expect('agentStep' in received[1]!).toBe(false); + }); + it('carries a snapshot with its pixels byte for byte', async () => { const data = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0, 255, 7]); const { remote } = overLoopback( diff --git a/packages/e2e/tests/unit/step-accounting.test.ts b/packages/e2e/tests/unit/step-accounting.test.ts index e2d56ef072..ea79288b97 100644 --- a/packages/e2e/tests/unit/step-accounting.test.ts +++ b/packages/e2e/tests/unit/step-accounting.test.ts @@ -26,7 +26,7 @@ function runtime(): AgentContext & { steps: StepRecorder } { } describe('step accounting operation contexts', () => { - it('marks actions as the agent\'s, and as the test\'s while a trace replays', () => { + it('marks actions as the agent\'s, and as the test\'s origin while a trace replays, always within the agent step', () => { const accounting = new StepAccounting(runtime(), { api: 'agent.act', timeoutMs: 10_000, @@ -37,11 +37,15 @@ describe('step accounting operation contexts', () => { expect(accounting.replayingTrace).toBe(false); expect(accounting.actionOperation().origin).toBe('agent'); expect(accounting.operation().origin).toBe('agent'); + expect(accounting.actionOperation().agentStep).toBe(true); accounting.replaying(true); // Both contexts: targeted actions take actionOperation(), navigation takes operation(). expect(accounting.replayingTrace).toBe(true); expect(accounting.actionOperation().origin).toBe('test'); expect(accounting.operation().origin).toBe('test'); + // A replayed action is still the agent step's, so an engine answers what the agent cannot see for it. + expect(accounting.actionOperation().agentStep).toBe(true); + expect(accounting.operation().agentStep).toBe(true); accounting.replaying(false); expect(accounting.replayingTrace).toBe(false); expect(accounting.actionOperation().origin).toBe('agent'); diff --git a/packages/web/src/browser.ts b/packages/web/src/browser.ts index b358b0cdcc..2edf857568 100644 --- a/packages/web/src/browser.ts +++ b/packages/web/src/browser.ts @@ -461,7 +461,7 @@ export function createBrowserFixture(surface: PlaywrightSurface, context: Engine run: (operation: OperationContext) => Promise, ): Promise => { const operation = context.operation(stepTimeout(options?.timeout, api) ?? context.timeouts.test); - return surface.guard(operation, 'navigation', run); + return surface.guardAction(operation, 'navigation', run); }; /** An assertion-style budget: the given timeout or the assertion timeout, clamped to the test. */ @@ -519,7 +519,7 @@ export function createBrowserFixture(surface: PlaywrightSurface, context: Engine const input: unknown = arg; // A handle, not a value: the boundary resolves to a string, which Playwright // keeps out of its in-page serializer when it is taken back from a handle. - const report = await surface.guard(context.operation(), 'evaluate', async (operation) => { + const report = await surface.guardAction(context.operation(), 'evaluate', async (operation) => { const page = surface.requirePage(); try { const handle = await page.evaluateHandle(evaluate, input); @@ -779,21 +779,21 @@ export function createBrowserFixture(surface: PlaywrightSurface, context: Engine }, keyboard: { down: (key) => - surface.guard(context.operation(), 'keyboard.down', () => surface.requirePage().keyboard.down(key), classifyInputError), + surface.guardAction(context.operation(), 'keyboard.down', () => surface.requirePage().keyboard.down(key), classifyInputError), up: (key) => - surface.guard(context.operation(), 'keyboard.up', () => surface.requirePage().keyboard.up(key), classifyInputError), + surface.guardAction(context.operation(), 'keyboard.up', () => surface.requirePage().keyboard.up(key), classifyInputError), press: (key) => - surface.guard(context.operation(), 'keyboard.press', () => surface.requirePage().keyboard.press(key), classifyInputError), + surface.guardAction(context.operation(), 'keyboard.press', () => surface.requirePage().keyboard.press(key), classifyInputError), type: (text) => - surface.guard(context.operation(), 'keyboard.type', () => surface.requirePage().keyboard.type(text), classifyInputError), + surface.guardAction(context.operation(), 'keyboard.type', () => surface.requirePage().keyboard.type(text), classifyInputError), }, mouse: { move: (x, y) => - surface.guard(context.operation(), 'mouse.move', () => surface.requirePage().mouse.move(x, y), classifyInputError), + surface.guardAction(context.operation(), 'mouse.move', () => surface.requirePage().mouse.move(x, y), classifyInputError), wheel: (deltaX, deltaY) => - surface.guard(context.operation(), 'mouse.wheel', () => surface.requirePage().mouse.wheel(deltaX, deltaY), classifyInputError), - down: () => surface.guard(context.operation(), 'mouse.down', () => surface.requirePage().mouse.down(), classifyInputError), - up: () => surface.guard(context.operation(), 'mouse.up', () => surface.requirePage().mouse.up(), classifyInputError), + surface.guardAction(context.operation(), 'mouse.wheel', () => surface.requirePage().mouse.wheel(deltaX, deltaY), classifyInputError), + down: () => surface.guardAction(context.operation(), 'mouse.down', () => surface.requirePage().mouse.down(), classifyInputError), + up: () => surface.guardAction(context.operation(), 'mouse.up', () => surface.requirePage().mouse.up(), classifyInputError), }, }; diff --git a/packages/web/src/dialogs.ts b/packages/web/src/dialogs.ts index 65a38b44b0..331494f503 100644 --- a/packages/web/src/dialogs.ts +++ b/packages/web/src/dialogs.ts @@ -2,10 +2,17 @@ * Native dialog routing for one attempt. Handlers are registered by the `web` * fixture; the surface owns the router so a dialog that fires before a test * ever touches `web` is still accounted for instead of silently dismissed. + * + * A dialog no handler takes fails the test's next step, unless the agent + * acted last: native dialogs are outside the page, so the agent never sees + * one and cannot answer it. The router answers it for the agent the way the + * action that opened it asked (an alert or a confirm accepted, a page left + * when the agent navigated away from unsaved changes) and tells the app log. + * A prompt asks for text the agent never sees, so it is dismissed. */ import type { Dialog as PwDialog } from 'playwright-core'; -import { EngineError } from 'e2e/engine'; +import { EngineError, type AppLogEntry } from 'e2e/engine'; import { ErrorLatch, isClassified, message } from './support.ts'; /** A native dialog as a test's handler sees it. */ @@ -26,14 +33,26 @@ interface Registration { readonly handler: DialogHandler; } +/** What the router asks its surface about a dialog no handler takes. */ +export interface UnhandledDialogs { + /** Whether the newest operation on the page was the agent's. */ + readonly agentActed: () => boolean; + /** The attempt's app log, told each dialog answered for the agent. */ + readonly log: (entry: AppLogEntry) => void; +} + export class DialogRouter { private registrations: Registration[] = []; /** * `latch` is shared with the surface: a dialog failure and a route-handler - * failure surface through one check at the next step. + * failure surface through one check at the next step. Without `unhandled`, + * every dialog no handler takes fails the next step. */ - constructor(private readonly latch: ErrorLatch = new ErrorLatch()) {} + constructor( + private readonly latch: ErrorLatch = new ErrorLatch(), + private readonly unhandled?: UnhandledDialogs, + ) {} /** * Registers a handler (the newest wins) and returns its idempotent @@ -62,6 +81,10 @@ export class DialogRouter { private async route(dialog: PwDialog): Promise { const handler = this.registrations.at(-1)?.handler; + if (handler === undefined && this.unhandled?.agentActed() === true) { + await this.answerForAgent(dialog, this.unhandled); + return; + } if (handler === undefined) { this.latch.latch( new EngineError( @@ -122,4 +145,27 @@ export class DialogRouter { if (!answered) await dialog.dismiss().catch(() => undefined); } } + + /** + * Answers a dialog the agent's action opened and names it in the app log. + * A dialog the page closed first, or a page that went away, leaves nothing + * to answer and nothing to tell. + */ + private async answerForAgent(dialog: PwDialog, unhandled: UnhandledDialogs): Promise { + const type = dialog.type(); + const accepted = type !== 'prompt'; + try { + if (accepted) await dialog.accept(); + else await dialog.dismiss(); + } catch { + return; + } + const text = dialog.message().replace(/\s+/g, ' ').trim(); + const quoted = text === '' ? '' : ` ${JSON.stringify(text)}`; + unhandled.log({ + source: 'system', + level: 'info', + text: `${type} dialog${quoted} ${accepted ? 'accepted' : 'dismissed'} for the agent`, + }); + } } diff --git a/packages/web/src/surface.ts b/packages/web/src/surface.ts index 96f9457516..76d6f2cc59 100644 --- a/packages/web/src/surface.ts +++ b/packages/web/src/surface.ts @@ -303,6 +303,8 @@ export class PlaywrightSurface { private artifactsDir = ''; /** Navigations the engine has in flight, which the app log leaves to the step that started them. */ private ownNavigations = 0; + /** Who made the newest action on the page: a dialog no handler takes is answered when it was the agent. */ + private actor: 'agent' | 'test' | undefined; /** One entry per navigation in flight, set to the URL of a document request a test route aborted during it. */ private readonly navigationAborts = new Set<{ url?: string }>(); /** What the attempt tells its trace beside the app log; absent between attempts. */ @@ -493,7 +495,8 @@ export class PlaywrightSurface { // `validateApp` admitted only grants. const granted = Object.keys(this.app.permissions ?? {}); this.latch = new ErrorLatch(); - const dialogs = new DialogRouter(this.latch); + this.actor = undefined; + const dialogs = new DialogRouter(this.latch, { agentActed: () => this.actor === 'agent', log: context.appLog }); this.routes = routes; this.initScripts = initScripts; this.dialogs = dialogs; @@ -714,11 +717,27 @@ export class PlaywrightSurface { } } + /** + * `guard` for an operation that acts on the page (a navigation, input, or + * a script) rather than reading it. Its author owns the dialogs that open + * after it until the next action: one no handler takes is answered when + * the author is an agent step, the step replayed from the cache included. + */ + guardAction( + operation: OperationContext, + label: string, + fn: (operation: OperationContext) => Promise, + translate?: (cause: unknown, label: string) => Error, + ): Promise { + this.actor = (operation.agentStep ?? operation.origin === 'agent') ? 'agent' : 'test'; + return this.guard(operation, label, fn, translate); + } + // --- steering: the session hooks --- /** Opens one URL the harness resolved; the attempt's page is created on first use. */ open(url: string, operation: OperationContext): Promise { - return this.acted(operation, this.guard(operation, 'navigation', (currentOperation) => + return this.acted(operation, this.guardAction(operation, 'navigation', (currentOperation) => this.navigating(async () => { const page = await this.ensurePage(); await page.goto(url, { waitUntil: 'load', timeout: currentOperation.timeoutMs }); @@ -727,7 +746,7 @@ export class PlaywrightSurface { } back(operation: OperationContext): Promise { - return this.acted(operation, this.guard(operation, 'navigation', (currentOperation) => + return this.acted(operation, this.guardAction(operation, 'navigation', (currentOperation) => this.navigating(async () => { await this.requirePage().goBack({ waitUntil: 'load', timeout: currentOperation.timeoutMs }); }), @@ -736,7 +755,7 @@ export class PlaywrightSurface { /** Sets the viewport for the agent's `resize` verb, as `browser.setViewport` does, and shows the screen after it. */ resize(size: ViewportSize, operation: OperationContext): Promise { - return this.acted(operation, this.guard(operation, 'resize', () => this.setViewport(size))); + return this.acted(operation, this.guardAction(operation, 'resize', () => this.setViewport(size))); } /** @@ -782,12 +801,12 @@ export class PlaywrightSurface { /** Restarts the document while retaining this attempt's context and storage. */ restart(operation: OperationContext): Promise { - return this.guard(operation, 'restart', () => this.requireSession().restart()); + return this.guardAction(operation, 'restart', () => this.requireSession().restart()); } /** Replaces the context with a clean one and opens its blank page. */ reset(operation: OperationContext): Promise { - return this.guard(operation, 'state reset', async (currentOperation) => { + return this.guardAction(operation, 'state reset', async (currentOperation) => { const session = this.requireSession(); await session.replace(undefined, currentOperation); await session.ensurePage(); @@ -927,7 +946,7 @@ export class PlaywrightSurface { * on the root acts on the document element the root stands for. */ perform(ref: NodeRef, action: LocatorAction, operation: OperationContext): Promise { - return this.acted(operation, this.guard( + return this.acted(operation, this.guardAction( operation, action.kind, (currentOperation) => { @@ -948,7 +967,7 @@ export class PlaywrightSurface { /** One pointer action at a viewport point in CSS pixels, with nothing resolved behind it; see `dispatchPointerAction`. */ performAt(point: ViewportPoint, action: PointerAction, operation: OperationContext): Promise { - return this.acted(operation, this.guard(operation, `${action.kind} at point`, (currentOperation) => { + return this.acted(operation, this.guardAction(operation, `${action.kind} at point`, (currentOperation) => { this.requireSession().requireObservation(); return dispatchPointerAction(this.requirePage(), point, action, currentOperation.signal); }, classifyInputError)); @@ -962,7 +981,7 @@ export class PlaywrightSurface { * scoped by the browser to the whole editing host, so it is opt-in. */ typeText(text: string, options: { readonly replace: boolean }, operation: OperationContext): Promise { - return this.acted(operation, this.guard(operation, 'keyboard.type', async (currentOperation) => { + return this.acted(operation, this.guardAction(operation, 'keyboard.type', async (currentOperation) => { const session = this.requireSession(); session.requireObservation(); const token = session.token(); @@ -986,7 +1005,7 @@ export class PlaywrightSurface { /** Sends one key to whatever holds focus, in the contract's key grammar Playwright shares. */ pressKey(key: string, operation: OperationContext): Promise { - return this.acted(operation, this.guard(operation, 'keyboard.press', () => { + return this.acted(operation, this.guardAction(operation, 'keyboard.press', () => { this.requireSession().requireObservation(); return this.requirePage().keyboard.press(key); }, classifyInputError)); diff --git a/packages/web/tests/integration/agent-dialogs.test.ts b/packages/web/tests/integration/agent-dialogs.test.ts new file mode 100644 index 0000000000..1d847aa242 --- /dev/null +++ b/packages/web/tests/integration/agent-dialogs.test.ts @@ -0,0 +1,182 @@ +/** + * Native dialogs no handler takes, in a real browser: after an agent step's + * own action, live or replayed, the engine answers them the way the action + * asked, so leaving a form with unsaved changes navigates and a confirm goes + * through, and the app log names each one. A dialog belongs to the newest + * action, never to a read: one a test's own step opened still fails the + * next step, as an unhandled dialog always has. + */ + +import { mkdtempSync, rmSync } from 'node:fs'; +import { createServer, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { AppLogEntry, EngineCleanupContext, EngineFixtureContext, OperationContext, SemanticNode } from 'e2e/engine'; +import { createBrowserFixture, type Browser } from '../../src/browser.ts'; +import { PlaywrightSurface } from '../../src/surface.ts'; +import { ignoreTrace, noSecrets } from '../helpers/secrets.ts'; + +const PAGES: Readonly> = { + '/form': `

Edit

`, + '/actions': `

Labels

idle

+ + + + `, + '/later': `

Reports

idle

+ + + `, + '/next': '

Next

', + '/api/save': 'ok', +}; + +const signal = new AbortController().signal; +const cleanup = (): EngineCleanupContext => ({ signal, timeoutMs: 30_000 }); +const operation = (origin: OperationContext['origin']): OperationContext => ({ signal, timeoutMs: 30_000, runId: 'dialogs', attemptId: 'a', origin }); + +/** Every node in document order. */ +function nodes(node: SemanticNode): SemanticNode[] { + return [node, ...(node.children ?? []).flatMap(nodes)]; +} + +describe('native dialogs no handler takes', () => { + let server: Server; + let url: string; + let surface: PlaywrightSurface; + let artifactsDir: string; + let log: AppLogEntry[]; + + beforeAll(async () => { + server = createServer((request, response) => { + const page = request.url === undefined ? undefined : PAGES[request.url]; + response.writeHead(page === undefined ? 404 : 200, { 'content-type': 'text/html' }); + response.end(page ?? ''); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + + /** Opens one attempt, runs `body` in it with the attempt's `browser` fixture, and tears it down. */ + async function attempt(body: (browser: Browser) => Promise): Promise { + surface = new PlaywrightSurface({}); + artifactsDir = mkdtempSync(path.join(tmpdir(), 'e2e-agent-dialogs-')); + log = []; + try { + await surface.init({ runId: 'dialogs', targetName: 'web', projectRoot: artifactsDir, app: { site: '127.0.0.1' }, + env: {}, headed: false, workerSlot: 0, signal, log: () => undefined }); + await surface.startAttempt({ attemptId: 'a', artifactsDir, signal, resolveSecret: noSecrets, ...ignoreTrace, appLog: (entry) => log.push(entry) }); + await body(createBrowserFixture(surface, { + operation: () => operation('test'), + app: { resolveUrl: (target: string) => new URL(target, url).href }, + timeouts: { test: 30_000, assertion: 5_000 }, + expectable: (target: object) => target, + fixture: (_name: string, target: object) => target, + } as unknown as EngineFixtureContext)); + } finally { + await surface.endAttempt(cleanup()).catch(() => undefined); + await surface.dispose(cleanup()); + rmSync(artifactsDir, { recursive: true, force: true }); + } + } + + /** The node `match` names on the current screen, observed as `origin`. */ + async function find(origin: OperationContext['origin'], match: (node: SemanticNode) => boolean): Promise { + const found = nodes((await surface.observe(operation(origin))).root).find(match); + if (found === undefined) throw new Error('no such node on the screen'); + return found; + } + + const button = (name: string) => (node: SemanticNode) => node.role === 'button' && node.name === name; + const status = async (origin: OperationContext['origin']) => + nodes((await surface.observe(operation(origin))).root).find((node) => node.role === 'status')?.text; + + it('lets the agent leave a form with unsaved changes', async () => { + await attempt(async () => { + await surface.open(`${url}/form`, operation('agent')); + const title = await find('agent', (node) => node.role === 'textbox'); + await surface.perform(title.ref, { kind: 'tap' }, operation('agent')); + await surface.perform(title.ref, { kind: 'fill', value: 'Draft', sensitive: false }, operation('agent')); + await surface.open(`${url}/next`, operation('agent')); + expect((await surface.observe(operation('agent'))).location).toBe(`${url}/next`); + expect(log).toEqual([{ source: 'system', level: 'info', text: 'beforeunload dialog accepted for the agent' }]); + }); + }); + + it("answers the agent's confirm and alert, dismisses its prompt, and names each in the app log", async () => { + await attempt(async () => { + await surface.open(`${url}/actions`, operation('agent')); + await surface.perform((await find('agent', button('Delete'))).ref, { kind: 'tap' }, operation('agent')); + expect(await status('agent')).toBe('deleted'); + await surface.perform((await find('agent', button('Rename'))).ref, { kind: 'tap' }, operation('agent')); + expect(await status('agent')).toBe('rename cancelled'); + await surface.perform((await find('agent', button('Save'))).ref, { kind: 'tap' }, operation('agent')); + expect(await status('agent')).toBe('saved'); + expect(log.map((entry) => entry.text)).toEqual([ + 'confirm dialog "Delete this label?" accepted for the agent', + 'prompt dialog "New name?" dismissed for the agent', + 'alert dialog "Saved" accepted for the agent', + ]); + }); + }); + + it('answers a dialog a replayed agent step opens, though the replay acts with the test origin', async () => { + await attempt(async () => { + await surface.open(`${url}/actions`, operation('test')); + const replayed: OperationContext = { ...operation('test'), agentStep: true }; + await surface.perform((await find('test', button('Delete'))).ref, { kind: 'tap' }, replayed); + expect(await status('test')).toBe('deleted'); + expect(log.map((entry) => entry.text)).toEqual(['confirm dialog "Delete this label?" accepted for the agent']); + }); + }); + + it("keeps a dialog the agent's action opened the agent's while a test route handler fetches in between", async () => { + await attempt(async (browser) => { + await browser.route('**/api/save', async (route) => { + const upstream = await route.fetch(); + await route.fulfill({ body: await upstream.text() }); + }); + await surface.open(`${url}/later`, operation('agent')); + await surface.perform((await find('agent', button('Save'))).ref, { kind: 'tap' }, operation('agent')); + await expect.poll(() => status('agent'), { timeout: 5_000 }).toBe('saved'); + expect(log.map((entry) => entry.text)).toEqual(['alert dialog "Saved" accepted for the agent']); + }); + }); + + it("fails a test's next step on a dialog it opened late, though an agent read the page in between", async () => { + await attempt(async () => { + await surface.open(`${url}/later`, operation('test')); + await surface.perform((await find('test', button('Submit later'))).ref, { kind: 'tap' }, operation('test')); + // An agent judgment polling the screen reads; it takes no dialog for the agent. + await surface.observe(operation('agent')); + await expect + .poll(() => surface.observe(operation('agent')).then(() => 'read', (error: { code?: string }) => error.code), { timeout: 5_000 }) + .toBe('INVALID_STATE'); + expect(await status('test')).toBe('kept'); + expect(log).toEqual([]); + }); + }); + + it("fails a test's next step on a dialog its own action opened, and dismisses the dialog", async () => { + await attempt(async () => { + await surface.open(`${url}/actions`, operation('test')); + await surface.perform((await find('test', button('Delete'))).ref, { kind: 'tap' }, operation('test')); + await expect(surface.observe(operation('test'))).rejects.toMatchObject({ + code: 'INVALID_STATE', + message: expect.stringContaining('unhandled confirm dialog: Delete this label?'), + }); + expect(await status('test')).toBe('kept'); + expect(log).toEqual([]); + }); + }); +}); diff --git a/packages/web/tests/unit/dialogs.test.ts b/packages/web/tests/unit/dialogs.test.ts index 47b9784f3a..ede414b5c0 100644 --- a/packages/web/tests/unit/dialogs.test.ts +++ b/packages/web/tests/unit/dialogs.test.ts @@ -4,18 +4,25 @@ */ import type { Dialog as PwDialog } from 'playwright-core'; -import { TestError } from 'e2e/engine'; +import { TestError, type AppLogEntry } from 'e2e/engine'; import { describe, expect, it, vi } from 'vitest'; import { DialogRouter } from '../../src/dialogs.ts'; import { ErrorLatch } from '../../src/support.ts'; -function fakeDialog(text = 'are you sure?') { +function fakeDialog(text = 'are you sure?', type = 'confirm') { const accept = vi.fn(async (_text?: string) => undefined); const dismiss = vi.fn(async () => undefined); - const dialog = { type: () => 'confirm', message: () => text, accept, dismiss } as unknown as PwDialog; + const dialog = { type: () => type, message: () => text, accept, dismiss } as unknown as PwDialog; return { dialog, accept, dismiss }; } +/** A router whose surface reports who acted last, with the app log it tells. */ +function routerFor(actor: 'agent' | 'test') { + const log: AppLogEntry[] = []; + const router = new DialogRouter(new ErrorLatch(), { agentActed: () => actor === 'agent', log: (entry) => log.push(entry) }); + return { router, log }; +} + describe('DialogRouter', () => { it('routes to the newest handler and falls back once it unsubscribes', async () => { const router = new DialogRouter(); @@ -152,6 +159,60 @@ describe('DialogRouter', () => { }); }); +describe('a dialog no handler takes', () => { + it.each([ + ['alert', 'Saved', 'alert dialog "Saved" accepted for the agent'], + ['confirm', 'Delete this label?', 'confirm dialog "Delete this label?" accepted for the agent'], + ['beforeunload', '', 'beforeunload dialog accepted for the agent'], + ] as const)('after an agent operation: accepts the %s, tells the app log, and fails nothing', async (type, text, line) => { + const { router, log } = routerFor('agent'); + const native = fakeDialog(text, type); + await router.dispatch(native.dialog); + expect(native.accept).toHaveBeenCalledOnce(); + expect(native.dismiss).not.toHaveBeenCalled(); + expect(log).toEqual([{ source: 'system', level: 'info', text: line }]); + expect(() => router.throwPending()).not.toThrow(); + }); + + it('after an agent operation: dismisses a prompt, whose text the agent cannot see to answer', async () => { + const { router, log } = routerFor('agent'); + const native = fakeDialog('New name?\n(max 20 characters)', 'prompt'); + await router.dispatch(native.dialog); + expect(native.dismiss).toHaveBeenCalledOnce(); + expect(native.accept).not.toHaveBeenCalled(); + expect(log).toEqual([{ source: 'system', level: 'info', text: 'prompt dialog "New name? (max 20 characters)" dismissed for the agent' }]); + expect(() => router.throwPending()).not.toThrow(); + }); + + it('after a test operation: dismisses it and latches INVALID_STATE, as with no surface to ask', async () => { + const { router, log } = routerFor('test'); + const native = fakeDialog('unexpected'); + await router.dispatch(native.dialog); + expect(native.dismiss).toHaveBeenCalledOnce(); + expect(log).toEqual([]); + expect(() => router.throwPending()).toThrowError(expect.objectContaining({ code: 'INVALID_STATE' })); + }); + + it("leaves a registered handler in charge after an agent operation", async () => { + const { router, log } = routerFor('agent'); + router.add('dismiss'); + const native = fakeDialog('Delete this label?'); + await router.dispatch(native.dialog); + expect(native.dismiss).toHaveBeenCalledOnce(); + expect(native.accept).not.toHaveBeenCalled(); + expect(log).toEqual([]); + }); + + it('tells nothing and fails nothing when the dialog is gone before the agent answer lands', async () => { + const { router, log } = routerFor('agent'); + const native = fakeDialog('Saved', 'alert'); + native.accept.mockRejectedValueOnce(new Error('Target page, context or browser has been closed')); + await router.dispatch(native.dialog); + expect(log).toEqual([]); + expect(() => router.throwPending()).not.toThrow(); + }); +}); + describe('ErrorLatch', () => { const budget = () => ({ timeoutMs: 1_000, signal: new AbortController().signal }); diff --git a/plugins/e2e/skills/e2e/references/mcp.md b/plugins/e2e/skills/e2e/references/mcp.md index b9e95d15d3..e7382d810f 100644 --- a/plugins/e2e/skills/e2e/references/mcp.md +++ b/plugins/e2e/skills/e2e/references/mcp.md @@ -95,6 +95,11 @@ The catalog, per session: | `stop_recording` | Stops it and returns the absolute path of each video file, under `/videos//` (`.e2e` by default), or the URL of a provider's own recording. | | Project tools | Every `defineTool` in the agent's `tools` that applies to the target's platform, under its own name; an engine pack such as `mobileTools` adds `open_app`, `swipe`, `alert`. | +A native dialog a grammar verb opens is answered as in `agent.act` (accepted, +a prompt dismissed), silently: its test line registers no handler, so a test +of that flow needs `browser.onDialog` before the action. A dialog `run_code` +opens fails the next call. + Project tools use their `toModelOutput` renderer for strings as well as structured results. Its `error-text` and `error-json` outputs set the MCP result's `isError: true`; treat them as failed calls. diff --git a/plugins/e2e/skills/e2e/references/writing-tests.md b/plugins/e2e/skills/e2e/references/writing-tests.md index e968ac7cab..e662983e04 100644 --- a/plugins/e2e/skills/e2e/references/writing-tests.md +++ b/plugins/e2e/skills/e2e/references/writing-tests.md @@ -520,7 +520,9 @@ the app opens itself (`target="_blank"`, `window.open`) is not followed: `alert`, `confirm`, `prompt`, or `beforeunload`, `accept` taking the prompt text; no handler, or one that neither accepts nor dismisses, fails the next step with `INVALID_STATE`. A throwing handler's unanswered dialog is - dismissed too, and its error fails the next step. + dismissed too, and its error fails the next step. A dialog an agent step's + own action opens is answered for the agent instead (accepted, a prompt + dismissed); scripted steps still need the handler. - `waitForDownload(() => trigger, { timeout? })`: returns `{ path, absolutePath, suggestedFilename }`; `absolutePath` names the saved runner-side file for reading its contents, including with a hosted browser. diff --git a/skills/e2e/references/mcp.md b/skills/e2e/references/mcp.md index b9e95d15d3..e7382d810f 100644 --- a/skills/e2e/references/mcp.md +++ b/skills/e2e/references/mcp.md @@ -95,6 +95,11 @@ The catalog, per session: | `stop_recording` | Stops it and returns the absolute path of each video file, under `/videos//` (`.e2e` by default), or the URL of a provider's own recording. | | Project tools | Every `defineTool` in the agent's `tools` that applies to the target's platform, under its own name; an engine pack such as `mobileTools` adds `open_app`, `swipe`, `alert`. | +A native dialog a grammar verb opens is answered as in `agent.act` (accepted, +a prompt dismissed), silently: its test line registers no handler, so a test +of that flow needs `browser.onDialog` before the action. A dialog `run_code` +opens fails the next call. + Project tools use their `toModelOutput` renderer for strings as well as structured results. Its `error-text` and `error-json` outputs set the MCP result's `isError: true`; treat them as failed calls. diff --git a/skills/e2e/references/writing-tests.md b/skills/e2e/references/writing-tests.md index e968ac7cab..e662983e04 100644 --- a/skills/e2e/references/writing-tests.md +++ b/skills/e2e/references/writing-tests.md @@ -520,7 +520,9 @@ the app opens itself (`target="_blank"`, `window.open`) is not followed: `alert`, `confirm`, `prompt`, or `beforeunload`, `accept` taking the prompt text; no handler, or one that neither accepts nor dismisses, fails the next step with `INVALID_STATE`. A throwing handler's unanswered dialog is - dismissed too, and its error fails the next step. + dismissed too, and its error fails the next step. A dialog an agent step's + own action opens is answered for the agent instead (accepted, a prompt + dismissed); scripted steps still need the handler. - `waitForDownload(() => trigger, { timeout? })`: returns `{ path, absolutePath, suggestedFilename }`; `absolutePath` names the saved runner-side file for reading its contents, including with a hosted browser. From 6823ff8512b5ce8c06f612111fa2b8356c6db929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 6 Nov 2026 22:43:32 +0100 Subject: [PATCH 2/4] fix(web): take dialog ownership when an action runs, and on viewport and media changes An action guard refuses before it runs no longer claims the next dialog, and browser.setViewport and browser.emulateMedia claim it like other page changes. Docs: an unhandled dialog fails the attempt at its end when no step follows, a run_code dialog fails the next statement first, and the skill warns that agent steps accept confirms unseen without a handler. --- docs/reference/mcp.mdx | 3 +- docs/reference/web.mdx | 5 ++- packages/web/src/browser.ts | 4 +- packages/web/src/surface.ts | 17 ++++--- .../tests/integration/agent-dialogs.test.ts | 44 +++++++++++++++++++ plugins/e2e/skills/e2e/references/mcp.md | 5 ++- .../skills/e2e/references/writing-tests.md | 8 ++-- skills/e2e/references/mcp.md | 5 ++- skills/e2e/references/writing-tests.md | 8 ++-- 9 files changed, 78 insertions(+), 21 deletions(-) diff --git a/docs/reference/mcp.mdx b/docs/reference/mcp.mdx index e11b126c4e..9d87da5458 100644 --- a/docs/reference/mcp.mdx +++ b/docs/reference/mcp.mdx @@ -278,7 +278,8 @@ changed on screen. confirm, and a leave-page dialog are accepted, and a prompt is dismissed. The result doesn't say so, and the test line registers no handler, so a test of that flow needs `browser.onDialog` first. `run_code` runs test - statements, so a dialog it opens fails the next call. + statements, so a dialog one opens fails the next statement, or the next + call when none follows. #### `dismiss_keyboard` diff --git a/docs/reference/web.mdx b/docs/reference/web.mdx index 6b544c33b1..998b5a56e2 100644 --- a/docs/reference/web.mdx +++ b/docs/reference/web.mdx @@ -857,8 +857,9 @@ await off(); `Dialog`. - A function handler must call `accept` or `dismiss`. One that does neither has the dialog dismissed, as an unhandled dialog does. -- A dialog no handler takes is dismissed and fails the next step, unless an - agent step's own action opened it, replayed from the cache or not. Then +- A dialog no handler takes is dismissed and fails the next step, or the + attempt at its end when no step follows, unless an agent step's own action + opened it, replayed from the cache or not. Then it's answered for the agent: an alert, a confirm, and a leave-page dialog are accepted, and a prompt is dismissed. The app log names each one. A dialog belongs to the newest action before it, never to a read. diff --git a/packages/web/src/browser.ts b/packages/web/src/browser.ts index 2edf857568..f3bf26c81b 100644 --- a/packages/web/src/browser.ts +++ b/packages/web/src/browser.ts @@ -743,7 +743,7 @@ export function createBrowserFixture(surface: PlaywrightSurface, context: Engine return surface.guard(context.operation(), 'clearCookies', () => surface.requireContext().clearCookies(options)); }, setViewport: (size) => - surface.guard(context.operation(), 'setViewport', async () => { + surface.guardAction(context.operation(), 'setViewport', async () => { await surface.setViewport(size); context.attachViewport({ width: size.width, height: size.height, scale: 1 }); }), @@ -758,7 +758,7 @@ export function createBrowserFixture(surface: PlaywrightSurface, context: Engine } // Read once, so a test that changes its object later changes nothing here. const requested = obj({ colorScheme: media.colorScheme, reducedMotion: media.reducedMotion }); - await surface.guard(context.operation(), 'emulateMedia', () => surface.emulateMedia(requested)); + await surface.guardAction(context.operation(), 'emulateMedia', () => surface.emulateMedia(requested)); }, // Async so the harness records the registration as a `browser.onDialog` step. async onDialog(handler: DialogHandler) { diff --git a/packages/web/src/surface.ts b/packages/web/src/surface.ts index 76d6f2cc59..4905fc2cee 100644 --- a/packages/web/src/surface.ts +++ b/packages/web/src/surface.ts @@ -718,10 +718,12 @@ export class PlaywrightSurface { } /** - * `guard` for an operation that acts on the page (a navigation, input, or - * a script) rather than reading it. Its author owns the dialogs that open - * after it until the next action: one no handler takes is answered when - * the author is an agent step, the step replayed from the cache included. + * `guard` for an operation that acts on the page (a navigation, input, a + * script, or a viewport or media change) rather than reading it. Once it + * runs, its author owns the dialogs that open after it until the next + * action: one no handler takes is answered when the author is an agent + * step, the step replayed from the cache included. An operation `guard` + * refuses before it runs takes nothing. */ guardAction( operation: OperationContext, @@ -729,8 +731,11 @@ export class PlaywrightSurface { fn: (operation: OperationContext) => Promise, translate?: (cause: unknown, label: string) => Error, ): Promise { - this.actor = (operation.agentStep ?? operation.origin === 'agent') ? 'agent' : 'test'; - return this.guard(operation, label, fn, translate); + const actor = (operation.agentStep ?? operation.origin === 'agent') ? 'agent' : 'test'; + return this.guard(operation, label, (current) => { + this.actor = actor; + return fn(current); + }, translate); } // --- steering: the session hooks --- diff --git a/packages/web/tests/integration/agent-dialogs.test.ts b/packages/web/tests/integration/agent-dialogs.test.ts index 1d847aa242..8195254083 100644 --- a/packages/web/tests/integration/agent-dialogs.test.ts +++ b/packages/web/tests/integration/agent-dialogs.test.ts @@ -33,6 +33,15 @@ const PAGES: Readonly> = { `, + '/sync': `

Sync

idle

+ + + + `, '/next': '

Next

', '/api/save': 'ok', }; @@ -82,6 +91,7 @@ describe('native dialogs no handler takes', () => { timeouts: { test: 30_000, assertion: 5_000 }, expectable: (target: object) => target, fixture: (_name: string, target: object) => target, + attachViewport: () => undefined, } as unknown as EngineFixtureContext)); } finally { await surface.endAttempt(cleanup()).catch(() => undefined); @@ -153,6 +163,40 @@ describe('native dialogs no handler takes', () => { }); }); + it("fails a test's next step on a dialog its viewport change opened, though an agent acted before it", async () => { + await attempt(async (browser) => { + await surface.open(`${url}/sync`, operation('agent')); + await surface.perform((await find('agent', button('Mark'))).ref, { kind: 'tap' }, operation('agent')); + await browser.setViewport({ width: 500, height: 600 }); + await expect + .poll(() => surface.observe(operation('agent')).then(() => 'read', (error: { code?: string }) => error.code), { timeout: 5_000 }) + .toBe('INVALID_STATE'); + expect(await status('test')).toBe('reverted'); + expect(log).toEqual([]); + }); + }); + + it('leaves a dialog with the agent when the next operation is refused before it runs', async () => { + await attempt(async (browser) => { + let failed!: () => void; + const routed = new Promise((resolve) => { + failed = resolve; + }); + await browser.route('**/api/fail', () => { + failed(); + throw new Error('route handler failed on purpose'); + }); + await surface.open(`${url}/sync`, operation('agent')); + await surface.perform((await find('agent', button('Sync'))).ref, { kind: 'tap' }, operation('agent')); + await routed; + await new Promise((resolve) => setTimeout(resolve, 50)); + // The handler's failure is pending, so this test action is refused before it runs and takes no dialog. + await expect(browser.evaluate('1')).rejects.toThrow(/route handler failed on purpose/); + await expect.poll(() => status('agent'), { timeout: 5_000 }).toBe('synced'); + expect(log.filter((entry) => entry.source === 'system').map((entry) => entry.text)).toEqual(['alert dialog "Synced" accepted for the agent']); + }); + }); + it("fails a test's next step on a dialog it opened late, though an agent read the page in between", async () => { await attempt(async () => { await surface.open(`${url}/later`, operation('test')); diff --git a/plugins/e2e/skills/e2e/references/mcp.md b/plugins/e2e/skills/e2e/references/mcp.md index e7382d810f..4de7c3db24 100644 --- a/plugins/e2e/skills/e2e/references/mcp.md +++ b/plugins/e2e/skills/e2e/references/mcp.md @@ -97,8 +97,9 @@ The catalog, per session: A native dialog a grammar verb opens is answered as in `agent.act` (accepted, a prompt dismissed), silently: its test line registers no handler, so a test -of that flow needs `browser.onDialog` before the action. A dialog `run_code` -opens fails the next call. +of that flow needs `browser.onDialog` before the action. A dialog a +`run_code` statement opens fails the next statement, or the next call when +none follows. Project tools use their `toModelOutput` renderer for strings as well as structured results. Its `error-text` and `error-json` outputs set the MCP diff --git a/plugins/e2e/skills/e2e/references/writing-tests.md b/plugins/e2e/skills/e2e/references/writing-tests.md index e662983e04..a2be121390 100644 --- a/plugins/e2e/skills/e2e/references/writing-tests.md +++ b/plugins/e2e/skills/e2e/references/writing-tests.md @@ -520,9 +520,11 @@ the app opens itself (`target="_blank"`, `window.open`) is not followed: `alert`, `confirm`, `prompt`, or `beforeunload`, `accept` taking the prompt text; no handler, or one that neither accepts nor dismisses, fails the next step with `INVALID_STATE`. A throwing handler's unanswered dialog is - dismissed too, and its error fails the next step. A dialog an agent step's - own action opens is answered for the agent instead (accepted, a prompt - dismissed); scripted steps still need the handler. + dismissed too, and its error fails the next step. With no handler, a + dialog an agent step's own action opens is answered for the agent + instead: a prompt is dismissed, the rest accepted unseen, so a destructive + confirm goes through. Register `onDialog('dismiss')` first to stop that; + scripted steps still need a handler. - `waitForDownload(() => trigger, { timeout? })`: returns `{ path, absolutePath, suggestedFilename }`; `absolutePath` names the saved runner-side file for reading its contents, including with a hosted browser. diff --git a/skills/e2e/references/mcp.md b/skills/e2e/references/mcp.md index e7382d810f..4de7c3db24 100644 --- a/skills/e2e/references/mcp.md +++ b/skills/e2e/references/mcp.md @@ -97,8 +97,9 @@ The catalog, per session: A native dialog a grammar verb opens is answered as in `agent.act` (accepted, a prompt dismissed), silently: its test line registers no handler, so a test -of that flow needs `browser.onDialog` before the action. A dialog `run_code` -opens fails the next call. +of that flow needs `browser.onDialog` before the action. A dialog a +`run_code` statement opens fails the next statement, or the next call when +none follows. Project tools use their `toModelOutput` renderer for strings as well as structured results. Its `error-text` and `error-json` outputs set the MCP diff --git a/skills/e2e/references/writing-tests.md b/skills/e2e/references/writing-tests.md index e662983e04..a2be121390 100644 --- a/skills/e2e/references/writing-tests.md +++ b/skills/e2e/references/writing-tests.md @@ -520,9 +520,11 @@ the app opens itself (`target="_blank"`, `window.open`) is not followed: `alert`, `confirm`, `prompt`, or `beforeunload`, `accept` taking the prompt text; no handler, or one that neither accepts nor dismisses, fails the next step with `INVALID_STATE`. A throwing handler's unanswered dialog is - dismissed too, and its error fails the next step. A dialog an agent step's - own action opens is answered for the agent instead (accepted, a prompt - dismissed); scripted steps still need the handler. + dismissed too, and its error fails the next step. With no handler, a + dialog an agent step's own action opens is answered for the agent + instead: a prompt is dismissed, the rest accepted unseen, so a destructive + confirm goes through. Register `onDialog('dismiss')` first to stop that; + scripted steps still need a handler. - `waitForDownload(() => trigger, { timeout? })`: returns `{ path, absolutePath, suggestedFilename }`; `absolutePath` names the saved runner-side file for reading its contents, including with a hosted browser. From 28a8ceafe4ca176b63b7377e94d01c706733edd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Wed, 11 Nov 2026 19:25:21 +0100 Subject: [PATCH 3/4] test(web): remove deferred dialog recording per review --- ...c4c71e08881c91c6b9c8deaa74a32431785e5.json | 63 ------------------- 1 file changed, 63 deletions(-) delete mode 100644 apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json diff --git a/apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json b/apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json deleted file mode 100644 index e004345098..0000000000 --- a/apps/web-benchmark/.e2e/cache/4e30b5754d86020de8d9df735efc4c71e08881c91c6b9c8deaa74a32431785e5.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "schemaVersion": "trace-1", - "createdAt": "2026-11-06T21:09:07.253Z", - "payload": { - "actions": [ - { - "name": "tap", - "summary": "tap button \"Submit report\"", - "target": { - "role": "button", - "name": "Submit report", - "testId": "submit-report-button" - } - } - ], - "executor": { - "name": "e2e-default-agent", - "version": "2" - }, - "recordedFor": { - "testId": "tests-agent/scenarios.e2e.ts::act%20completes%20deferred-dialog", - "targetId": "web", - "instructionDigest": "0f97cef062579bf174a4272ebd0c9023c90355e5341388a94c5bf29f99c1bd37", - "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "callIndex": 0, - "agent": "default" - }, - "recordedBy": { - "provider": "gateway", - "model": "openai/gpt-6-luna-fast" - }, - "summary": "Submitted the report and waited for the deferred confirmation; the page now shows “Report submitted after deferred confirmation.”", - "startPath": "/e/deferred-dialog", - "endPath": "/e/deferred-dialog", - "endAnchors": [ - { - "text": "Report submitted after deferred confirmation", - "testId": "success-message" - } - ], - "goneAnchors": [ - { - "text": "Submit the report; a confirmation dialog appears shortly after the click. Accept it." - }, - { - "role": "button", - "name": "Submit report", - "testId": "submit-report-button" - } - ], - "endWaitMs": 13352, - "keyedBy": { - "cacheSchema": "trace-1", - "policyVersion": "conservative/6", - "project": "6c340b4558007fd12144b6a4e0daf3abb724b92faa2d08705fa564444b68533c", - "platform": "web", - "engineName": "web", - "engineSpiVersion": 1, - "appIdentity": "dc587874e98340c0ead3b9555bef25c85bca9337c6c8e28ae9e991c982ef1202", - "agentContextDigest": "02e45116d4a18a6603260cbbc8fa198d311b6a1a01362cc9dd6aa341d16bb3cf" - } - } -} From 82d53467a9a26ee8acdb61464d908ead6a01dabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Thu, 12 Nov 2026 09:45:11 +0100 Subject: [PATCH 4/4] fix: document agent fallback and stabilize refusal assertion --- docs/reference/engine.mdx | 1 + packages/e2e/src/engine/contract.ts | 1 + packages/web/tests/integration/check-actions.test.ts | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 16e5f94f19..15e954706c 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -346,6 +346,7 @@ export interface OperationContext { * the replay cache repeats a choice it made in an earlier run, whose * `origin` is then `test`. An engine answers for the agent what the agent * cannot see, such as a native dialog its action opens. + * When absent, use `origin === 'agent'`: older cores send only `origin`. */ readonly agentStep?: boolean; } diff --git a/packages/e2e/src/engine/contract.ts b/packages/e2e/src/engine/contract.ts index a2c38b4c0d..25472ab097 100644 --- a/packages/e2e/src/engine/contract.ts +++ b/packages/e2e/src/engine/contract.ts @@ -55,6 +55,7 @@ export interface OperationContext { * the replay cache repeats a choice it made in an earlier run, whose * `origin` is then `test`. An engine answers for the agent what the agent * cannot see, such as a native dialog its action opens. + * When absent, use `origin === 'agent'`: older cores send only `origin`. */ readonly agentStep?: boolean; } diff --git a/packages/web/tests/integration/check-actions.test.ts b/packages/web/tests/integration/check-actions.test.ts index f792defbdc..e82acefd55 100644 --- a/packages/web/tests/integration/check-actions.test.ts +++ b/packages/web/tests/integration/check-actions.test.ts @@ -358,10 +358,12 @@ describe.each(KINDS)('a control its own label covers, through the %s target', (k await page.setContent(` `); + // A short deadline can expire before Playwright logs the visibility blocker. + // Assert the refusal and its effects rather than that timing-dependent diagnostic. for (const action of [{ kind: 'check' }, { kind: 'tap' }] as const) { expect(await perform(await targetOf(kind, '#c'), action, REFUSAL_TIMEOUT_MS)).toMatchObject({ code: 'NOT_ACTIONABLE', - message: expect.stringContaining('not visible'), + retryable: false, }); expect(await page.isChecked('#c')).toBe(false); expect(page.url()).not.toContain('#went');