diff --git a/.changeset/bump-deps-2026-11-14.md b/.changeset/bump-deps-2026-11-14.md index 4ed3199c0c..e6b9181322 100644 --- a/.changeset/bump-deps-2026-11-14.md +++ b/.changeset/bump-deps-2026-11-14.md @@ -1,6 +1,5 @@ --- "e2e": patch -"@e2e-dev/web": patch "@e2e-dev/smol": patch --- diff --git a/.changeset/exact-operation-timeout.md b/.changeset/exact-operation-timeout.md new file mode 100644 index 0000000000..be293b2c65 --- /dev/null +++ b/.changeset/exact-operation-timeout.md @@ -0,0 +1,6 @@ +--- +"e2e": patch +"@e2e-dev/core": patch +--- + +A step's own `timeout` reaches the engine exactly, so an error that names it, such as a download wait's, no longer reads a millisecond short. diff --git a/.changeset/init-web-hint.md b/.changeset/init-web-hint.md new file mode 100644 index 0000000000..04e2c98a15 --- /dev/null +++ b/.changeset/init-web-hint.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +`e2e init` labels the web engine `Chromium, Firefox` instead of `Playwright`. diff --git a/.changeset/resolve-label-hook.md b/.changeset/resolve-label-hook.md new file mode 100644 index 0000000000..47e75bb231 --- /dev/null +++ b/.changeset/resolve-label-hook.md @@ -0,0 +1,6 @@ +--- +"e2e": patch +"@e2e-dev/core": patch +--- + +`resolveExpression` takes `matches`, `hasText`, and `order` options, so an engine whose platform reads more than the tree carries (a browser's text, hidden, and state rules, and its match order) decides queries, text filters, and the order of matches by its own rules. diff --git a/.changeset/scroll-until-load-pause.md b/.changeset/scroll-until-load-pause.md new file mode 100644 index 0000000000..e50739587d --- /dev/null +++ b/.changeset/scroll-until-load-pause.md @@ -0,0 +1,6 @@ +--- +"e2e": patch +"@e2e-dev/core": patch +--- + +An agent scrolling to a text no longer gives up on a list that pauses while it loads its next page: the list counts as ended only once the screen stays unchanged for a second as well as three pages. diff --git a/.changeset/smol-prepare-browser.md b/.changeset/smol-prepare-browser.md new file mode 100644 index 0000000000..b89dac4477 --- /dev/null +++ b/.changeset/smol-prepare-browser.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/smol": patch +--- + +`prepare` gets a `browser` and `screen` on the warm browser's open tab as its second argument, so a sign-in reads like a test: `prepare: async (_cdpEndpoint, { browser, screen }) => { ... }`. smol detaches them once `prepare` resolves. The DevTools endpoint stays the first argument. Those calls may take 120 seconds in all; `prepareTimeout` raises it. Requires `@e2e-dev/web` 0.17.0 or newer. diff --git a/.changeset/vision-only-wait-pixels.md b/.changeset/vision-only-wait-pixels.md new file mode 100644 index 0000000000..7cb1dfc69b --- /dev/null +++ b/.changeset/vision-only-wait-pixels.md @@ -0,0 +1,6 @@ +--- +"e2e": patch +"@e2e-dev/core": patch +--- + +`agent.waitFor` with `vision: 'only'` compares only the screenshot between judgments, the one thing the judge is shown, so a screen read that fell back to pixels alone no longer costs another model call. diff --git a/.changeset/web-attach-browser.md b/.changeset/web-attach-browser.md new file mode 100644 index 0000000000..99c679abea --- /dev/null +++ b/.changeset/web-attach-browser.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/web": patch +--- + +Add `attachBrowser(cdpEndpoint, options?)`: `browser` and `screen` on a running Chromium's open tab in its default context, outside a test. `close()` detaches and leaves the browser, the tab, and its cookies and storage as they were. diff --git a/.changeset/web-bidi-only.md b/.changeset/web-bidi-only.md new file mode 100644 index 0000000000..31c2999cb6 --- /dev/null +++ b/.changeset/web-bidi-only.md @@ -0,0 +1,18 @@ +--- +"@e2e-dev/web": minor +--- + +**Breaking:** the web engine drives Chromium and Firefox over WebDriver BiDi and no longer uses Playwright. Tests, fixtures, and `web()` options stay the same, except: + +- WebKit is gone: `web({ browser: 'webkit' })` is `INVALID_CONFIG`. +- `surfaceOf` and `PlaywrightLiveSurface` are removed, so there is no raw Playwright `Page` or `BrowserContext` to reach for. +- `browser.locator` takes CSS and XPath, alone or in `>>` chains: no `text=`, `nth=`, or other Playwright engines. Chain `nth()` and screen queries instead. +- `connect.reconnectEndpoint` and `BrowserLease.reconnectEndpoint` are removed, and `reconnectEndpoint` in `connect` is `INVALID_CONFIG`: a dropped connection fails the attempt, and the next attempt attaches again through `cdpEndpoint`. +- With a `scope: 'attempt'` browser provider, `app.clearState()` and saved sessions throw `UNSUPPORTED_CAPABILITY`: the attempt runs in the fresh browser's own context. +- `setInputFiles` on a browser provider's browser sends the files' bytes to the page, up to 50 MB in all; a directory there throws `UNSUPPORTED_CAPABILITY`, and so does a download on a provider without `downloads`. +- On Firefox, `browser.emulateMedia({ reducedMotion })` (set `web({ reducedMotion })` instead), `route.continue({ url })`, and a drag that starts an HTML5 drag throw `UNSUPPORTED_CAPABILITY`. +- Browsers are new pinned builds: Chrome for Testing 157.0.8103.38 (the headless shell, or full Chrome when headed) and Firefox 159.0.1. They install into `E2E_BROWSERS_PATH`, else `e2e/browsers` in the user cache, with `npx @e2e-dev/web install [chromium|firefox] [--with-deps] [--headed]`, or on the first run. `PLAYWRIGHT_BROWSERS_PATH` is no longer read. +- Chromium video needs an ffmpeg with VP9: `E2E_FFMPEG_PATH`, else `ffmpeg` on the `PATH`, else a pinned build the first recording downloads into the browser cache (`npx @e2e-dev/web install ffmpeg` fetches it ahead). A run that records no video never downloads it. Windows on Arm has no pinned build: set `E2E_FFMPEG_PATH` there. +- The old `ms-playwright` browser cache is no longer used and can be deleted. +- Every browser and ffmpeg archive is checked against a pinned SHA-256 before it is unpacked. `E2E_BROWSERS_DOWNLOAD_HOST` names an `https` mirror (plain `http` only on `localhost`) serving the Chrome for Testing, Mozilla archive, and ffmpeg-static paths. Downloads honor `HTTPS_PROXY`, `HTTP_PROXY`, and `NO_PROXY` (on Node 22, with `NODE_USE_ENV_PROXY=1`). +- Concurrent installs into one cache wait for each other and say so; one whose owner died or stopped responding is taken over. Ctrl-C or SIGTERM during `e2e-web install` stops the download, removes the partial files, and exits 130 or 143. On Linux, `--with-deps` also installs `xz-utils`, which unpacking Firefox needs. diff --git a/.dev/skills/babysit/references/review-triage.md b/.dev/skills/babysit/references/review-triage.md index 56baefa479..8ec0c8040c 100644 --- a/.dev/skills/babysit/references/review-triage.md +++ b/.dev/skills/babysit/references/review-triage.md @@ -27,8 +27,8 @@ Reply with the one fact that disproves it, then resolve. Common shapes: - **Pre-existing.** `git blame` or `git log -L` shows the flagged lines predate the branch. A real, cheap issue gets a follow-up, not scope creep. -- **Misread library semantics.** Cite the library source or docs (Playwright, - the AI SDK, agent-device, node). +- **Misread library semantics.** Cite the library source or docs (the + WebDriver BiDi spec, chromium-bidi, the AI SDK, agent-device, node). - **Already guaranteed.** Name the shared helper, type, closed schema, or `e2e/engine` contract that makes it impossible. - **Intentional tradeoff.** Link the PR body line, the commit body, or the @@ -83,11 +83,11 @@ before any reviewer does. symlinks on Windows. 12. A new build input or benchmark dependency missing from `.github/filters.yml`, so the benchmark suites skip a change they cover. -13. Core naming an engine noun (`Web`, `browser`, `page`, `route`, - `playwright`) outside the `init` presets. +13. Core naming an engine noun (`Web`, `browser`, `page`, `route`, `bidi`) + outside the `init` presets. 14. A scripted rename of our API spelling (a fixture name, a call shape) that also rewrote another library's code: a Playwright `page.getByRole` - in a migration sample, a raw `Page` from `surfaceOf`. Grep the diff for + in a migration sample. Grep the diff for the old spelling on receivers that are not ours. 15. Negative-only security tests: dropping a result or reason makes absence and size checks pass. Assert the expected redacted value exists before diff --git a/.dev/skills/verify/SKILL.md b/.dev/skills/verify/SKILL.md index 4d5653ed36..613252f01b 100644 --- a/.dev/skills/verify/SKILL.md +++ b/.dev/skills/verify/SKILL.md @@ -18,8 +18,8 @@ repeat. ```bash pnpm install --frozen-lockfile -pnpm --filter @e2e-dev/web exec playwright-core install chromium pnpm build # every consumer below runs dist; rebuild after each change +node packages/web/dist/bin.js install chromium firefox ``` A stale `dist` is the most common false result. Rebuild before trusting any diff --git a/.dev/skills/writing-pr/SKILL.md b/.dev/skills/writing-pr/SKILL.md index 6745cbe61a..5d227ed097 100644 --- a/.dev/skills/writing-pr/SKILL.md +++ b/.dev/skills/writing-pr/SKILL.md @@ -13,7 +13,7 @@ diagrams and code carry the detail. ## Title - Conventional Commits, matching `git log`: - `fix(playwright): provision the browser in a runner-side prepare hook`. + `fix(web): provision the browser in a runner-side prepare hook`. PRs squash-merge with the number appended, so the title is the commit. - Mark spec or API breaks with `!` (`feat(config)!: ...`), same as the commit. - Name the outcome, not the activity. "install builds with the appPath diff --git a/.github/actions/chromium/action.yml b/.github/actions/chromium/action.yml index ce9cb4eefc..d8f94afecf 100644 --- a/.github/actions/chromium/action.yml +++ b/.github/actions/chromium/action.yml @@ -1,7 +1,8 @@ name: chromium description: >- - Installs Playwright's Chromium and its system packages, with the .deb files - apt downloads for them restored from and saved to the Actions cache. + Installs the Chrome for Testing build @e2e-dev/web pins and its system + packages, with the .deb files apt downloads for them restored from and + saved to the Actions cache. # `--with-deps` installs about 38 MB of packages from the Ubuntu mirror, and # the mirror often serves them at about 200 kB/s: the step takes 2 to 5 @@ -10,14 +11,14 @@ description: >- # the packages whose version changed since the cache was saved. inputs: only-shell: - description: Install only the headless shell, not the full browser. + description: Install only the headless shell, not also the full browser a headed run launches. default: 'true' runs: using: composite steps: # The prefix holds what decides the package list: the Ubuntu release and - # the Playwright version. The suffix is a digest of the .deb file names, + # the Chrome build. The suffix is a digest of the .deb file names, # so a key names one exact set of package versions. # The directory sits under /var/cache, not the runner's temp directory: # apt downloads as `_apt`, which cannot traverse /home/runner and would @@ -32,13 +33,14 @@ runs: run: | set -euo pipefail . /etc/os-release - PLAYWRIGHT="$(node -p "require('./packages/web/package.json').dependencies['playwright-core']")" + CHROME="$(sed -n "s/^export const CHROMIUM_BUILD = '\(.*\)';$/\1/p" packages/web/src/install/provision.ts)" + [ -n "$CHROME" ] || { echo "::error::CHROMIUM_BUILD not found in packages/web/src/install/provision.ts"; exit 1; } sudo install -d -o "$(id -u)" -g "$(id -g)" -m 755 "$ARCHIVES" sudo install -d -o _apt -g root -m 700 "$ARCHIVES/partial" printf 'Dir::Cache::Archives "%s/";\nAPT::Keep-Downloaded-Packages "true";\n' "$ARCHIVES" \ | sudo tee /etc/apt/apt.conf.d/99chromium-archives > /dev/null echo "archives=$ARCHIVES" >> "$GITHUB_OUTPUT" - echo "prefix=chromium-apt-$ID-$VERSION_ID-$(uname -m)-playwright-$PLAYWRIGHT" >> "$GITHUB_OUTPUT" + echo "prefix=chromium-apt-$ID-$VERSION_ID-$(uname -m)-chrome-$CHROME" >> "$GITHUB_OUTPUT" # No entry has exactly this key: restore-keys picks the newest entry with # the prefix. - name: Restore the archives @@ -48,11 +50,24 @@ runs: path: ${{ steps.apt.outputs.archives }}/*.deb key: ${{ steps.apt.outputs.prefix }} restore-keys: ${{ steps.apt.outputs.prefix }}- + # The install command is the engine's own CLI, so it needs the built + # package. A job that builds first skips this step. `e2e` and the engine + # are each other's dev dependencies, so a filtered build would order them + # arbitrarily, and the engine needs `e2e`'s types. + - name: Build the web engine + shell: bash + run: | + set -euo pipefail + [ -f packages/web/dist/bin.js ] || { pnpm --filter @e2e-dev/core run build && pnpm --filter e2e run build && pnpm --filter @e2e-dev/web run build; } + # The install runs apt-get through sudo itself. - name: Install Chromium shell: bash env: - ONLY_SHELL: ${{ inputs.only-shell == 'true' && '--only-shell' || '' }} - run: pnpm --filter @e2e-dev/web exec playwright-core install chromium --with-deps $ONLY_SHELL + ONLY_SHELL: ${{ inputs.only-shell }} + run: | + set -euo pipefail + node packages/web/dist/bin.js install chromium --with-deps + [ "$ONLY_SHELL" = 'true' ] || node packages/web/dist/bin.js install chromium --with-deps --headed # autoclean deletes the archives the mirror no longer serves, so the set # changes only when a package version does. - name: Name the archive set diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index 4f16e23388..cfb8f3dbd3 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -109,10 +109,10 @@ jobs: - uses: ./.github/actions/workspace-install with: agent-device-version: ${{ inputs.agent-device-version }} - - name: Install browsers - uses: ./.github/actions/chromium - name: Build run: pnpm run build + - name: Install browsers + uses: ./.github/actions/chromium - name: Run ${{ matrix.script }} continue-on-error: ${{ matrix.optional }} env: diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c228dbd097..4608a19433 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -77,10 +77,10 @@ jobs: - uses: ./.github/actions/workspace-install with: agent-device-version: ${{ inputs.agent-device-version }} - - name: Install browsers - uses: ./.github/actions/chromium - name: Build run: pnpm run build + - name: Install browsers + uses: ./.github/actions/chromium - name: Run test env: # The reporter posts the comment with the job's token. @@ -144,10 +144,10 @@ jobs: - uses: ./.github/actions/workspace-install with: agent-device-version: ${{ inputs.agent-device-version }} - - name: Install browsers - uses: ./.github/actions/chromium - name: Build run: pnpm run build + - name: Install browsers + uses: ./.github/actions/chromium - name: Run test:agent env: AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 6c8a225cfb..09e8a8d506 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -124,11 +124,9 @@ jobs: with: working-directory: ${{ matrix.tests }} agent-device-version: ${{ inputs.agent-device-version }} - # The runner downloads its own Chromium on first use; the system - # libraries it links against need root. - - name: Chromium system dependencies + - name: Chromium and its system dependencies working-directory: ${{ matrix.tests }} - run: npx playwright-core install-deps chromium + run: npx --no-install e2e-web install chromium --with-deps - name: Run the tests id: test working-directory: ${{ matrix.tests }} diff --git a/.github/workflows/spec.yml b/.github/workflows/spec.yml index ebda7af760..00af48f5f7 100644 --- a/.github/workflows/spec.yml +++ b/.github/workflows/spec.yml @@ -118,6 +118,9 @@ jobs: uses: ./.github/actions/chromium with: only-shell: false + # @e2e-dev/web's integration suite, locator parity included, runs on Firefox too. + - name: Install Firefox + run: node packages/web/dist/bin.js install firefox --with-deps - name: Unit and integration tests run: pnpm test @@ -176,11 +179,69 @@ jobs: AGENT_DEVICE_VERSION: ${{ inputs.agent-device-version }} run: node scripts/install-smoke.ts ${{ matrix.pnpm }} + # The testbed's own tests on Firefox. The tests ported from Playwright + # (tag playwright) assert Chromium's errors and run on Chromium in `check`. + firefox: + name: testbed (firefox) + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@9fd676a19091d4595eefd76e4bd31c97133911f1 # v4.2.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 26 + cache: pnpm + - uses: ./.github/actions/workspace-install + with: + agent-device-version: ${{ inputs.agent-device-version }} + - name: Build + run: pnpm run build + - name: Install Firefox + run: node packages/web/dist/bin.js install firefox --with-deps + - name: Testbed on Firefox + working-directory: apps/testbed + run: node node_modules/e2e/dist/cli/bin.js run --config e2e.firefox.config.ts --exclude-tag playwright + - name: Upload report + if: ${{ !cancelled() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: testbed-firefox + path: | + apps/testbed/.e2e/report.json + apps/testbed/.e2e/results + if-no-files-found: warn + retention-days: 7 + + # The web engine on Windows: its own installer, the watchdog that ends the + # browser's process tree, and a launch and attempt on each browser. + windows: + name: web (windows) + runs-on: windows-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@9fd676a19091d4595eefd76e4bd31c97133911f1 # v4.2.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 26 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm --filter @e2e-dev/core run build && pnpm --filter e2e run build && pnpm --filter @e2e-dev/web run build + - name: Install Chromium and Firefox + run: node packages/web/dist/bin.js install chromium firefox + # A Firefox launch takes 8 to 17 s on the Windows runner, so a test that + # launches twice needs more than the suite's 30 s. + - name: Launch, attempt, and close on each browser + working-directory: packages/web + run: npx vitest run --testTimeout=90000 --hookTimeout=90000 tests/integration/lifecycle.test.ts tests/integration/keyboard.test.ts tests/integration/tabs.test.ts + # The one check the ruleset requires from this workflow. A skipped job # passes (the change cannot reach it); a failed or cancelled one fails it. gate: name: spec gate - needs: [check, test, bun, install] + needs: [check, test, bun, install, firefox, windows] if: ${{ always() }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 5 diff --git a/AGENTS.md b/AGENTS.md index 99267b07ac..adcf061ef8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -158,14 +158,17 @@ suites that consume the built packages the way a user would. (peer). Each worker slot boots one warm browser machine from the pinned Playwright Ubuntu image; a Node TCP relay exposes DevTools, and SDK agent readiness allows machine startup without waiting for published ports. In - `attempt` scope every attempt runs in a copy-on-write branch. With `app`, + `attempt` scope (the provider's default, which `@e2e-dev/web` rejects + with `INVALID_CONFIG` for now) every attempt runs in a copy-on-write + branch. With `app`, the app under test runs in the same machine, so a branch also copies its running app and data on the machine. When maintaining this integration, update the provider's SDK dev dependency to a published version and raise its peer minimum only if the provider needs new SDK behavior. Keep the - Playwright image and Chromium version in sync with `@e2e-dev/web`. Run + image's Chromium close to the build `@e2e-dev/web` pins (`CHROMIUM_BUILD` + in `packages/web/src/install/provision.ts`). Run `pnpm --filter @e2e-dev/smol test:unit` and `pnpm check`, then verify live - machines in both attempt and worker scopes with an app running in the VM. + machines in worker scope with an app running in the VM. - `packages/limrun` — the Limrun hosted mobile device provider. Each worker leases one remote device, installs its app, and releases the device at exit. - `packages/decision` — the published `@e2e-dev/decision` package: a @@ -385,8 +388,8 @@ pnpm --filter @e2e-dev/testbed run test:headed - `pnpm typecheck` runs `build` first, then per-package `typecheck`. The package `typecheck` covers `tests/**`, which is what makes `tests/types/sdk-types.ts` a test. -- Integration tests need Chromium: `pnpm --filter @e2e-dev/web exec - playwright-core install chromium`. The web engine's `prepare` hook +- Integration tests need Chromium: `node packages/web/dist/bin.js install + chromium` after `pnpm build`. The web engine's `prepare` hook also installs a missing browser once per run, in the runner, before `plan` is emitted and the run's clock starts. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1676e087fe..c8b23e7dfd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -126,10 +126,10 @@ pnpm check # lint -> check:dead-code -> typecheck -> docs:check pnpm test # unit + integration ``` -Integration tests need Chromium: +Integration tests need Chromium. After `pnpm build`: ```sh -pnpm --filter @e2e-dev/web exec playwright-core install chromium +node packages/web/dist/bin.js install chromium ``` Docs are part of the change, not a follow-up. A behavior change updates its diff --git a/README.md b/README.md index a689c6c65a..7ba25f0c8c 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Flutter, and .NET MAUI, each a standalone project with a passing suite. | Package | What it does | | --- | --- | | [`e2e`](https://www.npmjs.com/package/e2e) | The SDK, runner, and CLI. | -| [`@e2e-dev/web`](https://www.npmjs.com/package/@e2e-dev/web) | Browser engine: Chromium, Firefox, and WebKit through Playwright. | +| [`@e2e-dev/web`](https://www.npmjs.com/package/@e2e-dev/web) | Browser engine: Chromium and Firefox over WebDriver BiDi. | | [`@e2e-dev/mobile`](https://www.npmjs.com/package/@e2e-dev/mobile) | iOS and Android engine: simulators and emulators through agent-device. | | [`@e2e-dev/decision`](https://e2e.com/docs/decision-models) | Decision-model executors for bounded semantic actions and assertions, and `decisionFilter()`, which leaves out the tests a pull request is unlikely to affect. | | [`@e2e-dev/acp`](https://e2e.com/docs/subscriptions#use-your-coding-agent) | Runs agent steps on your own coding agent (Claude Code or Codex) over the Agent Client Protocol. | diff --git a/SECURITY.md b/SECURITY.md index 912e28a0b1..8d8bc263d3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -148,7 +148,8 @@ full list, integrations and reporters included, is under only; deterministic suites make no model calls, and cached steps replay without one) - `readyUrl` probes against the app the runner starts -- Playwright browser downloads, once, when Chromium is missing +- browser downloads (Chrome for Testing, Firefox) and ffmpeg, once, when + missing - the app under test, and whatever that app itself loads `e2e run --ai-trace` writes every model call to a local file. Nothing uploads diff --git a/apps/testbed/e2e.firefox.config.ts b/apps/testbed/e2e.firefox.config.ts new file mode 100644 index 0000000000..98dfc89c8e --- /dev/null +++ b/apps/testbed/e2e.firefox.config.ts @@ -0,0 +1,10 @@ +import type { E2EConfig } from 'e2e'; +import { web } from '@e2e-dev/web'; +import config from './e2e.config.ts'; + +/** The deterministic suite on Firefox. */ +export default { + ...config, + filters: [], + targets: config.targets.map((target) => ({ ...target, engine: web({ browser: 'firefox' }) })), +} satisfies E2EConfig; diff --git a/apps/testbed/tests/board.e2e.ts b/apps/testbed/tests/board.e2e.ts index dd7541a8c6..ffe289f0aa 100644 --- a/apps/testbed/tests/board.e2e.ts +++ b/apps/testbed/tests/board.e2e.ts @@ -1,4 +1,6 @@ -import { test, expect } from 'e2e'; +import { test } from '@e2e-dev/web'; +import { expect } from 'e2e'; +import { browserName, failure } from './helpers.ts'; test('hover reveals the card menu action', async ({ app, screen }) => { await app.open('/board'); @@ -17,10 +19,16 @@ test('setInputFiles resolves paths from the project root', async ({ app, screen await expect(screen.getByLabel('Attachment state')).toHaveText('attachment.txt'); }); -test('dragging the card marks it done', async ({ app, screen }) => { +test('dragging the card marks it done', { requires: ['browser'] }, async ({ app, screen, browser }) => { await app.open('/board'); - await screen.getByText('Design review', { exact: true }).dragTo(screen.getByLabel('Done column')); + const drag = () => screen.getByText('Design review', { exact: true }).dragTo(screen.getByLabel('Done column')); + if (await browserName(browser) === 'firefox') { + // Firefox's remote agent drops no HTML5 drag (https://bugzilla.mozilla.org/show_bug.cgi?id=1515879). + expect(await failure(drag)).toHaveProperty('code', 'UNSUPPORTED_CAPABILITY'); + return; + } + await drag(); await expect(screen.getByLabel('Board state')).toHaveText('Design review is done'); }); diff --git a/apps/testbed/tests/browser.e2e.ts b/apps/testbed/tests/browser.e2e.ts index 2384655bf4..60c8d7434f 100644 --- a/apps/testbed/tests/browser.e2e.ts +++ b/apps/testbed/tests/browser.e2e.ts @@ -1,5 +1,6 @@ import { test } from '@e2e-dev/web'; import { expect } from 'e2e'; +import { browserName, failure } from './helpers.ts'; test.describe('browser fixture', { requires: ['browser'], tags: ['browser'] }, () => { test.beforeEach(async ({ app }) => { @@ -56,6 +57,11 @@ test('emulated media reaches the page\'s own @media rules, from the first paint await browser.emulateMedia({ colorScheme: 'dark' }); await app.open('/browser'); await expect(screen.getByLabel('Media')).toHaveText('dark, full motion'); + if (await browserName(browser) === 'firefox') { + // Firefox overrides only prefers-color-scheme over WebDriver BiDi; web({ reducedMotion }) sets it at launch. + expect(await failure(() => browser.emulateMedia({ reducedMotion: 'reduce' }))).toHaveProperty('code', 'UNSUPPORTED_CAPABILITY'); + return; + } await browser.emulateMedia({ reducedMotion: 'reduce' }); await expect(screen.getByLabel('Media')).toHaveText('dark, reduced motion'); await app.restart(); diff --git a/apps/testbed/tests/helpers.ts b/apps/testbed/tests/helpers.ts index 1d44aa11bf..b183f53a62 100644 --- a/apps/testbed/tests/helpers.ts +++ b/apps/testbed/tests/helpers.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import type { Locator, Point } from 'e2e'; +import type { Browser } from '@e2e-dev/web'; type Box = { x: number; y: number; width: number; height: number }; @@ -40,3 +41,8 @@ export function content(html: string): string { export function asset(name: string): string { return path.join(import.meta.dirname, '../app/playwright/assets', name); } + +/** The browser the open page runs in, read from its user agent. */ +export async function browserName(browser: Browser): Promise<'chromium' | 'firefox'> { + return (await browser.evaluate('navigator.userAgent')).includes('Firefox/') ? 'firefox' : 'chromium'; +} diff --git a/apps/testbed/tests/keyboard.e2e.ts b/apps/testbed/tests/keyboard.e2e.ts index af08bcfdbf..dfba38c0ca 100644 --- a/apps/testbed/tests/keyboard.e2e.ts +++ b/apps/testbed/tests/keyboard.e2e.ts @@ -16,7 +16,7 @@ import { test } from '@e2e-dev/web'; import { expect } from 'e2e'; -import { content, CROSS } from './helpers.ts'; +import { browserName, content, CROSS, failure } from './helpers.ts'; /** Records the last keydown in `window.lastEvent`, Playwright's `captureLastKeydown`. */ const captureLastKeydown = `() => { @@ -354,6 +354,11 @@ test.describe('page keyboard', { requires: ['browser'], tags: ['playwright'] }, test('should press audio and media control keys', async ({ browser }) => { await browser.goto(content('')); await browser.locator('input').focus(); + if (await browserName(browser) === 'firefox') { + // WebDriver defines no key values for the media keys (https://w3c.github.io/webdriver/#keyboard-actions), and Firefox takes key input over WebDriver BiDi only. + expect(await failure(() => browser.keyboard.press('MediaPlayPause'))).toHaveProperty('code', 'UNSUPPORTED_CAPABILITY'); + return; + } await browser.evaluate(captureLastKeydown); const mediaKeys = [ { key: 'AudioVolumeMute', code: 'AudioVolumeMute' }, @@ -443,7 +448,9 @@ test.describe('page keyboard', { requires: ['browser'], tags: ['playwright'] }, await browser.locator('textarea').pressSequentially('some text'); await browser.keyboard.press('Shift+Control+Alt+KeyB'); await browser.keyboard.press('Backspace'); - expect(await browser.evaluate("() => document.querySelector('textarea').value")).toBe('some '); + // Firefox on macOS runs no editing key binding for synthesized key events (microsoft/playwright#5721). + const firefox = await browserName(browser) === 'firefox'; + expect(await browser.evaluate("() => document.querySelector('textarea').value")).toBe(firefox ? 'some tex' : 'some '); }); test('should press the meta key', async ({ browser }) => { diff --git a/apps/testbed/tests/locator-misc-2.e2e.ts b/apps/testbed/tests/locator-misc-2.e2e.ts index 37331a9e49..d6dfa6d0b9 100644 --- a/apps/testbed/tests/locator-misc-2.e2e.ts +++ b/apps/testbed/tests/locator-misc-2.e2e.ts @@ -98,7 +98,7 @@ test.describe('locator misc 2', { requires: ['browser'], tags: ['playwright'] }, await promise; }); - test('locator.count should work with deleted Map in main world', { fail: 'locators resolve in Playwright\'s main world, which a page that replaces `Map` breaks; Playwright resolves its own in a utility world, but e2e\'s closed-shadow reach and locate reads need the main world' }, async ({ browser }) => { + test('locator.count should work with deleted Map in main world', async ({ browser }) => { await browser.goto('/empty.html'); await browser.evaluate('() => { window.Map = 1; }'); await browser.locator('#searchResultTableDiv .x-grid3-row').count(); diff --git a/apps/testbed/tests/pointer.e2e.ts b/apps/testbed/tests/pointer.e2e.ts index 8e22bb7fe6..1682d8f563 100644 --- a/apps/testbed/tests/pointer.e2e.ts +++ b/apps/testbed/tests/pointer.e2e.ts @@ -1,6 +1,6 @@ import { test } from '@e2e-dev/web'; import { expect } from 'e2e'; -import { boxOf, centerOf, failure } from './helpers.ts'; +import { boxOf, browserName, centerOf, failure } from './helpers.ts'; test('screen.tapAt taps a viewport point with no node behind it', async ({ app, screen }) => { await app.open('/pointer'); @@ -39,11 +39,16 @@ test('screen.swipe({ from, to }) moves the pointer along the path', async ({ app await expect(screen.getByLabel('Pad state')).toHaveText('swiped from 10,20 to 250,20'); }); -test('a path swipe drags and drops on the board', async ({ app, screen }) => { +test('a path swipe drags and drops on the board', { requires: ['browser'] }, async ({ app, screen, browser }) => { await app.open('/board'); const card = centerOf(await boxOf(screen.getByText('Design review', { exact: true }))); const done = centerOf(await boxOf(screen.getByLabel('Done column'))); + if (await browserName(browser) === 'firefox') { + // Firefox's remote agent drops no HTML5 drag (https://bugzilla.mozilla.org/show_bug.cgi?id=1515879). + expect(await failure(() => screen.swipe({ from: card, to: done }))).toHaveProperty('code', 'UNSUPPORTED_CAPABILITY'); + return; + } await screen.swipe({ from: card, to: done }); await expect(screen.getByLabel('Board state')).toHaveText('Design review is done'); diff --git a/docs/browser.mdx b/docs/browser.mdx index 34b8aa7014..50befd786e 100644 --- a/docs/browser.mdx +++ b/docs/browser.mdx @@ -6,9 +6,8 @@ description: Mock an API, handle a dialog, download a file, reach into an iframe --- The `browser` fixture of `@e2e-dev/web` mocks requests, handles dialogs, -downloads files, and runs code in the page. The engine runs Chromium, -Firefox, and WebKit through -[Playwright](https://github.com/microsoft/playwright). For navigation, +downloads files, and runs code in the page. The engine drives Chromium and +Firefox over WebDriver BiDi. For navigation, interactions, and checks, use `screen`, `app`, and `expect`. Import `test` from `@e2e-dev/web` to get `browser`: @@ -87,7 +86,7 @@ expect(csv).toContain('Invoice'); ``` e2e saves the file as a `download` artifact. Read it through -`file.absolutePath`, which works with hosted browsers too. `file.path` is +`file.absolutePath`. `file.path` is relative to the attempt's artifact directory, not to the project. ## Reach into an iframe @@ -188,7 +187,7 @@ browsers on one app are two targets, as in [One size per target](/web#one-size-per-target): ```ts title="Target fragment" -{ name: 'webkit', engine: web({ browser: 'webkit' }), app }, +{ name: 'firefox', engine: web({ browser: 'firefox' }), app }, ``` In CI, install the extra browser with the @@ -196,19 +195,20 @@ In CI, install the extra browser with the ```bash npm -npx @e2e-dev/web install webkit --with-deps +npx @e2e-dev/web install firefox --with-deps ``` ```bash pnpm -pnpm exec e2e-web install webkit --with-deps +pnpm exec e2e-web install firefox --with-deps ``` ```bash bun -bunx @e2e-dev/web install webkit --with-deps +bunx @e2e-dev/web install firefox --with-deps ``` -`connect` attaches to a remote Chromium over CDP instead of launching one, +WebKit is not supported. `connect` attaches to a remote Chromium over CDP +instead of launching one, or to another browser that speaks CDP, such as a running [Lightpanda](/integrations/lightpanda). To set sizes, see [Viewports](/web#viewports). @@ -270,8 +270,6 @@ export interface BrowserLease { readonly id: string; /** `ws://`, `wss://`, or `http://` DevTools endpoint to attach to over CDP. */ readonly cdpEndpoint: string; - /** Endpoint to reattach to the same browser after a transport drop; per-attempt leases only. */ - readonly reconnectEndpoint?: string; } ``` @@ -281,8 +279,8 @@ and a `log` that reaches the reporter. Read tokens from `request.env`, not `process.env`. A lease is JSON data. `release` gets back the object `acquire` returned, so -a provider can keep its own fields on it. Only `id`, `cdpEndpoint`, and -`reconnectEndpoint` travel to the worker, so key anything a worker uses, +a provider can keep its own fields on it. Only `id` and `cdpEndpoint` +travel to the worker, so key anything a worker uses, such as a recording, on `lease.id`. For an authenticated CDP connection, implement `cdpHeaders(lease, request)` @@ -374,8 +372,13 @@ Two workers lease two sessions in parallel and keep them for the run. `scope: 'worker'`, the default, keeps one browser per worker slot for the whole run. `scope: 'attempt'` gives each attempt, retries included, a fresh browser that no earlier test touched, for the cost of a session start per -attempt. A hosted session costs money while it runs, so keep `workers` at -what the suite can fill. +attempt. In worker scope each attempt runs in a new browser context, as in a +local run. In attempt scope it runs in the fresh browser's own context, so +what the provider set up there, such as a sign-in, reaches the test, and +`app.clearState()` and [sessions](/authentication) throw +`UNSUPPORTED_CAPABILITY`. `headers`, `basicAuth`, `userAgent`, `locale`, +and `timezoneId` work in both. A hosted session costs money while it runs, +so keep `workers` at what the suite can fill. A worker that dies (a crash, `process.exit`, a kill) can't release what it acquired. Implement `sweep` to have the runner release those leases when @@ -398,37 +401,20 @@ browser lives until the service ends it. - The side that acquired a lease releases it, exactly one time. - -The attempt runs in the browser's persistent default context, on its first -tab, so the remote browser must contain only that context. The limits of -[`connect.reconnectEndpoint`](/reference/web#recovering-a-cdp-transport) -apply: - -- No `headers`, `basicAuth`, `userAgent`, `locale`, or `timezoneId`. Each - is `INVALID_CONFIG`. -- No `app.clearState()`. It is `UNSUPPORTED_CAPABILITY`, naming the scope. -- No session state capture or restore. A run whose tests use a - [session](/authentication) on the target fails with `COLLECTION_ERROR` - before its setup test signs in. - -After a transport drop, the next operation reattaches through the lease's -`reconnectEndpoint`, else its `cdpEndpoint`, and checks that it is the same -browser. - - ### Downloads on the provider Implement `downloads`, and `browser.waitForDownload` works as it does -locally. A hosted browser saves a download to its own disk, where the -runner can't open it. With `downloads`, it saves the file under `dir`, and -the engine passes the finished file's path to `read`. `read` fetches the -bytes through the service's file API, in the worker, bounded by the call's -timeout. [Kernel](/integrations/kernel#downloads) serves downloads this -way. - -Without `downloads`, a download on a browser on another machine fails with -`ENGINE_FAILURE`, naming the provider. A provider whose browsers run on the -runner's machine needs nothing. +locally. A hosted browser saves a download to its own disk, where the runner +can't open it. With `downloads`, it saves each attempt's files in a folder +under `dir`, and the engine passes each finished file's path to `read`. +`read` fetches the bytes through the service's file API. +[Kernel](/integrations/kernel#downloads) and +[smol machines](/integrations/smol) serve downloads this way. + +Without `downloads`, `browser.waitForDownload` throws +`UNSUPPORTED_CAPABILITY`, naming the provider. `setInputFiles` on a +provider's browser sends the files' bytes to the page, up to 50 MB in all; a +directory throws `UNSUPPORTED_CAPABILITY`. ### Recording on the provider diff --git a/docs/ci/github-actions.mdx b/docs/ci/github-actions.mdx index d2a1aae0e2..7fc0782c2a 100644 --- a/docs/ci/github-actions.mdx +++ b/docs/ci/github-actions.mdx @@ -62,8 +62,8 @@ jobs: - run: pnpm install --frozen-lockfile - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pnpm-lock.yaml') }} + path: ~/.cache/e2e/browsers + key: e2e-browsers-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pnpm-lock.yaml') }} - name: Install browsers run: pnpm exec e2e-web install chromium --with-deps @@ -97,9 +97,9 @@ jobs: ``` `npx e2e` runs the version the lockfile installs. The -[install command](/reference/web#installing-browsers) downloads the browsers -for the engine's pinned Playwright version, with their system libraries; the -cache step skips the download until the lockfile changes. A +[install command](/reference/web#installing-browsers) downloads the browser +builds the engine pins, with their system libraries; the cache step skips the +download until the lockfile changes. A test that passed on retry keeps its failed attempt's evidence in the upload. ## Cache the timings history diff --git a/docs/executors.mdx b/docs/executors.mdx index ce09dd2cc8..9015eb7ce6 100644 --- a/docs/executors.mdx +++ b/docs/executors.mdx @@ -228,41 +228,6 @@ call leaves a gap that the executor fills on replay. Entries are keyed by the configured agent and its context, not the executor, so changing an agent's executor keeps its entries. -## Driving the page yourself - -On the web engine, `surfaceOf(handle)` from `@e2e-dev/web` gives you the -current attempt's `Page` and `BrowserContext`. Both accessors throw -`INVALID_STATE` outside an active attempt, and `page()` also when no page is -open. - -```ts -import type { E2EConfig, StepExecutor } from 'e2e'; -import { web, surfaceOf } from '@e2e-dev/web'; - -const engine = web(); -const own: StepExecutor = { - name: 'own-tools', - version: '1', - cache: 'off', - async runStep(ctx) { - const page = surfaceOf(engine)!.page(); // the page app.open() navigated - await ctx.budgets.runTool({ name: 'click', mutates: true }, () => - page.getByRole('button', { name: 'Save' }).click(), - ); - return { status: 'passed', summary: 'saved' }; - }, -}; - -export default { - targets: [{ engine, app: { url: 'http://localhost:3000' } }], - agents: { default: { executor: own } }, -} satisfies E2EConfig; -``` - -Direct Playwright calls skip action authorization and the replay cache, -and leave no artifacts. The step deadline and the verdict record still -apply. - ## Next diff --git a/docs/index.mdx b/docs/index.mdx index 7a8b4fa16f..829e839a7a 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -41,9 +41,9 @@ test('a member upgrades to Pro', async ({ app, agent, screen }) => { ## How it works - **Targets.** Each test runs one time per target, such as Chromium or an - iPhone simulator. The [web engine](/web) drives browsers through - Playwright. The [mobile engine](/mobile) drives iOS simulators and Android - emulators through agent-device. You can also + iPhone simulator. The [web engine](/web) drives Chromium and Firefox over + WebDriver BiDi. The [mobile engine](/mobile) drives iOS simulators and + Android emulators through agent-device. You can also [write your own](/writing-an-engine). - **Agent steps.** `agent.act` completes a goal. `agent.assert`, `agent.waitFor`, and `agent.extract` judge the screen. Use your own diff --git a/docs/integrations/kernel.mdx b/docs/integrations/kernel.mdx index 0c70003f06..83c9250ffc 100644 --- a/docs/integrations/kernel.mdx +++ b/docs/integrations/kernel.mdx @@ -8,7 +8,7 @@ description: Run web targets in Kernel's hosted Chromium with the kernel() brows `kernel()` from `@e2e-dev/kernel` is a [browser provider](/browser#hosted-browsers) for [Kernel](https://kernel.sh). -Each worker slot, or each attempt, gets its own hosted Chromium. Tests stay +Each worker slot gets its own hosted Chromium. Tests stay the same. Only the target's `browser` changes. ## Setup @@ -50,9 +50,8 @@ export default { Set `KERNEL_API_KEY` in the environment that `e2e run` starts in, such as a CI secret. `workers` sets how many browsers run at the same time. -Without the key, or on a Kernel error, the lease fails with that message. In -`worker` scope the run ends before any test. In `attempt` scope only the -attempt fails. +Without the key, or on a Kernel error, the lease fails with that message, +and the run ends before any test. ## Fill the screen @@ -78,7 +77,7 @@ and the rest of `BrowserCreateParams` in your `@onkernel/sdk`. It adds two: | Option | Default | Meaning | | -------- | ---------- | ------- | -| `scope` | `'worker'` | One browser per worker slot, or per attempt. See [Scopes](#scopes). | +| `scope` | `'worker'` | One browser per worker slot, or `'attempt'` for one per attempt. See [Scopes](#scopes). | | `replay` | `true` | A Kernel replay or the engine's screencast. See [Recordings](#recordings). | ## Scopes @@ -90,10 +89,9 @@ stays open behind it. A Kernel `profile` has no effect, because it applies only to the browser's default context. `scope: 'attempt'` leases a new browser for each attempt, retries included. -The attempt runs in its default context, on its first tab, so a `profile` -applies and the live view shows one tab. Each test pays for a browser start, -and the [limits of a per-attempt lease](/browser#hosted-browsers) apply, -sessions included. +The attempt runs in that browser's own context, so a `profile` applies, and +`app.clearState()` and [sessions](/authentication) throw +`UNSUPPORTED_CAPABILITY`. Each test pays for a browser start. ## Recordings @@ -114,7 +112,8 @@ Kernel replays only headed browsers, its default. With `headless: true` or `browser.waitForDownload` works as it does locally, in both scopes. The provider copies each file off the browser's disk into the attempt's -artifacts. +artifacts. `setInputFiles` sends the files' bytes to the page, up to 50 MB +in all; a directory throws `UNSUPPORTED_CAPABILITY`. ## Watch and clean up browsers @@ -132,7 +131,7 @@ itself dies first, Kernel deletes each browser after `timeout_seconds` idle, - How a browser provider leases browsers, and the limits of `'attempt'` scope. + How a browser provider leases browsers. Run the suite on GitHub Actions or another CI service. diff --git a/docs/integrations/lightpanda.mdx b/docs/integrations/lightpanda.mdx index ac8a6dcb92..e4a7bf2e46 100644 --- a/docs/integrations/lightpanda.mdx +++ b/docs/integrations/lightpanda.mdx @@ -107,9 +107,7 @@ when you run more. | `loadResources` | `[]` | Sub-resources the server fetches: `iframe`, `image`, `stylesheet`. See [Fetch more of the page](#fetch-more-of-the-page). | | `args` | `[]` | Further `lightpanda serve` flags, after the ones the provider sets. | -The provider leases one server per worker slot for the run; it has no -`scope: 'attempt'`, since Lightpanda does not expose the default context -identity a per-attempt lease reattaches by. A server that does not answer +The provider leases one server per worker slot for the run. A server that does not answer on its port within ten seconds fails the lease, and so does a binary that exits first, with its stderr in the message. @@ -163,9 +161,9 @@ tracks most of them. | ------- | ------------ | --- | | [Sessions](/authentication) (`test.setup` with `session.save`, `session:` on a test) | The setup passes and the session is saved, but a test that restores it opens signed out. | Cookies set on a context before its first page exists are dropped. Set them from the test with `browser.setCookies` after `app.open` instead. | | [Dialogs](/browser#handle-a-dialog) | `browser.onDialog` runs, then fails the test with `ENGINE_FAILURE`: `Page.handleJavaScriptDialog` reports no dialog. | Lightpanda answers `alert`, `confirm` (`false`), and `prompt` (`null`) itself before a client can. | -| [Downloads](/browser#download-a-file) | `browser.waitForDownload` times out. | No download events over CDP. | +| [Downloads](/browser#download-a-file) | `browser.waitForDownload` throws `UNSUPPORTED_CAPABILITY`. | The provider serves no [downloads](/browser#downloads-on-the-provider). | | [Iframes](/browser#reach-into-an-iframe) | `frameLocator` finds nothing. | Iframes are not loaded by default. `loadResources: ['iframe']` loads them; queries and fills then work, but a listener in the host page on an element the frame created does not fire. | -| Drags: `screen.swipe`, `locator.swipeTo`, and `browser.mouse` moves with a button down | `ENGINE_FAILURE` naming `Input.setInterceptDrags`. | Playwright needs that command for a drag and Lightpanda has not implemented it. | +| Drags: `screen.swipe`, `locator.swipeTo`, and `browser.mouse` moves with a button down | `ENGINE_FAILURE` naming `Input.setInterceptDrags`. | The engine needs that command for a drag and Lightpanda has not implemented it. | | `browser.setViewport` | The page measures the new size, but a `resize` listener never runs. | No `resize` event is dispatched. | | [Media emulation](/reference/web#media-emulation): `colorScheme`, `reducedMotion`, `browser.emulateMedia` | The page keeps reporting `light` and full motion. | `Emulation.setEmulatedMedia` does not change what `matchMedia` answers. | | Scrolling: `scrollIntoView`, `scrollUntilVisible`, and an action on a node "outside of the viewport" | The scroll is a no-op, and an actionability wait can time out. | Without layout there is no scroll position to move. The plain render stacks every block in one column, so more of a page sits below the fold than in Chrome, and nothing below it can be reached. | diff --git a/docs/integrations/smol.mdx b/docs/integrations/smol.mdx index 9cbeec4ff9..4e1e2bb32b 100644 --- a/docs/integrations/smol.mdx +++ b/docs/integrations/smol.mdx @@ -1,7 +1,7 @@ --- title: smol machines browsers sidebarTitle: smol machines -description: Run each web test attempt in its own branch of a warm Chromium microVM on your computer with the smol() browser provider from @e2e-dev/smol. +description: Run web tests in warm Chromium microVMs on your computer with the smol() browser provider from @e2e-dev/smol. "og:image": "https://e2e.com/docs/images/og/integrations/smol.png?v=93b61e0399" "twitter:image": "https://e2e.com/docs/images/og/integrations/smol.png?v=93b61e0399" --- @@ -9,18 +9,17 @@ description: Run each web test attempt in its own branch of a warm Chromium micr `smol()` from `@e2e-dev/smol` is a [browser provider](/browser#hosted-browsers) that runs Chromium in [smol machines](https://smolmachines.com) microVMs on your computer. Each -worker slot boots one browser machine. By default, each test attempt runs in -a copy-on-write branch of it, ready in about a second, so every retry starts -from the same browser state. The app's data resets only when the app -[runs in the machine](#run-the-app-in-the-machine). State on your computer -or in external services stays shared. +worker slot boots one browser machine, and each test attempt gets a branch +of it or a new browser context on it, depending on the +[scope](#choose-a-scope). State on your computer or in external services +stays shared. ## Set up the provider smol machines run on macOS on Apple Silicon, or on Linux with KVM (`/dev/kvm`). They need no account or API key. The first boot pulls the -Playwright Ubuntu image, so it takes longer. +Playwright Ubuntu image, which carries Chromium, so it takes longer. Install the provider and the smol machines SDK: @@ -79,48 +78,27 @@ it. Every attempt on that slot starts with the cookies, storage, and pages it left. It replaces [sessions](/authentication), which `'attempt'` scope can't use. -`prepare` gets the browser's DevTools endpoint. Work in the default context, -where attempts run. Disconnect before `prepare` resolves, and leave the -browser running. - - -Install the `playwright-core` version that `@e2e-dev/web` pins, 1.65.1 -today, and match it when you upgrade. pnpm doesn't let your config import -a dependency's dependency. - - -```bash npm -npm install --save-dev playwright-core@1.65.1 -``` - -```bash pnpm -pnpm add -D playwright-core@1.65.1 -``` - -```bash bun -bun add -d playwright-core@1.65.1 -``` - +`prepare` gets the browser's DevTools endpoint, and a `browser` and +`screen` on its open tab, the same ones a test gets. Pass `browser.goto` a +full URL. smol detaches them once `prepare` resolves; the browser keeps +running, with its cookies, storage, and pages. Another CDP client must +disconnect before `prepare` resolves and leave the browser running. +The `browser` and `screen` calls of one `prepare` may take 120 seconds in +all; raise that with `prepareTimeout`. + ```ts title="Provider options" -import { chromium } from 'playwright-core'; - smol({ hostPorts: [3000], - prepare: async (cdpEndpoint) => { - const browser = await chromium.connectOverCDP(cdpEndpoint); - const page = browser.contexts()[0]!.pages()[0]!; - await page.goto('http://localhost:3000/login'); - await page.getByLabel('Email').fill('qa@example.com'); - await page.getByLabel('Password').fill(process.env.QA_PASSWORD!); - await page.getByRole('button', { name: 'Sign in' }).click(); - await page.waitForURL('**/dashboard'); - await browser.close(); + prepare: async (_cdpEndpoint, { browser, screen }) => { + await browser.goto('http://localhost:3000/login'); + await screen.getByLabel('Email').fill('qa@example.com'); + await screen.getByLabel('Password').fill(process.env.QA_PASSWORD!); + await screen.getByRole('button', 'Sign in').tap(); + await browser.waitForURL('http://localhost:3000/dashboard'); }, }) ``` - -On a browser from `connectOverCDP`, `browser.close()` only disconnects. ## Run the app in the machine @@ -158,13 +136,16 @@ Set `app.url` to `http://localhost:` and declare no `app.command`. ## Choose a scope `scope: 'attempt'`, the default, gives every attempt its own branch of the -slot's warm browser, with the limits of a -[per-attempt lease](/browser#hosted-browsers). +slot's warm machine, so one test's changes to the machine, such as an app's +local database, never reach another. The attempt runs in the branched +browser's own context, with what `prepare` left. `app.clearState()` and +[sessions](/authentication) throw `UNSUPPORTED_CAPABILITY` here. Options +such as `headers` and `locale` work. `scope: 'worker'` keeps one machine per worker slot for the run. Each attempt gets a new browser context on it, as in a local run, so sessions -and options such as `headers` and `locale` work. Nothing is branched, and -`prepare` is an `INVALID_CONFIG`. +and `app.clearState()` work. Nothing is branched, and `prepare` is an +`INVALID_CONFIG`. ## Options @@ -172,6 +153,7 @@ and options such as `headers` and `locale` work. Nothing is branched, and | ----------- | ----------- | ------- | | `scope` | `'attempt'` | See [Choose a scope](#choose-a-scope). | | `prepare` | none | See [Start every test signed in](#start-every-test-signed-in). | +| `prepareTimeout` | `120000` | Milliseconds the `browser` and `screen` calls of one `prepare` may take in all. | | `hostPorts` | `[]` | See [Set up the provider](#set-up-the-provider). Not 80 or 9229. | | `app` | none | See [Run the app in the machine](#run-the-app-in-the-machine). | | `setup` | none | Shell script run as root in each machine before Chromium starts, such as `apt-get update && apt-get install -y fonts-noto-cjk` for more fonts. | @@ -181,7 +163,9 @@ and options such as `headers` and `locale` work. Nothing is branched, and ## What differs from a local browser The browser's back/forward cache is off, so going back reloads the page. -`browser.waitForDownload` works as it does locally. +`browser.waitForDownload` works as it does locally: the provider reads each +file off the machine's disk. `setInputFiles` sends the files' bytes to the +page, up to 50 MB in all; a directory throws `UNSUPPORTED_CAPABILITY`. ## Clean up leftover machines @@ -198,6 +182,6 @@ for example `smol machine ls`. What runs on your computer, and what can still leak. - How a browser provider leases browsers, and the limits of `'attempt'` scope. + How a browser provider leases browsers. diff --git a/docs/integrations/testmu.mdx b/docs/integrations/testmu.mdx index 1d3dd9d49e..3a7280755b 100644 --- a/docs/integrations/testmu.mdx +++ b/docs/integrations/testmu.mdx @@ -242,9 +242,10 @@ An attempt that records [video](/reference/config#video) gets: `ENGINE_FAILURE` when `launchTimeout` is over 60 seconds. - **Dropped browser connections.** A browser session cannot be resumed, so a dropped connection fails the attempt. -- **Browser downloads.** TestMu AI's browsers save downloads on their own - machines, so `browser.waitForDownload` fails with `ENGINE_FAILURE`. Assert - on the download link or the request instead. +- **Browser downloads.** TestMu AI's browsers run on other machines and + the provider serves no downloads, so `browser.waitForDownload` fails with + `UNSUPPORTED_CAPABILITY`. Assert on the download link or the request + instead. ## Next diff --git a/docs/migrate/playwright.mdx b/docs/migrate/playwright.mdx index 27b5d431ae..18793ac2e0 100644 --- a/docs/migrate/playwright.mdx +++ b/docs/migrate/playwright.mdx @@ -22,7 +22,7 @@ export const migrationPrompt = `Migrate this project's Playwright tests to e2e, 11. Summarize which tests moved, which stayed in Playwright, and why.`; e2e can run beside Playwright in one project while you move tests one file -at a time. `@e2e-dev/web` brings its own pinned `playwright-core`, so your +at a time. `@e2e-dev/web` does not use Playwright, so your `@playwright/test` version stays yours. e2e reads `e2e.config.ts` and finds `tests/**/*.e2e.ts` by default, so keep the two file patterns apart. @@ -293,7 +293,7 @@ purpose. Use the workaround, or keep the test in Playwright. | `test.setTimeout()`, `test.slow()`, `test.fixme()`, `test.describe.only()`, `test.fail()` on a describe | `{ timeout }` or `{ skip: 'reason' }` on the test or group; `test.only` and `test.fail()` on each test | | `failOnFlakyTests`, `use.navigationTimeout`, `--timeout`, `--quiet` | `run.summary.flaky` in `report.json`; `browser.goto(url, { timeout })`; `timeout` in the config | | The `blob` reporter and `merge-reports` | One report for each shard | -| The Playwright Docker image | The image for the `playwright-core` version `@e2e-dev/web` pins, or `npx @e2e-dev/web install chromium --with-deps` | +| The Playwright Docker image | Any Node.js image, with `npx @e2e-dev/web install chromium --with-deps` | | `test.use({ ... })` in a file or describe | Runner options on the group, engine options on a target, `browser.setViewport` at runtime | | `projects[].dependencies`, `teardown` | A top-level `test.setup` for sign-in; a script around the run | | `page.pause()` | `--step --headed` holds each step until a key | @@ -364,9 +364,9 @@ purpose. Use the workaround, or keep the test in Playwright. - Sessions last one run. `test.setup` saves state under a name and `{ session }` restores it. The file is encrypted and deleted at cleanup, so there is no `.auth/user.json` to reuse. -- Tests get no raw `Page`, because nothing on one is recorded, redacted, or - masked. `surfaceOf(engine)` is for - [custom executors](/executors#driving-the-page-yourself), not test code. +- Tests get no raw `Page`. The engine drives the browser over WebDriver + BiDi, not Playwright, and nothing on a raw page would be recorded, + redacted, or masked. ## Defaults that differ diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 1e72949340..7a585e70bf 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -66,7 +66,7 @@ need Xcode with an iOS simulator runtime, or the Android SDK with an emulator. ``` - Choose **Web (Playwright)** or **Mobile (iOS/Android)**, then a + Choose **Web (Chromium, Firefox)** or **Mobile (iOS/Android)**, then a subscription, a model provider, or **None** for tests without AI. The wizard writes a config and an example test, adds dependencies, and offers to set up your coding agent. It keeps existing files. diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx index 27576c4a78..85261ed28a 100644 --- a/docs/reference/cli.mdx +++ b/docs/reference/cli.mdx @@ -144,8 +144,8 @@ at once. Piped output and narrower terminals never show it. For a new project, the wizard asks for: -- **The engines.** Select one, both, or none. **Web** (Playwright, the - default) adds `@e2e-dev/web`. **Mobile (iOS/Android)** adds +- **The engines.** Select one, both, or none. **Web** (Chromium and + Firefox), the default, adds `@e2e-dev/web`. **Mobile (iOS/Android)** adds `@e2e-dev/mobile`, defaults to iOS on macOS and Android elsewhere, and opens Settings. Selecting none adds no engine, for HTTP tests or your own. - **A model provider or subscription for agent steps:** Vercel AI Gateway, @@ -175,7 +175,7 @@ field or lockfile, then the invoking package manager, then npm. | Creates | Content | | --- | --- | | `package.json` | Created as a private ESM package when missing; otherwise gets only the missing dependencies and scripts. Sibling packages (`@e2e-dev/web`, `@e2e-dev/mobile`) get the caret range of the version released with the CLI, or that exact version when the CLI is a prerelease. Adds a `test:e2e` script that runs `e2e run`, unless one exists | -| `e2e.config.ts` | One target per selected engine, with the app in its `app`. Playwright reads `APP_URL` (default `http://localhost:3000`) into `app.url` and shows `app.command` in a comment. Mobile sets `app: { bundleId: 'Settings' }` on iOS and `com.android.settings` on Android, and `workers: 1`. With a gateway, `agents.default` is `{ model, system }` | +| `e2e.config.ts` | One target per selected engine, with the app in its `app`. Web reads `APP_URL` (default `http://localhost:3000`) into `app.url` and shows `app.command` in a comment. Mobile sets `app: { bundleId: 'Settings' }` on iOS and `com.android.settings` on Android, and `workers: 1`. With a gateway, `agents.default` is `{ model, system }` | | `tests/example.e2e.ts` | An app-open test that passes against any page, plus a commented agent test. Device setup gets a Settings check instead. With both engines, `tests/web.e2e.ts` and `tests/mobile.e2e.ts` instead, each test with `requires: ['browser']` or `requires: ['device']` so the other target skips it | | `.gitignore` | Appends `node_modules/`, `.e2e/results/`, `.e2e/cache/`, `.e2e/sessions/`, `.e2e/report.json`, `.e2e/ai-trace.json`, `.e2e/timings.json`, `.e2e/junit.xml`, `.e2e/summary.md`, `.e2e/logs/`, and `.e2e/videos/`. Committing the cache is opt-in; see [commit the replay cache](/cache#commit-the-replay-cache) | | `.agents/skills/e2e/`, `.claude/skills/e2e/` | `SKILL.md` and one `references/.md` per `e2e guide` topic. With both chosen, the Claude Code directory is a relative symlink (`../../.agents/skills/e2e`) to the copy, the layout `npx skills add` produces | @@ -516,7 +516,7 @@ npx e2e run tests/signup.e2e.ts --tag smoke npx e2e run tests/signup.e2e.ts:12 npx e2e run tests/agent npx e2e run 'tests/**/*.smoke.e2e.ts' -npx e2e run --target web,webkit --tag smoke,billing --tag-mode all +npx e2e run --target web,firefox --tag smoke,billing --tag-mode all npx e2e run --tag smoke --exclude-tag slow npx e2e run --grep checkout --grep-invert '/refund/i' npx e2e run --last-failed @@ -534,7 +534,7 @@ pnpm exec e2e run tests/signup.e2e.ts --tag smoke pnpm exec e2e run tests/signup.e2e.ts:12 pnpm exec e2e run tests/agent pnpm exec e2e run 'tests/**/*.smoke.e2e.ts' -pnpm exec e2e run --target web,webkit --tag smoke,billing --tag-mode all +pnpm exec e2e run --target web,firefox --tag smoke,billing --tag-mode all pnpm exec e2e run --tag smoke --exclude-tag slow pnpm exec e2e run --grep checkout --grep-invert '/refund/i' pnpm exec e2e run --last-failed @@ -552,7 +552,7 @@ bunx e2e run tests/signup.e2e.ts --tag smoke bunx e2e run tests/signup.e2e.ts:12 bunx e2e run tests/agent bunx e2e run 'tests/**/*.smoke.e2e.ts' -bunx e2e run --target web,webkit --tag smoke,billing --tag-mode all +bunx e2e run --target web,firefox --tag smoke,billing --tag-mode all bunx e2e run --tag smoke --exclude-tag slow bunx e2e run --grep checkout --grep-invert '/refund/i' bunx e2e run --last-failed diff --git a/docs/reference/config.mdx b/docs/reference/config.mdx index 73bdfe65a5..c5b08da4e6 100644 --- a/docs/reference/config.mdx +++ b/docs/reference/config.mdx @@ -506,7 +506,6 @@ Each is `'grant'`, `'deny'`, or `'reset'`. | --- | --- | | `chromium` | `accelerometer`, `ambient-light-sensor`, `background-sync`, `camera`, `clipboard-read`, `clipboard-write`, `geolocation`, `gyroscope`, `local-fonts`, `local-network-access`, `magnetometer`, `microphone`, `midi`, `midi-sysex`, `notifications`, `payment-handler`, `screen-wake-lock`, `storage-access` | | `firefox` | `geolocation`, `local-network-access`, `notifications`, `persistent-storage`, `push`, `screen-wake-lock` | -| `webkit` | `camera`, `clipboard-read`, `geolocation`, `microphone`, `notifications`, `screen-wake-lock` | A browser provider or `connect` takes the `chromium` names: the engine drives it over CDP, Lightpanda included. @@ -889,7 +888,7 @@ engine. | Recorder | Files under the attempt's artifact directory | | --- | --- | -| Browser engine, Playwright screencast | `video/video.webm`, one segment per page the attempt showed, later ones as `video/video-part.webm`. [`web({ screencast })`](/reference/web#options) sets the frame size and quality | +| Browser engine screencast | `video/video.webm`, one segment per page the attempt showed, later ones as `video/video-part.webm`. [`web({ screencast })`](/reference/web#options) sets the frame size and quality | | Mobile engine, agent-device | `video/video.mp4` | | A provider's own recording ([`BrowserProvider.record`](/browser#recording-on-the-provider), [`DeviceProvider.record`](/mobile#hosted-devices)) | the file the provider wrote, as `video/`: Kernel's is `video/replay.mp4`. Or a link to a recording the service keeps, recorded with `url` in place of `path`; the runner never downloads it | diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 742f1ab3e0..ae4c91f008 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -1686,6 +1686,15 @@ async locate(expression) { - An engine whose tree is the whole truth (a device) calls it; a browser engine reproduces it. - A `selector` expression needs the `selector` option. +- The `matches` option `(node, query) => boolean | undefined` decides a + query by the platform's own rules, such as a browser's text and hidden + rules. Return `undefined` to keep the reference rules for that query. +- The `hasText` option `(node, pattern) => boolean` decides a + `filter({ hasText })`. Without it, the text of the node or any descendant + counts. +- The `order` option `(a, b) => number` sorts the matches, and the position + `nth` counts by, by the platform's own order, such as a browser listing + light-DOM matches before shadow-root ones. Without it, tree order. - A `frame` expression is `FRAME_NOT_FOUND`. - A `union` expression matches either branch in document order, without duplicate nodes. @@ -2186,7 +2195,7 @@ The factory runs once per attempt with an `EngineFixtureContext`: | Throws | When | | --- | --- | | `INVALID_CONFIG` | The factory returns a plain surface, not one declared through `context.fixture`, the first time a test reaches for it. | -| `APP_URL_REQUIRED` | `app.resolveUrl` on a target that declares no URL. | +| `APP_URL_REQUIRED` | `app.resolveUrl` with a relative URL on a target that declares no URL. | | `POLICY_DENIED` | `app.resolveUrl` with any scheme but `http:` and `https:` (`about:blank` aside). | ```ts diff --git a/docs/reference/environment.mdx b/docs/reference/environment.mdx index 03dae5ea73..7f05461001 100644 --- a/docs/reference/environment.mdx +++ b/docs/reference/environment.mdx @@ -171,8 +171,10 @@ What is sent and how to read it is on the [Telemetry](/telemetry) page. | Variable | Example | Reading | Effect | | --- | --- | --- | --- | -| `PLAYWRIGHT_BROWSERS_PATH` | `PLAYWRIGHT_BROWSERS_PATH=~/.cache/ms-playwright` | value | Playwright's browser cache, honored by the engine's `prepare` hook | -| `PLAYWRIGHT_SKIP_BROWSER_GC` | `PLAYWRIGHT_SKIP_BROWSER_GC=0` | value | Defaults to `1` for the engine's browser install, which keeps other browsers in the cache. `0` lets the install remove them | +| `E2E_BROWSERS_PATH` | `E2E_BROWSERS_PATH=/opt/e2e-browsers` | value | Where `@e2e-dev/web` installs and launches its browsers. Unset, the user cache. See [Installing browsers](/reference/web#installing-browsers) | +| `E2E_BROWSERS_DOWNLOAD_HOST` | `E2E_BROWSERS_DOWNLOAD_HOST=https://mirror.example.com` | value | A mirror `@e2e-dev/web` downloads browsers and ffmpeg from, serving the paths of `storage.googleapis.com`, `archive.mozilla.org`, and `github.com`. `https`, or `http` on `localhost` only. Each archive must still match its pinned SHA-256. Unset, those hosts. See [Installing browsers](/reference/web#installing-browsers) | +| `HTTPS_PROXY`, `HTTP_PROXY`, `NO_PROXY` | `HTTPS_PROXY=http://proxy.example.com:3128` | value | The proxy `@e2e-dev/web` downloads browsers and ffmpeg through, and the hosts it reaches directly. On Node 22 and early Node 24 releases, only with `NODE_USE_ENV_PROXY=1` | +| `E2E_FFMPEG_PATH` | `E2E_FFMPEG_PATH=/usr/bin/ffmpeg` | value | The ffmpeg that encodes Chromium video, with `libvpx-vp9`. Unset, `ffmpeg` on `PATH`, else a pinned build the first recording downloads (none on Windows on Arm) | | `KERNEL_API_KEY` | `KERNEL_API_KEY=` | value | The API key `kernel()` from `@e2e-dev/kernel` creates browsers with. Unset, the browser lease fails. See [Kernel](/integrations/kernel) | | `LIGHTPANDA_PATH` | `LIGHTPANDA_PATH=/opt/lightpanda` | value | The binary `lightpanda()` from `@e2e-dev/lightpanda` starts a CDP server with. Unset, `lightpanda` on `PATH`, `~/.lightpanda`, or `~/.local/bin`. See [Lightpanda](/integrations/lightpanda) | | [`EXPO_TOKEN`](#expo_token) | `EXPO_TOKEN=` | value | The Expo access token `expoSimulators()` from `@e2e-dev/eas` uses | diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index 1aeaf3e3fb..666ee1cd48 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -148,7 +148,7 @@ These appear in terminal output and in the report as plain string codes. | `TEST_AMBIGUOUS` | `e2e mcp`: `open_session`'s `test` selected several tests on the session's target. The message lists them as `file:line` | configuration | 2 | | `SESSION_TEST_UNSUPPORTED` | `e2e mcp`: `open_session`'s `test` selected a test a session cannot run alone: a serial group member, a test that starts from a setup test's `session`, or a test that runs as several agents. Run it with `e2e run` | configuration | 2 | | `UNSUPPORTED_ARTIFACT` | A [`video` mode](/reference/config#recording) set on a target, or on a test that will run there, records on some attempt, and the target's engine cannot record video. The message names the target, and the test for a test's mode. A mode from the config root or `--video` never raises it: it applies to the targets that can record, with a notice naming the rest | configuration | 2 | -| `UNSUPPORTED_CAPABILITY` | A fixture, action, or tier the target's engine does not declare: a contributed fixture it does not contribute, `screen` without `locate`, state restore without `state`, screenshots without `artifacts`, `app.clearState()` on a persistent browser context (the message names the mode), `agent.assert` `vision` with an executor that does not declare `vision: true`, or `app.startRecording()` while the test's video records | configuration | 2 | +| `UNSUPPORTED_CAPABILITY` | A fixture, action, or tier the target's engine does not declare: a contributed fixture it does not contribute, `screen` without `locate`, state restore without `state`, screenshots without `artifacts`, `app.clearState()` with a `scope: 'attempt'` browser provider, `agent.assert` `vision` with an executor that does not declare `vision: true`, `app.startRecording()` while the test's video records, or Chromium video on Windows on Arm without `E2E_FFMPEG_PATH` (`@e2e-dev/web`) | configuration | 2 | | `NO_TESTS` | Zero runnable ordinary test-target pairs. The message names the cause: no file matched the `tests` globs (it lists look-alike files), a positional matched nothing (it names the `!` entry that excludes a file the positional names), a file registered no tests, or filters excluded everything. Suppress with `--pass-with-no-tests`. When [test filters](/filters) leave nothing to run, the run passes instead (exit 0). Also `e2e mcp`'s `open_session` when its `test` selects no runnable test. | configuration | 2 | | `RUNNER_BUSY` | A suite overlapped another suite, an opening or open host, or a live attempt; or a host opened during a suite; or an `e2e mcp` `run_tests` started while another was running; or `e2e run --step` started without a terminal in a directory that has a stepped run going, or where a run killed while it started left its lock (the message names the file to remove). Await the suite and close existing hosts before you retry; end a stepped run with `e2e run --step stop`, or remove the lock file the message names. | configuration | 2 | | `RUNNER_NOT_INSTALLED` | `test`, `describe`, or a hook from `@e2e-dev/web` or `@e2e-dev/mobile` was called where `e2e` is not installed, as in a program on `@e2e-dev/core` alone. The engine factory from the same entry still works. Install `e2e` to write tests | configuration | 2 | @@ -169,7 +169,7 @@ These appear in terminal output and in the report as plain string codes. | `RUN_TIMEOUT` | The run spent its [`maxDuration`](/reference/config#maxduration) or `--max-duration`. The running tests end as `interrupted`, and the rest are skipped with cause `time-limit` | test | 1 | | `LAUNCH_TIMEOUT` | Engine init or attempt start exceeded `config.launchTimeout` | infrastructure | 3 | | `CLEANUP_TIMEOUT` | Engine attempt end exceeded `config.cleanupTimeout`, or a route or dialog handler the engine ran for the test was still running when the attempt settled | infrastructure | 3 | -| `BROWSER_INSTALL_FAILED` | Browser provisioning failed | infrastructure | 3 | +| `BROWSER_INSTALL_FAILED` | Browser or ffmpeg provisioning failed: a download error, an archive that does not match its pinned SHA-256, or an `E2E_BROWSERS_DOWNLOAD_HOST` that is not `https` (`http` only on `localhost`) | infrastructure | 3 | | `WORKER_INIT_FAILED` | A worker process never became ready | infrastructure | 3 | | `WORKER_EXIT` | A worker exited unexpectedly while running a file. The test in flight fails with `WORKER_CRASH`, and the file's other tests are skipped. Attempts that finished before the exit stay in the report, including a retry's earlier attempts and a serial group's finished members. The crash is recorded as the attempt after them, or on the last attempt when it came after it (an `afterAll`). When an unhandled rejection ends the worker (see `UNHANDLED_REJECTION`), the rejection is the run error, not this one | infrastructure | 3 | | `WORKER_CRASH` | A worker exited during a specific test | infrastructure | 3 | diff --git a/docs/reference/screen.mdx b/docs/reference/screen.mdx index dadabd4380..09b2d83ba4 100644 --- a/docs/reference/screen.mdx +++ b/docs/reference/screen.mdx @@ -161,8 +161,7 @@ Slate editor renders into, is a `textbox`. an unnamed `textbox`. A name from the host's own `aria-label`, `aria-labelledby`, `aria-placeholder`, or `data-placeholder` (Quill) stays. -`getByRole` resolves through Playwright's role selector, which has no such -rule. So `getByRole` finds an editor only when the host carries +`getByRole` follows Playwright's role rules, which have no such rule. So `getByRole` finds an editor only when the host carries `role="textbox"`; `getByLabel` and `getByTestId` reach it either way. Anchor a test on a label or a test id, not on the placeholder. diff --git a/docs/reference/security.mdx b/docs/reference/security.mdx index 4ab63493ac..736f5adfe7 100644 --- a/docs/reference/security.mdx +++ b/docs/reference/security.mdx @@ -123,9 +123,6 @@ lists. - `basicAuth` answers the origin that sends the `401`. A redirect to another origin, such as `http:` to `https:` or the apex to `www`, gets no answer unless you list the origin it lands on. -- On WebKit, `basicAuth` answers a `401` from any origin when an allowed - origin is on its scheme's default port (`https://staging.example.com`) or - has an IPv6 host. WebKit can't scope a credential to such an origin. - An extra is an `http:` or `https:` origin with no path, query, or fragment, such as `https://api.example.com`. Anything else, a `*` in the host, and port `0` are `INVALID_CONFIG` at config load. @@ -357,7 +354,7 @@ after download redaction, with the `redaction` label the report records. | Your model provider | Agent steps that call a model. Deterministic tests and fully replayed steps make no model call. | | A subscription provider's sign-in and model-list endpoints | `e2e login` and `e2e models` ([Subscriptions](/subscriptions)). | | The app's `readyUrl` | Readiness probes for an app the runner starts. | -| Playwright's browser download | One time, when the browser is missing. | +| `storage.googleapis.com` (Chrome for Testing), `archive.mozilla.org` (Firefox), and GitHub releases (ffmpeg) | One time, when the browser or ffmpeg is missing. | | The DevTools endpoint in `web({ connect })` | Every run of that target. | | The GitHub API | Only from the `@e2e-dev/github` reporter. | | The provider of the `model` you pass | Only from [`decisionFilter()`](/filters): the diff of what the commits change (a file only the working tree changes, by name alone), the test files, and the setup files that save their sessions, redacted as a [text download](#downloads) is, except a value a secret provider returns. | @@ -369,7 +366,7 @@ after download redaction, with the `redaction` label the report records. | `hub-cloud.browserstack.com` and `api-cloud.browserstack.com` (or `BROWSERSTACK_API_ENDPOINT`) | Only from [`browserstack()`](/integrations/browserstack). It signs in with `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY`. For each session, agent-device looks the device up in the App Automate API, uploads a local `app`, and runs the session on BrowserStack's Appium hub. For an `https` `app`, the provider has the API fetch the build once per run, and it asks the API for the session's video only when an attempt records one. | | `api.e2b.app` and its sandboxes (or `E2B_DOMAIN`, `E2B_API_URL`, `E2B_SANDBOX_URL`) | Only from [`e2b()`](/integrations/e2b#run-android-targets) and [`e2bBrowsers()`](/integrations/e2b#run-web-targets-in-browsers). It signs in with `E2B_API_KEY`. The engine reaches each sandbox's agent-device or Chromium through a tunnel on `127.0.0.1` ([How the sandboxes stay private](/integrations/e2b#how-the-sandboxes-stay-private)), and `e2b()` sends the `.apk` through it. Each Android sandbox downloads Node from `nodejs.org` if Node is absent, and installs agent-device from npm. | | The Expo API and the simulators' agent-device daemons | Only from [`expoSimulators()`](/integrations/eas), or its deprecated alias `easSimulators()`. It signs in with `EXPO_TOKEN`, else with the eas-cli login in `~/.expo/state.json`. On a CI runner, set `EXPO_TOKEN` or keep that login out of the runner's home. | -| `mcr.microsoft.com` | Only from [`smol()`](/integrations/smol), to pull the Playwright image. The machines then have unrestricted network access, and reach every port on your computer's loopback through `host.smolvm.internal`, not only `hostPorts` ([Set up the provider](/integrations/smol#set-up-the-provider)). | +| `mcr.microsoft.com` | Only from [`smol()`](/integrations/smol), to pull the Playwright Ubuntu image the machines run. The machines then have unrestricted network access, and reach every port on your computer's loopback through `host.smolvm.internal`, not only `hostPorts` ([Set up the provider](/integrations/smol#set-up-the-provider)). | | The package registry | When `e2e init` installs dependencies. | | The app under test | Every run, plus whatever the app itself loads. | diff --git a/docs/reference/test.mdx b/docs/reference/test.mdx index 7bebfec174..8956a69088 100644 --- a/docs/reference/test.mdx +++ b/docs/reference/test.mdx @@ -643,7 +643,7 @@ Per attempt: | Mistake | Fails | | --- | --- | | A value that is not a plain object of functions, an empty name, a core name (`agent`, `app`, `screen`, `platform`, `session`), or a name an earlier `extend` in the chain defined | At import, `COLLECTION_ERROR` naming the fixture | -| A name the target's engine contributes (`browser` on a Playwright target) | The attempt, in `beforeEach`, `TEST_SETUP_FAILED` | +| A name the target's engine contributes (`browser` on a web target) | The attempt, in `beforeEach`, `TEST_SETUP_FAILED` | | A definition that returns without calling `use` | The attempt, in `beforeEach`, `TEST_SETUP_FAILED` | | A definition that calls `use` twice | The attempt, at teardown, `TEST_SETUP_FAILED` | | A definition that throws before `use` | The attempt, in `beforeEach`, with that error | diff --git a/docs/reference/web.mdx b/docs/reference/web.mdx index 617c6ad49e..f1fab697c9 100644 --- a/docs/reference/web.mdx +++ b/docs/reference/web.mdx @@ -5,8 +5,8 @@ description: The browser engine, its options, and the browser fixture it contrib "twitter:image": "https://e2e.com/docs/images/og/reference/web.png?v=66fee465a4" --- -`@e2e-dev/web` is the browser engine. It drives Chromium, Firefox, and WebKit -through Playwright. A web target names it as its engine with `web()`, and it +`@e2e-dev/web` is the browser engine. It drives Chromium and Firefox over +WebDriver BiDi. A web target names it as its engine with `web()`, and it contributes the `browser` fixture for browser-only work. ```ts title="e2e.config.ts" @@ -74,7 +74,6 @@ web({ viewport: { width: 390, height: 844 }, locale: 'de-DE' }); | Throws | When | | --- | --- | | `INVALID_CONFIG` | An option `web()` does not take (`url` or any other app key included), or an unknown key inside `connect` or `basicAuth`. The message names the nearest known key for a likely typo (`viewprt` is `viewport`), else every key. | -| `INVALID_CONFIG` | `headers`, `basicAuth`, `userAgent`, `locale`, or `timezoneId` on a [persistent context](#recovering-a-cdp-transport) (`connect.reconnectEndpoint`, or a provider with `scope: 'attempt'`). | #### `browser` @@ -84,18 +83,21 @@ The browser to launch, or a provider that leases hosted browsers. web({ browser: 'firefox' }); // default: 'chromium' ``` -- `'chromium'`, `'firefox'`, or `'webkit'`. -- A `BrowserProvider` leases hosted browsers instead: one per worker slot for - the run, or one per attempt ([`WebOptions`](#weboptions)). The engine - drives each one as chromium over CDP, whether it is Chromium or another CDP +- `'chromium'` or `'firefox'`. [Installing browsers](#installing-browsers) + has the builds. +- A `BrowserProvider` leases hosted browsers instead, one per worker slot for + the run, or one per attempt with `scope: 'attempt'`. The engine drives + each one as chromium over CDP, whether it is Chromium or another CDP server such as [Lightpanda](/integrations/lightpanda). -- A provider with `scope: 'attempt'` rides a persistent context, with the - limits under [Recovering a CDP transport](#recovering-a-cdp-transport). +- With `scope: 'attempt'`, each attempt runs in the leased browser's own + context, so `app.clearState()` and saved sessions throw + `UNSUPPORTED_CAPABILITY`. - [Hosted browsers](/browser#hosted-browsers) has the `BrowserProvider` contract and an example provider. | Throws | When | | --- | --- | +| `INVALID_CONFIG` | Any other browser name, `'webkit'` included. | | `INVALID_CONFIG` | `browser: provider` together with `connect`. | #### `viewport` @@ -127,7 +129,7 @@ web({ screencast: { size: { width: 800, height: 450 }, quality: 60 } }); export interface WebScreencastOptions { /** Frame size in pixels; default the viewport's, or the window's under `viewport: null`. A smaller size makes a smaller file. */ readonly size?: ViewportSize; - /** JPEG quality of each captured frame, 0 through 100; Playwright's default when unset. */ + /** JPEG quality of each captured frame, 0 through 100; the browser's default when unset. */ readonly quality?: number; } ``` @@ -138,110 +140,34 @@ export interface WebScreencastOptions { #### `connect` -Attaches to a remote browser over CDP instead of launching one locally. +Attaches to a remote Chromium over CDP instead of launching one locally. ```ts web({ connect: { cdpEndpoint: async () => 'ws://127.0.0.1:9222' } }); ``` -`reconnectEndpoint` opts into a persistent context; see -[Recovering a CDP transport](#recovering-a-cdp-transport). +- `cdpEndpoint(signal)` resolves when a worker first needs the browser, and + again when an attempt starts after the connection dropped. +- Each attempt gets a new browser context on the remote browser. +- A connection that drops fails the attempt, and the next attempt attaches + again through `cdpEndpoint`. -Either resolver can return `{ endpoint, headers }` as a `CdpTarget` to authenticate the CDP handshake. These headers are separate from `web({ headers })`, which sends headers to the app. +`cdpEndpoint` can return `{ endpoint, headers }` as a `CdpTarget` to authenticate the CDP handshake. These headers are separate from `web({ headers })`, which sends headers to the app. ```ts interface WebConnectOptions { cdpEndpoint(signal: AbortSignal): string | CdpTarget | Promise; - reconnectEndpoint?(signal: AbortSignal): string | CdpTarget | Promise; } ``` | Throws | When | | --- | --- | -| `INVALID_CONFIG` | A `connect` without `cdpEndpoint`, or with one given as a string, at config load. Also `connect` beside a `browser` provider. | - - - -##### Recovering a CDP transport - -`connect.reconnectEndpoint(signal)` opts into recovery of the same remote -browser after its CDP transport disconnects. Omit it to keep isolated -contexts. - -The remote browser: - -- Every attempt, retries included, needs its own fresh browser from - `cdpEndpoint`. One long-lived browser cannot serve a suite. -- The remote must have only its default context. -- The host must delete the browser when the attempt ends. Engine cleanup - detaches its connection and does not terminate the host's browser. -- The attempt's first page is the browser's own first tab, navigated to - `about:blank`, not a second tab beside it. A page opened later, such as - the one `app.restart` opens, is a new tab. - -How recovery runs: - -- After a disconnect, the next operation calls `reconnectEndpoint` once, - within that operation's timeout and abort signal. The resolver must return - the existing browser's endpoint. -- The engine checks the persistent default context ID and the active tab's - target ID. It refuses a browser that does not expose the default context - ID. A missing active tab or a different context fails the attempt. A - matching URL does not select a replacement tab. The attempt's other tabs - are found by their target IDs; one that is gone is dropped. -- The engine installs secure-field masking before it publishes each CDP - binding. An incompatible Playwright version fails the connection instead of - exposing pixels. -- The original page's frames must keep their closed shadow DOM tracking - hooks. Existing closed roots cannot recover without a reload of the - application. Mask queries reject any untracked document. -- Endpoint resolution, CDP attachment, setup, and dispatch share one timeout. - Cancelling a concurrent waiter does not cancel the operation that started - recovery. Cancelling that recovery owner fails the shared recovery. +| `INVALID_CONFIG` | A `connect` without `cdpEndpoint`, or with one given as a string, at config load. Also `connect` beside a `browser` provider, or a `browser` other than chromium. | -| Throws | When | -| --- | --- | -| `ENGINE_FAILURE` | A document loaded without the closed shadow DOM hook while disconnected. | -| `OPERATION_TIMEOUT` | Endpoint resolution, attachment, setup, and dispatch exhausted their shared timeout. | -| `CANCELLED` | The caller or the attempt cancelled. | - -What survives: - -- Recovery keeps the page's unsaved state, storage, and the agent ledger. Dialog and route handlers are reattached. -- Video recording resumes. A segment lost during the disconnect is gone. -- Operations already dispatched are never retried. -- Recovery clears old node references. Observation-derived `tapAt` and - `doubleTapAt` calls, observation-root swipes, and focused engine keyboard input need a fresh - engine observation before they resume. -- An observation or locator read started before recovery cannot publish - references after the connection changes. Each attempt owns its binding and - recordings, so late cleanup cannot change a later attempt. -- The deterministic `browser.mouse` and `browser.keyboard` fixtures stay - available without an agent observation. Test code owns its input and can - read current geometry and focus with `browser.evaluate` before acting. -- Code that uses `surfaceOf(engine)` must read its accessors again after - recovery. Playwright objects it kept belong to the disconnected transport. - -A `browser` provider with `scope: 'attempt'` rides the same persistent -context, with the same limits. The lease's `cdpEndpoint` provisions the -attempt's browser, and its `reconnectEndpoint` (else `cdpEndpoint` again) -reconnects to it. A lease is fresh by the provider's contract, so the engine -does not compare its context with earlier attempts'. A hosted service may -restore browsers from one snapshot, which gives new browsers the same -context id. - -To reuse one sign-in across files, keep isolated contexts (no -`reconnectEndpoint`, no provider `scope: 'attempt'`) and +To reuse one sign-in across files, [sign in once in a setup test](/authentication#reuse-authenticated-sessions). A saved session carries it, not a shared browser. - -A persistent context does not support `headers`, `basicAuth`, `userAgent`, -`locale`, `timezoneId` (see [Options](#options)), `app.clearState`, or -session state capture and restore. `app.clearState` fails with -`UNSUPPORTED_CAPABILITY`. `app.restart` remains available. - - #### `headers` Headers added to every request that starts at the app's origin: the scheme, @@ -298,13 +224,6 @@ export interface WebBasicAuth { | --- | --- | | `INVALID_CONFIG` | A `:` in the user name, or an unknown key. | - -WebKit can't scope a credential to an origin on its scheme's default port -(`https://staging.example.com`) or with an IPv6 host. When the app's origin -or one in `credentialOrigins` is such an origin, WebKit answers a `401` from -any origin, as Playwright does by default. - - The secret protection is text only. A page that renders the password on screen is not masked in screenshots or model pixels @@ -371,7 +290,7 @@ web({ timezoneId: 'Europe/Berlin' }); // default: the machine's #### Media emulation `web({ colorScheme, reducedMotion })` sets the media features every page of -every attempt reports, as Playwright's context options of the same names do. +every attempt reports. `browser.emulateMedia(media)` changes them for one attempt. ```ts @@ -380,16 +299,13 @@ await browser.emulateMedia({ colorScheme: 'light' }); ``` - The page's `@media` rules and `matchMedia` read them from the first paint. -- A persistent context takes the options too: the engine sets them on each - page it opens. - `emulateMedia` applies for the rest of the attempt, on every open tab and every tab opened after them, through `app.restart()` and `app.clearState()`. - It works before the first navigation, so the next `app.open` loads with them. - A feature it leaves out keeps its value. -- A tab the app opens itself gets a test's change as it appears, on a - persistent context too, so a script it runs at load may still read the - configured values. +- A tab the app opens itself gets a test's change as it appears, so a + script it runs at load may still read the configured values. ```ts /** User preferences a page reads through `@media` rules and `matchMedia`. */ @@ -428,6 +344,21 @@ await app.open('/raffle'); export type WebInitScript = string | { readonly path: string } | (() => unknown); ``` +#### Limits + +| Limit | Why | +| --- | --- | +| No WebKit | WebKit has no WebDriver BiDi endpoint. | +| `connect` and browser providers are Chromium only | They attach over CDP. | +| `connect.reconnectEndpoint` is removed | A dropped connection is not recovered mid-attempt. The next attempt attaches again. | +| `app.clearState()` and saved sessions throw `UNSUPPORTED_CAPABILITY` with a `scope: 'attempt'` provider | The attempt runs in the fresh browser's own context, which they would replace. | +| Selectors are CSS and XPath, alone or in `>>` chains, with no `text=` or `nth=` engines | The page runs the selector itself. | +| On Chromium, an action that opens a `noopener` tab (a `target="_blank"` link) waits until the new tab's server answers | chromium-bidi holds every message behind the new tab's creation, which waits for its first response ([crbug 372842894](https://crbug.com/372842894)). | +| On Firefox, a drag that starts an HTML5 drag (a `draggable` element) throws `UNSUPPORTED_CAPABILITY` | [Firefox bug 1515879](https://bugzilla.mozilla.org/show_bug.cgi?id=1515879): Firefox drops it nowhere. Pointer drags work. | +| `route.continue({ url })` throws `UNSUPPORTED_CAPABILITY` on Firefox | [Firefox bug 2014825](https://bugzilla.mozilla.org/show_bug.cgi?id=2014825): the page's load of a rewritten URL fails. Use `route.fulfill`. | +| With a browser provider, `setInputFiles` sends the files' bytes to the page, up to 50 MB in all, and throws `UNSUPPORTED_CAPABILITY` for a directory; downloads throw it when the provider serves none | The browser's filesystem is another machine's. | +| A saved session throws `UNSUPPORTED_CAPABILITY` for an IndexedDB value that is not JSON or a `Date` | A `Blob` or class instance can't be restored. | + ### The browser fixture `browser` is the whole browser, showing one active tab. @@ -560,29 +491,30 @@ await browser.waitForURL(/\/orders\/\d+$/, { timeout: 15_000 }); #### `locator` -Creates a web-only locator from a Playwright selector. +Creates a web-only locator from a CSS or XPath selector. ```ts -const conditions = browser.locator( - '[data-testid="row"] >> nth=2 >> [data-testid="conditions"] > div', -); +const conditions = browser.locator('[data-testid="row"]').nth(2).getByTestId('conditions'); ``` -- The selector is CSS by default, XPath with `xpath=` or a leading `//`, and - takes `>>` chains, so the example scopes CSS to one row. +- The selector is CSS by default. `xpath=`, a leading `/`, or a leading `..` + makes it XPath, and `css=` forces CSS. +- CSS reaches into open shadow roots. +- `>>` chains CSS and XPath parts, each matched inside the last. There are + no `text=` or `nth=` engines: chain `nth` and the screen queries instead, + as in the example. - Use `filter({ has })` for an element that contains another. | Throws | When | | --- | --- | -| `INVALID_LOCATOR` | An empty selector, or a `*` capture before the last part (`*css=article >> text=Hello`). | +| `INVALID_LOCATOR` | An empty selector, or one that is not valid CSS or XPath. | Every `screen` query searches its scope and each closed shadow root the page -attached under it, with Playwright's own role, name, text, label, and -attribute matching and each node found once. A control the observed tree +attached under it, with Playwright's role, name, text, label, and +attribute rules and each node found once. A control the observed tree lists inside a closed root resolves and takes actions and reads. `locator`, -`frameLocator`, and `filter({ hasText })` keep Playwright's reach and stop at -a closed root; the [browser guide](/browser#shadow-dom) has the details. +`frameLocator`, and `filter({ hasText })` stop at a closed root; the [browser guide](/browser#shadow-dom) has the details. #### `frameLocator` @@ -642,7 +574,7 @@ const title = await browser.evaluate('document.title'); | --- | --- | | `INVALID_ARGUMENT` | `arg` or the return value is not a [`JsonValue`](#jsonvalue). A class instance, such as a `DOMRect` or an element, is not: return `rect.toJSON()` or the fields you need. | | `EVALUATE_FAILED` | The evaluated code threw, or the page replaced a JavaScript built-in such as `Map` that the engine needs. The error carries the message. | -| `ACTION_FAILED` | A Playwright timeout. | +| `OPERATION_TIMEOUT` | The evaluation did not settle in time. | | `ENGINE_FAILURE` | The page closed or the context was lost. | #### `route` @@ -959,8 +891,7 @@ await screen.getByRole('textbox', 'Verification code').fill(code); is. - `newTab()` opens a blank tab in the same browser context, with the same cookies and storage, and makes it active. Navigate it with `goto`. -- A tab whose opener closed before the tab loaded is listed too, on - Chromium. A page a test opens through `surfaceOf(engine)` is not. +- A tab whose opener closed before the tab loaded is listed too. - The screencast records the active tab, one video segment per change. A tab that closes itself keeps no segment, since its page closes before the recording can be saved. A provider's own recording covers every tab. @@ -996,14 +927,12 @@ const file = await browser.waitForDownload(() => - It registers the file as a `download` artifact and returns its relative `path`, the runner-side `absolutePath`, and `suggestedFilename`. -- Use `absolutePath` with file APIs to inspect the contents, including on a - hosted browser. -- On a hosted browser the file comes back through the provider's - [`downloads`](/browser#downloads-on-the-provider). +- Use `absolutePath` with file APIs to inspect the contents. - The wait includes the trigger's own time. | Throws | When | | --- | --- | +| `UNSUPPORTED_CAPABILITY` | The browser comes from a browser provider without [`downloads`](/browser#downloads-on-the-provider). | | `ACTION_FAILED` | The download did not complete in time. A trigger that resolved in time but started no download fails with `no download started within ms; the trigger resolved after ms`. | A trigger still running at the timeout fails with the step timeout. One that @@ -1042,17 +971,68 @@ await browser.mouse.up(); [`Browser`](#browser-interface) describes each method, `wheel` included. -## Installing browsers +### Functions + +| Function | Returns | +| --- | --- | +| [`attachBrowser`](#attachbrowser) | `Promise` | -The package pins `playwright-core` to an exact version, so the engine always -runs the Playwright it was tested against. An app that depends on Playwright -itself keeps its own copy. The two share a browser cache only when their -versions match. +#### `attachBrowser` -A missing browser downloads once per run, before the run's clock starts. To -install browsers as their own step, in CI or a container image, run the -package's command. It installs the browsers for the engine's pinned -Playwright: +`attachBrowser(cdpEndpoint: string, options?: AttachBrowserOptions)` attaches +to a Chromium that is already running, outside any test. It returns the same +`browser` and `screen` a test gets. + +```ts title="scripts/sign-in.ts" +import { attachBrowser } from '@e2e-dev/web'; + +await using attached = await attachBrowser('http://127.0.0.1:9222'); +const { browser, screen } = attached; +await browser.goto('http://localhost:3000/login'); +await screen.getByLabel('Email').fill('qa@example.com'); +await screen.getByRole('button', 'Sign in').tap(); +``` + +- `cdpEndpoint` is the DevTools HTTP endpoint or its `ws://` URL. Chromium + only. +- On macOS, start Chrome with `--use-mock-keychain`. Without it, Chrome for + Testing can wait on a Keychain prompt and load no page. +- It drives the first open tab in the browser's default context. When no + tab is open, it opens one. +- Relative URLs have no base. Pass `browser.goto` a full URL. +- `close()` detaches. The browser keeps running, with its tabs, cookies, and + storage. Only a tab the attach opened itself is closed. `close()` is + idempotent, runs on `await using`, and throws an `AggregateError` of what + failed. +- There is no `app`, so nothing restarts the tab, clears its state, or saves + a session. + +| Option | Type | Default | +| --- | --- | --- | +| `testIdAttribute` | `string` | `'data-testid'` | +| `timeout` | `number` | `120000` | +| `signal` | `AbortSignal` | - | + +- `timeout` is the budget in milliseconds for all operations together, + counted from the attach. +- `signal` cancels attaching and every operation. `close()` still detaches. + +```ts +interface AttachedBrowser extends AsyncDisposable { + readonly browser: Browser; + readonly screen: Screen; + /** Detaches. The browser keeps running, with its tabs, cookies, and storage. */ + close(): Promise; +} +``` + +## Installing browsers + +The engine runs pinned builds: Chrome for Testing 157.0.8103.38 for +`chromium` (the headless shell for headless runs, full Chrome for headed +ones) and Firefox 159.0.1 for `firefox`. A missing browser downloads once +per run, before the run's clock starts. To install browsers as their own +step, in CI or a container image, run the package's command: ```bash npm @@ -1068,63 +1048,34 @@ bunx @e2e-dev/web install chromium --with-deps ``` -- Name any of `chromium`, `firefox`, and `webkit`. With none named it +- Name `chromium`, `firefox`, `ffmpeg`, or several. With none named it installs `chromium`. +- `--headed` installs the full Chrome a headed run launches instead of the + headless shell; run both commands to have both. - `--with-deps` also installs the system libraries the browsers need on - Linux. -- The command exits with the installer's exit code. - -## Semantic capture recovery - -When the semantic reader times out, agent observations can recover with a -fresh viewport screenshot. - -- Playwright runs a new secure-field mask sweep across every reachable frame, - including open and closed shadow roots. Every mask probe must succeed. -- The observation reports unavailable semantics and clears earlier node - references. -- Recovery requires time left in the operation budget and permission from the - runner. `act` permits it; judgments require `vision: true` or `'only'`. -- Failed masking, cancellation, and hard engine errors still stop the - observation. - -When fallback is permitted, the reader reserves one quarter of the remaining -observation budget, capped at two seconds, for the screenshot. The deadline -covers the page walk, metadata and element readback, and nested frames. - -Semantic metadata crosses the Playwright connection as one JSON string, to -cut serialization overhead. The elements the reader walked stay in one -in-page array per document, and an action makes the handle for its node from -that array the first time it reaches for it. A reference still names the -element the walk read, not whatever sits at that position later. A page that -replaces `JSON.stringify`, or adds `toJSON` methods to object or array -prototypes, gets structured transfer. Cleanup does not wait for stalled -protocol replies. An abandoned reader cannot reuse node IDs or publish -references over a newer observation. - -## surfaceOf - -Returns the live Playwright surface of a `web()` handle, for a -[custom executor](/executors#driving-the-page-yourself) that drives the page -itself. - -```ts -import { web, surfaceOf } from '@e2e-dev/web'; - -const engine = web(); -const live = surfaceOf(engine); // PlaywrightLiveSurface | undefined -live!.page(); // the Page of the current attempt -live!.context(); // its BrowserContext -``` - -- It returns `undefined` for a handle from another package. -- Direct Playwright calls are not recorded as steps. -- After [CDP recovery](#recovering-a-cdp-transport), read the accessors - again. - -| Throws | When | -| --- | --- | -| `INVALID_STATE` | An accessor called outside an active attempt. | + Debian and Ubuntu, with `apt-get`, through `sudo` when not run as root. +- Browsers go to `E2E_BROWSERS_PATH`, else the user cache: + `~/Library/Caches/e2e/browsers` on macOS, `$XDG_CACHE_HOME/e2e/browsers` + or `~/.cache/e2e/browsers` on Linux, `%LOCALAPPDATA%\e2e\browsers` on + Windows. Every project on the machine shares it, and concurrent installs + wait for each other. +- Downloads come from `storage.googleapis.com` (Chrome for Testing), + `archive.mozilla.org` (Firefox), and `github.com` (ffmpeg). + `E2E_BROWSERS_DOWNLOAD_HOST` names a mirror that serves the same paths, + such as `https://mirror.example.com`; plain `http` works only on + `localhost`. Every archive must match its pinned SHA-256, mirrored or not. +- Downloads go through `HTTPS_PROXY` or `HTTP_PROXY`, minus the hosts in + `NO_PROXY`. On Node 22 and early Node 24 releases, also set + `NODE_USE_ENV_PROXY=1`. +- On Linux, Firefox unpacks with `tar` and `xz`; `--with-deps` installs + `xz-utils`. +- `chromium` video needs an ffmpeg with `libvpx-vp9`: `E2E_FFMPEG_PATH`, + else `ffmpeg` on the `PATH`, else a pinned build the first recording + downloads into the browser cache. Install it ahead with `install ffmpeg`. + Windows on Arm has no pinned build: set `E2E_FFMPEG_PATH` to record + video there. +- The command exits non-zero when an install fails. Ctrl-C or SIGTERM stops + the download, removes the partial files, and exits 130 or 143. ## Types @@ -1133,14 +1084,14 @@ The package exports these types for typing config helpers and fixtures: `WebBasicAuth`, `WebInitScript`, `BrowserName`, `BrowserProvider`, `BrowserProviderScope`, `BrowserRequest`, `BrowserLease`, `BrowserReleaseContext`, `BrowserProviderDownloads`, -`BrowserDownloadContext`, `PlaywrightLiveSurface`, `Browser`, `FrameScreen`, +`BrowserDownloadContext`, `Browser`, `FrameScreen`, `BrowserExpectation`, `WebRoute`, `RouteFulfillResponse`, `RouteContinueOverrides`, `WebResponse`, `Cookie`, `CookieFields`, -`CookieFilter`, `Dialog`, and `DialogHandler`. +`CookieFilter`, `Dialog`, `DialogHandler`, `AttachBrowserOptions`, and +`AttachedBrowser`. -The runtime exports are `web`, [`surfaceOf`](#surfaceof), and `test`, -`describe`, `beforeEach`, `afterEach`, `beforeAll`, and `afterAll`, typed -with the `browser` fixture. +The runtime exports are `web`, `attachBrowser`, and `test`, `describe`, `beforeEach`, +`afterEach`, `beforeAll`, and `afterAll` typed with the `browser` fixture. ### `WebOptions` @@ -1152,10 +1103,10 @@ with the `browser` fixture. */ export interface WebOptions { /** - * Browser to launch, `chromium` by default. A `BrowserProvider` leases - * hosted browsers instead: one per worker slot at `prepare`, or one per - * attempt, each attached to over CDP and released when its scope ends. A - * provider implies chromium and excludes `connect`. + * Browser to launch, `chromium` (default) or `firefox`, driven over + * WebDriver BiDi. A `BrowserProvider` leases hosted browsers instead, one + * per worker slot, each attached to over CDP and released when the run + * ends. A provider implies chromium and excludes `connect`. */ readonly browser?: BrowserName | BrowserProvider; /** @@ -1207,22 +1158,20 @@ export interface WebOptions { * The `User-Agent` every attempt's context sends and `navigator.userAgent` * reports, for an app that switches into a test mode on a marker in the * agent string. Defaults to the browser's own. A `user-agent` entry in - * `headers` beside it is `INVALID_CONFIG`, and so is a persistent context - * (`connect.reconnectEndpoint`, or a provider with `scope: 'attempt'`). + * `headers` beside it is `INVALID_CONFIG`. */ readonly userAgent?: string; /** * The locale every attempt's context runs in, a BCP 47 tag such as * `de-DE`: what `navigator.language`, `Intl` formatting, and the * `Accept-Language` header report. Defaults to the browser's own. An - * `accept-language` entry in `headers` beside it is `INVALID_CONFIG`, and - * so is a persistent context. + * `accept-language` entry in `headers` beside it is `INVALID_CONFIG`. */ readonly locale?: string; /** * The IANA time zone every attempt's context runs in, such as * `Europe/Berlin`: what `Date` and `Intl` resolve local time against. - * Defaults to the machine's. A persistent context is `INVALID_CONFIG`. + * Defaults to the machine's. */ readonly timezoneId?: string; /** The `prefers-color-scheme` every page reports. See [Media emulation](#media-emulation). */ diff --git a/docs/security.mdx b/docs/security.mdx index f916632533..67a76cf1f4 100644 --- a/docs/security.mdx +++ b/docs/security.mdx @@ -38,8 +38,7 @@ Redaction matches strings, and screenshots mask secure fields only: - **Request credentials.** `headers` go on every request that starts at the app's origin or one you list, even one a third-party page starts, and stay on its redirects to any origin. `basicAuth` answers a `401` from those - origins, and from any origin on WebKit when one of those is on a default - port or has an IPv6 host. + origins only. - **Committed recordings.** Cache entries store typed values verbatim, except secret fills. They aren't signed and replay with your permissions, so review them as you would test code. diff --git a/docs/web.mdx b/docs/web.mdx index 4f77d9ee4a..f33c512972 100644 --- a/docs/web.mdx +++ b/docs/web.mdx @@ -147,9 +147,7 @@ workers. Use `basicAuth` for HTTP basic authentication. The browser answers a `401` challenge from the origin of `app.url`, and from each origin in -`credentialOrigins`. WebKit answers any origin when one of those is on a -default port, such as an `https://` URL with no port, or has an IPv6 host. -Declare the password under `secrets` and pass `secrets.get(name)`: +`credentialOrigins`. Declare the password under `secrets` and pass `secrets.get(name)`: ```ts title="e2e.config.ts" import type { E2EConfig } from 'e2e'; @@ -195,7 +193,7 @@ export default { targets: [ { name: 'desktop', engine: web(), app }, { name: 'tablet', engine: web({ viewport: { width: 820, height: 1180 } }), app }, - { name: 'phone', engine: web({ browser: 'webkit', viewport: { width: 390, height: 844 } }), app }, + { name: 'phone', engine: web({ browser: 'firefox', viewport: { width: 390, height: 844 } }), app }, ], } satisfies E2EConfig; ``` diff --git a/packages/core/src/agent/action-dispatcher.ts b/packages/core/src/agent/action-dispatcher.ts index f93c68a24a..e9a3797be9 100644 --- a/packages/core/src/agent/action-dispatcher.ts +++ b/packages/core/src/agent/action-dispatcher.ts @@ -10,6 +10,7 @@ * and queue as `runTool`. */ +import { setTimeout as delay } from 'node:timers/promises'; import { isKeyModifier, KEY_MODIFIERS, type KeyModifier } from '../engine/contract.ts'; import type { LocatorAction, LocatorActionKind, SemanticNode, ViewportPoint, ViewportSize } from '../engine/surface.ts'; import { asEngineError, TestError } from '../internal/errors.ts'; @@ -85,6 +86,14 @@ const MAX_SCROLL_UNTIL_TEXT_CHARS = 200; * unchanged look is that as often as it is the end or the wrong list. */ const SCROLL_UNTIL_STILL_PAGES = 3; +/** + * How long the screen must also have stayed unchanged before the list counts + * as ended, the rest waited out without paging once the still pages are in: + * a list that loads its next page on a timer holds still while it waits, and + * an engine that pages fast would otherwise read that pause as the end where + * a slower one would not. + */ +const SCROLL_UNTIL_STILL_MS = HELD_STILL_MS; const SCROLL_UNTIL_WRONG_LIST_PAGES = 2; /** The engine action each node verb performs; also the name its engine event carries. */ @@ -581,6 +590,7 @@ export class ActionDispatcher { let paged: Pick, 'node' | 'spans'> = {}; let previous: PageReading | undefined; let still = 0; + let changedAt = Date.now(); let within = list; for (let screens = 0; ; screens += 1) { // Only the list being paged is read while it is on screen; once it is @@ -601,7 +611,25 @@ export class ActionDispatcher { if (screens >= MAX_SCROLL_UNTIL_SCREENS) throw notFound(`within ${String(MAX_SCROLL_UNTIL_SCREENS)} screens ${direction}`); const reading = observation.kind === 'semantic' ? readPage(observation) : undefined; still = reading !== undefined && previous !== undefined && !pageMoved(previous, reading) ? still + 1 : 0; - if (still >= SCROLL_UNTIL_STILL_PAGES) throw notFound(`before the screen stopped moving ${direction}, after ${String(screens)} screens`); + if (still === 0) changedAt = Date.now(); + if (still >= SCROLL_UNTIL_STILL_PAGES) { + // A list that loads its next page on a timer holds still while it + // waits: the rest of the stillness window is waited out with no + // more paging, and a screen that moved meanwhile goes on paging. + const rest = SCROLL_UNTIL_STILL_MS - (Date.now() - changedAt); + if (rest > 0) { + await delay(rest); + const later = await this.feed.look('held-still'); + const again = later.kind === 'semantic' ? readPage(later) : undefined; + if (reading !== undefined && again !== undefined && pageMoved(reading, again)) { + observation = later; + previous = undefined; + still = 0; + continue; + } + } + throw notFound(`before the screen stopped moving ${direction}, after ${String(screens)} screens`); + } if (still >= SCROLL_UNTIL_WRONG_LIST_PAGES && within !== undefined) { within = undefined; still = 0; diff --git a/packages/core/src/agent/observation.ts b/packages/core/src/agent/observation.ts index d6b6f9aa50..53f506bb3c 100644 --- a/packages/core/src/agent/observation.ts +++ b/packages/core/src/agent/observation.ts @@ -331,12 +331,14 @@ function judgeLocation(location: string, appOrigin: string | undefined): JudgedL /** * What a judgment is shown, as a wait compares it: the screen's shape, and - * the location unless the tree is withheld, since `vision: 'only'` is not - * shown it (`buildPrompt`). A clock in a device's screen title is no change, - * as in `observationShape`; an http(s) URL is compared as it is, since a time - * in its path or query names another page. + * the location. Under `vision: 'only'` the judgment is shown the pixels alone + * (`buildPrompt`), so only they are compared when there are any: a read that + * fell back to pixels with no tree is the same screen. A clock in a device's screen title + * is no change, as in `observationShape`; an http(s) URL is compared as it + * is, since a time in its path or query names another page. */ export function judgmentShape(observation: AgentObservation, options: { readonly treeWithheld: boolean }): string | undefined { + if (options.treeWithheld && observation.pixels !== undefined) return ``; const shape = observationShape(observation); const location = observation.judgedLocation; if (options.treeWithheld || location === undefined) return shape; diff --git a/packages/core/src/engine/index.ts b/packages/core/src/engine/index.ts index 9e2a081ed7..8dea02b00c 100644 --- a/packages/core/src/engine/index.ts +++ b/packages/core/src/engine/index.ts @@ -36,8 +36,8 @@ export type { ComputedStyleProperty, ComputedStyleValues } from './computed-styl export { ConfigurationError, InfrastructureError, TestError } from '../internal/errors.ts'; export { rejectUnknownKeys, rejectUnknownOptions } from '../internal/options.ts'; export { validateJsonValue, type JsonValueRules } from '../internal/json-value.ts'; -export { describePattern, matchesText, toTextPattern } from '../internal/text.ts'; -export { resolveExpression, type ResolveExpressionOptions } from './resolve.ts'; +export { describePattern, matchesText, MATCHES_TEXT_SOURCE, toTextPattern } from '../internal/text.ts'; +export { createResolver, resolveExpression, type ResolveExpressionOptions, type ResolverPlatform } from './resolve.ts'; export type { TextMatch } from '../types.ts'; export { raceAbort } from './timing.ts'; export { isProviderRecording, stopProviderRecording } from './recording.ts'; @@ -212,7 +212,7 @@ export interface EngineFixtureContext { readonly app: EngineAppInfo & { /** * Resolves a navigation target against the base URL. Throws - * `APP_URL_REQUIRED` when the target declares no URL and `POLICY_DENIED` + * `APP_URL_REQUIRED` for a relative URL when the target declares no URL and `POLICY_DENIED` * for any scheme but `http:` and `https:` (`about:blank` aside), so a * fixture never re-implements the rule the harness owns. */ diff --git a/packages/core/src/engine/resolve.ts b/packages/core/src/engine/resolve.ts index 9522ac901f..8976f71e4a 100644 --- a/packages/core/src/engine/resolve.ts +++ b/packages/core/src/engine/resolve.ts @@ -1,11 +1,13 @@ /** - * The reference locator semantics over a semantic tree: what `locate` - * answers for one `LocatorExpression` when the platform's tree is the whole - * truth. An engine over such a tree (a device's accessibility snapshot, an - * in-memory fake) calls this instead of interpreting the expression itself; - * an engine with a native query engine (a browser) reproduces these rules. - * One immediate pass, every match in document order; polling, strictness, - * and staleness stay with the runner. + * The reference locator semantics: what `locate` answers for one + * `LocatorExpression`. `createResolver` holds the rules every engine shares + * (unions, scopes, filters, positions, the innermost rule) over whatever its + * platform reads; it is self-contained, so a browser engine serializes it + * into the page and resolves there. `resolveExpression` runs it over a + * semantic tree, for an engine whose tree is the whole truth (a device's + * accessibility snapshot, an in-memory fake). One immediate pass, every + * match in the platform's order; polling, strictness, and staleness stay + * with the runner. */ import { matchesText } from '../internal/text.ts'; @@ -17,6 +19,171 @@ import { type TextPattern, } from './contract.ts'; +/** What the locator rules read of a platform's nodes. */ +export interface ResolverPlatform { + /** Every node, in the order matches come in and `nth` counts by. */ + readonly nodes: readonly N[]; + /** The node's parent among `nodes`, undefined for a root. */ + parent(node: N): N | undefined; + /** Whether `node` matches `query` by the platform's rules. */ + matches(node: N, query: SemanticQuery): boolean; + /** Whether a `filter({ hasText })` keeps `node`. */ + hasText(node: N, pattern: TextPattern): boolean; + /** The nodes among `candidates` a platform-native selector addresses, in their order. */ + selector(selector: string, candidates: readonly N[]): readonly N[]; + /** Throws the platform's error for a locator that cannot resolve. */ + fail(code: 'FRAME_NOT_FOUND', message: string): never; +} + +/** + * The locator rules over one platform: a function from an expression to the + * nodes it matches, in `platform.nodes` order, but a union's, which come in + * document order (each node before its children, a parent's children in + * `platform.nodes` order), as Playwright orders `or`. `text` and `label` queries + * answer with the innermost match, since a container echoing a descendant's + * text is not a second match; `scope` searches strict descendants of the + * scope's matches; `filter` keeps a match whose own or descendants' text + * answers `hasText` and whose descendants answer `has`, each `has` resolved + * among that match's descendants alone; `index` picks `first`, `last`, or a + * position. Linear in the nodes for every kind but `has`, which reads each + * match's subtree once. Serialized with `toString()`, so it closes over + * nothing of this module. + */ +export function createResolver(platform: ResolverPlatform): (expression: LocatorExpression) => N[] { + const INNERMOST_KINDS = new Set(['text', 'label']); + /** The tree the nodes form; built the first time a union or a `has` asks. */ + interface TreeIndex { + /** Each node's place in `platform.nodes`. */ + readonly position: Map; + /** Each node's children, in `platform.nodes` order. */ + readonly children: Map; + /** Each node's place in document order. */ + readonly rank: Map; + } + let tree: TreeIndex | undefined; + const treeIndex = (): TreeIndex => { + if (tree !== undefined) return tree; + const position = new Map(); + const children = new Map(); + const roots: N[] = []; + platform.nodes.forEach((node, index) => { + position.set(node, index); + const parent = platform.parent(node); + if (parent === undefined) { + roots.push(node); + return; + } + const siblings = children.get(parent); + if (siblings === undefined) children.set(parent, [node]); + else siblings.push(node); + }); + const rank = new Map(); + const stack = roots.toReversed(); + while (stack.length > 0) { + const node = stack.pop() as N; + rank.set(node, rank.size); + const below = children.get(node); + if (below !== undefined) for (let index = below.length - 1; index >= 0; index -= 1) stack.push(below[index] as N); + } + tree = { position, children, rank }; + return tree; + }; + + /** Drops every match that holds another match: each match marks its ancestors once. */ + const innermostOnly = (matches: readonly N[]): N[] => { + const holders = new Set(); + for (const match of matches) { + for (let at = platform.parent(match); at !== undefined && !holders.has(at); at = platform.parent(at)) holders.add(at); + } + return matches.filter((match) => !holders.has(match)); + }; + + /** The candidates that are strict descendants of a node in `ancestors`, each parent chain walked once. */ + const descendantsOf = (ancestors: readonly N[], candidates: readonly N[]): N[] => { + if (ancestors.length === 0) return []; + const scopes = new Set(ancestors); + const known = new Map(); + const isInside = (node: N): boolean => { + const chain: N[] = []; + let inside = false; + for (let at: N | undefined = node; at !== undefined; ) { + const answer = known.get(at); + if (answer !== undefined) { + inside = answer; + break; + } + chain.push(at); + at = platform.parent(at); + if (at !== undefined && scopes.has(at)) { + inside = true; + break; + } + } + for (let index = chain.length - 1; index >= 0; index -= 1) { + known.set(chain[index] as N, inside); + inside ||= scopes.has(chain[index] as N); + } + return known.get(node) === true; + }; + return candidates.filter(isInside); + }; + + /** The strict descendants of `node`, in `platform.nodes` order. */ + const subtreeOf = (node: N): N[] => { + const { position, children } = treeIndex(); + const out: N[] = []; + const stack = [...(children.get(node) ?? [])]; + while (stack.length > 0) { + const at = stack.pop() as N; + out.push(at); + stack.push(...(children.get(at) ?? [])); + } + return out.toSorted((a, b) => (position.get(a) ?? 0) - (position.get(b) ?? 0)); + }; + + const resolve = (expression: LocatorExpression, candidates: readonly N[]): N[] => { + switch (expression.kind) { + case 'union': { + const matches = new Set([...resolve(expression.left, candidates), ...resolve(expression.right, candidates)]); + const { rank } = treeIndex(); + return candidates.filter((node) => matches.has(node)).toSorted((a, b) => (rank.get(a) ?? 0) - (rank.get(b) ?? 0)); + } + case 'query': { + const pool = expression.scope === undefined ? candidates : descendantsOf(resolve(expression.scope, candidates), candidates); + const matches = pool.filter((node) => platform.matches(node, expression.query)); + return INNERMOST_KINDS.has(expression.query.kind) ? innermostOnly(matches) : matches; + } + case 'filter': { + const source = resolve(expression.source, candidates); + const { has, hasNot, hasText, hasNotText } = expression; + const pool = has === undefined && hasNot === undefined ? undefined : new Set(candidates); + return source.filter((node) => { + if (hasText !== undefined && !platform.hasText(node, hasText)) return false; + if (hasNotText !== undefined && platform.hasText(node, hasNotText)) return false; + if (pool !== undefined) { + const within = subtreeOf(node).filter((each) => pool.has(each)); + if (has !== undefined && resolve(has, within).length === 0) return false; + if (hasNot !== undefined && resolve(hasNot, within).length > 0) return false; + } + return true; + }); + } + case 'index': { + const source = resolve(expression.source, candidates); + const at = expression.index === 'first' ? 0 : expression.index === 'last' ? source.length - 1 : expression.index; + const picked = source[at]; + return picked === undefined ? [] : [picked]; + } + case 'selector': + return [...platform.selector(expression.selector, candidates)]; + case 'frame': + return platform.fail('FRAME_NOT_FOUND', `no nested document matches ${JSON.stringify(expression.selector)}: this surface has none to scope a query into`); + } + }; + + return (expression) => resolve(expression, platform.nodes); +} + export interface ResolveExpressionOptions { /** * Answers a `selector` expression: the nodes the platform-native selector @@ -34,31 +201,17 @@ export interface ResolveExpressionOptions { * * Query kinds: `role` compares the role exactly, then the name, the requested * states, and the heading level, and never matches a hidden node; `label` - * and `text` match the name (`text` the visible text too) and answer with - * the innermost match, since a container echoing a descendant's text is not - * a second match; `placeholder`, `displayValue`, and `testId` compare their - * one field. `visible` drops hidden nodes from any kind. `scope` searches - * strict descendants of the scope's matches; `filter` keeps a match whose - * own or descendants' name, text, or value matches `hasText`, and whose - * descendants answer `has`; `index` picks `first`, `last`, or a position. - * `frame` is `FRAME_NOT_FOUND`: a semantic tree has no nested documents. + * and `text` match the name (`text` the visible text too); `placeholder`, + * `displayValue`, and `testId` compare their one field. `visible` drops + * hidden nodes from any kind. A `hasText` filter reads the node's and its + * descendants' name, text, and value. `frame` is `FRAME_NOT_FOUND`: a + * semantic tree has no nested documents. */ export function resolveExpression( expression: LocatorExpression, nodes: readonly SemanticNode[], options: ResolveExpressionOptions = {}, ): SemanticNode[] { - const tree = indexTree(nodes); - return resolve(expression, tree.all, tree, options); -} - -/** Every node of a tree in document order, with the parent of each. */ -interface TreeIndex { - readonly all: readonly SemanticNode[]; - readonly parents: ReadonlyMap; -} - -function indexTree(roots: readonly SemanticNode[]): TreeIndex { const all: SemanticNode[] = []; const parents = new Map(); const walk = (node: SemanticNode, parent: SemanticNode | undefined): void => { @@ -66,118 +219,32 @@ function indexTree(roots: readonly SemanticNode[]): TreeIndex { if (parent !== undefined) parents.set(node, parent); for (const child of node.children ?? []) walk(child, node); }; - for (const root of roots) walk(root, undefined); - return { all, parents }; -} - -function resolve( - expression: LocatorExpression, - candidates: readonly SemanticNode[], - tree: TreeIndex, - options: ResolveExpressionOptions, -): SemanticNode[] { - switch (expression.kind) { - case 'union': { - const matches = new Set([ - ...resolve(expression.left, candidates, tree, options), - ...resolve(expression.right, candidates, tree, options), - ]); - return candidates.filter((node) => matches.has(node)); - } - case 'query': { - const pool = - expression.scope === undefined - ? candidates - : descendantsOf(resolve(expression.scope, candidates, tree, options), candidates, tree); - const matches = pool.filter((node) => matchesQuery(node, expression.query)); - return INNERMOST_KINDS.has(expression.query.kind) ? innermostOnly(matches, tree) : matches; - } - case 'filter': { - const source = resolve(expression.source, candidates, tree, options); - return source.filter((node) => { - if (expression.hasText !== undefined && !subtreeHasText(node, candidates, tree, expression.hasText)) { - return false; - } - if (expression.hasNotText !== undefined && subtreeHasText(node, candidates, tree, expression.hasNotText)) { - return false; - } - if (expression.has !== undefined || expression.hasNot !== undefined) { - const within = descendantsOf([node], candidates, tree); - if (expression.has !== undefined && resolve(expression.has, within, tree, options).length === 0) return false; - if (expression.hasNot !== undefined && resolve(expression.hasNot, within, tree, options).length > 0) return false; - } - return true; - }); - } - case 'index': { - const source = resolve(expression.source, candidates, tree, options); - const position = - expression.index === 'first' ? 0 : expression.index === 'last' ? source.length - 1 : expression.index; - const picked = source[position]; - return picked === undefined ? [] : [picked]; - } - case 'selector': { + for (const root of nodes) walk(root, undefined); + return createResolver({ + nodes: all, + parent: (node) => parents.get(node), + matches: matchesQuery, + hasText: subtreeHasText, + selector: (selector, candidates) => { if (options.selector === undefined) { throw new EngineError( 'UNSUPPORTED_CAPABILITY', - `selector ${JSON.stringify(expression.selector)} needs a platform selector language, and this surface has none`, + `selector ${JSON.stringify(selector)} needs a platform selector language, and this surface has none`, { retryable: false }, ); } - return [...options.selector(expression.selector, candidates)]; - } - case 'frame': - throw new EngineError( - 'FRAME_NOT_FOUND', - `no nested document matches ${JSON.stringify(expression.selector)}: this surface has none to scope a query into`, - { retryable: false }, - ); - } -} - -/** The query kinds a container answers for its descendants, so only the innermost match counts. */ -const INNERMOST_KINDS: ReadonlySet = new Set(['text', 'label']); - -/** - * Drops every match that contains another match, the way a browser's text - * selector answers with the innermost element. A device tree echoes text - * upwards: iOS reports a React Native Text host view and its StaticText child - * with the same label, and a container view inherits its descendants' labels, - * so without this rule every `getByText` on such a screen is ambiguous. - */ -function innermostOnly(matches: readonly SemanticNode[], tree: TreeIndex): SemanticNode[] { - return matches.filter((node) => !matches.some((other) => other !== node && isWithin(other, node, tree))); -} - -/** True when `node` is a strict descendant of `ancestor`. */ -function isWithin(node: SemanticNode, ancestor: SemanticNode, tree: TreeIndex): boolean { - for (let current = tree.parents.get(node); current !== undefined; current = tree.parents.get(current)) { - if (current === ancestor) return true; - } - return false; -} - -/** The candidates that are strict descendants of any node in `ancestors`, in document order. */ -function descendantsOf( - ancestors: readonly SemanticNode[], - candidates: readonly SemanticNode[], - tree: TreeIndex, -): SemanticNode[] { - if (ancestors.length === 0) return []; - return candidates.filter((node) => ancestors.some((ancestor) => isWithin(node, ancestor, tree))); + return options.selector(selector, candidates); + }, + fail: (code, message) => { + throw new EngineError(code, message, { retryable: false }); + }, + })(expression); } /** Whether the node's own name, text, or value, or a descendant's, matches `pattern`. */ -function subtreeHasText( - node: SemanticNode, - candidates: readonly SemanticNode[], - tree: TreeIndex, - pattern: TextPattern, -): boolean { - const textsOf = (entry: SemanticNode): (string | undefined)[] => [entry.name, entry.text, entry.value]; - const matchesAny = (entry: SemanticNode): boolean => - textsOf(entry).some((text) => text !== undefined && matchesText(text, pattern)); - return matchesAny(node) || descendantsOf([node], candidates, tree).some(matchesAny); +function subtreeHasText(node: SemanticNode, pattern: TextPattern): boolean { + if ([node.name, node.text, node.value].some((text) => text !== undefined && matchesText(text, pattern))) return true; + return (node.children ?? []).some((child) => subtreeHasText(child, pattern)); } /** The states a role query may require, in contract order. */ diff --git a/packages/core/src/internal/regexp.ts b/packages/core/src/internal/regexp.ts index e225fd6902..a8d0947695 100644 --- a/packages/core/src/internal/regexp.ts +++ b/packages/core/src/internal/regexp.ts @@ -5,25 +5,4 @@ export function escapeRegexpChar(ch: string): string { return /[.*+?^${}()|[\]\\]/.test(ch) ? `\\${ch}` : ch; } -/** Compiled wire patterns, bounded; assertion polls test the same pattern many times a second. */ -const MAX_CACHED_PATTERNS = 256; -const compiled = new Map(); - -/** - * Tests input against a wire regexp (source + flags). Compiled patterns are - * cached by source and flags; `lastIndex` is reset before every test, so - * sticky/global state can never leak between matches. - */ -export function testPattern(source: string, flags: string, input: string): boolean { - const key = `${flags}\n${source}`; - let pattern = compiled.get(key); - if (pattern === undefined) { - pattern = new RegExp(source, flags); - if (compiled.size >= MAX_CACHED_PATTERNS) { - compiled.delete(compiled.keys().next().value as string); - } - compiled.set(key, pattern); - } - pattern.lastIndex = 0; - return pattern.test(input); -} +export { testPattern } from './text.ts'; diff --git a/packages/core/src/internal/text.ts b/packages/core/src/internal/text.ts index fb8df7bdea..ce369f3310 100644 --- a/packages/core/src/internal/text.ts +++ b/packages/core/src/internal/text.ts @@ -4,10 +4,32 @@ import { types } from 'node:util'; import type { TextPattern } from '../engine/contract.ts'; import type { TextMatch } from '../types.ts'; import { sanitizeText, TestError } from './errors.ts'; -import { testPattern } from './regexp.ts'; export type { TextPattern }; +/** Compiled wire patterns, bounded; assertion polls test the same pattern many times a second. */ +const MAX_CACHED_PATTERNS = 256; +const compiled = new Map(); + +/** + * Tests input against a wire regexp (source + flags). Compiled patterns are + * cached by source and flags; `lastIndex` is reset before every test, so + * sticky/global state can never leak between matches. + */ +export function testPattern(source: string, flags: string, input: string): boolean { + const key = `${flags}\n${source}`; + let pattern = compiled.get(key); + if (pattern === undefined) { + pattern = new RegExp(source, flags); + if (compiled.size >= MAX_CACHED_PATTERNS) { + compiled.delete(compiled.keys().next().value as string); + } + compiled.set(key, pattern); + } + pattern.lastIndex = 0; + return pattern.test(input); +} + /** Caps prose at `maxChars`, marking the cut with an ellipsis; a cut never splits a character in two (a surrogate pair). */ export function bound(text: string, maxChars: number): string { if (text.length <= maxChars) return text; @@ -112,3 +134,16 @@ export function describePattern(pattern: TextPattern): string { if (pattern.kind === 'string') return JSON.stringify(pattern.value); return `/${pattern.source}/${pattern.flags}`; } + +/** + * `matchesText` and everything it calls as source that declares them, so an + * engine that matches in a page (a browser's locate) matches text by this + * one rule. Each function reads only this module's bindings, which the + * source declares too. + */ +export const MATCHES_TEXT_SOURCE = `const MAX_CACHED_PATTERNS = ${MAX_CACHED_PATTERNS}; +const compiled = new Map(); +${testPattern.toString()} +${normalizeText.toString()} +${compareText.toString()} +${matchesText.toString()}`; diff --git a/packages/core/src/internal/urls.ts b/packages/core/src/internal/urls.ts index 237ed24deb..6b3a8c5cd0 100644 --- a/packages/core/src/internal/urls.ts +++ b/packages/core/src/internal/urls.ts @@ -1,8 +1,7 @@ /** URL normalization and origin policy helpers. */ import { ConfigurationError } from './errors.ts'; -import { testPattern } from './regexp.ts'; -import { toTextPattern, withIgnoreCase } from './text.ts'; +import { testPattern, toTextPattern, withIgnoreCase } from './text.ts'; export interface NormalizedBaseUrl { /** Serialized base URL without trailing artifacts beyond the normalized path. */ @@ -191,7 +190,8 @@ export function sameSite(url: string | URL, site: string): boolean { /** * Compares a current URL to an expected string/regexp. - * Relative expected strings resolve against the base URL; string comparison is + * Relative expected strings resolve against the base URL, and match nothing + * without one; string comparison is * exact after WHATWG serialization; regexps test the complete serialized URL. * Anything else is `INVALID_ARGUMENT`, the `toTextPattern` rule. * `ignoreCase` is Playwright's: `true` compares a string case-insensitively @@ -200,7 +200,7 @@ export function sameSite(url: string | URL, site: string): boolean { export function urlMatches( current: string, expected: string | RegExp, - baseHref: string, + baseHref: string | undefined, ignoreCase?: boolean, ): boolean { const pattern = withIgnoreCase(toTextPattern(expected), ignoreCase); diff --git a/packages/core/src/locator/engine.ts b/packages/core/src/locator/engine.ts index 9e101e721a..c88c0b462c 100644 --- a/packages/core/src/locator/engine.ts +++ b/packages/core/src/locator/engine.ts @@ -150,7 +150,10 @@ export class LocatorEngine { /** Builds an operation context on a plain budget, capped by the remaining test timeout: 0 is an expired one. */ operation(timeoutMs?: number): OperationContext { - return this.operationWithin(this.capped(timeoutMs)); + const own = timeoutMs ?? this.options.actionTimeout; + // The call's own timeout, exactly, unless the running phase ends first: one read back off a deadline set + // a moment ago can come out a millisecond short, and the budget is the number an error names. + return this.options.budget.deadline.remaining() < own ? this.operationWithin(this.options.budget.deadline) : this.readOperation(own); } /** The reads of one wait on `deadline`: the last may run past it, up to the running phase's deadline. */ diff --git a/packages/core/src/run/fixtures.ts b/packages/core/src/run/fixtures.ts index 93bbc3e1f7..4b02c510d4 100644 --- a/packages/core/src/run/fixtures.ts +++ b/packages/core/src/run/fixtures.ts @@ -362,10 +362,7 @@ function fixtureContext( fixture: (name, surface, operations) => recorder.fixture(name, surface, operations), app: { ...engineAppInfo(app), - resolveUrl: (url) => { - requireAppUrl(environment.target); - return resolveNavigationUrl(url, app.base).url; - }, + resolveUrl: (url) => resolveNavigationUrl(url, app.base).url, }, // Getters: an `@e2e-dev/core` session's `setDefaultTimeout` moves them under a live fixture. timeouts: { diff --git a/packages/core/src/secrets.ts b/packages/core/src/secrets.ts index 052899178d..13bd8339d1 100644 --- a/packages/core/src/secrets.ts +++ b/packages/core/src/secrets.ts @@ -42,41 +42,64 @@ export function withConfigSecretReferences(load: () => T): T { return scope.run(true, load); } -/** Installed by the runner once config resolution completes. */ -export function setSecretRegistry(registry: SecretRegistry | undefined): void { +/** + * The registries secrets resolve against, in layers: the base a run worker + * installs, and the holds standalone workers and attempts stack on it, oldest + * first. Realm-shared like the installed registry, so every copy of e2e + * stacks on the same layers. + */ +interface RegistryLayers { + base: SecretRegistry | undefined; + readonly holds: { readonly registry: SecretRegistry; readonly live: boolean }[]; +} + +const layersSlot = realmSlot('e2e.secrets.layers.v1'); + +/** The realm's registry layers, created on first use. */ +function registryLayers(): RegistryLayers { + let layers = layersSlot.get(globalThis); + if (layers === undefined) { + layers = { base: undefined, holds: [] }; + layersSlot.set(globalThis, layers); + } + return layers; +} + +/** Installs the registry lookups resolve against: the newest live hold's, else the newest hold's, else the base. */ +function installTop(layers: RegistryLayers): void { + const registry = (layers.holds.findLast((hold) => hold.live) ?? layers.holds.at(-1))?.registry ?? layers.base; if (registry === undefined) registrySlot.delete(globalThis); else registrySlot.set(globalThis, registry); } -/** The registries standalone attempts hold, oldest first; the newest is the one installed. */ -const holds: { readonly registry: SecretRegistry; readonly live: boolean }[] = []; - -/** The newest live hold's registry, else the newest hold's, else none. */ -function installHeld(): void { - const newest = holds.findLast((hold) => hold.live) ?? holds.at(-1); - if (newest === undefined) registrySlot.delete(globalThis); - else registrySlot.set(globalThis, newest.registry); +/** Installed by the runner once config resolution completes: the base layer, under any hold still open. */ +export function setSecretRegistry(registry: SecretRegistry | undefined): void { + const layers = registryLayers(); + layers.base = registry; + installTop(layers); } /** - * Installs `registry` for a standalone worker or attempt until the returned - * release runs. Several may hold one at once, as live sessions do: releasing - * one installs the newest registry still held, and releasing the last clears - * the slot, so closing one session never leaves another resolving against - * none. A `live` hold is an attempt's: it outranks a worker that opens while - * the attempt runs, whose config would otherwise capture the attempt's handles. + * Stacks `registry` on the installed ones for a standalone worker or attempt + * until the returned release runs. Lookups resolve against the newest live + * hold, else the newest hold, else the base a run worker installed, so + * closing one session never leaves another resolving against none. A `live` + * hold is an attempt's: it outranks a worker that opens while the attempt + * runs, whose config would otherwise capture the attempt's handles. A + * registry with no secrets and no credentials (an engine opened with no + * config, as `attachBrowser` does) holds nothing, so it hides nothing. */ export function holdSecretRegistry(registry: SecretRegistry, options: { readonly live?: boolean } = {}): () => void { + if (registry.secrets.size === 0 && registry.credentials.size === 0) return () => undefined; + const layers = registryLayers(); const hold = { registry, live: options.live === true }; - holds.push(hold); - installHeld(); + layers.holds.push(hold); + installTop(layers); return () => { - const index = holds.indexOf(hold); + const index = layers.holds.indexOf(hold); if (index === -1) return; - holds.splice(index, 1); - // A registry a run installed since stays installed. - if (registrySlot.get(globalThis) !== registry) return; - installHeld(); + layers.holds.splice(index, 1); + installTop(layers); }; } diff --git a/packages/e2e/src/cli/init/engines.ts b/packages/e2e/src/cli/init/engines.ts index 6b8578fb75..56b48d6fc8 100644 --- a/packages/e2e/src/cli/init/engines.ts +++ b/packages/e2e/src/cli/init/engines.ts @@ -35,7 +35,7 @@ export function getEnginePresets() { { id: 'web', label: 'Web', - hint: 'Playwright', + hint: 'Chromium, Firefox', dependencies: siblingDependency('@e2e-dev/web'), imports: ["import { web } from '@e2e-dev/web';"], target: `{ diff --git a/packages/e2e/tests/helpers/fake-engine.ts b/packages/e2e/tests/helpers/fake-engine.ts index f62eabefc2..a5dca48f92 100644 --- a/packages/e2e/tests/helpers/fake-engine.ts +++ b/packages/e2e/tests/helpers/fake-engine.ts @@ -160,6 +160,8 @@ export interface FakeEngineBehavior { onCapture?(): void | Promise; /** Contributes a `gadget` fixture exercising every fixture-context facility. */ fixtures?: boolean; + /** The platform the engine declares; default none, so its target names one. */ + platform?: string; /** The secrets the engine declares, which `onStartAttempt` may resolve through its context. */ secrets?: readonly Secret[]; } @@ -252,6 +254,7 @@ export function createFakeEngine(behavior: FakeEngineBehavior = {}): FakeEngineH name: 'fake', version: '1.0.0', spiVersion: ENGINE_SPI_VERSION, + ...(behavior.platform === undefined ? {} : { platform: behavior.platform }), async init(info) { events.push('init'); inits.push(info); diff --git a/packages/e2e/tests/integration/agent-scroll-settle.test.ts b/packages/e2e/tests/integration/agent-scroll-settle.test.ts index 078a5b21a4..690feb487c 100644 --- a/packages/e2e/tests/integration/agent-scroll-settle.test.ts +++ b/packages/e2e/tests/integration/agent-scroll-settle.test.ts @@ -109,4 +109,44 @@ describe('the look after scrolling to text', () => { project.cleanup(); } }); + + it('keeps paging a list that holds still while it loads its next page, until the row arrives', async () => { + let found = false; + const fake = createFakeEngine({ + actions: ['tap', 'swipe'], + scene: (stage) => { + const ticket: ScriptedNode = { id: 'ticket', role: 'button', name: 'Golden Ticket', appearsAfterSwipes: Number.POSITIVE_INFINITY, rect: { x: 0, y: 20, width: 100, height: 40 } }; + // The next page lands on a timer, longer than three quick unchanged pages take. + stage.after(700, () => { ticket.appearsAfterSwipes = 0; }); + return [{ id: 'loading', role: 'status', name: 'Loading more' }, ticket]; + }, + }); + const executor: StepExecutor = { + name: 'scroll-loads', version: '1', + async runStep(context) { + await context.observe(); + await context.actions.scrollUntil('Golden Ticket', 'down'); + found = true; + return { status: 'passed', summary: 'reached the ticket' }; + }, + }; + const project = createProject({ + 'tests/scroll.e2e.ts': `import { test } from 'e2e'; +test('reaches the ticket', async ({ app, agent }) => { + await app.open(); + await agent.act('scroll to the golden ticket'); +});`, + }); + try { + const result = await runExisting(project, { + appUrl: FAKE_APP_URL, + config: { tests: 'tests/**/*.e2e.ts', targets: [{ name: 'device', platform: 'ios' as const, engine: fake.engine, app: FAKE_APP }], agents: { default: { executor } }, cache: 'off' as const }, + }); + const attempt = resultByTitle(result, 'reaches the ticket').attempts[0]; + expect(result.exitCode, JSON.stringify(attempt?.error ?? result.report.run.errors)).toBe(0); + expect(found).toBe(true); + } finally { + project.cleanup(); + } + }); }); diff --git a/packages/e2e/tests/integration/mcp-server.test.ts b/packages/e2e/tests/integration/mcp-server.test.ts index b75a8bb394..8b2e91bd29 100644 --- a/packages/e2e/tests/integration/mcp-server.test.ts +++ b/packages/e2e/tests/integration/mcp-server.test.ts @@ -326,7 +326,8 @@ describe('e2e mcp', { timeout: 120_000 }, () => { expect(stale.isError).toBe(true); expect(stale.text).toMatch(/^tap #n9999 failed: LOCATOR_NOT_FOUND: node #n9999 is not on the current screen/); expect(stale.text).not.toContain('Tapped'); - expect(stale.text).toContain('re-observe'); + // The page's late button may land before or after this look, so the screen comes back as changes or as unchanged. + expect(stale.text).toMatch(/\n\nScreen (changes|unchanged) since revision /); const before = await call('screenshot'); expect(before.isError, before.text).toBe(false); diff --git a/packages/e2e/tests/integration/web-platform.test.ts b/packages/e2e/tests/integration/web-platform.test.ts index 53161fb566..6c8d8089fb 100644 --- a/packages/e2e/tests/integration/web-platform.test.ts +++ b/packages/e2e/tests/integration/web-platform.test.ts @@ -57,7 +57,7 @@ test('deterministic queries and reads', async ({ app, screen, browser }) => { test('class assertions report the observed class on failure', async ({ app, browser }) => { await app.open(); try { - await expect(browser).toHaveClass(browser.locator('#class-card'), 'card inactive', { timeout: 300 }); + await expect(browser).toHaveClass(browser.locator('#class-card'), 'card inactive', { timeout: 2_000 }); throw new Error('toHaveClass unexpectedly passed'); } catch (error) { if (!(error instanceof Error) || !error.message.includes('class "card active"')) throw error; diff --git a/packages/e2e/tests/unit/agent-prompts.test.ts b/packages/e2e/tests/unit/agent-prompts.test.ts index 755fc6df4d..c569839965 100644 --- a/packages/e2e/tests/unit/agent-prompts.test.ts +++ b/packages/e2e/tests/unit/agent-prompts.test.ts @@ -112,4 +112,15 @@ describe('judgmentShape', () => { it('sees no location change when the tree is withheld, since the judgment is not shown it', () => { expect(shapeAt('https://app.test/a', true)).toBe(shapeAt('https://app.test/b', true)); }); + + it('compares only the pixels when the tree is withheld, so a read that fell back to pixels alone is the same screen', () => { + const root = { id: 'root', revision: 'r1' }; + const pixels = { data: new Uint8Array([1, 2, 3]), mediaType: 'image/png' as const, width: 800, height: 600, scale: 1 }; + const base = { root, truncated: false, revision: 'r1', capturedAt: '2026-01-01T00:00:00.000Z', viewport: { width: 800, height: 600 }, redaction: { secureNodeCount: 0, maskedRegionCount: 0 }, location: 'https://app.test/a', pixels }; + const options = { redact: REDACT, redactCut: REDACT, maxBytes: 4_096, appOrigin: 'https://app.test', pixelsAllowed: true }; + const semantic = prepareObservation({ ...base, kind: 'semantic', tree: { ref: root, role: 'document', name: 'Moves' } }, options); + const fallback = prepareObservation({ ...base, kind: 'pixels', location: 'https://app.test/b' }, options); + expect(judgmentShape(fallback, { treeWithheld: true })).toBe(judgmentShape(semantic, { treeWithheld: true })); + expect(judgmentShape(fallback, { treeWithheld: false })).not.toBe(judgmentShape(semantic, { treeWithheld: false })); + }); }); diff --git a/packages/e2e/tests/unit/cli-init.test.ts b/packages/e2e/tests/unit/cli-init.test.ts index 856e91172f..4e2b26c0ef 100644 --- a/packages/e2e/tests/unit/cli-init.test.ts +++ b/packages/e2e/tests/unit/cli-init.test.ts @@ -102,7 +102,7 @@ afterEach(() => { }); describe('e2e init', () => { - it('adds the runner, Playwright, and AI with --yes and does not install or prompt', async () => { + it('adds the runner, the web engine, and AI with --yes and does not install or prompt', async () => { expect((await init(dir, { yes: true })).exitCode).toBe(0); // The runner pins itself the way it pins engines: a caret on a stable version, exact on a canary. const runnerVersion = (JSON.parse(readFileSync(path.resolve(import.meta.dirname, '../../package.json'), 'utf8')) as { version: string }).version; @@ -270,7 +270,7 @@ describe('e2e init', () => { initialValues: ['web'], required: false, options: [ - expect.objectContaining({ value: 'web', label: 'Web', hint: 'Playwright' }), + expect.objectContaining({ value: 'web', label: 'Web', hint: 'Chromium, Firefox' }), expect.objectContaining({ value: 'mobile', label: 'Mobile (iOS/Android)', hint: 'agent-device' }), ], })); diff --git a/packages/e2e/tests/unit/locator-engine.test.ts b/packages/e2e/tests/unit/locator-engine.test.ts index 251bf66d78..56440a65e8 100644 --- a/packages/e2e/tests/unit/locator-engine.test.ts +++ b/packages/e2e/tests/unit/locator-engine.test.ts @@ -3,7 +3,7 @@ * complete EngineError -> runner taxonomy mapping. */ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { EngineError, type EngineErrorCode, @@ -404,6 +404,20 @@ describe('LocatorEngine step deadline', () => { expect(engine.operation(0).timeoutMs).toBe(1); }); + it('gives an operation its own timeout exactly, however the clock moves, and the phase what is left once it ends sooner', () => { + const budget = new AttemptBudget(new AbortController().signal, new Deadline(1_000)); + const { engine } = makeEngine({}, { actionTimeout: 100, budget }); + let now = Date.now(); + const clock = vi.spyOn(Date, 'now').mockImplementation(() => (now += 1)); + try { + expect(engine.operation(500).timeoutMs).toBe(500); + expect(engine.operation().timeoutMs).toBe(100); + expect(engine.operation(5_000).timeoutMs).toBeLessThan(1_000); + } finally { + clock.mockRestore(); + } + }); + it('refuses a negative, NaN, or infinite timeout before resolving', async () => { const { engine, calls } = makeEngine({}); for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { diff --git a/packages/e2e/tests/unit/package-imports.test.ts b/packages/e2e/tests/unit/package-imports.test.ts index 5a374cd40b..b857d4bb07 100644 --- a/packages/e2e/tests/unit/package-imports.test.ts +++ b/packages/e2e/tests/unit/package-imports.test.ts @@ -209,8 +209,8 @@ const STARTUP: readonly { readonly dir: string; readonly entry: string; readonly // A config imports these in every process, and only a worker that runs an ACP step or leases a Limrun device calls them. { dir: 'acp', entry: 'index.ts', lazy: ['@agentclientprotocol/sdk', '@modelcontextprotocol/server'] }, { dir: 'limrun', entry: 'index.ts', lazy: ['@limrun/api'] }, - // `e2e list`, and a run whose browsers a provider hosts, read the config without ever launching a browser. - { dir: 'web', entry: 'index.ts', lazy: ['playwright-core'] }, + // `e2e list`, and a run whose browsers a provider hosts, read the config without ever installing a browser or starting the BiDi mapper. + { dir: 'web', entry: 'index.ts', lazy: ['yauzl', 'chromium-bidi'] }, // A program that only drives a browser never decodes a PNG or runs the agent. { dir: 'core', entry: 'core/index.ts', lazy: ['@napi-rs/image', 'ai', 'zod'] }, ]; @@ -249,8 +249,8 @@ describe('the scan', () => { it('counts a list of type specifiers as a runtime import, since the output keeps `import {} from`', () => { expect(specifiers("import { type Tool, type ToolSet } from 'ai';")).toEqual(['ai']); expect(specifiers("import { type Tool } from 'ai';")).toEqual(['ai']); - expect(staticImports("import { type Browser } from 'playwright-core';")).toEqual(['playwright-core']); - expect(staticImports("import type { Browser } from 'playwright-core';")).toEqual([]); + expect(staticImports("import { type Browser } from 'webdriver-bidi-protocol';")).toEqual(['webdriver-bidi-protocol']); + expect(staticImports("import type { Browser } from 'webdriver-bidi-protocol';")).toEqual([]); }); it('objects to a devDependency, a bare builtin, and an optional peer outside its home', () => { diff --git a/packages/e2e/tests/unit/resolve-expression.test.ts b/packages/e2e/tests/unit/resolve-expression.test.ts index 4f6335528e..b3cfd604d1 100644 --- a/packages/e2e/tests/unit/resolve-expression.test.ts +++ b/packages/e2e/tests/unit/resolve-expression.test.ts @@ -7,8 +7,10 @@ import { describe, expect, it } from 'vitest'; import { + createResolver, resolveExpression, type LocatorExpression, + type ResolverPlatform, type SemanticNode, type SemanticQuery, type TextPattern, @@ -202,3 +204,62 @@ describe('resolveExpression', () => { ); }); }); + +describe('createResolver', () => { + /** A platform over numbered nodes, each one's parent at its index in `parents`, where a query's regexp value tests the node's number. */ + function numbered(parents: readonly (number | undefined)[], order?: readonly number[]): ResolverPlatform { + return { + nodes: order ?? parents.map((_parent, index) => index), + parent: (at) => parents[at], + matches: (at, asked) => asked.value.kind === 'regexp' && new RegExp(asked.value.source).test(String(at)), + hasText: () => false, + selector: () => [], + fail: (code, message) => { + throw new Error(`${code}: ${message}`); + }, + }; + } + + /** A query of `kind` whose value is the regexp `source`. */ + function digits(source: string, kind: SemanticQuery['kind'] = 'role'): LocatorExpression { + return { kind: 'query', query: { kind, value: { kind: 'regexp', source, flags: '' } } }; + } + + it('orders matches, and counts nth, by the platform order', () => { + const resolve = createResolver(numbered([undefined, 0, 0], [0, 2, 1])); + expect(resolve(digits('^[12]$'))).toEqual([2, 1]); + expect(resolve({ kind: 'index', source: digits('^[12]$'), index: 0 })).toEqual([2]); + }); + + it("resolves has among each match's own descendants, so a position counts inside it", () => { + // 0 holds 1 (holding 2 and 3) and 4 (holding 5). + const resolve = createResolver(numbered([undefined, 0, 1, 1, 0, 4])); + const firstLeaf: LocatorExpression = { kind: 'index', source: digits('^[235]$'), index: 'first' }; + expect(resolve({ kind: 'filter', source: digits('^[14]$'), has: firstLeaf })).toEqual([1, 4]); + expect(resolve({ kind: 'filter', source: digits('^[14]$'), has: digits('^5$') })).toEqual([4]); + expect(resolve({ kind: 'filter', source: digits('^[14]$'), hasNot: digits('^5$') })).toEqual([1]); + }); + + it('keeps innermost text matches and scopes to strict descendants of a deep, wide tree in linear time', () => { + // A chain 400 deep, then 20000 leaves under its last node. + const depth = 400; + const parents: (number | undefined)[] = [undefined]; + for (let index = 1; index < depth; index += 1) parents.push(index - 1); + for (let index = 0; index < 20_000; index += 1) parents.push(depth - 1); + const resolve = createResolver(numbered(parents)); + const started = performance.now(); + expect(resolve(digits('.', 'text'))).toHaveLength(20_000); + expect(resolve({ kind: 'query', query: { kind: 'role', value: { kind: 'regexp', source: '.', flags: '' } }, scope: digits('^0$') })).toHaveLength(parents.length - 1); + expect(performance.now() - started).toBeLessThan(1_000); + }); + + it('fails a frame through the platform', () => { + expect(() => createResolver(numbered([undefined]))({ kind: 'frame', selector: 'iframe', source: digits('0') })).toThrowError(/^FRAME_NOT_FOUND: /); + }); + + it('runs serialized, closing over nothing of its module', () => { + // oxlint-disable-next-line no-new-func -- the page revives the resolver from its source the same way + const revived = new Function(`return (${createResolver.toString()})`)() as typeof createResolver; + expect(revived(numbered([undefined, 0, 1]))(digits('.', 'label'))).toEqual([2]); + }); +}); diff --git a/packages/e2e/tests/unit/secrets.test.ts b/packages/e2e/tests/unit/secrets.test.ts index 61b9c484d1..70caf85d82 100644 --- a/packages/e2e/tests/unit/secrets.test.ts +++ b/packages/e2e/tests/unit/secrets.test.ts @@ -155,10 +155,47 @@ describe('holdSecretRegistry', () => { expect(resolves('admin-token')).toBe(false); }); + it('puts back the registry a run worker installed before the first hold', () => { + setSecretRegistry(withAdmin); + const release = holdSecretRegistry(withKey); + expect(resolves('api-key')).toBe(true); + release(); + expect(resolves('admin-token')).toBe(true); + setSecretRegistry(undefined); + }); + it('leaves a registry a run installed since in place', () => { const release = holdSecretRegistry(withKey); setSecretRegistry(withAdmin); release(); expect(resolves('admin-token')).toBe(true); }); + + it('hides nothing behind a hold with no secrets or credentials, as attachBrowser opens inside a worker', () => { + setSecretRegistry(withAdmin); + const release = holdSecretRegistry(registry([], []), { live: true }); + expect(resolves('admin-token')).toBe(true); + release(); + expect(resolves('admin-token')).toBe(true); + }); + + it('never puts back a finished run registry once overlapping holds are released', () => { + setSecretRegistry(withAdmin); + const releaseKey = holdSecretRegistry(withKey); + setSecretRegistry(undefined); + const releaseOther = holdSecretRegistry(registry([], [adminToken])); + releaseKey(); + releaseOther(); + expect(resolves('admin-token')).toBe(false); + expect(throws('never-configured')).toBe(false); + }); + + it('keeps open holds resolving when a run clears its registry', () => { + setSecretRegistry(withAdmin); + const release = holdSecretRegistry(withKey); + setSecretRegistry(undefined); + expect(resolves('api-key')).toBe(true); + release(); + expect(throws('never-configured')).toBe(false); + }); }); diff --git a/packages/lightpanda/src/provider.ts b/packages/lightpanda/src/provider.ts index c2d2045f3e..f8a43be72e 100644 --- a/packages/lightpanda/src/provider.ts +++ b/packages/lightpanda/src/provider.ts @@ -27,8 +27,8 @@ export interface LightpandaOptions { * Lightpanda browsers for `web({ browser: lightpanda() })`: one * `lightpanda serve` per worker slot, started on a free port when the * engine asks and stopped when it gives the lease back. Worker scope only: - * Lightpanda does not expose the default context identity a per-attempt - * lease reattaches by. A server already running is a `web({ connect })` + * each attempt already gets a new browser context on the slot's server, so a + * server per attempt would only add a start per test. A server already running is a `web({ connect })` * target, not a provider. */ export function lightpanda(options: LightpandaOptions = {}): BrowserProvider { diff --git a/packages/smol/package.json b/packages/smol/package.json index 5ef58b4559..aa88f38da1 100644 --- a/packages/smol/package.json +++ b/packages/smol/package.json @@ -52,7 +52,7 @@ "test:unit": "vitest run tests/unit" }, "peerDependencies": { - "@e2e-dev/web": ">=0.11.0 <1", + "@e2e-dev/web": ">=0.17.0 <1", "e2e": ">=0.15.0 <1", "smolmachines": ">=1.25.1 <2" }, diff --git a/packages/smol/src/index.ts b/packages/smol/src/index.ts index 6ce78221f7..7e0ee127e1 100644 --- a/packages/smol/src/index.ts +++ b/packages/smol/src/index.ts @@ -5,4 +5,4 @@ */ export { smol } from './provider.ts'; -export type { SmolApp, SmolOptions } from './provider.ts'; +export type { SmolApp, SmolOptions, SmolPrepareContext } from './provider.ts'; diff --git a/packages/smol/src/provider.ts b/packages/smol/src/provider.ts index ee19469517..8d59bc0ebd 100644 --- a/packages/smol/src/provider.ts +++ b/packages/smol/src/provider.ts @@ -3,7 +3,7 @@ import { createHash, randomBytes } from 'node:crypto'; import { createServer } from 'node:net'; import path from 'node:path'; -import type { BrowserLease, BrowserProvider, BrowserProviderScope, BrowserReleaseContext, BrowserRequest } from '@e2e-dev/web'; +import { attachBrowser, type AttachedBrowser, type BrowserLease, type BrowserProvider, type BrowserProviderScope, type BrowserReleaseContext, type BrowserRequest } from '@e2e-dev/web'; import { ConfigurationError } from 'e2e/engine'; import { smolMachines, type SmolMachine, type SmolMachines } from './machines.ts'; @@ -94,12 +94,15 @@ export interface SmolOptions { /** * Drives the warm browser once before any attempt branches it: sign in, * seed storage, open the app. Every attempt then starts from that state. - * It gets the browser's DevTools endpoint and must disconnect, not close - * the browser, before it resolves. Runs once per worker slot. `attempt` - * scope only: in `worker` scope every attempt gets a new browser context, - * which would never see it. + * It gets the browser's DevTools endpoint, and `browser` and `screen` on + * its open tab, which smol detaches once it resolves. Another CDP client + * must disconnect, not close the browser, before it resolves. Runs once per + * worker slot. `attempt` scope only: in `worker` scope every attempt gets a + * new browser context, which would never see it. */ - readonly prepare?: ((cdpEndpoint: string) => Promise) | undefined; + readonly prepare?: ((cdpEndpoint: string, attached: SmolPrepareContext) => Promise) | undefined; + /** How long the `browser` and `screen` calls of one `prepare` may take altogether, in milliseconds; default 120000. */ + readonly prepareTimeout?: number | undefined; /** * Runs the app under test inside each browser machine instead of on this * computer: its code never runs on the host, and every attempt gets its @@ -109,6 +112,9 @@ export interface SmolOptions { readonly app?: SmolApp | undefined; } +/** What `prepare` drives the warm browser with: the same `browser` and `screen` a test gets. */ +export type SmolPrepareContext = Pick; + /** A browser machine and the DevTools endpoint it publishes on this computer. */ interface Browser { readonly machine: SmolMachine; @@ -152,6 +158,18 @@ async function waitForCdp(endpoint: string, signal: AbortSignal): Promise throw new Error(`browser at ${endpoint} did not answer within ${CDP_READY_MS / 1000}s (${last})`); } +/** Runs `prepare` with `browser` and `screen` attached to the browser at `endpoint`, then detaches them. */ +async function prepareBrowser(prepare: NonNullable, endpoint: string, signal: AbortSignal, timeout: number | undefined): Promise { + const attached = await attachBrowser(endpoint, { signal, timeout }); + try { + await prepare(endpoint, { browser: attached.browser, screen: attached.screen }); + } catch (cause) { + await attached.close().catch(() => undefined); + throw cause; + } + await attached.close(); +} + /** Runs `command` in the background, past the end of the exec that starts it. */ function detached(command: string, log = '/dev/null'): string { return `nohup setsid ${command} ${log} 2>&1 &`; @@ -244,7 +262,7 @@ function appScript(app: SmolApp): string { * target, and named after both, so `sweep` deletes what a dead worker left. */ export function smol(options: SmolOptions = {}): BrowserProvider { - const { scope = 'attempt', setup, cpus = DEFAULT_CPUS, memoryMb = DEFAULT_MEMORY_MB, hostPorts = [], prepare, app } = options; + const { scope = 'attempt', setup, cpus = DEFAULT_CPUS, memoryMb = DEFAULT_MEMORY_MB, hostPorts = [], prepare, prepareTimeout, app } = options; for (const port of hostPorts) checkPort(`hostPorts entry`, port); if (app !== undefined) { checkPort('app.port', app.port); @@ -257,6 +275,9 @@ export function smol(options: SmolOptions = {}): BrowserProvider { throw new ConfigurationError('INVALID_CONFIG', `smol: port ${app.port} is both app.port and in hostPorts; the browser's localhost:${app.port} can be only one of them`); } } + if (prepareTimeout !== undefined && (typeof prepareTimeout !== 'number' || !Number.isFinite(prepareTimeout) || prepareTimeout <= 0)) { + throw new ConfigurationError('INVALID_CONFIG', `smol: prepareTimeout must be a positive number of milliseconds, got ${JSON.stringify(prepareTimeout)}`); + } const appSource = app === undefined ? undefined : path.resolve(app.source ?? '.'); if (prepare !== undefined && scope === 'worker') { // A worker-scope browser gives every attempt a new context, which never sees what prepare left in the default one. @@ -287,7 +308,7 @@ export function smol(options: SmolOptions = {}): BrowserProvider { await machine.shell(startScript(setup, hostPorts), request.signal); if (app !== undefined) await machine.shell(appScript(app), request.signal); await waitForCdp(endpoint, request.signal); - if (prepare !== undefined) await prepare(endpoint); + if (prepare !== undefined) await prepareBrowser(prepare, endpoint, request.signal, prepareTimeout); return { machine, endpoint }; } catch (cause) { await machines.delete(name).catch(() => false); diff --git a/packages/smol/tests/unit/smol.test.ts b/packages/smol/tests/unit/smol.test.ts index 70a9b94d05..7b72273357 100644 --- a/packages/smol/tests/unit/smol.test.ts +++ b/packages/smol/tests/unit/smol.test.ts @@ -83,11 +83,24 @@ const sdk = vi.hoisted(() => { vi.mock('smolmachines', () => ({ Machine: sdk.Machine })); +/** Each `attachBrowser` call: the endpoint, and whether the attached session was closed. */ +const attached = vi.hoisted(() => [] as { endpoint: string; timeout: number | undefined; browser: object; screen: object; closed: boolean }[]); + +vi.mock('@e2e-dev/web', async (importOriginal) => ({ + ...(await importOriginal()), + attachBrowser: async (endpoint: string, options: { timeout?: number } = {}) => { + const session = { endpoint, timeout: options.timeout, browser: { name: 'browser' }, screen: { name: 'screen' }, closed: false }; + attached.push(session); + return { browser: session.browser, screen: session.screen, close: async () => void (session.closed = true) }; + }, +})); + const fetched: string[] = []; beforeEach(() => { Object.assign(sdk.state, { created: [], scripts: [], branches: [], deleted: [], connected: [], connectedOptions: [], files: [], machines: [], exitCode: 0, deleteFailures: new Map(), branchFailure: undefined, createLockFailures: 0, blockAppShell: false, appShellEntered: undefined }); fetched.length = 0; + attached.length = 0; vi.stubGlobal('fetch', async (url: string) => { fetched.push(url); return new Response('{}'); @@ -160,6 +173,32 @@ describe('smol()', () => { expect(first.lines).toEqual([expect.stringMatching(new RegExp(`^browser ${lease1.id}, branched from ${warm} in \\d+ ms$`))]); }); + it('hands prepare the endpoint and a browser and screen attached to it, and detaches them after', async () => { + const calls: { endpoint: string; browser: unknown; screen: unknown; open: boolean }[] = []; + await smol({ + prepare: async (endpoint, { browser, screen }) => void calls.push({ endpoint, browser, screen, open: !attached[0]!.closed }), + }).acquire(request({ attemptId: 'a1' })); + const endpoint = `http://127.0.0.1:${hostPortOf(sdk.state.created[0]!.config.ports)}`; + expect(calls).toEqual([{ endpoint, browser: attached[0]!.browser, screen: attached[0]!.screen, open: true }]); + expect(attached).toEqual([expect.objectContaining({ endpoint, closed: true })]); + }); + + it('gives prepare the prepareTimeout as its attached session timeout', async () => { + await smol({ prepareTimeout: 600_000, prepare: async () => undefined }).acquire(request({ attemptId: 'a1' })); + expect(attached).toEqual([expect.objectContaining({ timeout: 600_000 })]); + }); + + it('refuses a prepareTimeout that is not a positive number of milliseconds', () => { + expect(() => smol({ prepareTimeout: 0 })).toThrow(/prepareTimeout must be a positive number/); + }); + + it('detaches and deletes the machine when prepare fails', async () => { + const provider = smol({ prepare: async () => { throw new Error('sign-in failed'); } }); + await expect(provider.acquire(request({ attemptId: 'a1' }))).rejects.toThrow('sign-in failed'); + expect(attached).toEqual([expect.objectContaining({ closed: true })]); + expect(sdk.state.deleted).toEqual([sdk.state.created[0]!.config.name]); + }); + it('runs setup before Chromium starts and relays each host port to the machine’s loopback', async () => { await smol({ setup: 'apt-get install -y fonts-noto-cjk', hostPorts: [3000, 4271] }).acquire(request({ attemptId: 'a1' })); const script = sdk.state.scripts[0]!.script; diff --git a/packages/web/README.md b/packages/web/README.md index a396090629..80b85a5fe1 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -1,12 +1,13 @@ # @e2e-dev/web -The browser engine for [`e2e`](https://www.npmjs.com/package/e2e), built on -[Playwright](https://playwright.dev). +The browser engine for [`e2e`](https://www.npmjs.com/package/e2e). It drives +Chromium and Firefox over [WebDriver BiDi](https://w3c.github.io/webdriver-bidi/): +Firefox natively, Chromium through the chromium-bidi mapper. `e2e` ships no engine of its own: every target names the engine that drives it, and this package is the one for browsers. It implements the public -`@e2e-dev/core/engine` contract (`e2e/engine` to the runner), so a device or -desktop engine plugs into the same seam with no privilege either way. +`e2e/engine` contract, so a device or desktop engine plugs into the same +seam with no privilege either way. ## Install @@ -14,19 +15,6 @@ desktop engine plugs into the same seam with no privilege either way. npm install --save-dev e2e @e2e-dev/web ``` -To drive a browser from a program without the test runner, install -[`@e2e-dev/core`](https://www.npmjs.com/package/@e2e-dev/core) instead of `e2e`, and -import `web` from `@e2e-dev/web/engine`: - -```bash -npm install @e2e-dev/core @e2e-dev/web -``` - -This package depends on `playwright-core` pinned to an exact version, so the -engine always runs the Playwright it was tested against. An app that depends -on Playwright itself keeps its own copy; the two share a browser cache only -when their versions match. - ```ts title="e2e.config.ts" import type { E2EConfig } from 'e2e'; import { web } from '@e2e-dev/web'; @@ -44,7 +32,7 @@ with `readyUrl` to poll, default `url`), `environment` (`test`, `staging`, session key when the origin is ephemeral). `web({ url })` and the other old app options are `INVALID_CONFIG`. A target with `app.url` is a `browser` target, so a test can `requires: ['browser']`. Browser options: -`browser` (`chromium`, `firefox`, `webkit`; default `chromium`; or a +`browser` (`chromium` or `firefox`; default `chromium`; or a `BrowserProvider` that leases hosted browsers, see below), `viewport` (`{ width, height }`; default 1280x720), `testIdAttribute` (the attribute `getByTestId` and a node's `testId` read; default `data-testid`), `userAgent` @@ -54,16 +42,16 @@ such as `de-DE` and `Europe/Berlin`), `colorScheme` and `reducedMotion` (the `prefers-color-scheme` and `prefers-reduced-motion` every page reports, from first paint; `browser.emulateMedia` changes them in a test), `initScripts` (scripts every document runs before the page's own), and -`connect` — attach to a remote browser over CDP instead of launching a local -one. +`connect` (attach to a remote browser over CDP instead of launching a local +one). ### Attaching to a remote browser (`connect`) Set `connect.cdpEndpoint` to attach over the Chrome DevTools Protocol rather than launch. The resolver is async and called at worker init, and again at the start of any attempt that finds the session dropped, so a hosted browser whose -endpoint is provisioned per run — a cloud session URL that is not known at -config load — resolves each time it is needed. The resolver receives an +endpoint is provisioned per run (a cloud session URL that is not known at +config load) resolves each time it is needed. The resolver receives an `AbortSignal` that fires when the init or attempt is cancelled or exceeds its budget; a browser that connects after that is detached, never used. CDP attach is chromium-only, and disposing the engine detaches the session without @@ -75,19 +63,9 @@ engine: web({ }); ``` -For transport recovery, add `connect.reconnectEndpoint(signal)`. This opts -into the remote browser's persistent default context. `cdpEndpoint` then -provisions a fresh, dedicated browser for every attempt, including retries; -`reconnectEndpoint` must return that same browser after a disconnect. The -engine verifies the default context ID and active page target ID, clears stale -references, and never repeats a dispatched operation. A missing target fails -the attempt. The host owns deleting the remote browser after cleanup. - -This mode does not support `headers`, `basicAuth`, `userAgent`, `locale`, `timezoneId`, context reset, or session -state capture and restore. Recording resumes after reconnect, but a segment -lost during the disconnect remains unavailable. See the -[Playwright reference](../../docs/reference/web.mdx#recovering-a-cdp-transport) -for the lifecycle and ownership contract. +Each attempt gets a new browser context on the remote browser. A dropped +connection fails the attempt, and the next attempt attaches again through +`cdpEndpoint`. ### Leasing hosted browsers (`browser`) @@ -96,10 +74,12 @@ it provisioned. Pass a `BrowserProvider` as `browser` instead and the engine asks for a browser when one is needed and gives it back when its scope ends, on every exit path. `scope: 'worker'` (the default) leases one browser per worker slot in `prepare` and releases them in `finish`; a worker whose browser -drops leases a replacement itself. `scope: 'attempt'` leases a fresh browser in -every `startAttempt` and releases it in `endAttempt`, reattaching through the -lease's `reconnectEndpoint` after a transport drop, with the limits of -`connect.reconnectEndpoint`. A provider implies chromium and excludes +drops leases a replacement itself. `scope: 'attempt'` leases a fresh browser +per attempt instead, and the attempt runs in that browser's own context, so +`app.clearState()` and saved sessions throw `UNSUPPORTED_CAPABILITY`. +Downloads work when the provider implements `downloads` and throw +`UNSUPPORTED_CAPABILITY` otherwise; `setInputFiles` on a +provider's browser sends the files' bytes, up to 50 MB in all. A provider implies chromium and excludes `connect`; no vendor ships in the package. See [Hosted browsers](../../docs/browser.mdx#hosted-browsers) for the contract and an example against a generic session API. @@ -146,11 +126,21 @@ cacheable: npx @e2e-dev/web install chromium --with-deps ``` -It downloads the browsers for the Playwright version this package pins, the -ones the engine launches. Under pnpm, run the bin by name: +It downloads the builds this package pins, each checked against its SHA-256: +Chrome for Testing for `chromium` (the headless shell; `--headed` installs +the full Chrome a headed run launches instead, so run both commands to have +both) and Firefox for `firefox`. Browsers go to `E2E_BROWSERS_PATH`, else the +user cache (`~/Library/Caches/e2e/browsers`, `~/.cache/e2e/browsers`, or +`%LOCALAPPDATA%\e2e\browsers`). Under pnpm, run the bin by name: `pnpm exec e2e-web install chromium --with-deps`. -`--with-deps` also installs the system libraries a slim container image lacks. +Chromium video needs an ffmpeg with VP9: `E2E_FFMPEG_PATH`, else a system +`ffmpeg`, else a pinned build the first recording downloads. `install ffmpeg` +fetches it ahead of time. + +`--with-deps` also installs the system libraries a slim Debian or Ubuntu image +lacks. `E2E_BROWSERS_DOWNLOAD_HOST` names a mirror, and downloads honor +`HTTPS_PROXY`, `HTTP_PROXY`, and `NO_PROXY`. ## Documentation diff --git a/packages/web/package.json b/packages/web/package.json index 2f721769ad..7135062b0f 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,7 +1,7 @@ { "name": "@e2e-dev/web", "version": "0.16.0", - "description": "Web engine for e2e agentic end-to-end testing, built on Playwright", + "description": "Web engine for e2e agentic end-to-end testing: Chromium and Firefox over WebDriver BiDi", "keywords": [ "e2e", "end-to-end", @@ -10,8 +10,9 @@ "web-testing", "browser-testing", "browser-automation", - "playwright", - "chromium" + "webdriver-bidi", + "chromium", + "firefox" ], "license": "Apache-2.0", "contributors": [ @@ -59,7 +60,8 @@ "test:unit": "vitest run tests/unit" }, "dependencies": { - "playwright-core": "1.65.1" + "chromium-bidi": "159.0.8147-0", + "yauzl": "3.4.0" }, "peerDependencies": { "@e2e-dev/core": ">=0.23.2 <1", @@ -74,9 +76,12 @@ "@e2e-dev/core": "workspace:*", "@napi-rs/image": "1.16.0", "@types/node": "26.6.6", + "@types/yauzl": "3.4.0", + "devtools-protocol": "0.0.1734725", "e2e": "workspace:*", "typescript": "7.0.2", - "vitest": "5.1.0" + "vitest": "5.1.0", + "webdriver-bidi-protocol": "1.0.2" }, "engines": { "node": "^22.22.3 || >=24.8.0" diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts deleted file mode 100644 index 2d6090c88c..0000000000 --- a/packages/web/src/actions.ts +++ /dev/null @@ -1,506 +0,0 @@ -/** Locator action dispatch for the Playwright engine. */ - -import type { ElementHandle, Page } from 'playwright-core'; -import { Deadline, EngineError, pollCondition, type KeyModifier, type LocatorAction, type NodeRef, type PointerAction, type ViewportPoint } from '@e2e-dev/core/engine'; -import { - asActionable, - isClassified, - isNavigationRace, - isPwTimeout, - message, - nearestPixel, - performElementSwipe, - performPointDrag, - performPointerDrag, - performViewportSwipe, - POST_DISPATCH_PATTERN, - translatePwError, - targetPage, - type ActionTarget, -} from './support.ts'; -import type { ConnectionBudget } from './operation-budget.ts'; - -/** How long a long press holds the button when the action names no duration. */ -const DEFAULT_LONG_PRESS_MS = 500; - -/** The keys a click holds, as Playwright's `modifiers` takes them; the contract spells them the same. */ -function heldKeys(action: Extract): { modifiers?: KeyModifier[] } { - return action.modifiers === undefined ? {} : { modifiers: [...action.modifiers] }; -} - -/** Playwright's words for an element handle whose element left the DOM. */ -const DETACHED_PATTERN = /element (is |was )?(detached|not attached)/i; - -/** - * Sets a checkbox, switch, or radio to `checked` with one click, as - * Playwright's `check` does, apart from the reads after the click. Playwright - * reads the state once; here it is polled until the operation's deadline, so - * a control that commits its new state after an await or a timer passes, and - * one that never changes fails at the deadline. A control that is gone by - * the first read (an app that swaps a picked radio for its selected view, or - * navigates on change) took the click, so the action is done, where - * Playwright reports it detached as if the click never happened. One that - * goes later, still in its old state, may have been replaced for any reason, - * so it fails rather than passing on a click the app rejected. Whatever took - * its place is not read: the next observation or assertion shows it, as it - * does after a tap. - * The reads and the click share one element, so the state before and after - * the click is one control's. - */ -async function setChecked(target: ActionTarget, checked: boolean, operation: ConnectionBudget): Promise { - const deadline = new Deadline(operation.timeoutMs); - // Playwright reads a timeout of 0 as no timeout at all. - const remaining = (): number => Math.max(1, deadline.remaining()); - const element = - target.kind === 'element' ? target.element : await target.locator.elementHandle({ timeout: operation.timeoutMs }); - const verb = checked ? 'check' : 'uncheck'; - try { - if ((await element.isChecked()) === checked) return; - const radio = checked ? undefined : await radioKind(element); - if (radio === 'native') { - throw new EngineError('NOT_ACTIONABLE', 'uncheck cannot clear a radio button; select another radio in its group', { - retryable: false, - }); - } - await clickControl({ kind: 'element', element }, verb, { timeout: remaining(), signal: operation.signal }); - const hint = radio === 'aria' ? '; a radio usually clears only when another radio in its group is selected' : ''; - const unchanged = (detail = ''): EngineError => - new EngineError('NOT_ACTIONABLE', `${verb} clicked the control but its checked state did not change${detail}${hint}`, { - retryable: false, - }); - let firstRead = true; - await pollCondition({ - deadline, - signal: operation.signal, - negated: false, - evaluate: async () => { - const first = firstRead; - firstRead = false; - try { - return (await element.isChecked()) === checked; - } catch (cause) { - if (!DETACHED_PATTERN.test(message(cause)) && !isNavigationRace(cause)) throw cause; - if (first) return true; - throw unchanged(' before the control was replaced'); - } - }, - onTimeout: () => unchanged(), - }); - } finally { - if (target.kind === 'locator') void element.dispose().catch(() => undefined); - } -} - -/** - * What the page throws while a checkbox or radio and every one of its labels - * are unrendered, so a click could reach none of them yet. - */ -const UNRENDERED = 'the control and its labels are not visible'; - -/** - * Runs in the page. A component library (Element Plus) hides a native - * checkbox or radio under its label's styled spans with `z-index: -1`, or - * gives it no box at all, so Playwright refuses the input itself (` - * intercepts pointer events`, `element is not visible`) though a user's click - * there reaches it through the label. Returns that label, or null to click the - * control itself. The choice reads the control and its labels only, past - * anything else on top (a loading mask, a modal): Playwright's checks on the - * label wait that out, or fail on it. Throws `UNRENDERED` while none of them - * is rendered (a dialog not open yet), since nothing tells the two apart then. - */ -function pickStandIn(node: Element, unrendered: string): HTMLLabelElement | null { - if (!(node instanceof HTMLInputElement) || (node.type !== 'checkbox' && node.type !== 'radio') || node.matches(':disabled')) { - return null; - } - const labels = [...(node.labels ?? [])]; - const root = node.getRootNode(); - if (labels.length === 0 || (!(root instanceof Document) && !(root instanceof ShadowRoot))) return null; - const INTERACTIVE = 'a,button,input,select,textarea,summary,[contenteditable],[role="button"],[role="link"]'; - const hasBox = ({ width, height }: DOMRect): boolean => width > 0 && height > 0; - const centre = ({ left, top, width, height }: DOMRect): [number, number] => [left + width / 2, top + height / 2]; - const topmost = (x: number, y: number): Element | undefined => - root.elementsFromPoint(x, y).find((hit) => hit === node || labels.some((label) => label.contains(hit))); - const pressesInteractive = (label: HTMLLabelElement, hit: Element | undefined): boolean => - label.contains(hit?.closest(INTERACTIVE) ?? null); - node.scrollIntoView({ block: 'nearest', inline: 'nearest' }); - const box = node.getBoundingClientRect(); - if (!hasBox(box)) { - const shown = labels.filter((label) => hasBox(label.getBoundingClientRect()) && label.checkVisibility({ visibilityProperty: true })); - if (shown.length === 0) throw new Error(unrendered); - return ( - shown.find((label) => { - label.scrollIntoView({ block: 'nearest', inline: 'nearest' }); - return !pressesInteractive(label, topmost(...centre(label.getBoundingClientRect()))); - }) ?? null - ); - } - const [x, y] = centre(box); - const hit = topmost(x, y); - if (hit === undefined) throw new Error(unrendered); - const label = labels.find((own) => own.contains(hit)); - if (label === undefined || pressesInteractive(label, hit)) return null; - const outer = label.getBoundingClientRect(); - return x >= outer.left && x <= outer.right && y >= outer.top && y <= outer.bottom ? label : null; -} - -/** - * Runs in the page: where a click on the label lands, the centre of its - * control relative to the label, or none for a control with no box, which - * takes the label's centre. - */ -function controlPoint(label: HTMLLabelElement): { x: number; y: number } | null { - const box = label.control?.getBoundingClientRect(); - if (box === undefined || box.width === 0 || box.height === 0) return null; - const outer = label.getBoundingClientRect(); - return { x: box.left + box.width / 2 - outer.left, y: box.top + box.height / 2 - outer.top }; -} - -/** - * Clicks a control with Playwright's actionability checks, through its own - * label when `pickStandIn` names one. The checks then run on the label, so - * anything else on top, such as a modal, still fails NOT_ACTIONABLE. The - * choice is read again until the control or a label renders, within the - * deadline. - */ -async function clickControl( - target: ActionTarget, - verb: 'tap' | 'check' | 'uncheck', - options: { timeout: number; signal: AbortSignal; modifiers?: KeyModifier[] }, -): Promise { - const { signal, ...click } = options; - const deadline = new Deadline(options.timeout); - const remaining = (): number => Math.max(1, deadline.remaining()); - let label: ElementHandle | undefined; - let firstRead = true; - await pollCondition({ - deadline, - signal, - negated: false, - evaluate: async () => { - const first = firstRead; - firstRead = false; - try { - const handle = - target.kind === 'locator' - ? await target.locator.evaluateHandle(pickStandIn, UNRENDERED, { timeout: remaining() }) - : await target.element.evaluateHandle(pickStandIn, UNRENDERED); - label = handle.asElement() ?? undefined; - if (label === undefined) void handle.dispose().catch(() => undefined); - return true; - } catch (cause) { - if (message(cause).includes(UNRENDERED)) return undefined; - // A read after one that found nothing rendered times out only at the deadline, which ends the poll. - if (!first && isPwTimeout(cause)) return undefined; - throw cause; - } - }, - onTimeout: () => - new EngineError('NOT_ACTIONABLE', `${verb} did not become actionable in time: ${UNRENDERED}`, { retryable: false }), - }); - // The operation abandons this call when cancelled; a read still in flight then must not lead to a click. - if (label === undefined) { - signal.throwIfAborted(); - await asActionable(target).click({ ...click, timeout: remaining() }); - return; - } - try { - const position = await label.evaluate(controlPoint); - signal.throwIfAborted(); - await label.click({ ...click, timeout: remaining(), ...(position === null ? {} : { position }) }); - } finally { - void label.dispose().catch(() => undefined); - } -} - -/** - * Which kind of radio a checkable element is, if any. A native radio can be - * selected but never cleared by a click. An ARIA radio is clicked like any - * control, as Playwright does: its script decides what a click does. - */ -function radioKind(element: ElementHandle): Promise<'native' | 'aria' | undefined> { - return element.evaluate((node) => { - if (node instanceof HTMLInputElement && node.type === 'radio') return 'native'; - return node.getAttribute('role') === 'radio' ? 'aria' : undefined; - }); -} - -/** - * Attaches files to a file input, or to a control that opens a native file - * chooser when clicked, such as a button over a hidden input. A file input, - * or a node inside a `