From f1b86d7dd0002cc67c67c4cad3f66568d40c8364 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 20:05:01 -0700 Subject: [PATCH 001/864] test(interaction): real touch input for browser interaction proofs Adds internal/interaction-harness (@astryxdesign/interaction-harness) and playwright.interaction.config.ts: phone-sized touch projects in Chromium and WebKit for *.interaction.browser.spec.ts specs. Touch scripts are data (tap, drag with hold and rest, flick with an exact release velocity and velocity profiles, hold-then-drag, pinch, rotate, any path, a second finger mid-gesture), played as real engine input: CDP Input.dispatchTouchEvent with scripted timestamps in Chromium, WebKit's own input command in WebKit. The harness's self-test proves the timestamps, a native fling and multi-touch delivery; a pr-interaction CI job runs the proofs against each PR's Storybook build, and a container script runs the WebKit project where host libraries are missing. --- .github/scripts/ci-test-routing.test.mjs | 49 +++ .github/workflows/ci.yml | 88 ++++ .github/workflows/deploy.yml | 3 + internal/README.md | 13 +- internal/interaction-harness/README.md | 67 +++ internal/interaction-harness/package.json | 21 + .../scripts/webkit-container.mjs | 72 ++++ .../src/harness.interaction.browser.spec.ts | 168 ++++++++ internal/interaction-harness/src/index.ts | 17 + .../interaction-harness/src/scripts.test.ts | 142 +++++++ internal/interaction-harness/src/scripts.ts | 395 ++++++++++++++++++ internal/interaction-harness/src/story.ts | 103 +++++ internal/interaction-harness/src/touch.ts | 295 +++++++++++++ internal/interaction-harness/tsconfig.json | 4 + package.json | 1 + playwright.interaction.config.ts | 77 ++++ pnpm-lock.yaml | 15 + 17 files changed, 1524 insertions(+), 6 deletions(-) create mode 100644 internal/interaction-harness/README.md create mode 100644 internal/interaction-harness/package.json create mode 100644 internal/interaction-harness/scripts/webkit-container.mjs create mode 100644 internal/interaction-harness/src/harness.interaction.browser.spec.ts create mode 100644 internal/interaction-harness/src/index.ts create mode 100644 internal/interaction-harness/src/scripts.test.ts create mode 100644 internal/interaction-harness/src/scripts.ts create mode 100644 internal/interaction-harness/src/story.ts create mode 100644 internal/interaction-harness/src/touch.ts create mode 100644 internal/interaction-harness/tsconfig.json create mode 100644 playwright.interaction.config.ts diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs index dcb44405376a1..1e419f13d18f6 100644 --- a/.github/scripts/ci-test-routing.test.mjs +++ b/.github/scripts/ci-test-routing.test.mjs @@ -517,3 +517,52 @@ describe('deploy.yml push gating', () => { } }); }); + +describe('ci.yml interaction proofs', () => { + const workflow = load('ci.yml'); + const job = workflow.jobs['pr-interaction']; + const step = name => job.steps.find(candidate => candidate.name === name); + + it('runs the interaction config in Chromium against the exact PR head', () => { + expect(KNOWN_RUNNERS.has(job['runs-on'])).toBe(true); + expect(job.needs).toEqual( + expect.arrayContaining(['check-scope', 'build-storybook']), + ); + const checkout = job.steps.find(candidate => + candidate.uses?.startsWith('actions/checkout@'), + ); + expect(checkout?.with?.ref).toBe( + '${{ github.event.pull_request.head.sha }}', + ); + const run = step('Run interaction proofs').run; + expect(run).toContain('--config playwright.interaction.config.ts'); + expect(run).toContain('--project chromium'); + const packageJson = JSON.parse(read('package.json')); + expect(packageJson.scripts['test:interaction']).toBe( + 'playwright test --config playwright.interaction.config.ts', + ); + }); + + it('runs whenever the harness, its config, a component or a story changes', () => { + const scope = step('Check interaction scope').run; + expect(scope).toContain('.github/scripts/pr-scope-touches.sh'); + for (const path of [ + 'internal/interaction-harness/', + 'playwright\\.interaction\\.config\\.ts', + 'packages/[^/]+/src/', + 'apps/storybook/', + ]) { + expect(scope).toContain(path); + } + // Unknown scope (exit 2) runs the proofs; only "out of scope" skips them. + expect(scope).toContain('if [ "$scope" = 1 ]; then'); + }); + + it('typechecks the harness in both workflows', () => { + for (const file of ['ci.yml', 'deploy.yml']) { + expect(runLines(load(file).jobs.typecheck), file).toContain( + '@astryxdesign/interaction-harness typecheck', + ); + } + }); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d19b9322e0b0..28db4356c3fc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1171,6 +1171,12 @@ jobs: if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' run: pnpm -F @astryxdesign/native typecheck:types + # The interaction harness is consumed as TypeScript source by Vitest and + # Playwright, which do not typecheck; this is the only gate over it. + - name: Typecheck interaction harness + if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' + run: pnpm -F @astryxdesign/interaction-harness typecheck + # Sandbox preview build — the single slowest piece of PR CI (~4min), run as # its own job so it overlaps build-storybook instead of serializing after it. # PR-only: merge_group runs never deploy previews, and docsite-only PRs skip @@ -3437,6 +3443,88 @@ jobs: retention-days: 30 delete-merged: true + # Touch interaction and animation proofs: the harness in + # internal/interaction-harness and every *.interaction.browser.spec.ts. + # Real touch input, composited frames, rendering traces and presented + # animation values only exist in a real engine, and the specs read stories + # from this PR's own Storybook build. The harness's self-test runs with + # them, so a change to an instrument is proven by its negative controls + # before a component spec relies on it. Chromium only on these runners; + # see the harness README for running WebKit. + pr-interaction: + needs: [check-scope, build-storybook] + if: github.event_name == 'pull_request' && needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' + # Frame evidence measures timing: four cores, one Playwright worker. + runs-on: 4-core-ubuntu + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Verify evidence checkout + env: + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA" + + # The helper comes from the base, since an older PR head lacks it. + # Exit 1 means out of scope; 0 in scope; 2 unknown, so run. + - name: Check interaction scope + id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + scope=2 + helper="$(mktemp)" + if git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA" \ + && git show "$BASE_SHA:.github/scripts/pr-scope-touches.sh" > "$helper"; then + scope=0 + bash "$helper" "$BASE_SHA" '^(internal/interaction-harness/|playwright\.interaction\.config\.ts$|packages/[^/]+/src/|apps/storybook/)' || scope=$? + fi + if [ "$scope" = 1 ]; then + echo "Neither the interaction harness nor a component changed; skipping the interaction proofs." + echo 'run=false' >> "$GITHUB_OUTPUT" + else + echo 'run=true' >> "$GITHUB_OUTPUT" + fi + + - name: Setup Node and pnpm + if: steps.scope.outputs.run == 'true' + uses: ./.github/actions/setup + with: + install: 'false' + + - name: Download Storybook artifact + if: steps.scope.outputs.run == 'true' + uses: actions/download-artifact@v8 + with: + name: storybook-${{ needs.build-storybook.outputs.short_hash }} + path: apps/storybook/dist + + - name: Install Playwright + if: steps.scope.outputs.run == 'true' + run: pnpm install --frozen-lockfile && npx playwright install chromium + + - name: Run interaction proofs + if: steps.scope.outputs.run == 'true' + run: | + pnpm exec playwright test \ + --config playwright.interaction.config.ts \ + --project chromium \ + --output interaction-proof-output + + # Storyboards (PNG contact sheets) and curve traces, for review. + - name: Upload interaction evidence + if: always() && steps.scope.outputs.run == 'true' + uses: actions/upload-artifact@v7 + with: + name: interaction-evidence + path: test-results/interaction-evidence + if-no-files-found: ignore + retention-days: 7 + # Build package test owner. The plugin's production CSS partition is a Build # integration contract: source inspection cannot prove which declaration # paints after Vite emits the final bundle. The `test-build` name makes that diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3eaee6914b595..f3b572c1dd629 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -132,6 +132,9 @@ jobs: - name: Typecheck accessibility spec-test contracts run: pnpm -F @astryxdesign/a11y-spec typecheck + - name: Typecheck interaction harness + run: pnpm -F @astryxdesign/interaction-harness typecheck + # Stable join for the three post-merge validation lanes. test: needs: [test-ui, test-node, typecheck] diff --git a/internal/README.md b/internal/README.md index 101fd8b07c28f..650fbd245d1a6 100644 --- a/internal/README.md +++ b/internal/README.md @@ -4,9 +4,10 @@ Internal tooling packages not published to npm. -| Directory | Role | Purpose | -| ------------- | ------- | ----------------------------------------------------------------------- | -| `a11y-spec/` | Testing | Reusable accessibility spec-test contracts and their evidence harnesses | -| `scripts/` | Tooling | One-off automation scripts (Crowdin screenshot upload, etc.) | -| `test-utils/` | Testing | Shared testing utilities and setup for Vitest | -| `vibe-tests/` | Testing | LLM vibeability test harness for component system | +| Directory | Role | Purpose | +| ---------------------- | ------- | --------------------------------------------------------------------------------- | +| `a11y-spec/` | Testing | Reusable accessibility spec-test contracts and their evidence harnesses | +| `interaction-harness/` | Testing | Playwright instruments for touch interaction and animation proofs and storyboards | +| `scripts/` | Tooling | One-off automation scripts (Crowdin screenshot upload, etc.) | +| `test-utils/` | Testing | Shared testing utilities and setup for Vitest | +| `vibe-tests/` | Testing | LLM vibeability test harness for component system | diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md new file mode 100644 index 0000000000000..a0b56d854c6b0 --- /dev/null +++ b/internal/interaction-harness/README.md @@ -0,0 +1,67 @@ +# @astryxdesign/interaction-harness + +Playwright instruments for touch interaction proofs: real touch input. +**Internal and unpublished.** Specs live next to their component as +`*.interaction.browser.spec.ts` and run under +[`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on +a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device +pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. + +The harness proves its own instruments in +[`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts), +on fixture pages written inline. + +## Running + +```sh +pnpm storybook:build # component specs read stories from the built Storybook +pnpm exec playwright install chromium webkit +pnpm test:interaction --project chromium # everything, Chromium +pnpm test:interaction # one spec, both engines +``` + +On Linux distributions whose system libraries are older than Playwright's +WebKit build needs, run the WebKit project inside the official Playwright +image (podman or docker; the repository is mounted at the same path): + +```sh +node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] +``` + +## Instruments + +| Module | What it gives a spec | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | + +## Real touch + +Untrusted DOM events are not input: they start no native scroll, claim no +gesture, and read `isTrusted: false`. Chromium input goes through CDP +`Input.dispatchTouchEvent` with an explicit timestamp per event: the page's +`event.timeStamp` is the scripted time to the microsecond, whatever the +dispatch latency, and the events go through the compositor's gesture detection +(native scrolling, flings with velocity from those timestamps, `touch-action`, +`pointercancel`). WebKit input goes through the engine's own input command; see +the limits below. Read positions a frame after a move: Chromium delivers +`pointermove` aligned to animation frames. + +## Engine limits (measured with Playwright 1.61) + +| | Chromium 149 | WebKit 26.5 (Linux build) | +| ------------------------------------------------------ | ------------ | ----------------------------------------------------------------------------------- | +| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) | +| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; real-time pacing keeps velocities within ~1 ms | +| Sub-pixel touch positions | yes | no: integer CSS px | +| Native scroll, fling, pinch-zoom from injected touches | yes | no | +| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | +| Input log | yes | yes | + +What WebKit cannot show here — native scroll hand-off, flings, and anything +iOS-only — needs Mobile Safari itself, on a device or the iOS simulator. + +The WebKit input command is reached through Playwright's in-process server +object, so it needs a locally launched browser (the default), not `connect()`. +Firefox has no touch input path and no project. diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json new file mode 100644 index 0000000000000..08b1d02f61634 --- /dev/null +++ b/internal/interaction-harness/package.json @@ -0,0 +1,21 @@ +{ + "name": "@astryxdesign/interaction-harness", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Internal: Playwright helpers for touch interaction proofs (real touch input). Never published.", + "main": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./scripts": "./src/scripts.ts" + }, + "scripts": { + "typecheck": "tsc --project tsconfig.json --noEmit" + }, + "devDependencies": { + "@astryxdesign/a11y-spec": "workspace:*", + "@playwright/test": "^1.61.1", + "typescript": "^6.0.3", + "vitest": "^4.1.11" + } +} diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs new file mode 100644 index 0000000000000..32cc0452396df --- /dev/null +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * Runs the interaction proofs' WebKit project inside the official Playwright + * container image, for hosts where WebKit cannot launch (Playwright's Linux + * WebKit build needs newer system libraries than some distributions ship). + * + * node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] + * + * The repository is mounted at the same path, so node_modules, the built + * Storybook and the evidence directory are the host's own. The image tag + * matches the installed playwright, so its browsers match the client. + * Uses podman when present, else docker. `--network=host` lets the specs' + * local Storybook server and the browser reach each other. + */ +import {spawnSync} from 'node:child_process'; +import fs from 'node:fs'; +import {createRequire} from 'node:module'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const root = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../..', +); +const require = createRequire(path.join(root, 'package.json')); +const {version} = JSON.parse( + fs.readFileSync(require.resolve('playwright/package.json'), 'utf8'), +); +const image = + process.env.PLAYWRIGHT_IMAGE ?? + `mcr.microsoft.com/playwright:v${version}-noble`; +const engine = ['podman', 'docker'].find( + cmd => spawnSync(cmd, ['--version'], {stdio: 'ignore'}).status === 0, +); +if (engine == null) { + console.error('webkit-container: neither podman nor docker is installed'); + process.exit(1); +} +const args = process.argv.slice(2); +const env = [ + 'ASTRYX_STORYBOOK_DIR', + 'ASTRYX_INTERACTION_EVIDENCE', + 'INTERACTION_WORKERS', + 'CI', +] + .filter(name => process.env[name] != null) + .flatMap(name => ['-e', `${name}=${process.env[name]}`]); +const command = [ + 'run', + '--rm', + '--network=host', + '--ipc=host', + '-v', + `${root}:${root}`, + '-w', + root, + ...env, + image, + 'node', + path.join(root, 'node_modules/@playwright/test/cli.js'), + 'test', + '--config', + 'playwright.interaction.config.ts', + '--project', + 'webkit', + ...args, +]; +console.log(`${engine} ${command.join(' ')}`); +const result = spawnSync(engine, command, {stdio: 'inherit'}); +process.exit(result.status ?? 1); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts new file mode 100644 index 0000000000000..10d41e668c9bf --- /dev/null +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -0,0 +1,168 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file harness.interaction.browser.spec.ts + * @input Fixture pages written inline (no Storybook), real Chromium/WebKit + * @output Proof that each touch instrument delivers real engine input: + * scripted timestamps reach the page; a flick flings natively (Chromium); + * a second finger and a pinch arrive as distinct pointers. + * @position The harness's own test. A component spec trusts these + * instruments; this file is why it may. + */ + +import {expect, test, type Page} from '@playwright/test'; +import { + drag, + flick, + installInputLog, + merge, + pinch, + readInputLog, + tap, + touchDriver, + withFinger, +} from './index'; + +const FIXTURE = ` + +
+
+
${Array.from({length: 200}, (_, i) => `
row ${i}
`).join('')}
`; + +// A real navigation (not setContent, whose document.open() erases the window +// listeners the input log installs). +async function fixture(page: Page) { + await installInputLog(page); + await page.route('http://harness.test/', route => + route.fulfill({contentType: 'text/html', body: FIXTURE}), + ); + await page.goto('http://harness.test/'); +} + +test.describe('touch input', () => { + test('the page receives trusted touches and pointers at the scripted positions and times', async ({ + page, + browserName, + }) => { + await fixture(page); + const driver = await touchDriver(page); + const script = drag( + {x: 100, y: 100}, + {x: 100, y: 220}, + {durationMs: 200, hz: 60, restMs: 120}, + ); + await driver.play(script); + const log = await readInputLog(page); + const moves = log.filter(r => r.type === 'pointermove'); + const down = log.find(r => r.type === 'pointerdown'); + const up = log.find(r => r.type === 'pointerup'); + expect(down?.isTrusted).toBe(true); + expect(down?.pointerType).toBe('touch'); + expect(up, JSON.stringify(log.map(r => r.type))).toBeDefined(); + // (Not `cancelable`: with only passive listeners an engine may dispatch + // touchmove as non-blocking, which reads cancelable false.) + expect( + log.filter(r => r.type === 'touchmove').every(r => r.isTrusted), + ).toBe(true); + expect(moves.at(-1)?.y).toBe(220); + if (browserName === 'chromium') { + // The scripted 16.67 ms spacing arrives exactly, and the 120 ms rest + // before the lift is in the timestamps too. + const deltas = moves + .slice(1) + .map((m, i) => m.timeStamp - moves[i].timeStamp); + for (const delta of deltas) { + expect(delta).toBeCloseTo(1000 / 60, 0); + } + expect( + (up?.timeStamp ?? NaN) - (moves.at(-1)?.timeStamp ?? NaN), + ).toBeCloseTo(120, 0); + } else { + // WebKit stamps the real dispatch time: real-time pacing keeps it close. + const deltas = moves + .slice(1) + .map((m, i) => m.timeStamp - moves[i].timeStamp); + const mean = deltas.reduce((a, b) => a + b, 0) / deltas.length; + expect(mean).toBeGreaterThan(14); + expect(mean).toBeLessThan(20); + } + }); + + test('a flick flings a native scroller, farther for a faster finger', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'injected WebKit touches drive no native scrolling (README: engine limits)', + ); + await fixture(page); + const driver = await touchDriver(page); + const scroller = page.locator('#scroller'); + const distanceFor = async (velocity: number) => { + await scroller.evaluate(el => (el.scrollTop = 0)); + await driver.play( + flick( + {x: 200, y: 680}, + {velocity: {x: 0, y: -velocity}, durationMs: 60, hz: 120}, + ), + ); + await expect + .poll( + async () => { + const a = await scroller.evaluate(el => el.scrollTop); + await page.waitForTimeout(150); + return a === (await scroller.evaluate(el => el.scrollTop)); + }, + {timeout: 8_000}, + ) + .toBe(true); + return scroller.evaluate(el => el.scrollTop); + }; + const slow = await distanceFor(800); + const fast = await distanceFor(2400); + expect(slow).toBeGreaterThan(48); // it coasted past the finger's own travel + expect(fast).toBeGreaterThan(slow * 1.8); + }); + + test('a second finger and a pinch arrive as distinct touch pointers', async ({ + page, + }) => { + await fixture(page); + const driver = await touchDriver(page); + const first = drag( + {x: 80, y: 80}, + {x: 80, y: 200}, + {durationMs: 300, hz: 60}, + ); + const second = withFinger(tap({x: 300, y: 120}, {holdMs: 100}), 1, 120); + await driver.play(merge(first, second)); + const downs = (await readInputLog(page, {clear: true})).filter( + r => r.type === 'pointerdown', + ); + expect(downs).toHaveLength(2); + expect(new Set(downs.map(d => d.pointerId)).size).toBe(2); + + await driver.play( + pinch({ + center: {x: 200, y: 150}, + fromDistance: 60, + toDistance: 200, + durationMs: 200, + }), + ); + const log = await readInputLog(page); + expect(log.filter(r => r.type === 'pointerdown')).toHaveLength(2); + expect( + Math.max( + ...log.filter(r => r.type === 'touchmove').map(r => r.touches ?? 0), + ), + ).toBe(2); + }); +}); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts new file mode 100644 index 0000000000000..3216b3c084322 --- /dev/null +++ b/internal/interaction-harness/src/index.ts @@ -0,0 +1,17 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file index.ts + * @input Re-exports the harness modules + * @output The `@astryxdesign/interaction-harness` entry point for Playwright + * specs. Pure helpers are also reachable without Playwright through + * `@astryxdesign/interaction-harness/math` and `/scripts` (unit tests). + * @position Imported only from `*.interaction.browser.spec.ts` files (run by + * playwright.interaction.config.ts) and from this package's own tests. + * + * SYNC: When a module is added, export it here and describe it in README.md. + */ + +export * from './scripts'; +export * from './story'; +export * from './touch'; diff --git a/internal/interaction-harness/src/scripts.test.ts b/internal/interaction-harness/src/scripts.test.ts new file mode 100644 index 0000000000000..b85d4b526de55 --- /dev/null +++ b/internal/interaction-harness/src/scripts.test.ts @@ -0,0 +1,142 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import {describe, expect, it} from 'vitest'; +import { + drag, + flick, + holdThenDrag, + merge, + path, + pinch, + releaseVelocity, + rotate, + tap, + toTouchEvents, + withFinger, +} from './scripts'; + +describe('touch scripts', () => { + it('tap presses and lifts in place', () => { + expect( + toTouchEvents(tap({x: 5, y: 6}, {holdMs: 80})).map(e => [e.t, e.type]), + ).toEqual([ + [0, 'touchStart'], + [80, 'touchEnd'], + ]); + }); + + it('drag moves at 60 Hz, holds, rests and lifts', () => { + const script = drag( + {x: 0, y: 0}, + {x: 0, y: 100}, + {durationMs: 100, hz: 60, holdMs: 500, restMs: 150}, + ); + const events = toTouchEvents(script); + expect(events[0]).toMatchObject({t: 0, type: 'touchStart'}); + const moves = events.filter(e => e.type === 'touchMove'); + expect(moves).toHaveLength(6); + expect(moves[0].t).toBeCloseTo(516.667, 3); + expect(moves.at(-1)).toMatchObject({ + t: 600, + fingers: [{id: 0, x: 0, y: 100}], + }); + expect(events.at(-1)).toMatchObject({ + t: 750, + type: 'touchEnd', + fingers: [], + }); + // A rest of 100 ms or more releases nothing in motion. + expect(releaseVelocity(script)).toEqual({x: 0, y: 0}); + }); + + it('flick releases at exactly the scripted velocity', () => { + const script = flick( + {x: 10, y: 10}, + {velocity: {x: 0, y: 1500}, durationMs: 80, hz: 120}, + ); + expect(releaseVelocity(script).y).toBeCloseTo(1500, 6); + const last = script.at(-2)!; + expect(last.fingers[0].y).toBeCloseTo(10 + 1500 * 0.08, 6); + // The lift lands a quarter frame after the last move, still moving. + expect(script.at(-1)!.t - last.t).toBeCloseTo(1000 / 120 / 4, 6); + }); + + it('flick follows a velocity profile', () => { + const script = flick( + {x: 0, y: 0}, + {velocity: {x: 1000, y: 0}, durationMs: 100, hz: 100, profile: p => p}, + ); + // Linear ramp: distance = v * T / 2. + expect(script.at(-2)!.fingers[0].x).toBeCloseTo(50, 6); + }); + + it('pinch and rotate keep two fingers about the centre', () => { + const spread = pinch({ + center: {x: 100, y: 100}, + fromDistance: 40, + toDistance: 200, + durationMs: 100, + }); + const end = spread.at(-2)!.fingers; + expect(Math.hypot(end[1].x - end[0].x, end[1].y - end[0].y)).toBeCloseTo( + 200, + 6, + ); + const turn = rotate({ + center: {x: 0, y: 0}, + distance: 100, + fromAngle: 0, + toAngle: 90, + durationMs: 100, + }); + expect(turn.at(-2)!.fingers[1].x).toBeCloseTo(0, 6); + expect(turn.at(-2)!.fingers[1].y).toBeCloseTo(50, 6); + expect(toTouchEvents(spread)[0]).toMatchObject({ + type: 'touchStart', + changed: [{id: 0}, {id: 1}], + }); + }); + + it('merge lands a second finger mid-drag and lifts it first', () => { + const first = drag({x: 0, y: 0}, {x: 0, y: 90}, {durationMs: 300, hz: 10}); + const second = withFinger(tap({x: 50, y: 50}, {holdMs: 100}), 1, 120); + const events = toTouchEvents(merge(first, second)); + const starts = events.filter(e => e.type === 'touchStart'); + expect( + starts.map(e => [e.t, e.changed.map(f => f.id), e.fingers.length]), + ).toEqual([ + [0, [0], 1], + [120, [1], 2], + ]); + const ends = events.filter(e => e.type === 'touchEnd'); + expect(ends[0]).toMatchObject({t: 220, changed: [{id: 1}]}); + expect(ends[0].fingers.map(f => f.id)).toEqual([0]); + expect(ends.at(-1)!.fingers).toEqual([]); + expect(() => merge(first, first)).toThrow(/two scripts/); + }); + + it('path follows any curve; holdThenDrag holds before the first move', () => { + const arc = path( + p => ({x: 100 * Math.cos(p * Math.PI), y: 100 * Math.sin(p * Math.PI)}), + {durationMs: 100, hz: 40}, + ); + const last = arc.at(-2)!.fingers[0]; + expect(last.x).toBeCloseTo(-100, 6); + expect(last.y).toBeCloseTo(0, 6); + expect(arc.at(-1)!.fingers).toEqual([]); + const held = holdThenDrag( + {x: 0, y: 0}, + {x: 30, y: 0}, + {durationMs: 50, hz: 20, holdMs: 500}, + ); + const moves = toTouchEvents(held).filter(e => e.type === 'touchMove'); + expect(moves[0].t).toBeCloseTo(550, 6); + expect(moves.at(-1)!.fingers[0].x).toBeCloseTo(30, 6); + }); + + it('refuses a script that leaves a finger down', () => { + expect(() => + toTouchEvents([{t: 0, fingers: [{id: 0, x: 0, y: 0}]}]), + ).toThrow(/lifted/); + }); +}); diff --git a/internal/interaction-harness/src/scripts.ts b/internal/interaction-harness/src/scripts.ts new file mode 100644 index 0000000000000..111d9e01234aa --- /dev/null +++ b/internal/interaction-harness/src/scripts.ts @@ -0,0 +1,395 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file scripts.ts + * @input Nothing: pure functions + * @output Touch scripts — timed lists of the fingers that are down — and the + * builders that make them: `tap`, `drag`, `flick`, `holdThenDrag`, `pinch`, + * `rotate`, `path`, `withFinger`, `merge`, plus `toTouchEvents`, which turns + * a script into the per-event sequence a touch driver dispatches. + * @position The input half of every interaction proof. A script is data, so a + * test states exactly what the finger does — positions and timestamps — and + * a unit test can check a builder without a browser. + * + * A script is a list of frames ordered by time. Each frame lists every finger + * that is down at that moment; a finger missing from a frame has lifted. The + * last frame is always empty (every finger up). Times are milliseconds from + * the gesture's start; coordinates are CSS px in the viewport. + * + * SYNC: Keep ./scripts.test.ts covering every builder. + */ + +export interface Point { + readonly x: number; + readonly y: number; +} + +export interface Finger extends Point { + /** Stable per finger for the whole gesture. */ + readonly id: number; +} + +export interface TouchFrame { + /** Milliseconds from the start of the gesture. */ + readonly t: number; + /** Every finger that is down at `t`. Empty = all lifted. */ + readonly fingers: readonly Finger[]; +} + +export type TouchScript = readonly TouchFrame[]; + +/** One dispatchable event: what changed at `t`, and the fingers down after it. */ +export interface TouchEventStep { + readonly t: number; + readonly type: 'touchStart' | 'touchMove' | 'touchEnd' | 'touchCancel'; + /** Fingers down after this event (CDP semantics: the full active set). */ + readonly fingers: readonly Finger[]; + /** Fingers this event pressed, moved or released. */ + readonly changed: readonly Finger[]; +} + +const DEFAULT_HZ = 60; + +function frameTimes(durationMs: number, hz: number): number[] { + const step = 1000 / hz; + const count = Math.max(1, Math.round(durationMs / step)); + return Array.from( + {length: count}, + (_, i) => +(((i + 1) * durationMs) / count).toFixed(3), + ); +} + +function lerp(a: Point, b: Point, p: number): Point { + return {x: a.x + (b.x - a.x) * p, y: a.y + (b.y - a.y) * p}; +} + +/** A press and lift in place. */ +export function tap( + at: Point, + {holdMs = 60, id = 0}: {holdMs?: number; id?: number} = {}, +): TouchScript { + return [ + {t: 0, fingers: [{id, ...at}]}, + {t: holdMs, fingers: []}, + ]; +} + +export interface DragOptions { + /** Time from the first move to the last, ms. */ + readonly durationMs: number; + /** Move samples per second (iOS reports touches at 60 Hz, 120 Hz on ProMotion). */ + readonly hz?: number; + /** Rest before the first move (press-and-hold), ms. */ + readonly holdMs?: number; + /** Rest after the last move before the lift, ms: >= 100 makes the release velocity 0. */ + readonly restMs?: number; + /** Progress easing over the move, default linear (constant velocity). */ + readonly easing?: (progress: number) => number; + readonly id?: number; +} + +/** Press at `from`, move to `to`, lift. Constant velocity unless `easing` says otherwise. */ +export function drag( + from: Point, + to: Point, + options: DragOptions, +): TouchScript { + const { + durationMs, + hz = DEFAULT_HZ, + holdMs = 0, + restMs = 0, + easing = (p: number) => p, + id = 0, + } = options; + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...from}]}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({ + t: holdMs + t, + fingers: [{id, ...lerp(from, to, easing(t / durationMs))}], + }); + } + // A real finger reports no move while it rests: the lift simply comes later. + const last = frames[frames.length - 1].t; + frames.push({t: last + Math.max(restMs, 1000 / hz / 4), fingers: []}); + return frames; +} + +/** `drag` after a press-and-hold (a long press that turns into a drag). */ +export function holdThenDrag( + from: Point, + to: Point, + options: DragOptions & {readonly holdMs: number}, +): TouchScript { + return drag(from, to, options); +} + +export interface FlickOptions { + /** Velocity at the lift, px/s. */ + readonly velocity: Point; + /** How long the finger moves before the lift, ms. */ + readonly durationMs?: number; + readonly hz?: number; + /** + * Velocity over the move as a fraction of the final velocity (0..1 → 0..∞), + * default constant. `p => p` is a finger accelerating linearly into the lift. + */ + readonly profile?: (progress: number) => number; + readonly id?: number; +} + +/** + * A throw: the finger moves with the given velocity profile and lifts while + * still moving (the lift lands a quarter frame after the last move, as on a + * device), so the release velocity is exactly `velocity`. + */ +export function flick(from: Point, options: FlickOptions): TouchScript { + const { + velocity, + durationMs = 80, + hz = 120, + profile = () => 1, + id = 0, + } = options; + const step = 1000 / hz; + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...from}]}]; + let x = from.x; + let y = from.y; + let previous = 0; + for (const t of frameTimes(durationMs, hz)) { + // Integrate the profile over (previous, t] with the midpoint rule. + const scale = profile((previous + t) / 2 / durationMs); + x += (velocity.x * scale * (t - previous)) / 1000; + y += (velocity.y * scale * (t - previous)) / 1000; + previous = t; + frames.push({t, fingers: [{id, x, y}]}); + } + frames.push({t: previous + step / 4, fingers: []}); + return frames; +} + +/** Any single-finger path: `at(progress)` for progress 0..1 over `durationMs`. */ +export function path( + at: (progress: number) => Point, + { + durationMs, + hz = DEFAULT_HZ, + holdMs = 0, + restMs = 0, + id = 0, + }: Omit, +): TouchScript { + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...at(0)}]}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({t: holdMs + t, fingers: [{id, ...at(t / durationMs)}]}); + } + frames.push({ + t: frames[frames.length - 1].t + Math.max(restMs, 1000 / hz / 4), + fingers: [], + }); + return frames; +} + +export interface PinchOptions { + readonly center: Point; + /** Finger separation at the start and at the end, px. */ + readonly fromDistance: number; + readonly toDistance: number; + /** Angle of the line between the fingers, degrees (0 = horizontal). */ + readonly fromAngle?: number; + readonly toAngle?: number; + readonly durationMs: number; + readonly hz?: number; + readonly restMs?: number; +} + +/** Two fingers (ids 0 and 1) pressed together, spread or pinched and turned, lifted together. */ +export function pinch(options: PinchOptions): TouchScript { + const { + center, + fromDistance, + toDistance, + fromAngle = 0, + toAngle = fromAngle, + durationMs, + hz = DEFAULT_HZ, + restMs = 0, + } = options; + const at = (p: number): Finger[] => { + const distance = fromDistance + (toDistance - fromDistance) * p; + const angle = ((fromAngle + (toAngle - fromAngle) * p) * Math.PI) / 180; + const dx = (Math.cos(angle) * distance) / 2; + const dy = (Math.sin(angle) * distance) / 2; + return [ + {id: 0, x: center.x - dx, y: center.y - dy}, + {id: 1, x: center.x + dx, y: center.y + dy}, + ]; + }; + const frames: TouchFrame[] = [{t: 0, fingers: at(0)}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({t, fingers: at(t / durationMs)}); + } + frames.push({t: durationMs + Math.max(restMs, 1000 / hz / 4), fingers: []}); + return frames; +} + +/** Two fingers turning around `center` at a fixed separation. */ +export function rotate( + options: Omit & { + readonly distance: number; + }, +): TouchScript { + return pinch({ + ...options, + fromDistance: options.distance, + toDistance: options.distance, + }); +} + +/** Re-labels a single-finger script's finger, and shifts it in time. */ +export function withFinger( + script: TouchScript, + id: number, + delayMs = 0, +): TouchScript { + return script.map(frame => ({ + t: frame.t + delayMs, + fingers: frame.fingers.map(f => ({...f, id})), + })); +} + +/** + * Plays several scripts at once (each with its own finger ids): at every time + * any of them changes, each finger is where its own script last put it. + * Used for a second finger that lands mid-gesture. + */ +export function merge(...scripts: TouchScript[]): TouchScript { + const times = [...new Set(scripts.flatMap(s => s.map(f => f.t)))].sort( + (a, b) => a - b, + ); + const stateAt = (script: TouchScript, t: number): readonly Finger[] => { + let current: readonly Finger[] = []; + for (const frame of script) { + if (frame.t > t) { + break; + } + current = frame.fingers; + } + return current; + }; + const owner = new Map(); + scripts.forEach((script, index) => { + for (const frame of script) { + for (const f of frame.fingers) { + if ((owner.get(f.id) ?? index) !== index) { + throw new Error(`merge: finger ${f.id} is used by two scripts`); + } + owner.set(f.id, index); + } + } + }); + return times.map(t => ({t, fingers: scripts.flatMap(s => stateAt(s, t))})); +} + +/** + * Turns frames into events: a finger that appears is a `touchStart`, one that + * disappears a `touchEnd`, one that moved a `touchMove`. A frame that both + * moves some fingers and lifts others yields the move first, then the lift. + * Unchanged frames yield nothing (a resting finger reports nothing). + */ +export function toTouchEvents(script: TouchScript): TouchEventStep[] { + const events: TouchEventStep[] = []; + let down = new Map(); + for (const frame of script) { + const next = new Map(frame.fingers.map(f => [f.id, f])); + const pressed = frame.fingers.filter(f => !down.has(f.id)); + const released = [...down.values()].filter(f => !next.has(f.id)); + const moved = frame.fingers.filter(f => { + const before = down.get(f.id); + return before != null && (before.x !== f.x || before.y !== f.y); + }); + if (moved.length > 0) { + const after = [...down.values()].map(f => next.get(f.id) ?? f); + events.push({ + t: frame.t, + type: 'touchMove', + fingers: after, + changed: moved, + }); + down = new Map(after.map(f => [f.id, f])); + } + if (released.length > 0) { + const remaining = [...down.values()].filter(f => next.has(f.id)); + events.push({ + t: frame.t, + type: 'touchEnd', + fingers: remaining, + changed: released.map(f => down.get(f.id) ?? f), + }); + down = new Map(remaining.map(f => [f.id, f])); + } + if (pressed.length > 0) { + const all = [...down.values(), ...pressed]; + events.push({ + t: frame.t, + type: 'touchStart', + fingers: all, + changed: pressed, + }); + down = new Map(all.map(f => [f.id, f])); + } + } + if (down.size > 0) { + throw new Error( + 'toTouchEvents: a script must end with every finger lifted', + ); + } + return events; +} + +/** + * The velocity (px/s) of finger `id` at its lift: the slope over the last + * `windowMs` of moves, or zero when the finger rested `windowMs` or longer + * before lifting (a finger that rested released nothing in motion). + */ +export function releaseVelocity( + script: TouchScript, + id = 0, + windowMs = 100, +): Point { + const samples = script.flatMap(frame => + frame.fingers + .filter(f => f.id === id) + .map(f => ({t: frame.t, x: f.x, y: f.y})), + ); + if (samples.length < 2) { + return {x: 0, y: 0}; + } + const lift = script.find( + frame => frame.t > samples[0].t && !frame.fingers.some(f => f.id === id), + ); + // The last sample that actually moved: a resting finger reports nothing. + let lastMove = samples.length - 1; + while ( + lastMove > 0 && + samples[lastMove].x === samples[lastMove - 1].x && + samples[lastMove].y === samples[lastMove - 1].y + ) { + lastMove -= 1; + } + const last = samples[lastMove]; + if (lift != null && lift.t - last.t >= windowMs) { + return {x: 0, y: 0}; + } + const recent = samples + .slice(0, lastMove + 1) + .filter(s => s.t >= last.t - windowMs); + const first = recent[0]; + const dt = last.t - first.t; + return dt <= 0 + ? {x: 0, y: 0} + : { + x: ((last.x - first.x) / dt) * 1000, + y: ((last.y - first.y) / dt) * 1000, + }; +} diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts new file mode 100644 index 0000000000000..9e4323436ff27 --- /dev/null +++ b/internal/interaction-harness/src/story.ts @@ -0,0 +1,103 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file story.ts + * @input Uses a Playwright `Page` and test object, and the a11y-spec package's + * static server over a built Storybook + * @output `phone` (the touch-phone browser context every interaction proof + * runs in), `withStorybook` (serve the built Storybook for a spec file), and + * `openStory` (load one story and wait until it has rendered). + * @position Stories are the reproduction path, as for every other browser + * check in this repository: a scenario used for verification is checked in, + * not built in a one-off page. + * + * The phone is an iPhone 17 Pro in Safari: 402 × 874 pt screen, 402 × 714 pt + * visible page with the toolbars shown, 3 device pixels per point, touch, + * mobile viewport rules. Measured on the iOS 26.2 simulator + * (`innerWidth`/`innerHeight`, `screen`, `devicePixelRatio`). + * + * SYNC: Keep `phone` equal to the projects in playwright.interaction.config.ts. + */ + +import type {Page, TestType} from '@playwright/test'; +import { + DEFAULT_STORYBOOK_DIR, + serveStorybook, + type StaticServer, +} from '@astryxdesign/a11y-spec/storybook'; + +export const phone = { + viewport: {width: 402, height: 714}, + screen: {width: 402, height: 874}, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, +} as const; + +/** + * Serves the built Storybook for the spec file's tests (`ASTRYX_STORYBOOK_DIR` + * overrides the default `apps/storybook/dist`). Returns its origin getter. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any fixture set +export function withStorybook(test: TestType): () => string { + let server: StaticServer | undefined; + test.beforeAll(async () => { + server = await serveStorybook( + process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR, + ); + }); + test.afterAll(async () => { + await server?.close(); + }); + return () => { + if (server == null) { + throw new Error( + 'withStorybook: the Storybook server is not running (call inside a test)', + ); + } + return server.origin; + }; +} + +/** Loads a story in the page and waits for its first render, its fonts and two frames. */ +export async function openStory( + page: Page, + origin: string, + storyId: string, + args?: Readonly>, +): Promise { + const params = new URLSearchParams({id: storyId, viewMode: 'story'}); + if (args != null) { + params.set( + 'args', + Object.entries(args) + .map(([k, v]) => `${k}:${v}`) + .join(';'), + ); + } + await page.goto(`${origin}/iframe.html?${params.toString()}`); + await page + .locator('#storybook-root > *') + .first() + .waitFor({state: 'attached'}); + await page.evaluate(async () => { + await document.fonts.ready; + await new Promise(resolve => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ); + }); +} + +/** Resolves after `count` animation frames have been presented in the page. */ +export async function frames(page: Page, count = 2): Promise { + await page.evaluate( + n => + new Promise(resolve => { + let left = n; + const tick = () => + --left <= 0 ? resolve() : requestAnimationFrame(tick); + requestAnimationFrame(tick); + }), + count, + ); +} diff --git a/internal/interaction-harness/src/touch.ts b/internal/interaction-harness/src/touch.ts new file mode 100644 index 0000000000000..56cdf8aa50289 --- /dev/null +++ b/internal/interaction-harness/src/touch.ts @@ -0,0 +1,295 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file touch.ts + * @input Uses a Playwright `Page` (Chromium or WebKit) and a touch script from + * ./scripts + * @output `touchDriver(page)` — plays touch scripts as REAL browser touch input + * — and `installInputLog` / `readInputLog`, the page-side record of what the + * page actually received. + * @position The input half of every interaction proof. Untrusted DOM events + * are not input: they never start native scrolling, never claim a gesture, + * and a component can tell them apart (`isTrusted`). Both drivers here go + * through the engine's own input pipeline instead. + * + * Chromium: CDP `Input.dispatchTouchEvent` with an explicit `timestamp` per + * event. The page's `event.timeStamp` IS that timestamp, whatever the + * dispatch latency, so a velocity profile reaches the page exactly. The + * events go through the compositor's gesture detection: native scrolling, + * flings (velocity from those timestamps), `touch-action`, pointer events + * and `pointercancel` all behave as for a finger. + * + * WebKit (Playwright's Linux build): the engine's own inspector command + * `Input.dispatchTouchEvent` on the page proxy. Touches and the pointer + * events derived from them are trusted and cancelable, multi-touch works. + * Limits, measured with WebKit 26.5: no timestamp parameter (the page sees + * the real dispatch time, so velocity comes from real-time pacing, ±1 ms); + * integer coordinates only; no native scrolling, fling or pinch-zoom from + * these touches; a `touchstart` lists only the fingers that landed in it + * (pointer events are exact); no click synthesised from a raw touch sequence (use + * `tap`, which dispatches WebKit's tap gesture and does produce the + * compatibility mouse events and click). The command is reached through + * Playwright's in-process server object, so it needs a locally launched + * browser (the default), not a remote `connect()`. + * + * SYNC: Keep the engine limits above equal to the README's table and to the + * harness self-test (../src/harness.interaction.browser.spec.ts). + */ + +import type {CDPSession, Page} from '@playwright/test'; +import { + toTouchEvents, + type Finger, + type Point, + type TouchEventStep, + type TouchScript, +} from './scripts'; + +export type Engine = 'chromium' | 'webkit'; + +export interface PlayOptions { + /** + * Pace dispatch on the wall clock so each event leaves at its scripted time + * (default true). Required in WebKit for a velocity to mean anything; in + * Chromium the page sees the scripted timestamps either way, but native + * behaviors that run on the compositor clock (a fling) want real pacing. + */ + readonly realtime?: boolean; + /** Called after each event is dispatched, before the next one (storyboards). */ + readonly onEvent?: ( + step: TouchEventStep, + index: number, + ) => Promise | void; +} + +export interface PlayedEvent extends TouchEventStep { + /** Wall-clock milliseconds (epoch) at which the event was dispatched. */ + readonly dispatchedAt: number; + /** The timestamp handed to the engine, epoch ms (Chromium only). */ + readonly timestamp: number | null; +} + +export interface TouchDriver { + readonly engine: Engine; + /** True when the page's `event.timeStamp` follows the script exactly. */ + readonly scriptedTimestamps: boolean; + /** True when these touches can scroll, fling and pinch-zoom natively. */ + readonly drivesNativeGestures: boolean; + /** Plays a script; resolves when its last event has been dispatched. */ + play(script: TouchScript, options?: PlayOptions): Promise; + /** A tap through the engine's tap path (WebKit synthesises click only from this). */ + tap(at: Point): Promise; +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** Waits until `performance.now()` reaches `deadline` with ~0.1 ms precision. */ +async function until(deadline: number): Promise { + const remaining = deadline - performance.now(); + if (remaining > 3) { + await sleep(remaining - 2); + } + while (performance.now() < deadline) { + // spin: setTimeout alone is ±1-4 ms late + } +} + +async function playSteps( + script: TouchScript, + options: PlayOptions, + dispatch: (step: TouchEventStep, timestamp: number) => Promise, + stampFor: (t: number) => number, +): Promise { + const {realtime = true, onEvent} = options; + const steps = toTouchEvents(script); + const played: PlayedEvent[] = []; + const origin = performance.now(); + for (const [index, step] of steps.entries()) { + if (realtime) { + await until(origin + step.t); + } + const timestamp = stampFor(step.t); + const dispatchedAt = Date.now(); + await dispatch(step, timestamp); + played.push({...step, dispatchedAt, timestamp}); + await onEvent?.(step, index); + } + return played; +} + +async function chromiumDriver(page: Page): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const touchPoints = (fingers: readonly Finger[]) => + fingers.map(f => ({ + x: f.x, + y: f.y, + id: f.id, + radiusX: 1, + radiusY: 1, + force: 1, + })); + return { + engine: 'chromium', + scriptedTimestamps: true, + drivesNativeGestures: true, + async play(script, options = {}) { + // Stamps follow the wall clock the page's own clock is on: with real-time + // pacing an event's stamp and its dispatch agree to ~0.1 ms. + const base = Date.now(); + return playSteps( + script, + options, + async (step, timestamp) => { + // CDP diffs each event's active points against the previous event's: + // a missing point is released, a new one pressed. touchEnd must carry + // no points, so lifting one finger of several is a touchMove that + // lists the fingers still down. + const partialLift = + step.type === 'touchEnd' && step.fingers.length > 0; + await cdp.send('Input.dispatchTouchEvent', { + type: partialLift ? 'touchMove' : step.type, + touchPoints: + step.type === 'touchEnd' && !partialLift + ? [] + : touchPoints(step.fingers), + timestamp: timestamp / 1000, + }); + }, + t => base + t, + ); + }, + async tap(at) { + await page.touchscreen.tap(at.x, at.y); + }, + }; +} + +interface WebKitPageProxySession { + send(method: string, params?: Record): Promise; + on(event: string, listener: (payload: unknown) => void): unknown; + off?(event: string, listener: (payload: unknown) => void): unknown; +} + +/** + * Playwright's server-side WebKit page session, reached through the in-process + * connection. Exported for helpers that use WebKit's own inspector protocol. + */ +export function webkitPageSession(page: Page): WebKitPageProxySession { + const connection = ( + page as unknown as {_connection?: {toImpl?: (o: unknown) => unknown}} + )._connection; + const impl = connection?.toImpl?.(page) as + {delegate?: {_pageProxySession?: WebKitPageProxySession}} | undefined; + const session = impl?.delegate?._pageProxySession; + if (session == null) { + throw new Error( + "WebKit touch input needs Playwright's in-process server (a locally launched browser); this page has none.", + ); + } + return session; +} + +async function webkitDriver(page: Page): Promise { + const session = webkitPageSession(page); + // WebKit's command takes integer CSS px and applies the event's phase to + // every point it lists: a start lists only the fingers that land (listing a + // finger already down presses it again), an end only the fingers that lift + // (an empty list releases nothing), a move every finger down. + const points = (fingers: readonly Finger[]) => + fingers.map(f => ({x: Math.round(f.x), y: Math.round(f.y), id: f.id})); + return { + engine: 'webkit', + scriptedTimestamps: false, + drivesNativeGestures: false, + async play(script, options = {}) { + return playSteps( + script, + options, + async step => { + await session.send('Input.dispatchTouchEvent', { + type: step.type, + touchPoints: points( + step.type === 'touchMove' ? step.fingers : step.changed, + ), + }); + }, + () => Date.now(), + ).then(played => played.map(p => ({...p, timestamp: null}))); + }, + async tap(at) { + await page.touchscreen.tap(Math.round(at.x), Math.round(at.y)); + }, + }; +} + +/** A touch driver for the page's engine. Firefox is not supported (no touch input path). */ +export async function touchDriver(page: Page): Promise { + const name = page.context().browser()?.browserType().name(); + if (name === 'chromium') { + return chromiumDriver(page); + } + if (name === 'webkit') { + return webkitDriver(page); + } + throw new Error( + `touchDriver: no real touch input path for ${name ?? 'this browser'}`, + ); +} + +export interface InputRecord { + readonly type: string; + /** The event's own `timeStamp`, ms since the page's time origin. */ + readonly timeStamp: number; + readonly isTrusted: boolean; + readonly cancelable: boolean; + readonly defaultPrevented: boolean; + readonly pointerId?: number; + readonly pointerType?: string; + readonly x?: number; + readonly y?: number; + readonly touches?: number; + readonly target: string; +} + +const INPUT_LOG_SCRIPT = `(() => { + if (window.__harnessInput) return; + const log = []; + window.__harnessInput = log; + const describe = node => node instanceof Element + ? node.tagName.toLowerCase() + (node.id ? '#' + node.id : '') + (node.getAttribute('aria-label') ? '[' + node.getAttribute('aria-label') + ']' : '') + : String(node && node.nodeName); + const types = ['pointerdown', 'pointermove', 'pointerup', 'pointercancel', 'touchstart', 'touchmove', + 'touchend', 'touchcancel', 'click', 'contextmenu', 'gotpointercapture', 'lostpointercapture']; + for (const type of types) { + // Bubble phase on window, so defaultPrevented reflects the page's own handlers. + window.addEventListener(type, event => { + const record = {type, timeStamp: event.timeStamp, isTrusted: event.isTrusted, cancelable: event.cancelable, + defaultPrevented: event.defaultPrevented, target: describe(event.target)}; + if ('pointerId' in event) Object.assign(record, {pointerId: event.pointerId, pointerType: event.pointerType, x: event.clientX, y: event.clientY}); + if (event.touches) Object.assign(record, {touches: event.touches.length, x: event.changedTouches[0] && event.changedTouches[0].clientX, y: event.changedTouches[0] && event.changedTouches[0].clientY}); + log.push(record); + }, {passive: true}); + } +})()`; + +/** Records every pointer/touch/click the page receives (install before navigation). */ +export async function installInputLog(page: Page): Promise { + await page.addInitScript(INPUT_LOG_SCRIPT); +} + +/** What the page received since navigation (or since the last `clear`). */ +export async function readInputLog( + page: Page, + {clear = false}: {clear?: boolean} = {}, +): Promise { + return page.evaluate(shouldClear => { + const log = + (window as unknown as {__harnessInput?: InputRecord[]}).__harnessInput ?? + []; + const copy = log.slice(); + if (shouldClear) { + log.length = 0; + } + return copy; + }, clear); +} diff --git a/internal/interaction-harness/tsconfig.json b/internal/interaction-harness/tsconfig.json new file mode 100644 index 0000000000000..596e2cf729a9f --- /dev/null +++ b/internal/interaction-harness/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src"] +} diff --git a/package.json b/package.json index 9052c34ad02d0..a4ca9f2ae830e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "test:coverage": "vitest --coverage", "test:a11y-contract": "playwright test --config playwright.config.ts --project chromium", "test:docsite-browser": "playwright test --config apps/docsite/playwright.config.ts", + "test:interaction": "playwright test --config playwright.interaction.config.ts", "check:package-boundaries": "node scripts/check-package-boundaries.js", "check:repo": "pnpm check:visual-owner && pnpm check:sync && pnpm check:neutral-palette-refs && pnpm check:stone-palette-refs && pnpm check:gothic-palette-refs && pnpm check:matcha-palette-refs && pnpm check:butter-palette-refs && pnpm check:y2k-palette-refs && pnpm check:data-tokens && pnpm check:token-docs && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-runtime && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract", "check:visual-owner": "node scripts/check-visual-workflow-owner.mjs", diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts new file mode 100644 index 0000000000000..357a5dc3b7642 --- /dev/null +++ b/playwright.interaction.config.ts @@ -0,0 +1,77 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file playwright.interaction.config.ts + * @input Uses @playwright/test + * @output Touch interaction proofs on a phone-sized touch screen, in Chromium + * and WebKit. + * @position Run with `pnpm test:interaction`. Separate from `pnpm test` (no + * browser needed there) and from playwright.config.ts (accessibility + * contracts on desktop profiles). + * + * Specs are named `*.interaction.browser.spec.ts` and read stories from a + * built Storybook, like the accessibility specs: + * + * pnpm storybook:build + * pnpm exec playwright install chromium webkit + * pnpm test:interaction --project chromium + * pnpm test:interaction harness.interaction.browser.spec.ts + * + * A spec that needs Chromium-only evidence (CDP tracing, scripted touch + * timestamps) skips itself in WebKit with the reason; see the harness README. + * Where WebKit cannot launch on the host (missing system libraries on some + * Linux distributions), internal/interaction-harness/scripts/webkit-container.mjs + * runs the WebKit project inside the official Playwright image. + * + * SYNC: Keep the phone profile here equal to `phone` in + * internal/interaction-harness/src/story.ts. + */ + +import {defineConfig} from '@playwright/test'; + +// The phone: an iPhone 17 Pro in Safari (402 × 874 pt screen, 402 × 714 pt +// page with the toolbars shown, 3 device px per point), measured on the iOS +// 26.2 simulator. Inline rather than imported: the config loads as CommonJS and +// the harness package is ESM. +const phone = { + viewport: {width: 402, height: 714}, + screen: {width: 402, height: 874}, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, +} as const; + +const isCI = (process.env.CI ?? '') !== ''; + +function workers(): number { + const raw = process.env.INTERACTION_WORKERS; + const count = raw ? Number(raw) : 1; + if (!Number.isInteger(count) || count < 1) { + throw new Error( + `INTERACTION_WORKERS must be a positive integer, got "${raw}"`, + ); + } + return count; +} + +export default defineConfig({ + testDir: '.', + testMatch: [ + 'internal/interaction-harness/src/**/*.interaction.browser.spec.ts', + 'packages/*/src/**/*.interaction.browser.spec.ts', + ], + // Playwright wipes its output directory per run; evidence (storyboards, + // curve traces) goes to test-results/interaction-evidence instead. + outputDir: 'test-results/interaction-output', + // Frame evidence measures timing: one worker by default, so a proof never + // shares the CPU with another browser. + workers: workers(), + fullyParallel: false, + timeout: 60_000, + forbidOnly: isCI, + reporter: isCI ? [['list'], ['github']] : [['list']], + projects: [ + {name: 'chromium', use: {browserName: 'chromium', ...phone}}, + {name: 'webkit', use: {browserName: 'webkit', ...phone}}, + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2e4d844ccbfb..c93bc6ef3838a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,21 @@ importers: specifier: ^8.62.1 version: 8.62.1(eslint@10.6.0(jiti@2.7.0))(typescript@6.0.3) + internal/interaction-harness: + devDependencies: + '@astryxdesign/a11y-spec': + specifier: workspace:* + version: link:../a11y-spec + '@playwright/test': + specifier: ^1.61.1 + version: 1.61.1 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.11 + version: 4.1.11(@types/node@25.9.4)(@vitest/coverage-v8@4.1.11)(happy-dom@20.11.1)(jsdom@27.4.0)(vite@8.1.3(@types/node@25.9.4)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.0)(yaml@2.9.0)) + internal/test-utils: dependencies: '@testing-library/jest-dom': From 4076bf3816ef9f56c772e2970a6268e6bbb5a73b Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 19:54:42 -0700 Subject: [PATCH 002/864] spec(AST-094): touch presentations follow UIKit --- .../specs/AST-094-touch-presentations/spec.md | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 docs/specs/AST-094-touch-presentations/spec.md diff --git a/docs/specs/AST-094-touch-presentations/spec.md b/docs/specs/AST-094-touch-presentations/spec.md new file mode 100644 index 0000000000000..d81f92cc41ea0 --- /dev/null +++ b/docs/specs/AST-094-touch-presentations/spec.md @@ -0,0 +1,293 @@ +--- +schema_version: 4 +template_version: 2 +kind: system-spec +id: spec:AST-094 +authority: draft +archive_reason: null +superseded_by: null +approved_by: null +approved_at: null +phase: proposed +owners: [vjeux] +affects_architecture: + [architecture:interaction-modality, architecture:theme-tokens] +affects_families: [family:buttons, family:overlay-dismissal] +affects_contributing: [] +affects_consumer_docs: [] +--- + +# Touch presentations follow UIKit system spec + +## Intent + +A person using an Astryx product on a phone gets the controls, sheets, menus, +lists and navigation they know from iOS: the same physics, the same timings, +the same gesture thresholds and the same feel. Every component's touch +presentation follows UIKit on a named reference release, and its motion keeps +running at display rate while the page's JavaScript is busy. + +The reference is UIKit on iOS 26.2 today. Each number a touch presentation +uses names the iOS version it was taken from; moving the reference to a newer +release re-measures the table, and differences on older releases are noted, +not targeted. + +## Non-goals + +- Pointer (desktop) presentations. Their timing and easing stay with the + theme's motion tokens (`architecture:theme-tokens`). +- WebKit embedded in a native app and the native Swift package. This record + governs the web components themselves. +- Scroll physics. Scrolling stays native on every platform: on iOS an overflow + area or the page is a UIKit scroll view already. +- Which components get a touch presentation, and when. Each component's own + record owns its presentation. +- Equivalent internal implementations remain valid when they satisfy this + contract. The modules, files and algorithms of the motion and gesture + foundations belong to their architecture records. + +## Requirements + +### Physics + +- **FR1 — One table of UIKit values.** Every spring, duration, deceleration + rate, rubber-band coefficient, gesture threshold and haptic moment a touch + presentation uses MUST come from one shared table of UIKit values. A + component MUST NOT carry a number of its own. +- **FR2 — Every value has a source.** Each value in the table MUST name its + source next to it: Apple's documentation, a WWDC session, WebKit's own + source where WebKit states it matches UIKit, or a measurement of UIKit on + the iOS simulator naming the app, the iOS version, the method and the date. + A value without one of these MUST be labelled a guess and name the + measurement that will replace it. A decompilation of UIKit MAY aim a + measurement and MUST NOT be a source. +- **FR3 — UIKit's physics.** Springs MUST follow UIKit's spring model + (response and damping ratio, mass, stiffness and damping, and the release + velocity), with overdamping clamped to critical damping wherever UIKit + clamps it. A momentum coast MUST decay like a UIKit scroll view (0.998 per + millisecond, 0.99 for the fast rate), a drag past an edge MUST resist with + UIKit's rubber band `(1 − 1 ÷ (x·c ÷ d + 1))·d` with the coefficient its use + measures (0.55 for a scroll view and a sheet), and a release MUST pick its + rest point by projecting its velocity the way its UIKit counterpart does: + with a deceleration rate, or with a measured projection time. +- **FR4 — Physics are not themable in touch presentations.** A touch + presentation MUST read its motion from the table on every theme. Themes keep + color, shape, type and spacing; the theme's duration and easing tokens keep + governing pointer presentations. **Proposed; pending owner review:** this + narrows `architecture:theme-tokens` (owners: cixzhang, imdreamrunner) to + pointer presentations, and departs from the draft `design:motion` DR3 + (owners: ernestt, cixzhang) wherever UIKit settles with an overshoot or + stretches with a rubber band. + +### Motion + +- **FR5 — Motion runs on the compositor.** Every release and every + programmatic animation of a touch presentation MUST animate `transform`, the + individual `translate`, `scale` and `rotate` properties, or `opacity` + (a dialog's `::backdrop` included) through Web Animations that the browser + runs off the main thread, + and MUST keep advancing at display rate through a main-thread block. It MUST + NOT animate layout properties, and MUST NOT step a frame from script when the + compositor can run the curve. +- **FR6 — Motion is interruptible.** A finger MUST be able to catch any + animation mid-flight at the value on screen. That value and its velocity + MUST come from the curve, not from a layout read. A new target MUST keep the + current velocity, and a release MUST hand its velocity to the spring that + follows. +- **FR7 — Exits animate as UIKit's do.** A touch presentation MUST play the + exit its UIKit counterpart plays. **Proposed; pending owner review:** this + departs from the draft `spec:AST-090` FR1 (exits off by default) and FR10 + (reduced motion closes at once) for touch presentations (owners: ernestt, + cixzhang). +- **FR8 — Reduce Motion substitutes, never jumps.** Under + `prefers-reduced-motion: reduce`, a touch presentation MUST behave as its + UIKit counterpart does with Reduce Motion and Prefer Cross-Fade Transitions + on: a transition that travels becomes a cross-fade carrying the same change + of state. An entrance lands in place and fades in, an exit fades out in + place, and any other move fades through. A navigation push or pop + cross-fades over 0.5 s on the ease-in-out curve; a sheet, a full-screen + modal or an alert fades on its own presentation spring and drops its travel + and scale; menus open and close on UIKit's Reduce Motion spring. Motion that + follows a finger, such as a throw, MUST keep its motion. **Proposed; pending + owner review:** this is the minimally moving form the draft `design:motion` + DR5 asks for, for touch presentations (owners: ernestt, cixzhang); the + alternative is UIKit's Reduce Motion alone, which keeps the slides of + navigation, sheets and modals. + +### Gestures + +- **FR9 — Fingers are tracked by recognizers.** A drag MUST be tracked by the + shared gesture recognizers, writing one transform per move, with no + component render per move and no layout read after the gesture begins. + Native scrolling MUST be used only where the component is a scroll view + (lists, scroll areas, the page). A component MAY track a finger with native + scroll snapping only with simulator evidence that it tracks through a + main-thread block, ends on UIKit's release curve, ends correctly when the + finger lifts during a block, and lets a finger catch the content mid-flight. +- **FR10 — A second finger never breaks the first.** A second finger MUST NOT + cancel or redirect a drag a recognizer has claimed. Page pinch-zoom MUST stay + the browser's whenever no pinch recognizer claims the fingers and no drag is + claimed (WCAG 1.4.4). +- **FR11 — Two press profiles.** Press feedback MUST follow one of two UIKit + profiles. A control (UIControl) highlights on touch-down outside scroll + views, keeps tracking within its expanded bounds (70 pt), turns its highlight + off and on as the finger drags out and back, and activates on a lift inside. + A cell (a table or collection cell) highlights 150 ms after touch-down inside + a scroll view and drops the highlight when the scroll wins. The 150 ms delay + applies only inside scroll views. **Proposed; pending owner review:** this + amends `design:user-states` DR1 and DR3 for touch presentations (owners: + ernestt, cixzhang), and the press treatment of `family:buttons` (owner: + cixzhang). +- **FR12 — Long press and rubber band are per use.** A long press MUST use the + thresholds its UIKit counterpart uses: 0.5 s and 10 pt for a plain long + press, the measured values for context menus and menus. A rubber band MUST + use the coefficient its use measures. +- **FR13 — One touch-presentation resolver.** Whether a component shows its + touch presentation MUST come from one shared resolver: touch behaviors key + on a coarse pointer, layout on the available size, and a presentation that + depends on both (a sheet instead of a popover) asks for both. A component + MUST NOT write its own media query for it. **Proposed; pending owner + review:** the resolver sits beside the shared modality state of + `architecture:interaction-modality` (owner: cixzhang). +- **FR14 — Nested scrolling follows UIKit.** Who gets a drag that starts + inside a scroll area MUST be decided once, when the drag begins. If the + area sits at its edge and the drag moves toward that edge, the surrounding + surface (a sheet) takes the drag; otherwise the area keeps it for the whole + gesture and rubber-bands at its edge. The drag MUST NOT change hands midway, + in either direction. + **Proposed; pending owner review:** this amends `spec:AST-025` FR14 (scroll + chaining by default) for touch presentations (owner: cixzhang). +- **FR15 — Recognizers arbitrate like UIKit's.** Recognizers that see the + same touch sequence MUST be exclusive by default: the first to recognize + wins and the others fail, unless both allow simultaneous recognition. A + recognizer that requires another to fail MUST hold its recognition until + that one fails, and MUST fail if that one recognizes. When several recognize + on the same event, the innermost element's recognizer MUST win. +- **FR16 — Gestures keep the event clock.** Recognizers MUST read time from the + input events' timestamps. Durations (a long press, the interval between + taps) MUST be deadlines on that clock, so a blocked main thread delays the + report and never the moment reported. +- **FR17 — UIKit's readings.** A recognizer's thresholds and readings MUST + follow UIKit's and come from the table. A pan begins at 10 pt of travel on + either axis, and its translation excludes that travel; its velocity is + UIKit's estimate, 0.2 × the newest event's instantaneous velocity + 0.8 × + the previous event's, never decayed through a rest. +- **FR18 — `touch-action` leaves the browser the rest.** An element with touch + recognizers MUST get a `touch-action` that leaves the browser exactly what + its recognizers do not use: the other axis for an axis-limited pan or swipe, + `manipulation` for taps and long presses, everything for a one-way pan or a + pan that shares its axis with native scrolling. `pinch-zoom` MUST stay + unless a pinch or rotation recognizer is attached. +- **FR19 — Claims are made in time and only by an owner.** A recognizer that + begins MUST claim its touches by cancelling `touchmove` through a + non-passive listener on its element. A `touchmove` of a touch no recognizer + owns MUST NOT be cancelled. A recognizer that competes with native scrolling + MUST decide within 10 px of travel, before the browser commits to the + scroll. +- **FR20 — The browser can take a touch.** `pointercancel`, an uncancelable + `touchmove`, or a native scroll under the touch MUST count as the browser + taking the touch: possible recognizers that share it fail, active ones are + cancelled, and the view's press is cancelled without a release. +- **FR21 — Clicks follow the gesture.** When a gesture takes a touch, its + `touchend` MUST be cancelled, and no click or compatibility mouse event + reaches the page. A click held for a recognizer that may still recognize (a + tap that a double tap may claim) MUST be replayed on its target if that + recognizer fails. Activation with `detail === 0` (keyboard, assistive + technology) MUST always pass. A tap MUST NOT wait for the browser's + double-tap delay. +- **FR22 — iOS Safari keeps its edge swipe.** Safari's own back swipe from the + screen edge MUST stay the browser's; a touch presentation never overrides it. + +### Haptics + +- **FR23 — Haptics where UIKit plays them.** A touch presentation MUST play a + haptic only at the moments its UIKit counterpart does. In iOS Safari the only + haptic a page can produce is a light impact from a real tap landing on a + `