diff --git a/.gitignore b/.gitignore index be01bda0..03b22c74 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ dist/ *.tsbuildinfo .wrangler/ apps/web/worker-configuration.d.ts +apps/web/.vitest-app-baseline.json +devpost-01-landing-1440x900.png +docs/submission-assets/public-2026-07-22/ diff --git a/Dockerfile.runner b/Dockerfile.runner index 9d97bbc1..994da756 100644 --- a/Dockerfile.runner +++ b/Dockerfile.runner @@ -43,6 +43,14 @@ RUN pnpm install --frozen-lockfile --filter @counterlab/hosted-runner... \ && node --check /out/runner.mjs \ && mkdir -p /out/codex \ && cp -a /build/node_modules/.pnpm/@openai+codex@0.144.4-linux-x64/node_modules/@openai/codex/vendor/x86_64-unknown-linux-musl/. /out/codex/ \ + && mkdir -p \ + /out/runtime-rootfs/dev/pts \ + /out/runtime-rootfs/dev/shm \ + /out/runtime-rootfs/dev/mqueue \ + /out/runtime-rootfs/sys/fs/cgroup \ + /out/runtime-rootfs/counterlab-runtime \ + /out/runtime-rootfs/etc \ + && touch /out/runtime-rootfs/etc/hosts \ && chmod -R a+rX /out FROM python:3.13-slim-trixie@sha256:bffeb7bd6a85767587059c6ba23e1e9122078e3aa3fa836099171b9bb5a9bb00 @@ -69,8 +77,12 @@ ENV DEBIAN_FRONTEND=noninteractive \ COPY --from=node-builder /usr/local/bin/node /usr/local/bin/node -RUN chmod 0555 /usr/local/bin/node \ +RUN apt-get update \ + && apt-get install --yes --no-install-recommends bubblewrap=0.11.0-2+deb13u1 \ + && chmod 0555 /usr/local/bin/node \ && test -x /usr/local/bin/node \ + && test "$(/usr/bin/bwrap --version)" = "bubblewrap 0.11.0" \ + && test -x /usr/bin/bwrap \ && test -x /usr/sbin/groupadd \ && test -x /usr/sbin/useradd \ && test -x /usr/bin/setpriv \ @@ -86,6 +98,7 @@ COPY fixtures/public/fraud_rare_event.csv ./fixtures/public/fraud_rare_event.csv COPY --from=python-builder /wheelhouse/ /opt/counterlab-wheelhouse/ COPY --from=node-builder /out/runner.mjs /app/runner.mjs COPY --from=node-builder /out/codex/ /opt/codex/ +COPY --from=node-builder /out/runtime-rootfs/ / RUN chmod -R a+rX /opt/codex \ && ln -s /opt/codex/bin/codex /usr/local/bin/codex \ @@ -104,6 +117,26 @@ RUN chmod -R a+rX /opt/codex \ && chown -R root:root /app \ && chmod -R a=rX /app \ && chmod 555 /app/runner.mjs \ + && chown root:root \ + /usr/lib \ + /usr/lib64 \ + /usr/lib/x86_64-linux-gnu \ + /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 \ + /dev/pts \ + /dev/shm \ + /dev/mqueue \ + /sys/fs/cgroup \ + /counterlab-runtime \ + && chmod 0755 \ + /usr/lib \ + /usr/lib64 \ + /usr/lib/x86_64-linux-gnu \ + /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 \ + /dev/pts \ + /dev/shm \ + /dev/mqueue \ + /sys/fs/cgroup \ + /counterlab-runtime \ && chown -R counterlab-codex:counterlab-codex /work/jobs /run/counterlab-codex \ && chmod 700 /work/jobs /run/counterlab-codex diff --git a/README.md b/README.md index 95cb0782..3dadab3a 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,21 @@ misconceptions in supported Jupyter notebooks. Its released Subject Packs cover entity leakage and class imbalance/metric choice; it is not a generic notebook copilot or an unrestricted code runner. -**Live judge surface:** +**Live judge surface:** The v6.1 learner UX is implemented locally on -`feat/learner-ux-v6.1`; it has not been pushed, deployed, or browser-qualified. -The public URL represents its separately recorded deployment and must not be -treated as running this branch. +`feat/learner-ux-v6.1` and has current local CloakBrowser evidence. It has not +yet been pushed, deployed, or qualified as one exact public source/image/Worker +tuple. The public URL represents its separately recorded historical deployment +and must not be treated as running this branch. + +The first screen states the operating contract directly: CounterLab is a +**belief debugger, not a tutor or notebook linter**. Its visible sequence is +`State claim → lock Prediction → controlled test runs → result passes +verification`. GPT-5.6 proposes a bounded belief frame, Runtime Codex compiles +an allowlisted experiment plan, fixed kernels compute values, and the frozen +verifier decides whether evidence may be released. No live call begins on the +landing screen. ## Learner journey @@ -24,7 +33,8 @@ CounterLab presents one six-stage path while preserving the existing scientific state machine and authority gates: 1. **Question** — state a claim or attach a supported notebook. Sample prompts - are optional, and Judge Mode remains a secondary route. + are optional. **Start verified sample** and **Judge Mode** are the first + visible cold-user paths; claim testing remains available immediately below. 2. **Prediction** — review exact notebook evidence and the sanitized packet, confirm that the two-model comparison captures the learner's view, then seal an immutable categorical prediction and confidence. @@ -48,11 +58,11 @@ local verification record is in ## Judge Mode -| Path | Mode | Meaning | Secrets required | -| --------------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | -| Verified sample | `sample` | Bundled approved leakage lesson evidence; clearly labelled, credential-free, and separate from live authority | No | -| Live notebook | `live_notebook` | Artifact-specific GPT analyst plus the authenticated Container runner, typed bounded artifacts, fixed scorer/kernel, frozen verifier, and copied-notebook patch | Server-side model and runner configuration | -| Verified replay | `replay` | Read-only reconstruction of stored events and payloads; persistently labelled and unable to make a new model call | No | +| Path | Session mode | GPT-5.6 / Runtime Codex | Numerical source | Mutability | May claim | Must not claim | +| --------------- | ----------------- | -------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| Verified sample | `sample_lesson` | No live call; reviewed Subject Pack framing | Integrity-checked bundled fixed-kernel fixture | Learner inputs may advance a new Sample session; the verified fixture is immutable | The scoped fixed Sample demonstrates the recorded mechanism | A live run, artifact-specific analysis, or newly calculated browser result | +| Live notebook | `live_notebook` | Server-side calls may occur only after supported upload, packet approval, and capability admission | Fixed scorer, kernel, and frozen verifier bound to that artifact | Explicit persisted state transitions; original upload remains read-only and patching targets a copy | The bounded artifact-specific result and provenance that actually occurred | Authority when capability is absent, an unsupported notebook result, or a Sample/Replay fallback | +| Verified replay | `verified_replay` | No new model or compiler call | Stored validated events and signed or integrity-bound payloads | Read-only | A reconstruction of the specific recorded verified session | A current rerun, a new configuration, or live capability | Sample is not live and cannot enter a live session. Replay does not make a new model call or borrow sample authority. The rejected compiler run authorized no @@ -62,6 +72,14 @@ backed by a versioned, precomputed fixed-kernel fixture. It does not add an arbitrary sample configuration matrix, and the browser never computes a substitute authoritative result. +Visible source labels preserve that separation: learner-authored framing is +`Your input`; reviewed fixed framing is `Reviewed Subject Pack draft`; +`AI-suggested draft` appears only with genuine model provenance. During a live +test, `Generated planning`, `Fixed testing`, and `Verified result` are derived +from the real runner job kind and result-ready evidence. Artifact surfaces say +`Bundled sample artifact`, `Uploaded notebook`, or `Replay artifact`; a +route/session mode mismatch withholds proof instead of guessing. + ## Supported notebook contract CounterLab accepts `nbformat` 4 `.ipynb` files up to 10 MiB containing the @@ -86,15 +104,36 @@ The server defaults to `OPENAI_MODEL=gpt-5.6` and safety identifier, and resolves every evidence reference locally. Invalid, unsupported, or unresolved output cannot advance state. +For supported live Belief Specs, the current source then runs a bounded +Learning Director over the same approved evidence. It may make at most four +read-only tool calls across three stateless Responses turns, ask at most one +optional fixed-choice clarification, and select only registered presentation +scaffolds, scene recipes, Boundary views, and candidate references. The fixed +question and answer labels are local UI copy; the model cannot author them or +veto learner-owned Belief Spec confirmation. Invalid Director output is omitted +while the deterministic learner flow remains available. It uses `store: false` +and the same hashed safety identifier. CounterLab persists only the validated +decision, tool hashes/durations, configured model and prompt identifiers, and +token totals; it does not persist provider response IDs or private reasoning. +The Learning Director cannot select the final experiment, compute values, +verify evidence, grade transfer, unlock Repair, or issue proof. + `OPENAI_BASE_URL` optionally selects a compatible Responses API endpoint. It is server-only and may be configured as an HTTPS host root, a `/v1` base, or the full `/v1/responses` endpoint; CounterLab normalizes all three to the SDK base and never returns the endpoint in health, events, evidence, or browser state. -Before a live call, Studio shows the exact sanitized packet: concept routing, -claim, schema summary, support state, and bounded evidence excerpts. The learner -must approve that packet; sensitive-looking excerpts require a second explicit -confirmation. Editing the claim or artifact invalidates the approval. +Before a live call, Studio shows the exact sanitized packet envelope for both +the Belief Analyst and Learning Director: concept routing, claim, schema +summary, support state, bounded evidence excerpts, Subject Pack version, and +registered candidate references. One approval hash binds that envelope to the +resulting Belief Spec and any persisted presentation decision. Privacy +inventory covers every declared schema field before the visible 64-field cap; +declared-sensitive names and common identifier patterns are normalized and +redacted before truncation. The learner must approve that packet; +sensitive-looking or suppressed fields require a second explicit confirmation. +Editing the claim or artifact invalidates the approval. This remains heuristic +redaction, not a guarantee of complete de-identification. ### Runtime Codex generates @@ -152,12 +191,34 @@ The complete authority matrix is in ## Architecture +Evidence release follows one bounded authority chain: + +```text +Question / supported artifact + │ + ▼ +sanitized packet ──► GPT-5.6 Belief Spec + bounded presentation plan + │ + ▼ +Runtime Codex typed plan ──► fixed candidate scorer + │ + ▼ +fixed kernel result ──► frozen verifier + │ + ▼ +verified UI + Boundary Map + Proof Capsule +``` + +The proposal and presentation layers never supply numerical truth. A result +cannot cross the final boundary until the learner has sealed a Prediction and +the fixed evidence has passed verification. + ```text Vite React Studio │ typed requests + reconnectable public events Cloudflare Worker ── D1 sessions/jobs/event chain │ └─ R2 private inputs/artifacts/results/patches/proofs - ├─ Responses API (optional live analyst) + ├─ Responses API (optional live analyst + Learning Director) ├─ stored sample/replay path └─ Container-backed Durable Object ├─ Codex App Server over stdio JSONL @@ -190,6 +251,14 @@ Required for local reproduction: Node 22+, pnpm 11.13.1, Python 3.12+, and the locked dependencies. A Docker-compatible engine is required for local Cloudflare Container development and for the advanced local adapter proof. +The two ready-to-use supported notebook fixtures are: + +- `fixtures/notebooks/customer_churn_leakage.ipynb` +- `fixtures/notebooks/fraud_class_imbalance.ipynb` + +Use them for the entity-leakage and class-imbalance live paths respectively; +do not treat them as evidence that arbitrary notebooks are supported. + ```bash cp .env.example .env.local ./scripts/clean-demo.sh @@ -266,10 +335,10 @@ The current measured artifact is - random accuracy 0.984722, group accuracy 0.594444, ablation accuracy 0.673611; - group entity overlap 0; - canonical result hash - `2501654264b9aa85b39fca944e585ff9b04263b83e182bc186d1f16464fee3b0`; -- 12/12 published critical mutations detected; + `a6ae7652e04e4d70196f991c63b8f7bcb3b76f8c4ab833d3ce2b626df0ab6c94`; +- 14/14 published leakage critical mutations detected; - class-imbalance majority accuracy 0.989333 with 0 rare-class recall, and - 12/12 imbalance mutations detected; + 15/15 published imbalance mutations detected; - held-out intake/routing 10/10 and fixed full-loop completion 7/8; the Random Forest patch is explicitly outside the registered non-sample patch contract; @@ -288,7 +357,10 @@ Map, fixed transfer, patch lock, copied-notebook repair, Reasoning Diff v2, Proof Capsule v2, resumable six-stage learner navigation, and privacy-safe interaction evidence. Historical production evidence separately covers the legacy sample/replay and hosted leakage/imbalance paths identified above. The -v6.1 CloakBrowser journey is specified but has not been executed. +v6.1 landing, Judge, proof-navigation, keyboard, and fixed-Sample journeys have +been executed locally with CloakBrowser at mobile and desktop sizes. Those +mutable-development checks are non-qualifying; the full matrix against one +exact public release is still pending. Failed honestly: the first live Codex run used an unsupported SDK argument; two repairs then left an unexpected `__pycache__`, so that run remained rejected and @@ -310,20 +382,30 @@ The recorded local host App Server replay could inspect global skill files outside its generation directory, so that replay remains labelled `PARTIAL`. The hosted source-free Plan path is separate: credentials are staged only for App Server initialization and revoked before the generated turn; generated -child commands receive neither model credentials nor signing authority. The -Container launch uses a fixed non-root UID, `setpriv --no-new-privs`, and write -constraints, but it has no -mount namespace or filesystem read allowlist. Hosted filesystem generation -read isolation is explicitly `PARTIAL`; formal sandbox proof is not claimed. +child commands receive neither model credentials nor signing authority. Current +source refuses a live launch unless the exact runner reports a pinned +Bubblewrap boundary with an allowlisted runtime mount set, a writable +generation workspace, missing repository/held-out/verifier paths, a fixed +non-root UID, and `no-new-privs`. That is source-level admission logic, not production evidence: +until a newly built exact image passes the sentinel and release qualification, +hosted live authority remains unavailable and no `OS_ENFORCED` deployment claim +is made. See [docs/THREAT_MODEL.md](docs/THREAT_MODEL.md). ## How Codex accelerated the build -Codex helped implement and test the monorepo, then CounterLab used Codex itself -as a bounded runtime compiler. The real rejected trace exposed two product -contract defects: incomplete public SDK documentation and transient bytecode in -an exact-file workspace. The public contract was strengthened; a separate later -run verified successfully. +Build-time Codex helped map the monorepo, turn audit findings into focused +regressions, implement bounded UI and release slices, triage cross-language +failures, review authority drift, and keep the evidence documents synchronized +with executed checks. Changes were integrated as small independently reviewed +commits; a source test, deployment receipt, model call, or learner result was +never treated as successful without direct evidence. + +That engineering role is separate from Runtime Codex inside CounterLab. Runtime +Codex acts only as a bounded Plan compiler. The real rejected trace exposed two +product-contract defects: incomplete public SDK documentation and transient +bytecode in an exact-file workspace. The public contract was strengthened; a +separate later run verified successfully. Human decisions remained the product thesis, learner authority, concept scope, fixture parameters, evidence contracts, verifier invariants, transfer design, diff --git a/apps/web/e2e/browser-authority.ts b/apps/web/e2e/browser-authority.ts index edb33110..a0148fe6 100644 --- a/apps/web/e2e/browser-authority.ts +++ b/apps/web/e2e/browser-authority.ts @@ -17,6 +17,27 @@ export type BrowserAuthority = type BrowserEnvironment = Readonly>; +export function validateCloakCdpEndpoint(configured: string): string { + const endpoint = new URL(configured); + if (!["http:", "https:", "ws:", "wss:"].includes(endpoint.protocol)) { + throw new Error( + "CLOAK_CDP_ENDPOINT must use an http(s) or ws(s) CDP endpoint", + ); + } + if (endpoint.username !== "" || endpoint.password !== "") { + throw new Error("CLOAK_CDP_ENDPOINT must not contain URL credentials"); + } + if ( + !new Set(["127.0.0.1", "localhost", "[::1]"]).has(endpoint.hostname) || + endpoint.port === "" + ) { + throw new Error( + "CLOAK_CDP_ENDPOINT must use an explicit port on a loopback host", + ); + } + return configured; +} + export function resolveBrowserAuthority( environment: BrowserEnvironment, ): BrowserAuthority { @@ -34,7 +55,7 @@ export function resolveBrowserAuthority( return { kind: "cloak", evidenceLabel: "CLOAK_CDP_ENDPOINT", - endpoint, + endpoint: validateCloakCdpEndpoint(endpoint), }; } if (requestedAuthority !== STOCK_CHROMIUM_DESIGN_REVIEW) { diff --git a/apps/web/e2e/cloak-test.ts b/apps/web/e2e/cloak-test.ts index 4c8a09e2..1f201f8d 100644 --- a/apps/web/e2e/cloak-test.ts +++ b/apps/web/e2e/cloak-test.ts @@ -8,7 +8,23 @@ import { test as base } from "@playwright/test"; import { currentBrowserAuthorityLabel, resolveBrowserAuthority, + validateCloakCdpEndpoint, } from "./browser-authority"; +import { + classifyExpectedHttpErrorObservation, + classifyExpectedHttpResourceConsoleObservation, + JOURNEY_OBSERVATION_ATTACHMENT, + journeyIdForParts, + summarizeExpectedHttpErrors, + summarizeExpectedHttpResourceConsoleErrors, + summarizeRequestFailures, + type ExpectedHttpErrorObservation, +} from "./qualification-reporter"; +import { + ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS, + REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID, + REQUIRED_CLOAK_JOURNEY_VIEWPORT_BY_ID, +} from "../../../scripts/submission-publication-evidence"; export { currentBrowserAuthorityLabel }; @@ -16,6 +32,19 @@ const repositoryRoot = realpathSync( fileURLToPath(new URL("../../..", import.meta.url)), ); const runtimeParent = resolve(repositoryRoot, "apps/web/test-results/runtime"); +type AllowedExpectedHttpError = + (typeof ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS)[number]; +const allowedExpectedHttpErrors: ReadonlySet = new Set( + ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS, +); + +function expectedBrowserOrigin(): string { + const configuredBaseUrl = process.env.COUNTERLAB_E2E_BASE_URL; + const baseUrl = + configuredBaseUrl ?? + `http://127.0.0.1:${process.env.COUNTERLAB_E2E_PORT ?? "5173"}`; + return new URL(baseUrl).origin; +} function assertNoSymlinkEscape(candidate: string, label: string): void { const relativePath = relative(repositoryRoot, candidate); @@ -88,20 +117,11 @@ export function runtimeOutputPath(candidate: string, label: string): string { return resolved; } -function validatedCloakEndpoint(configured: string): string { - const endpoint = new URL(configured); - if (!["http:", "https:", "ws:", "wss:"].includes(endpoint.protocol)) { - throw new Error( - "CLOAK_CDP_ENDPOINT must use an http(s) or ws(s) CDP endpoint", - ); - } - if (endpoint.username !== "" || endpoint.password !== "") { - throw new Error("CLOAK_CDP_ENDPOINT must not contain URL credentials"); - } - return endpoint.toString(); -} +type CounterLabAutomaticFixtures = { + counterlabJourneyObservation: void; +}; -export const test = base.extend({ +export const test = base.extend({ browser: [ async ({ playwright }, use) => { const authority = resolveBrowserAuthority(process.env); @@ -113,7 +133,7 @@ export const test = base.extend({ assertContainedPath(artifactsDir, "Playwright browser artifacts"); if (authority.kind === "cloak") { const browser = await playwright.chromium.connectOverCDP( - validatedCloakEndpoint(authority.endpoint), + validateCloakCdpEndpoint(authority.endpoint), { artifactsDir, timeout: 30_000, @@ -168,6 +188,140 @@ export const test = base.extend({ }, { scope: "worker" }, ], + counterlabJourneyObservation: [ + async ({ page }, use, testInfo) => { + const journeyId = journeyIdForParts(testInfo.file, testInfo.titlePath); + const expectedViewport = + REQUIRED_CLOAK_JOURNEY_VIEWPORT_BY_ID[ + journeyId as keyof typeof REQUIRED_CLOAK_JOURNEY_VIEWPORT_BY_ID + ]; + if (expectedViewport === undefined) { + throw new Error( + `Unregistered browser qualification journey: ${journeyId}`, + ); + } + const [width, height] = expectedViewport + .split("x") + .map((part) => Number.parseInt(part, 10)); + await page.setViewportSize({ width: width!, height: height! }); + const expectedOrigin = expectedBrowserOrigin(); + const observedConsoleErrors: (ExpectedHttpErrorObservation | null)[] = []; + const observedHttpErrors: (ExpectedHttpErrorObservation | null)[] = []; + const failedRequests: string[] = []; + page.on("console", (message) => { + if (message.type() !== "error") return; + observedConsoleErrors.push( + classifyExpectedHttpResourceConsoleObservation({ + expectedOrigin, + locationUrl: message.location().url, + text: message.text(), + }), + ); + }); + page.on("pageerror", () => { + observedConsoleErrors.push(null); + }); + page.on("response", (response) => { + if (response.status() < 400) return; + observedHttpErrors.push( + classifyExpectedHttpErrorObservation({ + expectedOrigin, + method: response.request().method(), + status: response.status(), + url: response.url(), + }), + ); + }); + page.on("requestfailed", (request) => { + try { + const url = new URL(request.url()); + failedRequests.push( + `${request.method().toUpperCase()} ${url.pathname}`, + ); + } catch { + failedRequests.push("UNPARSEABLE"); + } + }); + const browserVersion = + page.context().browser()?.version().trim() || "unavailable"; + + await use(); + + const expectedHttpAnnotations = testInfo.annotations.filter( + (annotation) => annotation.type === "counterlab-expected-http-errors", + ); + const expectedHttpAnnotationValues = expectedHttpAnnotations.map( + (annotation) => annotation.description ?? "", + ); + const expectedHttpErrorResponses = expectedHttpAnnotationValues.filter( + (value): value is AllowedExpectedHttpError => + allowedExpectedHttpErrors.has(value), + ); + const requiredHttpErrorResponses = + REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID[ + journeyId as keyof typeof REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID + ]; + const httpErrorSummary = summarizeExpectedHttpErrors( + observedHttpErrors.map((observation) => observation?.signature ?? null), + expectedHttpAnnotationValues, + ); + const httpResourceConsoleSummary = + summarizeExpectedHttpResourceConsoleErrors( + observedConsoleErrors, + httpErrorSummary.matched + ? observedHttpErrors.filter( + (observation): observation is ExpectedHttpErrorObservation => + observation !== null, + ) + : [], + ); + const expectedFailureAnnotations = testInfo.annotations.filter( + (annotation) => + annotation.type === "counterlab-expected-request-failures", + ); + const expectedRequestFailures = expectedFailureAnnotations.map( + (annotation) => annotation.description ?? "", + ); + const requestFailureSummary = summarizeRequestFailures( + failedRequests, + expectedRequestFailures, + ); + const viewport = page.viewportSize(); + await testInfo.attach(JOURNEY_OBSERVATION_ATTACHMENT, { + body: Buffer.from( + JSON.stringify({ + schemaVersion: "4", + authority: currentBrowserAuthorityLabel(), + viewport: viewport ?? { width: 0, height: 0 }, + totalConsoleErrors: httpResourceConsoleSummary.totalCount, + expectedHttpResourceConsoleErrors: + httpResourceConsoleSummary.expectedCount, + unexpectedConsoleErrors: httpResourceConsoleSummary.unexpectedCount, + expectedHttpErrorResponses, + observedHttpErrorResponses: + httpErrorSummary.observedHttpErrorResponses, + unexpectedHttpErrorResponses: httpErrorSummary.unexpectedCount, + expectedHttpResponsesMatched: + httpErrorSummary.matched && + expectedHttpAnnotationValues.length === + expectedHttpErrorResponses.length && + JSON.stringify(expectedHttpErrorResponses) === + JSON.stringify(requiredHttpErrorResponses), + expectedRequestFailures, + expectedFailedRequests: requestFailureSummary.expectedCount, + unexpectedFailedRequests: requestFailureSummary.unexpectedCount, + observedRequestFailures: + requestFailureSummary.observedRequestFailures, + observedFailedRequests: requestFailureSummary.observedCount, + expectedRequestFailuresMatched: requestFailureSummary.matched, + browserVersion, + }), + ), + contentType: "application/json", + }); + }, + { auto: true }, + ], }); export { expect } from "@playwright/test"; diff --git a/apps/web/e2e/judged-flow.spec.ts b/apps/web/e2e/judged-flow.spec.ts index 114c8d2f..496c6927 100644 --- a/apps/web/e2e/judged-flow.spec.ts +++ b/apps/web/e2e/judged-flow.spec.ts @@ -15,6 +15,7 @@ import { ReasoningDiffV2Schema, } from "@counterlab/contracts"; import { createHash } from "node:crypto"; +import type { Request } from "@playwright/test"; import { lstat, readFile, realpath, writeFile } from "node:fs/promises"; import { isAbsolute, relative, resolve } from "node:path"; @@ -25,11 +26,14 @@ import { snapshotBrowserPerformanceEvidence, type BrowserPerformanceEvidence, } from "./performance-evidence"; +import { privateSessionHeaders } from "./private-session"; const claim = "The 98 percent random split accuracy proves this model generalizes to customers it has never seen."; const revision = "When rows repeat an entity, hold out whole entities and remove identity-derived features before claiming generalization."; +const prePredictionResultLanguage = + /59\.4%|\bdeceptive\b|\bfairer test\b|\bverified result\b|\bevidence verdict\b|\bsupported hypothesis\b|\bthe fix\b|\bremove customer(?:_| )id\b|\bkeep each customer's rows together\b|\bproves?\b/i; const imbalanceNotebookPath = new URL( "../../../fixtures/notebooks/fraud_class_imbalance.ipynb", import.meta.url, @@ -150,9 +154,11 @@ async function writeLiveSmokeEvidence( if (sessionId === null) { throw new Error("Live smoke evidence requires a persisted session ID"); } + const ownerHeaders = await privateSessionHeaders(page, sessionId); const completedSessionResponse = await page.request.get( `/api/sessions/${encodeURIComponent(sessionId)}`, + { headers: ownerHeaders }, ); expect(completedSessionResponse.ok()).toBe(true); const completedSessionPayload = (await completedSessionResponse.json()) as { @@ -270,7 +276,7 @@ async function writeLiveSmokeEvidence( const duplicatePublicationResponse = await page.request.post( `/api/sessions/${encodeURIComponent(sessionId)}/replays`, - { data: {} }, + { data: {}, headers: ownerHeaders }, ); expect(duplicatePublicationResponse.status()).toBe(200); const duplicatePublicationPayload = @@ -285,6 +291,7 @@ async function writeLiveSmokeEvidence( const activeStatusResponse = await page.request.get( `/api/sessions/${encodeURIComponent(sessionId)}/replays/status`, + { headers: ownerHeaders }, ); expect(activeStatusResponse.ok()).toBe(true); const activeStatusPayload = (await activeStatusResponse.json()) as { @@ -509,6 +516,7 @@ async function writeLiveSmokeEvidence( const revokedStatusResponse = await page.request.get( `/api/sessions/${encodeURIComponent(sessionId)}/replays/status`, + { headers: ownerHeaders }, ); expect(revokedStatusResponse.ok()).toBe(true); await expect(revokedStatusResponse.json()).resolves.toMatchObject({ @@ -520,7 +528,7 @@ async function writeLiveSmokeEvidence( const duplicateRevocationResponse = await page.request.post( `/api/sessions/${encodeURIComponent(sessionId)}/replays/revoke`, - { data: {} }, + { data: {}, headers: ownerHeaders }, ); expect(duplicateRevocationResponse.status()).toBe(200); await expect(duplicateRevocationResponse.json()).resolves.toMatchObject({ @@ -529,7 +537,7 @@ async function writeLiveSmokeEvidence( const republishRevokedResponse = await page.request.post( `/api/sessions/${encodeURIComponent(sessionId)}/replays`, - { data: {} }, + { data: {}, headers: ownerHeaders }, ); expect(republishRevokedResponse.status()).toBe(409); await expect(republishRevokedResponse.json()).resolves.toMatchObject({ @@ -660,12 +668,13 @@ async function resetWithBrowserFailureObservation( return failures; } -function expectWithinComprehensionBudget( - startedAt: number, +async function expectWithinComprehensionBudget( + page: Page, maximumMs: number, label: string, ) { - expect(Date.now() - startedAt, label).toBeLessThanOrEqual(maximumMs); + const navigationElapsedMs = await page.evaluate(() => performance.now()); + expect(navigationElapsedMs, label).toBeLessThanOrEqual(maximumMs); } function expectNoBrowserFailures(failures: BrowserFailureLog) { @@ -704,7 +713,6 @@ async function expectBeliefBreakInFirstViewport( page: Page, surface: Locator, modeLabel: RegExp, - startedAt: number, ) { const mechanism = surface.getByRole("region", { name: "Verified sample belief-break mechanism", @@ -792,6 +800,14 @@ async function expectBeliefBreakInFirstViewport( await expect(mechanism).toBeVisible(); for (const stage of comprehensionStages) { + await Promise.all( + stage.required.map(({ locator }) => expect(locator).toBeVisible()), + ); + await expectWithinComprehensionBudget( + page, + stage.budgetMs, + `Judge ${stage.label} must be inspectable within ${stage.budgetMs / 1_000} seconds`, + ); for (const required of stage.required) { await expectEntirelyInFirstViewport( page, @@ -799,15 +815,19 @@ async function expectBeliefBreakInFirstViewport( required.label, ); } - expectWithinComprehensionBudget( - startedAt, - stage.budgetMs, - `Judge ${stage.label} must be inspectable within ${stage.budgetMs / 1_000} seconds`, - ); } } -async function captureBeliefBreakScreenshot(page: Page, fileName: string) { +async function captureBeliefBreakScreenshot( + page: Page, + fileName: string, + options: { + performanceBudgetScope?: + "navigation-first-fold" | "post-seal-spa-observation"; + } = {}, +) { + const performanceBudgetScope = + options.performanceBudgetScope ?? "navigation-first-fold"; const configuredDirectory = process.env.COUNTERLAB_E2E_BELIEF_BREAK_EVIDENCE_DIR ?? resolve(requiredRuntimeRoot(), "evidence", "belief-break"); @@ -816,7 +836,9 @@ async function captureBeliefBreakScreenshot(page: Page, fileName: string) { ); await page.screenshot({ path: destination, fullPage: false }); const performance = await snapshotBrowserPerformanceEvidence(page); - expectFirstFoldPerformanceBudgets(performance); + if (performanceBudgetScope === "navigation-first-fold") { + expectFirstFoldPerformanceBudgets(performance); + } const evidenceDestination = await ensureRuntimeParent( resolve(configuredDirectory, fileName.replace(/\.png$/u, ".json")), ); @@ -827,6 +849,10 @@ async function captureBeliefBreakScreenshot(page: Page, fileName: string) { browserAuthority: currentBrowserAuthorityLabel(), capturedAt: new Date().toISOString(), performance, + performanceBudget: { + enforced: performanceBudgetScope === "navigation-first-fold", + scope: performanceBudgetScope, + }, screenshot: fileName, }, null, @@ -892,10 +918,50 @@ async function startInstant(page: Page) { ).toBeVisible(); } +function waitForPostResponse(page: Page, routeSuffix: string) { + return page.waitForResponse((response) => { + const request = response.request(); + return ( + request.method() === "POST" && + new URL(response.url()).pathname.endsWith(routeSuffix) + ); + }); +} + +function trackNetworkQuiescence(page: Page): () => Promise { + const active = new Set(); + page.on("request", (request) => active.add(request)); + page.on("requestfinished", (request) => active.delete(request)); + page.on("requestfailed", (request) => active.delete(request)); + + return async () => { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (active.size === 0) { + await page.waitForTimeout(500); + if (active.size === 0) return; + } else { + await page.waitForTimeout(50); + } + } + throw new Error( + `Browser network did not become quiescent (${active.size} request(s) remain)`, + ); + }; +} + async function commitAndOpenResult(page: Page) { await page.getByLabel(/Remain near 98%/i).check(); + const compileFinished = waitForPostResponse(page, "/lab/compile"); await page.getByRole("button", { name: /Seal my prediction/i }).click(); - await page.getByRole("button", { name: /Run the fair test/i }).click(); + expect((await compileFinished).ok()).toBe(true); + const runFinished = waitForPostResponse(page, "/lab/run"); + await page + .getByRole("button", { + name: /Run the fair test|Reveal verified sample result/i, + }) + .click(); + expect((await runFinished).ok()).toBe(true); await expect( page.getByRole("heading", { name: /Compare the verified result/i }), ).toBeVisible(); @@ -914,6 +980,7 @@ async function authorResultInterpretation(page: Page) { ).toBeVisible(); await expect(page.getByRole("tab", { name: /Explore/i })).toBeEnabled(); await expect(page.getByRole("tab", { name: /Boundary/i })).toBeEnabled(); + await page.waitForLoadState("networkidle"); } async function selectLeakageTransferEvidence(page: Page) { @@ -1315,17 +1382,49 @@ for (const viewport of beliefBreakViewports) { test(`${viewport.name} Landing keeps the unprimed Question and fair-test promise in the first viewport`, async ({ page, }) => { - const navigationStartedAt = Date.now(); await page.setViewportSize(viewport); const failures = await resetWithBrowserFailureObservation(page); + const heading = page.getByRole("heading", { + name: /What result are you trying to understand/i, + }); + const artifactSafetySummary = page.getByText( + /Ask a question or attach a supported notebook.*never runs its cells/i, + ); + const audienceDifference = page.getByText( + /Built for ML learners: seal a Prediction, then let one controlled test answer—not AI prose/i, + ); + const composer = page.getByLabel("Your question or claim"); + const submit = page.getByRole("button", { name: /Test this claim/i }); + const promptStarters = page.getByRole("group", { + name: /Prompt starters/i, + }); + const preview = page.locator( + '[data-presentation="strip"][data-result-visibility="locked"]', + ); + await Promise.all( + [ + heading, + artifactSafetySummary, + audienceDifference, + composer, + submit, + promptStarters, + preview, + ].map((locator) => expect(locator).toBeVisible()), + ); + await expectWithinComprehensionBudget( + page, + 10_000, + "Landing Question and fair-test promise must become inspectable within ten seconds", + ); + const skipLink = page.getByRole("link", { name: /Skip to main content/i, }); await skipLink.focus(); await expect(skipLink).toBeFocused(); await page.keyboard.press("Tab"); - const composer = page.getByLabel("Your question or claim"); await expect(composer).toBeFocused(); await expectEntirelyInFirstViewport( page, @@ -1334,37 +1433,52 @@ for (const viewport of beliefBreakViewports) { ); await expectEntirelyInFirstViewport( page, - page.getByText( - /For learners testing whether a notebook result means what they think it means/i, - ), - "learner audience statement", + heading, + "Question-first heading", + ); + await expectEntirelyInFirstViewport( + page, + artifactSafetySummary, + "artifact safety summary", ); - const preview = page.getByRole("complementary", { - name: /One changed variable.*Everything else held fixed/i, - }); + await expect(submit).toBeDisabled(); + await expectEntirelyInFirstViewport( + page, + submit, + "primary Question action", + ); + await expectEntirelyInFirstViewport( + page, + promptStarters, + "learner prompt starters", + ); + if (viewport.width <= 620) { + const submitBox = await submit.boundingBox(); + expect(submitBox, "mobile primary Question action bounds").not.toBeNull(); + expect(submitBox!.width).toBeGreaterThanOrEqual(130); + } + await expectEntirelyInFirstViewport( page, preview, "pre-Prediction fair-test promise", ); + await expect(preview).toContainText(/Result locked until Prediction/i); await expect(preview).toContainText( - /Result hidden until your Prediction is sealed/i, + /Familiar rows.*change who counts as new.*Unseen customers/i, ); await expect(preview).toContainText( - /Name the claim.*Lock your expectation.*Change one thing/i, + /Same model, features, preprocessing, sample size, metric, and seed/i, + ); + await expect(page.locator("body")).not.toContainText( + prePredictionResultLanguage, ); - await expect(page.locator("body")).not.toContainText("59.4%"); await expect( page.getByRole("region", { name: "Verified sample belief-break mechanism", }), ).toHaveCount(0); - expectWithinComprehensionBudget( - navigationStartedAt, - 10_000, - "Landing Question and fair-test promise must become inspectable within ten seconds", - ); await expectNoHorizontalOverflow(page); await captureBeliefBreakScreenshot( page, @@ -1377,7 +1491,6 @@ for (const viewport of beliefBreakViewports) { test(`${viewport.name} Judge Mode shows the honest fixed-sample belief break in the first viewport`, async ({ page, }) => { - const navigationStartedAt = Date.now(); const failures = observeBrowserFailures(page); await installBrowserPerformanceEvidence(page); await page.setViewportSize(viewport); @@ -1391,7 +1504,6 @@ for (const viewport of beliefBreakViewports) { page, preview, /Completed fixed sample.*not a live result/i, - navigationStartedAt, ); await expect( preview.getByText( @@ -1459,7 +1571,11 @@ for (const viewport of beliefBreakViewports) { await expect( page.getByRole("heading", { name: /The fair test is ready/i }), ).toBeVisible(); - await page.getByRole("button", { name: /Run the fair test/i }).click(); + await page + .getByRole("button", { + name: /Run the fair test|Reveal verified sample result/i, + }) + .click(); await expect( page.getByRole("heading", { name: /Compare the verified result/i }), ).toBeVisible(); @@ -1485,6 +1601,7 @@ for (const viewport of beliefBreakViewports) { await captureBeliefBreakScreenshot( page, `belief-break-sample-post-seal-${viewport.name}.png`, + { performanceBudgetScope: "post-seal-spa-observation" }, ); await page.waitForLoadState("networkidle"); expectNoBrowserFailures(failures); @@ -1492,7 +1609,7 @@ for (const viewport of beliefBreakViewports) { } test("Judge Mode distinguishes every authority path", async ({ page }) => { - const healthResponse = await page.request.get("/api/health"); + const healthResponse = await page.request.get("/api/health?readiness=probe"); expect(healthResponse.ok()).toBe(true); const health = (await healthResponse.json()) as { data?: { @@ -1511,7 +1628,7 @@ test("Judge Mode distinguishes every authority path", async ({ page }) => { health.data.liveCodex === "configured" && health.data.liveKernel === "configured" && health.data.sandbox === "credential-and-privilege-boundary" && - health.data.generationFilesystemReadIsolation === "PARTIAL" && + health.data.generationFilesystemReadIsolation === "OS_ENFORCED" && health.data.release?.status === "bound"; await page.goto("/judge"); await expect(page).toHaveURL(/\/judge$/); @@ -1520,7 +1637,8 @@ test("Judge Mode distinguishes every authority path", async ({ page }) => { name: /see a verified belief break in ten seconds/i, }), ).toBeVisible(); - await expect(page.getByText("Sample lesson")).toBeVisible(); + await page.getByRole("button", { name: /Check live readiness/i }).click(); + await expect(page.getByText("Sample lesson", { exact: true })).toBeVisible(); await expect(page.getByText("Live notebook analysis")).toBeVisible(); await expect( page.getByText("Verified replay", { exact: true }), @@ -1532,7 +1650,10 @@ test("Judge Mode distinguishes every authority path", async ({ page }) => { ); } else { await expect( - page.getByText(/live authority is unavailable/i), + page.getByRole("status").filter({ + hasText: + /live authority (?:is|remains) unavailable|live authority did not pass the latest readiness check/i, + }), ).toBeVisible(); await expect(page.getByRole("link", { name: /run live/i })).toHaveCount(0); } @@ -1572,14 +1693,22 @@ test("Judge Mode distinguishes every authority path", async ({ page }) => { test("the first visit explains the lesson before asking for technical knowledge", async ({ page, }) => { + await page.setViewportSize({ width: 768, height: 1024 }); + expect(page.viewportSize()).toEqual({ width: 768, height: 1024 }); await reset(page); + await revealLandingNavigation(page); await expect( page.getByRole("heading", { name: "What result are you trying to understand?", }), ).toBeVisible(); await expect( - page.getByText(/No account needed.*never run the cells/i), + page.getByText(/reads the evidence and never runs its cells/i), + ).toBeVisible(); + await expect( + page.getByText( + /No account needed.*Sample, live, and replay stay clearly labelled/i, + ), ).toBeVisible(); await expect(page.getByLabel("Your question or claim")).toBeInViewport(); await expect(page.getByLabel("Attach notebook")).toBeVisible(); @@ -1612,7 +1741,7 @@ test("the first visit explains the lesson before asking for technical knowledge" }); for (const viewport of [ - { name: "wide desktop", width: 1440, height: 900 }, + { name: "wide desktop", width: 1920, height: 1080 }, { name: "compact desktop", width: 1280, height: 720 }, { name: "mobile", width: 390, height: 844 }, ] as const) { @@ -1620,6 +1749,10 @@ for (const viewport of [ page, }) => { await page.setViewportSize(viewport); + expect(page.viewportSize()).toEqual({ + width: viewport.width, + height: viewport.height, + }); await reset(page); await revealLandingNavigation(page); @@ -1753,6 +1886,8 @@ test("Try Instantly persists the verified learning loop and exports a valid proo test("the lesson keeps one learner decision in focus at a time", async ({ page, }) => { + await page.setViewportSize({ width: 1366, height: 768 }); + expect(page.viewportSize()).toEqual({ width: 1366, height: 768 }); await startInstant(page); await commitAndOpenResult(page); @@ -1784,7 +1919,9 @@ test("the lesson keeps one learner decision in focus at a time", async ({ await page.getByRole("button", { name: /Check transfer/i }).click(); await expect( - page.getByRole("heading", { name: /You applied the rule correctly/i }), + page.getByRole("heading", { + name: /This fixed forecasting transfer passed/i, + }), ).toBeVisible(); expect(await page.evaluate(() => window.scrollY)).toBeLessThan(24); await expect( @@ -1796,7 +1933,10 @@ test("the lesson keeps one learner decision in focus at a time", async ({ expect(await page.evaluate(() => window.scrollY)).toBeLessThan(24); await expect(page.locator("pre.diff")).not.toBeVisible(); await expect( - page.getByRole("button", { name: "Download proof record", exact: true }), + page.getByRole("button", { + name: "Download repaired notebook", + exact: true, + }), ).toBeVisible(); }); @@ -1808,8 +1948,11 @@ test("prediction is immutable and results do not exist before commitment", async window.localStorage.getItem("counterlab.sessionId"), ); expect(sessionId).not.toBeNull(); + const ownerHeaders = await privateSessionHeaders(page, sessionId!); - const before = await page.request.get(`/api/sessions/${sessionId}`); + const before = await page.request.get(`/api/sessions/${sessionId}`, { + headers: ownerHeaders, + }); expect(before.ok()).toBe(true); expect((await before.json()).data.verifiedResult).toBeUndefined(); @@ -1824,12 +1967,15 @@ test("prediction is immutable and results do not exist before commitment", async `/api/sessions/${sessionId}/prediction`, { data: { choice: "Accuracy remains near 98%", confidence: 100 }, + headers: ownerHeaders, }, ); expect(overwrite.status()).toBe(409); expect((await overwrite.json()).error.code).toBe("ILLEGAL_TRANSITION"); - const committed = await page.request.get(`/api/sessions/${sessionId}`); + const committed = await page.request.get(`/api/sessions/${sessionId}`, { + headers: ownerHeaders, + }); expect(committed.ok()).toBe(true); const committedPayload = await committed.json(); expect(committedPayload.data.prediction.confidence).toBe(88); @@ -1864,23 +2010,39 @@ test("the fixed sample keeps approved framing and can return home", async ({ test("refresh restores the question and confirmed Prediction phases", async ({ page, }) => { + const waitForNetworkQuiescence = trackNetworkQuiescence(page); await reset(page); + const sampleSessionCreated = waitForPostResponse( + page, + "/api/sample/sessions", + ); await page.getByRole("button", { name: /Try verified sample/i }).click(); + expect((await sampleSessionCreated).ok()).toBe(true); + await expect( + page.getByRole("heading", { name: /What do you think the score means/i }), + ).toBeVisible(); + await waitForNetworkQuiescence(); await page.reload(); await expect( page.getByRole("heading", { name: /What do you think the score means/i }), ).toBeVisible(); + const beliefProposed = waitForPostResponse(page, "/belief-test"); await page.getByRole("button", { name: /Compare two explanations/i }).click(); + expect((await beliefProposed).ok()).toBe(true); + await waitForNetworkQuiescence(); await page.reload(); await expect( page.getByRole("heading", { name: /Does your current explanation capture what you mean/i, }), ).toBeVisible(); + const beliefConfirmed = waitForPostResponse(page, "/belief-test/confirm"); await page .getByRole("button", { name: /Yes, this captures my view/i }) .click(); + expect((await beliefConfirmed).ok()).toBe(true); + await waitForNetworkQuiescence(); await page.reload(); await expect( page.getByRole("heading", { @@ -1892,7 +2054,11 @@ test("refresh restores the question and confirmed Prediction phases", async ({ test("refresh restores the current lesson and the committed prediction", async ({ page, }) => { + const waitForNetworkQuiescence = trackNetworkQuiescence(page); + const beliefConfirmed = waitForPostResponse(page, "/belief-test/confirm"); await startInstant(page); + expect((await beliefConfirmed).ok()).toBe(true); + await waitForNetworkQuiescence(); await page.reload(); await expect( page.getByRole("heading", { @@ -1901,17 +2067,27 @@ test("refresh restores the current lesson and the committed prediction", async ( ).toBeVisible(); await page.getByLabel(/Fall materially/i).check(); await page.getByLabel(/Confidence/i).fill("88"); + const compileFinished = waitForPostResponse(page, "/lab/compile"); await page.getByRole("button", { name: /Seal my prediction/i }).click(); + expect((await compileFinished).ok()).toBe(true); + await waitForNetworkQuiescence(); await page.reload(); await expect( page.getByRole("heading", { name: /The fair test is ready/i }), ).toBeVisible(); - await page.getByRole("button", { name: /Run the fair test/i }).click(); + const runFinished = waitForPostResponse(page, "/lab/run"); + await page + .getByRole("button", { + name: /Run the fair test|Reveal verified sample result/i, + }) + .click(); + expect((await runFinished).ok()).toBe(true); await expect( page.getByRole("heading", { name: /Compare the verified result/i }), ).toBeVisible(); await authorResultInterpretation(page); + await waitForNetworkQuiescence(); await page.reload(); await expect( @@ -1925,7 +2101,10 @@ test("refresh restores the current lesson and the committed prediction", async ( ).toBeVisible(); await authorResultInterpretation(page); + const revisionRecorded = waitForPostResponse(page, "/revision"); await recordRevision(page); + expect((await revisionRecorded).ok()).toBe(true); + await waitForNetworkQuiescence(); await page.reload(); await expect( page.getByRole("heading", { name: /Try your rule on forecasting/i }), @@ -1933,16 +2112,24 @@ test("refresh restores the current lesson and the committed prediction", async ( await page.getByLabel(/Time-ordered holdout/i).check(); await page.getByLabel(/Centered rolling target/i).check(); await selectLeakageTransferEvidence(page); + const transferEvaluated = waitForPostResponse(page, "/transfer"); await page.getByRole("button", { name: /Check transfer/i }).click(); + expect((await transferEvaluated).ok()).toBe(true); await expect( page.locator(".eyebrow", { hasText: "Transfer passed" }), ).toBeVisible(); + await waitForNetworkQuiescence(); await page.reload(); await expect( - page.getByRole("heading", { name: /You applied the rule correctly/i }), + page.getByRole("heading", { + name: /This fixed forecasting transfer passed/i, + }), ).toBeVisible(); + const patchStarted = waitForPostResponse(page, "/patch/compile"); await page.getByRole("button", { name: /Verify notebook patch/i }).click(); + expect((await patchStarted).ok()).toBe(true); await waitForSamplePatch(page); + await waitForNetworkQuiescence(); await page.reload(); await expect( page.getByRole("heading", { @@ -1953,7 +2140,11 @@ test("refresh restores the current lesson and the committed prediction", async ( test("a rejected test releases no result and remains recoverable after refresh", async ({ page, -}) => { +}, testInfo) => { + testInfo.annotations.push({ + type: "counterlab-expected-http-errors", + description: "409 POST /api/sessions/:sessionId/lab/compile", + }); await page.route("**/api/sessions/*/lab/compile", async (route) => { await route.fulfill({ status: 409, @@ -1985,7 +2176,10 @@ test("a rejected test releases no result and remains recoverable after refresh", const sessionId = await page.evaluate(() => window.localStorage.getItem("counterlab.sessionId"), ); - const stored = await page.request.get(`/api/sessions/${sessionId}`); + expect(sessionId).not.toBeNull(); + const stored = await page.request.get(`/api/sessions/${sessionId}`, { + headers: await privateSessionHeaders(page, sessionId!), + }); expect(stored.ok()).toBe(true); expect((await stored.json()).data.verifiedResult).toBeUndefined(); @@ -2026,9 +2220,11 @@ test("local hints and Theater views never request a model or new result", async "true", ); await expect( - page.getByRole("img", { - name: /Verified accuracy comparison.*familiar rows.*new customers/i, - }), + page + .locator('[data-trusted-visual-id="verified_sample_belief_break_v1"]') + .getByRole("region", { + name: "Verified sample belief-break mechanism", + }), ).toBeVisible(); const hint = page.getByLabel("Contextual help"); @@ -2089,10 +2285,21 @@ test("Replay remains visibly labelled and read-only after refresh", async ({ }) => { await reset(page); await page.getByRole("button", { name: /Watch verified replay/i }).click(); - const replayBanner = page.getByLabel("Legacy replay status"); + const replayBanner = page.getByLabel("Replay status"); await expect(replayBanner).toContainText( "Verified replay · read-only stored evidence", ); + await expect( + page.getByRole("heading", { name: /Replay verified session/i }), + ).toBeVisible(); + const continueReplay = page.getByRole("button", { + name: /Continue replay/i, + }); + await expect(continueReplay).toBeVisible(); + await continueReplay.click(); + await expect(page.getByLabel("Legacy replay status")).toContainText( + "Verified replay · read-only stored evidence", + ); await expect( page.getByRole("heading", { name: /Inspect the result without changing its history/i, @@ -2101,9 +2308,7 @@ test("Replay remains visibly labelled and read-only after refresh", async ({ await expect( page.getByRole("table", { name: /Stored fixed-kernel comparison/i }), ).toBeVisible(); - await expect( - page.getByRole("button", { name: /Continue replay/i }), - ).toHaveCount(0); + await expect(continueReplay).toHaveCount(0); await expect( page.getByRole("button", { name: /Show me what happened/i }), ).toHaveCount(0); @@ -2122,7 +2327,7 @@ test("Replay remains visibly labelled and read-only after refresh", async ({ test("missing live capabilities are stated without claiming a model call", async ({ page, }) => { - await page.route("**/api/health", async (route) => { + await page.route("**/api/health*", async (route) => { await route.fulfill({ status: 200, contentType: "application/json", @@ -2155,7 +2360,7 @@ test("missing live capabilities are stated without claiming a model call", async test("configured reasoning cannot start without a qualified hosted runner", async ({ page, }) => { - await page.route("**/api/health", async (route) => { + await page.route("**/api/health*", async (route) => { await route.fulfill({ status: 200, contentType: "application/json", @@ -2184,13 +2389,45 @@ test("configured reasoning cannot start without a qualified hosted runner", asyn page.getByText(/qualified hosted runner is needed/i), ).toBeVisible(); await expect( - page.getByRole("button", { name: /Continue with my notebook/i }), + page.getByLabel(/Attach a supported notebook/i), ).not.toBeVisible(); await expect(page.locator("body")).not.toContainText( /OPENAI|GPT-|https?:\/\//i, ); }); +test("partial generation isolation cannot expose live notebook upload", async ({ + page, +}) => { + await page.route("**/api/health*", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + ok: true, + data: { + platform: "cloudflare-workers", + sample: "available", + replay: "available", + liveGpt: "configured", + liveCodex: "configured", + liveKernel: "configured", + readiness: "ready", + sandbox: "credential-and-privilege-boundary", + generationFilesystemReadIsolation: "PARTIAL", + requestId: "e2e-health-partial-isolation", + }, + }), + }); + }); + + await openLiveSetup(page); + await expect( + page.getByText(/generation filesystem read isolation is partial/i), + ).toBeVisible(); + await expect(page.locator('input[type="file"]')).toHaveCount(0); +}); + test("unsupported notebooks are parsed without execution and cannot advance", async ({ page, }) => { @@ -2285,7 +2522,11 @@ test("the judged path is keyboard operable with reduced motion", async ({ await expect( page.getByRole("heading", { name: /The fair test is ready/i }), ).toBeVisible(); - await page.getByRole("button", { name: /Run the fair test/i }).focus(); + await page + .getByRole("button", { + name: /Run the fair test|Reveal verified sample result/i, + }) + .focus(); await page.keyboard.press("Enter"); await expect( page.getByRole("heading", { name: /Compare the verified result/i }), @@ -2301,9 +2542,11 @@ test("the judged path is keyboard operable with reduced motion", async ({ page.getByText(/Interpretation recorded locally/i), ).toBeVisible(); - const comparison = page.getByRole("img", { - name: /Verified accuracy comparison.*familiar rows.*new customers/i, - }); + const comparison = page + .locator('[data-trusted-visual-id="verified_sample_belief_break_v1"]') + .getByRole("region", { + name: "Verified sample belief-break mechanism", + }); await expect(comparison).toBeVisible(); const observeTab = page.getByRole("tab", { name: /Observe/i }); await observeTab.focus(); @@ -2312,7 +2555,7 @@ test("the judged path is keyboard operable with reduced motion", async ({ await page.keyboard.press("ArrowRight"); await expect(page.getByRole("tab", { name: /Boundary/i })).toBeFocused(); await expect(page.getByRole("tabpanel")).toContainText( - /Verified sample boundary/i, + /Verified sample exploration/i, ); await page.getByRole("button", { name: /Reveal the map/i }).click(); await expect( @@ -2382,7 +2625,7 @@ test("a configured hosted runner completes an untouched leakage notebook", async liveCodex: "configured", liveKernel: "configured", sandbox: "credential-and-privilege-boundary", - generationFilesystemReadIsolation: "PARTIAL", + generationFilesystemReadIsolation: "OS_ENFORCED", release: { status: "bound" }, }); @@ -2391,10 +2634,8 @@ test("a configured hosted runner completes an untouched leakage notebook", async page.getByText(/Hosted notebook runner is ready/i), ).toBeVisible(); await page - .getByRole("button", { name: /Continue with my notebook/i }) - .click(); - - await page.locator('input[type="file"]').setInputFiles(leakageNotebookPath); + .getByLabel(/Attach a supported notebook/i) + .setInputFiles(leakageNotebookPath); await expect( page.getByText(/leakage-rows-pipeline.ipynb/i).first(), ).toBeVisible(); @@ -2621,7 +2862,7 @@ test("a configured hosted runner completes an untouched class-imbalance notebook liveCodex: "configured", liveKernel: "configured", sandbox: "credential-and-privilege-boundary", - generationFilesystemReadIsolation: "PARTIAL", + generationFilesystemReadIsolation: "OS_ENFORCED", release: { status: "bound" }, }); @@ -2633,10 +2874,8 @@ test("a configured hosted runner completes an untouched class-imbalance notebook page.getByText(/Hosted notebook runner is ready/i), ).toBeVisible(); await page - .getByRole("button", { name: /Continue with my notebook/i }) - .click(); - - await page.locator('input[type="file"]').setInputFiles(imbalanceNotebookPath); + .getByLabel(/Attach a supported notebook/i) + .setInputFiles(imbalanceNotebookPath); await expect( page.getByText(/fraud_class_imbalance.ipynb/i).first(), ).toBeVisible(); @@ -2700,10 +2939,10 @@ test("a configured hosted runner completes an untouched class-imbalance notebook "When positive cases are rare, compare against the majority baseline and choose class-specific metrics and a threshold that match deployment cost.", ); await page.getByRole("button", { name: /Try it on defects/i }).click(); - await page.getByLabel(/Lower threshold based on missed-defect cost/i).check(); - await page.getByLabel(/Missing a defect is the costly error/i).check(); - await page.getByLabel(/Confusion matrix shows misses/i).check(); - await page.getByLabel(/Prevalence changes precision/i).check(); + await page.getByLabel(/Reject the accuracy-only conclusion/i).check(); + await page.getByLabel(/Defect recall and PR-AUC/i).check(); + await page.getByLabel(/zero true positives/i).check(); + await page.getByLabel(/Defects are only 1%/i).check(); await page.getByRole("button", { name: /Check transfer/i }).click(); await expect(page.locator(".imbalance-transfer-pass .eyebrow")).toHaveText( "Transfer passed", diff --git a/apps/web/e2e/mobile-public-replay.spec.ts b/apps/web/e2e/mobile-public-replay.spec.ts index ef7581d8..ae0a6939 100644 --- a/apps/web/e2e/mobile-public-replay.spec.ts +++ b/apps/web/e2e/mobile-public-replay.spec.ts @@ -105,6 +105,54 @@ async function expectReadOnlyReplay(page: Page): Promise { name: /Continue replay|Show me what happened|Run fair test|Lock my answer|Check my answer|Verify notebook patch/i, }), ).toHaveCount(0); + + if (await legacyHeading.isVisible()) { + const tableRegion = page.getByRole("region", { + name: "Stored fixed-kernel comparison values", + }); + await expect(tableRegion).toBeVisible(); + const dimensions = await tableRegion.evaluate((element) => ({ + clientWidth: element.clientWidth, + scrollWidth: element.scrollWidth, + })); + expect(dimensions.scrollWidth).toBeGreaterThan(dimensions.clientWidth); + await tableRegion.evaluate((element) => { + element.scrollLeft = 0; + }); + await tableRegion.focus(); + await expect(tableRegion).toBeFocused(); + await tableRegion.press("ArrowRight"); + await expect + .poll(() => tableRegion.evaluate((element) => element.scrollLeft)) + .toBeGreaterThan(0); + const finalValue = page.getByRole("cell").last(); + await finalValue.scrollIntoViewIfNeeded(); + await expect(finalValue).toBeVisible(); + } +} + +async function enterReadOnlyReplay(page: Page): Promise { + const replayStatus = page.getByRole("complementary", { + name: "Replay status", + }); + const finalReplayStatus = page + .getByRole("complementary", { name: "Legacy replay status" }) + .or(page.getByRole("complementary", { name: "Verified replay mode" })); + await expect(replayStatus.or(finalReplayStatus)).toBeVisible({ + timeout: 30_000, + }); + + if (await replayStatus.isVisible()) { + await expect(replayStatus).toContainText( + "Verified replay · read-only stored evidence", + ); + await expect( + page.getByRole("heading", { name: /Replay verified session/i }), + ).toBeVisible(); + + await page.getByRole("button", { name: /Continue replay/i }).click(); + } + await expectReadOnlyReplay(page); } async function readRootOverflow(page: Page): Promise { @@ -194,7 +242,7 @@ test.describe("public mobile replay routing", () => { expect(response!.request().redirectedFrom()).toBeNull(); expectConfiguredPublicOrigin(response!.url()); await expect(page).toHaveURL(new RegExp(`${replayPath}$`, "u")); - await expectReadOnlyReplay(page); + await enterReadOnlyReplay(page); await page.waitForLoadState("networkidle"); await page.reload({ waitUntil: "domcontentloaded" }); @@ -249,10 +297,10 @@ test.describe("public mobile replay routing", () => { await expect(replayLink).toHaveAttribute("href", replayPath); await replayLink.click(); await expect(page).toHaveURL(new RegExp(`${replayPath}$`, "u")); - await expectReadOnlyReplay(page); + await enterReadOnlyReplay(page); await page.waitForLoadState("networkidle"); - await page.goBack({ waitUntil: "domcontentloaded" }); + await page.goBack({ waitUntil: "commit" }); await expect(page).toHaveURL(/\/judge$/u); await expect( page.getByRole("heading", { @@ -261,7 +309,7 @@ test.describe("public mobile replay routing", () => { ).toBeVisible(); await page.waitForLoadState("networkidle"); - await page.goForward({ waitUntil: "domcontentloaded" }); + await page.goForward({ waitUntil: "commit" }); await expect(page).toHaveURL(new RegExp(`${replayPath}$`, "u")); await expectReadOnlyReplay(page); const overflow = await readRootOverflow(page); diff --git a/apps/web/e2e/performance-evidence.ts b/apps/web/e2e/performance-evidence.ts index 99202653..7c0bca50 100644 --- a/apps/web/e2e/performance-evidence.ts +++ b/apps/web/e2e/performance-evidence.ts @@ -1,7 +1,13 @@ import type { Page } from "@playwright/test"; export const BROWSER_PERFORMANCE_EVIDENCE_SCHEMA = - "counterlab.browser-performance.v1" as const; + "counterlab.browser-performance.v2" as const; + +export type InteractionToNextPaintStatus = + | "measured" + | "awaiting-interaction" + | "unsupported" + | "observation-limit-exceeded"; export interface BrowserPerformanceEvidence { schemaVersion: typeof BROWSER_PERFORMANCE_EVIDENCE_SCHEMA; @@ -13,6 +19,7 @@ export interface BrowserPerformanceEvidence { longTasks: boolean; navigationTiming: boolean; resourceTiming: boolean; + interactionToNextPaint: boolean; }; largestContentfulPaintMs: number | null; cumulativeLayoutShift: number | null; @@ -27,6 +34,8 @@ export interface BrowserPerformanceEvidence { count: number; transferBytes: number; } | null; + interactionToNextPaintMs: number | null; + interactionToNextPaintStatus: InteractionToNextPaintStatus; } type ObservedMetric = keyof BrowserPerformanceEvidence["support"]; @@ -46,12 +55,14 @@ interface BrowserPerformanceState { navigationTtfbMs: number | null; resourceCount: number; resourceTransferBytes: number; + interactionDurations: Map; + interactionObservationLimitExceeded: boolean; observers: PerformanceObserver[]; drainObservers: Array<() => void>; } interface PerformanceEvidenceWindow extends Window { - __counterlabBrowserPerformanceV1?: BrowserPerformanceState; + __counterlabBrowserPerformanceV2?: BrowserPerformanceState; } interface LayoutShiftPerformanceEntry extends PerformanceEntry { @@ -59,6 +70,10 @@ interface LayoutShiftPerformanceEntry extends PerformanceEntry { value?: number; } +interface InteractionPerformanceEntry extends PerformanceEntry { + interactionId?: unknown; +} + /** * Runs inside the browser before application code. Keep this function * self-contained because Playwright serializes it into the target page. @@ -66,14 +81,20 @@ interface LayoutShiftPerformanceEntry extends PerformanceEntry { export function initializeBrowserPerformanceEvidence(): void { const browserWindow = window as PerformanceEvidenceWindow; if ( - browserWindow.__counterlabBrowserPerformanceV1?.schemaVersion === - "counterlab.browser-performance.v1" + browserWindow.__counterlabBrowserPerformanceV2?.schemaVersion === + "counterlab.browser-performance.v2" ) { return; } + // A release-qualification journey should never approach this ceiling. The + // bound prevents an unexpectedly long-lived page from retaining unbounded + // interaction identifiers; crossing it invalidates INP instead of returning + // a partial metric. + const maximumObservedInteractions = 10_000; + const state: BrowserPerformanceState = { - schemaVersion: "counterlab.browser-performance.v1", + schemaVersion: "counterlab.browser-performance.v2", support: { largestContentfulPaint: false, cumulativeLayoutShift: false, @@ -81,6 +102,7 @@ export function initializeBrowserPerformanceEvidence(): void { longTasks: false, navigationTiming: false, resourceTiming: false, + interactionToNextPaint: false, }, largestContentfulPaintMs: null, cumulativeLayoutShift: 0, @@ -94,10 +116,12 @@ export function initializeBrowserPerformanceEvidence(): void { navigationTtfbMs: null, resourceCount: 0, resourceTransferBytes: 0, + interactionDurations: new Map(), + interactionObservationLimitExceeded: false, observers: [], drainObservers: [], }; - browserWindow.__counterlabBrowserPerformanceV1 = state; + browserWindow.__counterlabBrowserPerformanceV2 = state; if (typeof PerformanceObserver === "undefined") return; @@ -108,13 +132,14 @@ export function initializeBrowserPerformanceEvidence(): void { entryType: string, metric: ObservedMetric, processEntries: (entries: PerformanceEntry[]) => void, + options: { durationThreshold?: number } = {}, ): void => { if (!PerformanceObserver.supportedEntryTypes.includes(entryType)) return; try { const observer = new PerformanceObserver((list) => { processEntries(list.getEntries()); }); - observer.observe({ type: entryType, buffered: true }); + observer.observe({ type: entryType, buffered: true, ...options }); state.support[metric] = true; state.observers.push(observer); state.drainObservers.push(() => { @@ -210,6 +235,57 @@ export function initializeBrowserPerformanceEvidence(): void { state.resourceTransferBytes += entry.transferSize; } }); + + const eventTimingConstructor = ( + globalThis as typeof globalThis & { + PerformanceEventTiming?: { prototype?: object }; + } + ).PerformanceEventTiming; + if ( + eventTimingConstructor?.prototype !== undefined && + "interactionId" in eventTimingConstructor.prototype + ) { + observe( + "event", + "interactionToNextPaint", + (entries) => { + for (const entry of entries as InteractionPerformanceEntry[]) { + const interactionId = entry.interactionId; + if ( + typeof interactionId !== "number" || + !Number.isSafeInteger(interactionId) || + interactionId <= 0 || + !finiteNonNegative(entry.duration) + ) { + continue; + } + + const normalizedInteractionId = interactionId; + const previousDuration = state.interactionDurations.get( + normalizedInteractionId, + ); + if (previousDuration !== undefined) { + state.interactionDurations.set( + normalizedInteractionId, + Math.max(previousDuration, entry.duration), + ); + continue; + } + if (state.interactionDurations.size >= maximumObservedInteractions) { + state.interactionObservationLimitExceeded = true; + continue; + } + state.interactionDurations.set( + normalizedInteractionId, + entry.duration, + ); + } + }, + // Event Timing's lowest useful threshold is one frame. Capturing from + // 16 ms gives the percentile estimator the broadest practical sample. + { durationThreshold: 16 }, + ); + } } /** Installs collection for future navigations and the currently loaded page. */ @@ -222,15 +298,39 @@ export async function installBrowserPerformanceEvidence( function snapshotBrowserPerformanceEvidenceInPage(): BrowserPerformanceEvidence { const browserWindow = window as PerformanceEvidenceWindow; - const state = browserWindow.__counterlabBrowserPerformanceV1; - if (state?.schemaVersion !== "counterlab.browser-performance.v1") { + const state = browserWindow.__counterlabBrowserPerformanceV2; + if (state?.schemaVersion !== "counterlab.browser-performance.v2") { throw new Error("Browser performance evidence collection is not installed"); } for (const drainObserver of state.drainObservers) drainObserver(); + let interactionToNextPaintMs: number | null = null; + let interactionToNextPaintStatus: InteractionToNextPaintStatus; + if (!state.support.interactionToNextPaint) { + interactionToNextPaintStatus = "unsupported"; + } else if (state.interactionObservationLimitExceeded) { + interactionToNextPaintStatus = "observation-limit-exceeded"; + } else if (state.interactionDurations.size === 0) { + interactionToNextPaintStatus = "awaiting-interaction"; + } else { + const longestFirst = [...state.interactionDurations.values()].sort( + (left, right) => right - left, + ); + // INP approximates the 98th percentile by ignoring one high outlier for + // every 50 interactions. Entries sharing an interactionId were already + // collapsed to that interaction's maximum duration. + const candidateIndex = Math.min( + longestFirst.length - 1, + Math.floor(longestFirst.length / 50), + ); + interactionToNextPaintMs = longestFirst[candidateIndex] ?? null; + interactionToNextPaintStatus = + interactionToNextPaintMs === null ? "awaiting-interaction" : "measured"; + } + return { - schemaVersion: "counterlab.browser-performance.v1", + schemaVersion: "counterlab.browser-performance.v2", // Never persist origin, credentials, query parameters, or fragments. pathname: window.location.pathname || "/", support: { ...state.support }, @@ -259,6 +359,8 @@ function snapshotBrowserPerformanceEvidenceInPage(): BrowserPerformanceEvidence transferBytes: state.resourceTransferBytes, } : null, + interactionToNextPaintMs, + interactionToNextPaintStatus, }; } diff --git a/apps/web/e2e/private-session.ts b/apps/web/e2e/private-session.ts new file mode 100644 index 00000000..5c452904 --- /dev/null +++ b/apps/web/e2e/private-session.ts @@ -0,0 +1,20 @@ +import type { Page } from "./cloak-test"; + +const ownerCapabilityPattern = /^cl_owner_[A-Za-z0-9_-]{43}$/u; + +export async function privateSessionHeaders( + page: Page, + sessionId: string, +): Promise<{ authorization: string }> { + const storageKey = `counterlab.ownerCapability.${encodeURIComponent(sessionId)}`; + const capability = await page.evaluate( + (key) => window.localStorage.getItem(key), + storageKey, + ); + if (capability === null || !ownerCapabilityPattern.test(capability)) { + throw new Error( + `Private session ${sessionId} has no valid browser-held owner capability`, + ); + } + return { authorization: `Bearer ${capability}` }; +} diff --git a/apps/web/e2e/qualification-reporter.ts b/apps/web/e2e/qualification-reporter.ts new file mode 100644 index 00000000..aff3303a --- /dev/null +++ b/apps/web/e2e/qualification-reporter.ts @@ -0,0 +1,604 @@ +import { + basename, + dirname, + isAbsolute, + relative, + resolve, + sep, +} from "node:path"; +import { mkdir, writeFile } from "node:fs/promises"; + +import type { + FullResult, + Reporter, + TestCase, + TestResult, + TestStep, +} from "@playwright/test/reporter"; +import { z } from "zod"; + +import { + ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS, + ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES, + CloakBrowserRawRunSchema, + PublicationReleaseBindingSchema, + REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID, + REQUIRED_CLOAK_EXPECTED_REQUEST_FAILURES_BY_ID, + REQUIRED_CLOAK_JOURNEY_VIEWPORT_BY_ID, + REQUIRED_CLOAK_JOURNEY_IDS, + REQUIRED_CLOAK_VIEWPORTS, +} from "../../../scripts/submission-publication-evidence"; +import { containedNewOutputFile } from "../../../scripts/repository-cli-paths"; + +export const JOURNEY_OBSERVATION_ATTACHMENT = + "counterlab-journey-observation" as const; + +const PageObservationSchema = z + .object({ + schemaVersion: z.literal("4"), + authority: z.enum(["CLOAK_CDP_ENDPOINT", "stock-chromium-design-review"]), + viewport: z + .object({ + width: z.number().int().positive().max(8_192), + height: z.number().int().positive().max(8_192), + }) + .strict(), + totalConsoleErrors: z.number().int().nonnegative(), + expectedHttpResourceConsoleErrors: z.number().int().nonnegative(), + unexpectedConsoleErrors: z.number().int().nonnegative(), + expectedHttpErrorResponses: z + .array(z.enum(ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS)) + .max(ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS.length), + observedHttpErrorResponses: z + .array(z.enum(ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS)) + .max(ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS.length), + unexpectedHttpErrorResponses: z.number().int().nonnegative(), + expectedHttpResponsesMatched: z.boolean(), + expectedRequestFailures: z + .array(z.enum(ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES)) + .max(ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES.length), + expectedFailedRequests: z.number().int().nonnegative(), + unexpectedFailedRequests: z.number().int().nonnegative(), + observedRequestFailures: z + .array(z.enum(ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES)) + .max(ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES.length), + observedFailedRequests: z.number().int().nonnegative(), + expectedRequestFailuresMatched: z.boolean(), + browserVersion: z.string().trim().min(1).max(128), + }) + .strict(); + +export type JourneyPageObservation = z.infer; + +export interface CapturedJourney { + id: string; + status: TestResult["status"]; + expectedStatus: TestCase["expectedStatus"]; + attempt: number; + durationMs: number; + assertionCount: number; + viewport: string; + totalConsoleErrors: number; + expectedHttpResourceConsoleErrors: number; + unexpectedConsoleErrors: number; + expectedHttpErrorResponses: readonly (typeof ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS)[number][]; + observedHttpErrorResponses: readonly (typeof ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS)[number][]; + unexpectedHttpErrorResponses: number; + expectedRequestFailures: readonly (typeof ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES)[number][]; + expectedFailedRequests: number; + unexpectedFailedRequests: number; + observedRequestFailures: readonly (typeof ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES)[number][]; + observedFailedRequests: number; + browserVersion: string; + browserAuthority: + "CLOAK_CDP_ENDPOINT" | "stock-chromium-design-review" | "unavailable"; + telemetryValid: boolean; +} + +type RawRunAuthority = "CLOAKBROWSER" | "STOCK_CHROMIUM_DESIGN_REVIEW"; + +interface BuildRawRunInput { + authority: RawRunAuthority; + baseUrl: string; + completedAt: string; + journeys: readonly CapturedJourney[]; + playwrightStatus: FullResult["status"]; + qualificationRequested: boolean; + release: z.infer | null; + rootErrors?: number; + startedAt: string; +} + +interface QualificationReporterOptions { + authority: RawRunAuthority; + baseUrl: string; + outputFile: string; + qualificationRequested: boolean; + releaseBinding: z.infer | null; + repositoryRoot: string; + runtimeRoot: string; +} + +const expectedProductionOrigin = + "https://counterlab.cserules.workers.dev" as const; +const allowedExpectedRequestFailures: ReadonlySet = new Set( + ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES, +); +const allowedExpectedHttpErrors: ReadonlySet = new Set( + ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS, +); + +export function summarizeRequestFailures( + observed: readonly string[], + expected: readonly string[], +): { + expectedCount: number; + matched: boolean; + observedCount: number; + observedRequestFailures: readonly (typeof ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES)[number][]; + unexpectedCount: number; +} { + const expectedValid = + expected.every((failure) => allowedExpectedRequestFailures.has(failure)) && + new Set(expected).size === expected.length; + const remaining = [...observed]; + for (const expectedFailure of expected) { + const index = remaining.indexOf(expectedFailure); + if (index >= 0) remaining.splice(index, 1); + } + return { + expectedCount: expected.length, + matched: + expectedValid && + observed.length === expected.length && + remaining.length === 0, + observedCount: observed.length, + observedRequestFailures: observed.filter( + ( + failure, + ): failure is (typeof ALLOWED_CLOAK_EXPECTED_REQUEST_FAILURES)[number] => + allowedExpectedRequestFailures.has(failure), + ), + unexpectedCount: remaining.length, + }; +} + +type AllowedExpectedHttpError = + (typeof ALLOWED_CLOAK_EXPECTED_HTTP_ERRORS)[number]; + +export interface ExpectedHttpErrorObservation { + signature: AllowedExpectedHttpError; + ephemeralCorrelationKey: string; +} + +function normalizedExpectedHttpPath(pathname: string): string { + if (pathname === "/api/artifacts") return pathname; + if (/^\/api\/sessions\/[^/]+\/lab\/compile$/u.test(pathname)) { + return "/api/sessions/:sessionId/lab/compile"; + } + return pathname; +} + +export function classifyExpectedHttpErrorObservation(input: { + expectedOrigin: string; + method: string; + status: number; + url: string; +}): ExpectedHttpErrorObservation | null { + let url: URL; + try { + url = new URL(input.url); + } catch { + return null; + } + if ( + url.origin !== input.expectedOrigin || + url.search !== "" || + url.hash !== "" + ) { + return null; + } + const signature = `${input.status} ${input.method.toUpperCase()} ${normalizedExpectedHttpPath(url.pathname)}`; + if (!allowedExpectedHttpErrors.has(signature)) return null; + return { + signature: signature as AllowedExpectedHttpError, + ephemeralCorrelationKey: `${input.status} ${input.method.toUpperCase()} ${url.origin}${url.pathname}`, + }; +} + +export function classifyExpectedHttpError(input: { + expectedOrigin: string; + method: string; + status: number; + url: string; +}): AllowedExpectedHttpError | null { + return classifyExpectedHttpErrorObservation(input)?.signature ?? null; +} + +export function classifyExpectedHttpResourceConsoleObservation(input: { + expectedOrigin: string; + locationUrl: string; + text: string; +}): ExpectedHttpErrorObservation | null { + const match = + /^Failed to load resource: the server responded with a status of (\d{3})(?:\s|$)/u.exec( + input.text, + ); + if (match?.[1] === undefined) return null; + const status = Number.parseInt(match[1], 10); + for (const method of ["POST"] as const) { + const classified = classifyExpectedHttpErrorObservation({ + expectedOrigin: input.expectedOrigin, + method, + status, + url: input.locationUrl, + }); + if (classified !== null) return classified; + } + return null; +} + +export function classifyExpectedHttpResourceConsoleError(input: { + expectedOrigin: string; + locationUrl: string; + text: string; +}): AllowedExpectedHttpError | null { + return ( + classifyExpectedHttpResourceConsoleObservation(input)?.signature ?? null + ); +} + +export function summarizeExpectedHttpErrors( + observed: readonly (AllowedExpectedHttpError | null)[], + expected: readonly string[], +): { + expectedCount: number; + matched: boolean; + observedCount: number; + observedHttpErrorResponses: readonly AllowedExpectedHttpError[]; + unexpectedCount: number; +} { + const expectedValid = + expected.every((error) => allowedExpectedHttpErrors.has(error)) && + new Set(expected).size === expected.length; + const remaining = [...observed]; + for (const expectedError of expected) { + const index = remaining.indexOf(expectedError as AllowedExpectedHttpError); + if (index >= 0) remaining.splice(index, 1); + } + return { + expectedCount: expected.length, + matched: + expectedValid && + observed.length === expected.length && + remaining.length === 0, + observedCount: observed.length, + observedHttpErrorResponses: observed.filter( + (error): error is AllowedExpectedHttpError => error !== null, + ), + unexpectedCount: remaining.length, + }; +} + +export function summarizeExpectedHttpResourceConsoleErrors( + observed: readonly (ExpectedHttpErrorObservation | null)[], + matchedHttpErrors: readonly ExpectedHttpErrorObservation[], +): { + expectedCount: number; + totalCount: number; + unexpectedCount: number; +} { + const available = [...matchedHttpErrors]; + let expectedCount = 0; + for (const error of observed) { + if (error === null) continue; + const index = available.findIndex( + (candidate) => + candidate.ephemeralCorrelationKey === error.ephemeralCorrelationKey && + candidate.signature === error.signature, + ); + if (index >= 0) { + available.splice(index, 1); + expectedCount += 1; + } + } + return { + expectedCount, + totalCount: observed.length, + unexpectedCount: observed.length - expectedCount, + }; +} + +export function journeyIdForTest( + test: Pick, +): string { + return journeyIdForParts(test.location.file, test.titlePath()); +} + +export function journeyIdForParts( + file: string, + titlePath: readonly string[], +): string { + const fileName = basename(file); + const fileStem = fileName.replace(/\.spec\.[cm]?[jt]sx?$/u, ""); + const titles = titlePath.filter( + (title) => title !== "" && basename(title) !== fileName, + ); + return [fileStem, ...titles].join("::"); +} + +export function countPlaywrightAssertions( + steps: readonly Pick[], +): number { + return steps.reduce( + (count, step) => + count + + (step.category === "fixture" || step.category === "hook" + ? 0 + : (step.category === "expect" ? 1 : 0) + + countPlaywrightAssertions(step.steps)), + 0, + ); +} + +export function readJourneyObservation( + result: Pick, +): JourneyPageObservation { + const observations = result.attachments.filter( + (attachment) => attachment.name === JOURNEY_OBSERVATION_ATTACHMENT, + ); + if (observations.length !== 1) { + throw new Error("Expected exactly one journey observation attachment"); + } + const observation = observations[0]!; + if ( + observation.contentType !== "application/json" || + observation.body === undefined || + observation.path !== undefined + ) { + throw new Error("Journey observation must be in-memory JSON"); + } + return PageObservationSchema.parse( + JSON.parse(observation.body.toString("utf8")) as unknown, + ); +} + +function sortedJourneys( + journeys: readonly CapturedJourney[], +): CapturedJourney[] { + const registryOrder = new Map( + REQUIRED_CLOAK_JOURNEY_IDS.map((id, index) => [id, index]), + ); + return [...journeys].sort((left, right) => { + const leftIndex = registryOrder.get(left.id) ?? Number.MAX_SAFE_INTEGER; + const rightIndex = registryOrder.get(right.id) ?? Number.MAX_SAFE_INTEGER; + return leftIndex - rightIndex || left.id.localeCompare(right.id); + }); +} + +function exactJourneyRegistry(journeys: readonly CapturedJourney[]): boolean { + const identifiers = journeys.map((journey) => journey.id); + return ( + identifiers.length === REQUIRED_CLOAK_JOURNEY_IDS.length && + new Set(identifiers).size === identifiers.length && + JSON.stringify([...identifiers].sort()) === + JSON.stringify([...REQUIRED_CLOAK_JOURNEY_IDS].sort()) + ); +} + +function exactViewportCoverage(journeys: readonly CapturedJourney[]): boolean { + const observed = new Set(journeys.map((journey) => journey.viewport)); + return ( + REQUIRED_CLOAK_VIEWPORTS.every((viewport) => observed.has(viewport)) && + journeys.every( + (journey) => + REQUIRED_CLOAK_JOURNEY_VIEWPORT_BY_ID[ + journey.id as keyof typeof REQUIRED_CLOAK_JOURNEY_VIEWPORT_BY_ID + ] === journey.viewport, + ) + ); +} + +export function buildCloakBrowserRawRun(input: BuildRawRunInput) { + const expectedJourneyAuthority = + input.authority === "CLOAKBROWSER" + ? "CLOAK_CDP_ENDPOINT" + : "stock-chromium-design-review"; + const exactCleanRun = + input.qualificationRequested && + input.authority === "CLOAKBROWSER" && + input.baseUrl === expectedProductionOrigin && + input.release !== null && + input.release.productionOrigin === input.baseUrl && + input.playwrightStatus === "passed" && + (input.rootErrors ?? 0) === 0 && + exactJourneyRegistry(input.journeys) && + exactViewportCoverage(input.journeys) && + input.journeys.every( + (journey) => + journey.status === "passed" && + journey.expectedStatus === "passed" && + journey.attempt === 0 && + journey.durationMs > 0 && + journey.assertionCount > 0 && + journey.unexpectedConsoleErrors === 0 && + JSON.stringify(journey.expectedHttpErrorResponses) === + JSON.stringify( + REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID[ + journey.id as keyof typeof REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID + ], + ) && + JSON.stringify(journey.observedHttpErrorResponses) === + JSON.stringify(journey.expectedHttpErrorResponses) && + journey.unexpectedHttpErrorResponses === 0 && + JSON.stringify(journey.expectedRequestFailures) === + JSON.stringify( + REQUIRED_CLOAK_EXPECTED_REQUEST_FAILURES_BY_ID[ + journey.id as keyof typeof REQUIRED_CLOAK_EXPECTED_REQUEST_FAILURES_BY_ID + ], + ) && + journey.expectedRequestFailures.length === + journey.expectedFailedRequests && + JSON.stringify(journey.observedRequestFailures) === + JSON.stringify(journey.expectedRequestFailures) && + journey.observedFailedRequests === journey.expectedFailedRequests && + journey.unexpectedFailedRequests === 0 && + journey.browserVersion.trim() !== "" && + journey.browserVersion !== "unavailable" && + journey.browserAuthority === expectedJourneyAuthority && + journey.telemetryValid, + ) && + new Set(input.journeys.map((journey) => journey.browserVersion)).size === 1; + + return CloakBrowserRawRunSchema.parse({ + schemaVersion: "4", + kind: "cloakbrowser-raw-run", + status: input.qualificationRequested + ? exactCleanRun + ? "PASSED" + : "FAILED" + : "NON_QUALIFYING", + authority: input.authority, + qualificationRequested: input.qualificationRequested, + baseUrl: input.baseUrl, + release: input.release, + startedAt: input.startedAt, + completedAt: input.completedAt, + playwrightVersion: "1.61.1", + playwrightStatus: input.playwrightStatus, + rootErrors: input.rootErrors ?? 0, + journeys: sortedJourneys(input.journeys), + privacy: { + containsSecrets: false, + containsRawNotebook: false, + containsPersonalData: false, + }, + }); +} + +function assertReporterOutput(outputFile: string, runtimeRoot: string): string { + if ( + runtimeRoot.trim() === "" || + !isAbsolute(runtimeRoot) || + !isAbsolute(outputFile) + ) { + throw new Error( + "Qualification reporter paths require an absolute repository runtime root", + ); + } + const resolvedRoot = resolve(runtimeRoot); + const resolvedOutput = resolve(outputFile); + const pathFromRoot = relative(resolvedRoot, resolvedOutput); + if ( + pathFromRoot === "" || + pathFromRoot === ".." || + pathFromRoot.startsWith(`..${sep}`) || + isAbsolute(pathFromRoot) + ) { + throw new Error( + "Qualification reporter output must stay beneath the E2E runtime root", + ); + } + return resolvedOutput; +} + +export default class CounterLabQualificationReporter implements Reporter { + private readonly capturedJourneys: CapturedJourney[] = []; + private readonly startedAt = new Date().toISOString(); + private rootErrors = 0; + + constructor(private readonly options: QualificationReporterOptions) {} + + printsToStdio(): boolean { + return false; + } + + onError(): void { + this.rootErrors += 1; + } + + onTestEnd(test: TestCase, result: TestResult): void { + let observation: JourneyPageObservation | null = null; + try { + observation = readJourneyObservation(result); + } catch { + observation = null; + } + this.capturedJourneys.push({ + id: journeyIdForTest(test), + status: result.status, + expectedStatus: test.expectedStatus, + attempt: result.retry, + durationMs: Math.max(0, Math.round(result.duration)), + assertionCount: countPlaywrightAssertions(result.steps), + viewport: + observation === null + ? "0x0" + : `${observation.viewport.width}x${observation.viewport.height}`, + totalConsoleErrors: observation?.totalConsoleErrors ?? 0, + expectedHttpResourceConsoleErrors: + observation?.expectedHttpResourceConsoleErrors ?? 0, + unexpectedConsoleErrors: observation?.unexpectedConsoleErrors ?? 0, + expectedHttpErrorResponses: observation?.expectedHttpErrorResponses ?? [], + observedHttpErrorResponses: observation?.observedHttpErrorResponses ?? [], + unexpectedHttpErrorResponses: + observation?.unexpectedHttpErrorResponses ?? 0, + expectedRequestFailures: observation?.expectedRequestFailures ?? [], + expectedFailedRequests: observation?.expectedFailedRequests ?? 0, + unexpectedFailedRequests: observation?.unexpectedFailedRequests ?? 0, + observedRequestFailures: observation?.observedRequestFailures ?? [], + observedFailedRequests: observation?.observedFailedRequests ?? 0, + browserVersion: observation?.browserVersion ?? "unavailable", + browserAuthority: observation?.authority ?? "unavailable", + telemetryValid: + observation !== null && + observation.expectedHttpResponsesMatched && + observation.expectedRequestFailuresMatched, + }); + } + + async onEnd( + result: FullResult, + ): Promise<{ status?: FullResult["status"] } | undefined> { + if ( + this.capturedJourneys.length === 0 && + !this.options.qualificationRequested + ) { + return undefined; + } + try { + const report = buildCloakBrowserRawRun({ + authority: this.options.authority, + baseUrl: this.options.baseUrl, + completedAt: new Date().toISOString(), + journeys: this.capturedJourneys, + playwrightStatus: result.status, + qualificationRequested: this.options.qualificationRequested, + release: this.options.releaseBinding, + rootErrors: this.rootErrors, + startedAt: this.startedAt, + }); + const outputFile = assertReporterOutput( + this.options.outputFile, + this.options.runtimeRoot, + ); + await mkdir(dirname(outputFile), { recursive: true }); + await containedNewOutputFile( + this.options.repositoryRoot, + outputFile, + "CloakBrowser raw run evidence", + ); + await writeFile(outputFile, `${JSON.stringify(report, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + if (this.options.qualificationRequested && report.status !== "PASSED") { + return { status: "failed" }; + } + return undefined; + } catch { + return { status: "failed" }; + } + } +} diff --git a/apps/web/e2e/recovery-and-intake.spec.ts b/apps/web/e2e/recovery-and-intake.spec.ts new file mode 100644 index 00000000..ae6d7a9c --- /dev/null +++ b/apps/web/e2e/recovery-and-intake.spec.ts @@ -0,0 +1,486 @@ +import { expect, test, type Page } from "./cloak-test"; +import { privateSessionHeaders } from "./private-session"; +import { SAMPLE_LEAKAGE_QUESTION } from "../shared/sample-authority"; + +const liveQuestion = + "Does this notebook score show that the model generalizes to customers it has never seen?"; +const supportedNotebookPath = new URL( + "../../../evals/held-out/notebooks/leakage-rows-pipeline.ipynb", + import.meta.url, +).pathname; +const supportedNotebookName = "leakage-rows-pipeline.ipynb"; +const exactLiveJourney = process.env.COUNTERLAB_E2E_LIVE === "1"; + +const readyHealth = { + ok: true, + data: { + platform: "cloudflare-workers", + sample: "available", + replay: "available", + liveGpt: "configured", + liveCodex: "configured", + liveKernel: "configured", + readiness: "ready", + sandbox: "credential-and-privilege-boundary", + generationFilesystemReadIsolation: "OS_ENFORCED", + release: { + status: "bound", + workerVersionId: "11111111-2222-3333-4444-555555555555", + workerVersionTag: `git-${"a".repeat(40)}`, + workerEvidenceCommit: "a".repeat(40), + runnerSourceCommit: "b".repeat(40), + runnerImageDigest: `sha256:${"c".repeat(64)}`, + generationIsolationEvidenceSha256: "5".repeat(64), + generationIsolationProbeSha256: "6".repeat(64), + releaseCheckGenerationIsolationEvidenceSha256: "7".repeat(64), + releaseCheckGenerationIsolationProbeSha256: "6".repeat(64), + releaseCheckGenerationIsolationVerifiedAt: + "2026-07-19T05:31:00.000+05:30", + timeoutCleanupReceiptSha256: "d".repeat(64), + aggregateLimitEvidenceSha256: "9".repeat(64), + runtimePolicySha256: "e".repeat(64), + proofDependencyManifestSha256: "f".repeat(64), + workerArtifactClassification: "PROCESS_BOUND_PARTIAL", + workerArtifactManifestSha256: "1".repeat(64), + workerBundleSha256: "2".repeat(64), + clientAssetsSha256: "3".repeat(64), + clientAssetCount: 27, + clientPublicAssetsSha256: "4".repeat(64), + clientPublicAssetCount: 25, + viteVersion: "8.1.4", + wranglerVersion: "4.110.0", + }, + requestId: "e2e-ready-intake", + }, +} as const; + +async function reset(page: Page): Promise { + await page.goto("/", { waitUntil: "domcontentloaded" }); + await page.evaluate(() => { + window.localStorage.clear(); + window.sessionStorage.clear(); + }); + await page.reload({ waitUntil: "domcontentloaded" }); + await expect( + page.getByRole("heading", { + name: "What result are you trying to understand?", + }), + ).toBeVisible(); +} + +async function storedSessionId(page: Page): Promise { + return page.evaluate(() => + window.localStorage.getItem("counterlab.sessionId"), + ); +} + +async function openSampleModelDuel(page: Page): Promise { + await reset(page); + await page.getByRole("button", { name: /Try verified sample/i }).click(); + await expect( + page.getByRole("heading", { name: /What do you think the score means/i }), + ).toBeVisible(); + await page.getByRole("button", { name: /Compare two explanations/i }).click(); + await expect( + page.getByRole("heading", { + name: /Does your current explanation capture what you mean/i, + }), + ).toBeVisible(); +} + +async function installReadyHealth(page: Page): Promise { + await page.route("**/api/health*", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(readyHealth), + }); + }); +} + +async function openReadyLiveIntake( + page: Page, + options: { requireExactLiveRelease?: boolean } = {}, +): Promise { + if (options.requireExactLiveRelease !== true) { + await installReadyHealth(page); + } + await reset(page); + await page.getByLabel("Your question or claim").fill(liveQuestion); + await page.getByRole("button", { name: /Test this claim/i }).click(); + await expect( + page.getByRole("heading", { + name: /Start with evidence that matches your question/i, + }), + ).toBeVisible(); + await page + .getByRole("button", { name: /Check live notebook tools/i }) + .click(); + await expect( + page.getByRole("heading", { name: "Test my notebook" }), + ).toBeVisible(); + await expect( + page.getByText("Source-bound hosted notebook runner is ready"), + ).toBeVisible(); + await expect(page.getByLabel(/Attach a supported notebook/i)).toBeEnabled(); +} + +function isPostTo(requestUrl: string, method: string, suffix: string): boolean { + return method === "POST" && new URL(requestUrl).pathname.endsWith(suffix); +} + +for (const response of [ + { + action: "Reject", + terminalState: "REJECTED_BY_LEARNER", + }, + { + action: "Not enough evidence", + terminalState: "INSUFFICIENT_EVIDENCE", + }, +] as const) { + test(`${response.action} preserves the claim and resubmits only in a fresh session`, async ({ + page, + }) => { + const proposalSessionIds: string[] = []; + page.on("request", (request) => { + if (!isPostTo(request.url(), request.method(), "/belief-test")) return; + const segments = new URL(request.url()).pathname.split("/"); + const encodedSessionId = segments.at(-2); + if (encodedSessionId !== undefined) { + proposalSessionIds.push(decodeURIComponent(encodedSessionId)); + } + }); + + await openSampleModelDuel(page); + const closedSessionId = await storedSessionId(page); + expect(closedSessionId).toMatch(/^session_/u); + await expect(page).toHaveURL( + new RegExp(`/session/${encodeURIComponent(closedSessionId!)}$`, "u"), + ); + const preservedClaim = ( + await page.locator("blockquote").first().innerText() + ).trim(); + expect(preservedClaim).toBe(SAMPLE_LEAKAGE_QUESTION); + + await page.getByText("More ways to respond", { exact: true }).click(); + await page + .getByRole("button", { name: response.action, exact: true }) + .click(); + await expect( + page.getByText("Your response closed that explanation."), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: /Compare two explanations/i }), + ).toBeDisabled(); + await expect(page.locator("body")).not.toContainText( + /REJECTED_BY_LEARNER|INSUFFICIENT_EVIDENCE/u, + ); + const terminalResponse = await page.request.get( + `/api/sessions/${encodeURIComponent(closedSessionId!)}`, + { + headers: await privateSessionHeaders(page, closedSessionId!), + }, + ); + expect(terminalResponse.ok()).toBe(true); + expect((await terminalResponse.json()).data.state).toBe( + response.terminalState, + ); + + await page.reload({ waitUntil: "domcontentloaded" }); + await expect( + page.getByText("Your response closed that explanation."), + ).toBeVisible(); + await expect( + page.getByText(SAMPLE_LEAKAGE_QUESTION, { exact: true }), + ).toBeVisible(); + expect(await storedSessionId(page)).toBe(closedSessionId); + + await page + .getByRole("button", { name: /Revise in a new investigation/i }) + .click(); + await expect + .poll(() => storedSessionId(page), { + message: "the replacement investigation should install a fresh session", + }) + .not.toBe(closedSessionId); + await expect( + page.getByRole("heading", { name: /What do you think the score means/i }), + ).toBeVisible(); + const freshSessionId = await storedSessionId(page); + expect(freshSessionId).toMatch(/^session_/u); + expect(freshSessionId).not.toBe(closedSessionId); + await expect(page).toHaveURL( + new RegExp(`/session/${encodeURIComponent(freshSessionId!)}$`, "u"), + ); + expect( + await page.evaluate(() => + window.localStorage.getItem("counterlab.claim"), + ), + ).toBe(preservedClaim); + await expect( + page.getByRole("button", { name: /Compare two explanations/i }), + ).toBeEnabled(); + + await page + .getByRole("button", { name: /Compare two explanations/i }) + .click(); + await expect( + page.getByRole("heading", { + name: /Does your current explanation capture what you mean/i, + }), + ).toBeVisible(); + expect(proposalSessionIds).toEqual([closedSessionId, freshSessionId]); + + await page.goBack({ waitUntil: "domcontentloaded" }); + await expect(page).toHaveURL( + new RegExp(`/session/${encodeURIComponent(closedSessionId!)}$`, "u"), + ); + await expect( + page.getByText("Your response closed that explanation."), + ).toBeVisible(); + await expect( + page.getByText(SAMPLE_LEAKAGE_QUESTION, { exact: true }), + ).toBeVisible(); + expect(await storedSessionId(page)).toBe(closedSessionId); + + await page.goForward({ waitUntil: "domcontentloaded" }); + await expect(page).toHaveURL( + new RegExp(`/session/${encodeURIComponent(freshSessionId!)}$`, "u"), + ); + await expect( + page.getByRole("heading", { + name: /Does your current explanation capture what you mean/i, + }), + ).toBeVisible(); + expect(await storedSessionId(page)).toBe(freshSessionId); + await page + .getByRole("button", { name: /Yes, this captures my view/i }) + .click(); + await expect( + page.getByRole("heading", { + name: /Seal what you expect before the result appears/i, + }), + ).toBeVisible(); + expect(proposalSessionIds).toEqual([closedSessionId, freshSessionId]); + }); +} + +test("live intake cannot continue without an uploaded supported artifact", async ({ + page, +}) => { + await openReadyLiveIntake(page); + + expect(await storedSessionId(page)).toBeNull(); + await expect(page.getByText(/Notebook intake passed/i)).toHaveCount(0); + await expect( + page.getByRole("button", { name: /Compare two explanations/i }), + ).toHaveCount(0); + await expect(page.locator("body")).not.toContainText(/Preparing artifact/i); +}); + +test("malformed live upload remains at intake and creates no session", async ({ + page, +}, testInfo) => { + testInfo.annotations.push({ + type: "counterlab-expected-http-errors", + description: "422 POST /api/artifacts", + }); + let sessionCreationRequests = 0; + page.on("request", (request) => { + if (isPostTo(request.url(), request.method(), "/api/live/sessions")) { + sessionCreationRequests += 1; + } + }); + await openReadyLiveIntake(page); + + await page.getByLabel(/Attach a supported notebook/i).setInputFiles({ + name: "broken.ipynb", + mimeType: "application/x-ipynb+json", + buffer: Buffer.from("not-json"), + }); + + await expect(page.getByRole("alert")).toContainText(/not valid JSON/i); + await expect( + page.getByRole("heading", { name: "Test my notebook" }), + ).toBeVisible(); + await expect(page.getByLabel(/Attach a supported notebook/i)).toBeEnabled(); + expect(sessionCreationRequests).toBe(0); + expect(await storedSessionId(page)).toBeNull(); +}); + +test("unsupported live upload is refused without creating a session", async ({ + page, +}) => { + let sessionCreationRequests = 0; + page.on("request", (request) => { + if (isPostTo(request.url(), request.method(), "/api/live/sessions")) { + sessionCreationRequests += 1; + } + }); + await openReadyLiveIntake(page); + const unsupportedNotebook = { + cells: [ + { + cell_type: "code", + execution_count: null, + metadata: {}, + outputs: [], + source: ["%load_ext custom_extension\n", "import requests\n"], + }, + ], + metadata: {}, + nbformat: 4, + nbformat_minor: 5, + }; + + await page.getByLabel(/Attach a supported notebook/i).setInputFiles({ + name: "unsupported.ipynb", + mimeType: "application/x-ipynb+json", + buffer: Buffer.from(JSON.stringify(unsupportedNotebook)), + }); + + await expect(page.getByText(/UNSUPPORTED_MAGIC/i)).toBeVisible(); + await expect(page.getByText(/EXTERNAL_NETWORK_DEPENDENCY/i)).toBeVisible(); + await expect( + page.getByRole("button", { name: /Compare two explanations/i }), + ).toBeDisabled(); + expect(sessionCreationRequests).toBe(0); + expect(await storedSessionId(page)).toBeNull(); +}); + +test("a supported live upload creates one source-bound session", async ({ + page, +}) => { + test.skip( + !exactLiveJourney, + "Set COUNTERLAB_E2E_LIVE=1 only against an exact release-ready live API.", + ); + let uploadRequests = 0; + let sessionCreationRequests = 0; + const duplicateKeyWarnings: string[] = []; + page.on("console", (message) => { + if ( + message.type() === "error" && + /same key|keys should be unique/i.test(message.text()) + ) { + duplicateKeyWarnings.push(message.text()); + } + }); + page.on("request", (request) => { + if (isPostTo(request.url(), request.method(), "/api/artifacts")) { + uploadRequests += 1; + } + if (isPostTo(request.url(), request.method(), "/api/live/sessions")) { + sessionCreationRequests += 1; + } + }); + await openReadyLiveIntake(page, { requireExactLiveRelease: true }); + + await page + .getByLabel(/Attach a supported notebook/i) + .setInputFiles(supportedNotebookPath); + + await expect( + page.getByRole("heading", { name: /What do you think the score means/i }), + ).toBeVisible(); + await expect(page.getByText(supportedNotebookName)).toBeVisible(); + await expect( + page.getByRole("button", { name: /Compare two explanations/i }), + ).toBeEnabled(); + expect(await storedSessionId(page)).toMatch(/^session_/u); + expect(uploadRequests).toBe(1); + expect(sessionCreationRequests).toBe(1); + expect(duplicateKeyWarnings).toEqual([]); +}); + +test("an interrupted upload accepts the same file on retry", async ({ + page, +}, testInfo) => { + test.skip( + !exactLiveJourney, + "Set COUNTERLAB_E2E_LIVE=1 only against an exact release-ready live API.", + ); + testInfo.annotations.push({ + type: "counterlab-expected-request-failures", + description: "POST /api/artifacts", + }); + let uploadAttempts = 0; + await page.route("**/api/artifacts", async (route) => { + if (route.request().method() !== "POST") { + await route.continue(); + return; + } + uploadAttempts += 1; + if (uploadAttempts === 1) { + await route.abort("connectionreset"); + return; + } + await route.continue(); + }); + await openReadyLiveIntake(page, { requireExactLiveRelease: true }); + const upload = page.getByLabel(/Attach a supported notebook/i); + + await upload.setInputFiles(supportedNotebookPath); + await expect(page.getByRole("alert")).toContainText( + /could not reach the API/i, + ); + await expect(upload).toBeEnabled(); + await upload.setInputFiles(supportedNotebookPath); + + await expect( + page.getByRole("heading", { name: /What do you think the score means/i }), + ).toBeVisible(); + expect(uploadAttempts).toBe(2); + expect(await storedSessionId(page)).toMatch(/^session_/u); +}); + +test("a lost private-session response retries without re-uploading", async ({ + page, +}, testInfo) => { + test.skip( + !exactLiveJourney, + "Set COUNTERLAB_E2E_LIVE=1 only against an exact release-ready live API.", + ); + testInfo.annotations.push({ + type: "counterlab-expected-request-failures", + description: "POST /api/live/sessions", + }); + let uploadRequests = 0; + let sessionCreationAttempts = 0; + page.on("request", (request) => { + if (isPostTo(request.url(), request.method(), "/api/artifacts")) { + uploadRequests += 1; + } + }); + await page.route("**/api/live/sessions", async (route) => { + sessionCreationAttempts += 1; + if (sessionCreationAttempts === 1) { + await route.abort("connectionreset"); + return; + } + await route.continue(); + }); + await openReadyLiveIntake(page, { requireExactLiveRelease: true }); + + await page + .getByLabel(/Attach a supported notebook/i) + .setInputFiles(supportedNotebookPath); + await expect(page.getByRole("alert")).toContainText( + /could not reach the API/i, + ); + await expect( + page.getByRole("button", { name: /Retry private session setup/i }), + ).toBeEnabled(); + await expect(page.getByText(supportedNotebookName)).toBeVisible(); + + await page + .getByRole("button", { name: /Retry private session setup/i }) + .click(); + await expect( + page.getByRole("heading", { name: /What do you think the score means/i }), + ).toBeVisible(); + expect(uploadRequests).toBe(1); + expect(sessionCreationAttempts).toBe(2); + expect(await storedSessionId(page)).toMatch(/^session_/u); +}); diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts index 6bf51bb8..2b62cfb4 100644 --- a/apps/web/playwright.config.ts +++ b/apps/web/playwright.config.ts @@ -1,10 +1,23 @@ import { defineConfig } from "@playwright/test"; -import { existsSync, lstatSync, realpathSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + existsSync, + fstatSync, + lstatSync, + openSync, + readFileSync, + realpathSync, +} from "node:fs"; import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { resolveBrowserAuthority } from "./e2e/browser-authority"; +import { DeploymentReceiptV7Schema } from "../../packages/scientific-engine-registry/src/index.js"; +import { PublicationReleaseBindingSchema } from "../../scripts/submission-publication-evidence.js"; const repositoryRoot = realpathSync(resolve(import.meta.dirname, "../..")); +const maximumDeploymentReceiptBytes = 5 * 1_024 * 1_024; const runtimeParent = resolve(import.meta.dirname, "test-results/runtime"); const configuredRuntimeRoot = process.env.COUNTERLAB_E2E_RUNTIME_ROOT; if (configuredRuntimeRoot !== undefined && !isAbsolute(configuredRuntimeRoot)) { @@ -49,8 +62,13 @@ assertNoSymlinkTraversal(runtimeRoot, "COUNTERLAB_E2E_RUNTIME_ROOT"); const outputDir = join(runtimeRoot, "playwright-output"); const resultsFile = join(runtimeRoot, "evidence/results.json"); +const qualificationRunFile = join( + runtimeRoot, + "evidence/cloakbrowser-raw-run.json", +); assertNoSymlinkTraversal(outputDir, "Playwright output directory"); assertNoSymlinkTraversal(resultsFile, "Playwright results file"); +assertNoSymlinkTraversal(qualificationRunFile, "CloakBrowser raw run evidence"); const browserAuthority = resolveBrowserAuthority(process.env); @@ -90,6 +108,90 @@ const baseURL = remoteBaseURL ?? `http://127.0.0.1:${port}`; const staticDesignReview = browserAuthority.kind === "stock-chromium-design-review" && process.env.COUNTERLAB_E2E_STATIC_CLIENT === "true"; +const localAdmissionKey = + process.env.COUNTERLAB_ADMISSION_KEY?.trim() || + ["counterlab", "local", "e2e", "admission", "only", "000000"].join("-"); +const qualificationSetting = process.env.COUNTERLAB_BROWSER_QUALIFICATION; +if ( + qualificationSetting !== undefined && + qualificationSetting !== "true" && + qualificationSetting !== "false" +) { + throw new Error( + "COUNTERLAB_BROWSER_QUALIFICATION must be exactly true or false", + ); +} +const qualificationRequested = qualificationSetting === "true"; +if ( + qualificationRequested && + (browserAuthority.kind !== "cloak" || + remoteBaseURL !== "https://counterlab.cserules.workers.dev") +) { + throw new Error( + "CloakBrowser qualification requires the exact public CounterLab origin", + ); +} +const deploymentReceiptSetting = process.env.COUNTERLAB_E2E_DEPLOYMENT_RECEIPT; +let qualificationReleaseBinding: ReturnType< + typeof PublicationReleaseBindingSchema.parse +> | null = null; +if (qualificationRequested) { + if ( + deploymentReceiptSetting === undefined || + deploymentReceiptSetting.trim() === "" + ) { + throw new Error( + "CloakBrowser qualification requires COUNTERLAB_E2E_DEPLOYMENT_RECEIPT", + ); + } + const requestedReceipt = resolve(repositoryRoot, deploymentReceiptSetting); + if (!isContained(repositoryRoot, requestedReceipt, false)) { + throw new Error("CloakBrowser deployment receipt escaped the repository"); + } + assertNoSymlinkTraversal(requestedReceipt, "CloakBrowser deployment receipt"); + const physicalReceipt = realpathSync(requestedReceipt); + if ( + !isContained(repositoryRoot, physicalReceipt, false) || + !lstatSync(physicalReceipt).isFile() + ) { + throw new Error( + "CloakBrowser deployment receipt must be a physical repository file", + ); + } + const receiptDescriptor = openSync( + physicalReceipt, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + let deploymentBytes: Buffer; + try { + const metadata = fstatSync(receiptDescriptor); + if (!metadata.isFile() || metadata.size > maximumDeploymentReceiptBytes) { + throw new Error( + "CloakBrowser deployment receipt is not a bounded regular file", + ); + } + deploymentBytes = readFileSync(receiptDescriptor); + } finally { + closeSync(receiptDescriptor); + } + const deployment = DeploymentReceiptV7Schema.parse( + JSON.parse(deploymentBytes.toString("utf8")) as unknown, + ); + qualificationReleaseBinding = PublicationReleaseBindingSchema.parse({ + deploymentReceiptSha256: createHash("sha256") + .update(deploymentBytes) + .digest("hex"), + productionOrigin: deployment.productionOrigin, + workerEvidenceCommit: deployment.workerEvidenceCommit, + runnerSourceCommit: deployment.runnerSourceCommit, + containerImageDigest: deployment.containerImageDigest, + workerVersionId: deployment.workerVersionId, + }); +} else if (deploymentReceiptSetting !== undefined) { + throw new Error( + "COUNTERLAB_E2E_DEPLOYMENT_RECEIPT is allowed only for qualification", + ); +} export default defineConfig({ testDir: "./e2e", @@ -98,7 +200,25 @@ export default defineConfig({ forbidOnly: true, retries: process.env.CI ? 1 : 0, outputDir, - reporter: [["line"], ["json", { outputFile: resultsFile }]], + reporter: [ + ["line"], + ["json", { outputFile: resultsFile }], + [ + "./e2e/qualification-reporter.ts", + { + authority: + browserAuthority.kind === "cloak" + ? "CLOAKBROWSER" + : "STOCK_CHROMIUM_DESIGN_REVIEW", + baseUrl: baseURL, + outputFile: qualificationRunFile, + qualificationRequested, + releaseBinding: qualificationReleaseBinding, + repositoryRoot, + runtimeRoot, + }, + ], + ], use: { baseURL, browserName: "chromium", @@ -129,6 +249,13 @@ export default defineConfig({ ? "node ../../scripts/serve-built-client.mjs" : `./node_modules/.bin/vite --host 127.0.0.1 --port ${port}`, cwd: import.meta.dirname, + ...(staticDesignReview + ? {} + : { + env: { + COUNTERLAB_ADMISSION_KEY: localAdmissionKey, + }, + }), url: baseURL, reuseExistingServer: false, timeout: 120_000, diff --git a/apps/web/src/App.test.tsx b/apps/web/src/App.test.tsx index b54ed4a1..51e10e71 100644 --- a/apps/web/src/App.test.tsx +++ b/apps/web/src/App.test.tsx @@ -3,6 +3,7 @@ import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + ArtifactManifestSchema, migrateBeliefTestV1ToV2, VerifiedResultSetSchema, type EvidenceEvent, @@ -11,7 +12,7 @@ import { import imbalanceResultText from "../../../fixtures/held-out/imbalance_epistemic_competing_v2.json?raw"; import { SAMPLE_LEAKAGE_QUESTION } from "../shared/sample-authority"; -import { App } from "./App"; +import { App, notebookEvidenceReferences } from "./App"; import { SessionViewSchema } from "./api"; import { publicReplayFixture, @@ -21,6 +22,9 @@ import { activeRunnerRegistryKey } from "./features/learner/activeRunnerRegistry import { requireBundledSampleResult, sampleArtifact } from "./sample"; const sampleResult = requireBundledSampleResult(); +const PRE_PREDICTION_RESULT_LANGUAGE = + /59\.4%|\bdeceptive\b|\bfairer test\b|\bverified result\b|\bevidence verdict\b|\bsupported hypothesis\b|\bthe fix\b|\bremove customer(?:_| )id\b|\bkeep each customer's rows together\b|\bproves?\b/i; +const LANDING_FIXED_SAMPLE_VALUES = /98\.5%|59\.4%/i; const verifiedImbalanceResult = VerifiedResultSetSchema.parse( JSON.parse(imbalanceResultText), @@ -165,6 +169,7 @@ const livePreview = { }, ], }, + learningDirectorPacket: {}, }; const sensitiveLivePreview = { @@ -203,6 +208,41 @@ const liveResult = { })), }; +const epistemicReportHash = "9".repeat(64); +const evidenceVerdictBase = { + schemaVersion: "1" as const, + irHash: "8".repeat(64), + technicalReportHash: "7".repeat(64), + verifierVersion: "epistemic-verifier-v1", +}; +const supportingEvidenceVerdict = { + ...evidenceVerdictBase, + kind: "SUPPORTS" as const, + hypothesisId: "competing" as const, + scope: "unseen customers in this supplied notebook", + resultHash: liveResult.resultHash, +}; +const supportingImbalanceEvidenceVerdict = { + ...evidenceVerdictBase, + kind: "SUPPORTS" as const, + hypothesisId: "competing" as const, + scope: "rare events in this supplied notebook", + resultHash: verifiedImbalanceResult.resultHash, +}; +const inconclusiveEvidenceVerdict = { + ...evidenceVerdictBase, + kind: "INCONCLUSIVE" as const, + reasonCode: "OBSERVED_GAP_WITHIN_TOLERANCE" as const, + scope: "unseen customers in this supplied notebook", + resultHash: liveResult.resultHash, +}; +const rejectedEvidenceVerdict = { + ...evidenceVerdictBase, + kind: "REJECTED" as const, + findingIds: ["result_binding_mismatch"], + resultReleased: false as const, +}; + const liveLabSceneView = { schemaVersion: "1" as const, scene: { @@ -418,6 +458,7 @@ function installApi( options: { liveGpt?: "configured" | "server-key-required"; runner?: "configured" | "local-runner-required"; + generationIsolation?: "PARTIAL" | "OS_ENFORCED"; rejectLiveBelief?: boolean; cancelOffline?: boolean; beliefTest?: typeof liveBeliefTest | typeof imbalanceBeliefTest; @@ -451,6 +492,7 @@ function installApi( restartSessionExtra?: Record; omitRestoredPrediction?: boolean; replay?: ReturnType; + replayFailure?: boolean; eventsHandler?: ( sessionId: string, requestIndex: number, @@ -491,7 +533,9 @@ function installApi( options.runner === "configured" ? "credential-and-privilege-boundary" : "local-runner-required", - generationFilesystemReadIsolation: "PARTIAL", + generationFilesystemReadIsolation: + options.generationIsolation ?? + (options.runner === "configured" ? "OS_ENFORCED" : "PARTIAL"), ...(options.runner === "configured" ? { release: { @@ -501,6 +545,12 @@ function installApi( workerEvidenceCommit: "a".repeat(40), runnerSourceCommit: "b".repeat(40), runnerImageDigest: `sha256:${"c".repeat(64)}`, + generationIsolationEvidenceSha256: "5".repeat(64), + generationIsolationProbeSha256: "6".repeat(64), + releaseCheckGenerationIsolationEvidenceSha256: "7".repeat(64), + releaseCheckGenerationIsolationProbeSha256: "6".repeat(64), + releaseCheckGenerationIsolationVerifiedAt: + "2026-07-19T05:31:00.000+05:30", timeoutCleanupReceiptSha256: "d".repeat(64), aggregateLimitEvidenceSha256: "9".repeat(64), runtimePolicySha256: "e".repeat(64), @@ -607,6 +657,23 @@ function installApi( activeArtifactId = uploadedArtifact.artifactId; const restoredExtra = options.restoredSessionExtra ?? {}; const restoredState = options.restoredSessionState ?? "LAB_COMPILING"; + const restoredResult = restoredExtra.verifiedResult as + { resultHash?: unknown } | undefined; + const inferredV2Evidence = + restoredExtra.beliefSpec !== undefined && + typeof restoredResult?.resultHash === "string" && + restoredExtra.evidenceVerdict === undefined + ? { + evidenceVerdict: { + ...evidenceVerdictBase, + kind: "SUPPORTS" as const, + hypothesisId: "competing" as const, + scope: "The restored fixture's bounded evaluation scope.", + resultHash: restoredResult.resultHash, + }, + epistemicReportHash, + } + : {}; const restoredStateRequiresPrediction = ![ "INGESTED", "INSUFFICIENT_EVIDENCE", @@ -628,6 +695,7 @@ function installApi( ...(activePrediction === null ? {} : { prediction: activePrediction }), + ...inferredV2Evidence, ...restoredExtra, }), ); @@ -882,6 +950,7 @@ function installApi( ); } if (path.startsWith("/api/replays/")) { + if (options.replayFailure) throw new TypeError("network unavailable"); const replayId = decodeURIComponent(path.slice("/api/replays/".length)); if (options.replay?.replayId === replayId) { return response(options.replay); @@ -1013,6 +1082,38 @@ async function openSampleModelDuel(user: ReturnType) { } describe("CounterLab judged flow", () => { + it("assigns distinct evidence identities to metrics from one output", () => { + const outputHash = "d".repeat(64); + const references = notebookEvidenceReferences( + ArtifactManifestSchema.parse({ + ...uploadedArtifact, + cells: [ + { + index: 1, + type: "code", + sourceSha256: "b".repeat(64), + sourceExcerpt: "accuracy and roc_auc", + executionCount: 1, + outputHashes: [outputHash], + symbols: [], + metricCandidates: [ + { name: "accuracy", value: 0.98, outputIndex: 0 }, + { name: "roc_auc", value: 0.97, outputIndex: 0 }, + ], + }, + ], + }), + ); + + expect(references.map(({ id }) => id)).toEqual([ + `${outputHash}-0-accuracy`, + `${outputHash}-0-roc_auc`, + ]); + expect(new Set(references.map(({ id }) => id)).size).toBe( + references.length, + ); + }); + it("keeps Judge Mode on its own refresh-safe route", async () => { const user = userEvent.setup(); const fetcher = installApi({ @@ -1027,7 +1128,7 @@ describe("CounterLab judged flow", () => { expect( await screen.findByRole("heading", { - name: /see a verified belief break in ten seconds/i, + name: /inspect a verified belief break from question to proof/i, }), ).toBeInTheDocument(); expect(window.location.pathname).toBe("/judge"); @@ -1070,17 +1171,35 @@ describe("CounterLab judged flow", () => { ).toBeInTheDocument(); expect( screen.getByText( - /for learners testing whether a notebook result means what they think it means/i, + /counterlab is a belief debugger—not a tutor or notebook linter/i, ), ).toBeInTheDocument(); expect( screen.getByText( - /evidence-first learning.*seal a Prediction.*fixed evidence.*not AI prose/i, + /attach a supported notebook.*reads the evidence and never runs its cells.*seal a prediction.*one controlled test answer—not ai prose/i, ), ).toBeInTheDocument(); expect( screen.getByRole("link", { name: /skip to main content/i }), ).toHaveAttribute("href", "#main-content"); + const fastPaths = screen.getByLabelText( + /fastest ways to see counterlab work/i, + ); + const skipLink = screen.getByRole("link", { + name: /skip to main content/i, + }); + skipLink.focus(); + expect(skipLink).toHaveFocus(); + await user.tab(); + expect( + within(fastPaths).getByRole("button", { + name: /^start verified sample/i, + }), + ).toHaveFocus(); + await user.tab(); + expect( + within(fastPaths).getByRole("link", { name: /judge mode/i }), + ).toHaveFocus(); await user.tab(); expect( screen.getByRole("textbox", { name: /your question or claim/i }), @@ -1090,14 +1209,23 @@ describe("CounterLab judged flow", () => { screen.getByRole("button", { name: /try verified sample/i }), ).toBeEnabled(); expect( - screen.getByRole("button", { name: /start verified sample lesson/i }), + screen.getByRole("button", { name: /^start verified sample/i }), ).toBeEnabled(); expect( screen.getByRole("button", { name: /watch verified replay/i }), ).toBeEnabled(); - expect(screen.getByRole("link", { name: /judge mode/i })).toHaveAttribute( - "href", - "/judge", + expect( + within(fastPaths).getByRole("link", { name: /judge mode/i }), + ).toHaveAttribute("href", "/judge"); + expect( + screen.getByLabelText(/how the opening sequence works/i), + ).toHaveTextContent( + /state claim.*lock prediction.*controlled test runs.*result passes verification/i, + ); + expect( + screen.getByLabelText(/authority roles for a live notebook run/i), + ).toHaveTextContent( + /gpt-5\.6.*runtime codex.*fixed kernel.*frozen verifier/i, ); expect(screen.getByText(/no account needed/i)).toBeInTheDocument(); expect( @@ -1107,7 +1235,7 @@ describe("CounterLab judged flow", () => { screen.getByText(/fixed kernels calculate the result/i), ).toBeInTheDocument(); expect( - screen.getByText(/read the evidence and never run the cells/i), + screen.getByText(/reads the evidence and never runs its cells/i), ).toBeInTheDocument(); expect( screen.getByRole("navigation", { @@ -1123,21 +1251,16 @@ describe("CounterLab judged flow", () => { expect( screen.getByRole("button", { name: /test this claim/i }), ).toBeDisabled(); - expect( - screen.getByText(/result hidden until your Prediction is sealed/i), - ).toBeInTheDocument(); - expect( - screen.getByText( - /name the claim.*lock your expectation.*change one thing/i, - ), - ).toBeInTheDocument(); - expect( - screen.getByLabelText(/fair-test mechanism with result locked/i), - ).toHaveTextContent(/familiar-row test.*unseen-entity test/i); - expect(document.body).not.toHaveTextContent(/98\.5%|59\.4%/i); - expect( - screen.getByLabelText(/fair-test mechanism with result locked/i), - ).toHaveAttribute("data-result-visibility", "locked"); + const lockedPreview = screen.getByLabelText( + /fair-test preview with result locked/i, + ); + expect(lockedPreview).toHaveTextContent( + /familiar rows.*change who counts as new.*unseen customers/i, + ); + expect(lockedPreview).toHaveAttribute("data-presentation", "strip"); + expect(lockedPreview).toHaveAttribute("data-result-visibility", "locked"); + expect(document.body).not.toHaveTextContent(LANDING_FIXED_SAMPLE_VALUES); + expect(document.body).not.toHaveTextContent(PRE_PREDICTION_RESULT_LANGUAGE); const proofSummary = screen.getByText("Evidence & proof"); await user.click(screen.getByRole("link", { name: /how proof works/i })); expect(proofSummary.closest("details")).toHaveAttribute("open"); @@ -1148,7 +1271,7 @@ describe("CounterLab judged flow", () => { ); await user.click( screen.getByRole("button", { - name: /why did my model score highly but fail/i, + name: /does this evaluation match how the model will be used/i, }), ); expect( @@ -1214,6 +1337,15 @@ describe("CounterLab judged flow", () => { ).toBeInTheDocument(); expect(screen.queryByLabelText(/your claim/i)).not.toBeInTheDocument(); expect(screen.getByText(SAMPLE_LEAKAGE_QUESTION)).toBeInTheDocument(); + expect(screen.getByLabelText(/saved original question/i)).toHaveTextContent( + question, + ); + expect(window.localStorage.getItem("counterlab.sampleOriginQuestion")).toBe( + question, + ); + expect( + window.localStorage.getItem("counterlab.sampleOriginSessionId"), + ).toBe("session_ui"); expect( screen.getByText(/does not analyze or sign a custom claim/i), ).toBeInTheDocument(); @@ -1229,7 +1361,108 @@ describe("CounterLab judged flow", () => { expect( screen.getByText(/pre-authored for this fixed sample/i), ).toBeInTheDocument(); - expect(document.body).not.toHaveTextContent(question); + expect(screen.getByLabelText(/saved original question/i)).toHaveTextContent( + /does not claim to answer or sign your original question/i, + ); + expect(screen.getByText(question)).toBeInTheDocument(); + }); + + it("does not attribute an origin question to a different sample session", async () => { + window.history.replaceState({}, "", "/session/session_ui"); + window.localStorage.setItem( + "counterlab.sampleOriginQuestion", + "Will an older score hold for another population?", + ); + window.localStorage.setItem( + "counterlab.sampleOriginSessionId", + "different-session", + ); + installApi({ + restoredSessionState: "INGESTED", + restoredSessionExtra: { + mode: { kind: "sample_lesson", sampleId: "leakage-01" }, + }, + }); + + render(); + + expect( + await screen.findByRole("heading", { + name: /what do you think the score means/i, + }), + ).toBeInTheDocument(); + expect( + screen.queryByLabelText(/saved original question/i), + ).not.toBeInTheDocument(); + expect(document.body).not.toHaveTextContent( + "Will an older score hold for another population?", + ); + }); + + it("checks exact live readiness before the evidence chooser sends a notebook", async () => { + const user = userEvent.setup(); + const fetcher = installApi(); + render(); + + await user.type( + screen.getByRole("textbox", { name: /your question or claim/i }), + "Will this score hold for new customers?", + ); + await user.click(screen.getByRole("button", { name: /test this claim/i })); + await user.upload( + screen.getByLabelText(/attach a supported notebook/i), + new File(["{}"], uploadedArtifact.fileName, { + type: "application/json", + }), + ); + + expect( + await screen.findByRole("heading", { name: /test my notebook/i }), + ).toBeInTheDocument(); + expect( + fetcher.mock.calls.filter( + ([path]) => String(path) === "/api/health?readiness=probe", + ), + ).toHaveLength(1); + expect( + fetcher.mock.calls.some(([path]) => String(path) === "/api/artifacts"), + ).toBe(false); + expect( + fetcher.mock.calls.some( + ([path]) => String(path) === "/api/live/sessions", + ), + ).toBe(false); + }); + + it("uploads from the evidence chooser only after exact live readiness passes", async () => { + const user = userEvent.setup(); + const fetcher = installApi({ liveGpt: "configured", runner: "configured" }); + render(); + + await user.type( + screen.getByRole("textbox", { name: /your question or claim/i }), + "Will this score hold for new customers?", + ); + await user.click(screen.getByRole("button", { name: /test this claim/i })); + await user.upload( + screen.getByLabelText(/attach a supported notebook/i), + new File(["{}"], uploadedArtifact.fileName, { + type: "application/json", + }), + ); + + expect( + await screen.findByRole("heading", { + name: /what do you think the score means/i, + }), + ).toBeInTheDocument(); + const requestedPaths = fetcher.mock.calls.map(([path]) => String(path)); + expect(requestedPaths.indexOf("/api/health?readiness=probe")).toBeLessThan( + requestedPaths.indexOf("/api/artifacts"), + ); + expect(requestedPaths.indexOf("/api/artifacts")).toBeLessThan( + requestedPaths.indexOf("/api/live/sessions"), + ); }); it("restores a saved question on the refresh-safe new investigation route", async () => { @@ -1275,6 +1508,28 @@ describe("CounterLab judged flow", () => { expect(document.body).not.toHaveTextContent(/local runner required/i); }); + it("keeps live notebook upload disabled when generation read isolation is partial", async () => { + const user = userEvent.setup(); + installApi({ + liveGpt: "configured", + runner: "configured", + generationIsolation: "PARTIAL", + }); + render(); + + await openLiveSetup(user); + + expect( + await screen.findByText(/qualified hosted runner is needed/i), + ).toBeInTheDocument(); + expect(document.body).toHaveTextContent( + /generation filesystem read isolation is partial.*not accepted as live authority/i, + ); + expect( + screen.queryByLabelText(/attach a supported notebook/i), + ).not.toBeInTheDocument(); + }); + it("does not continue live setup until a supported notebook has been uploaded", async () => { const user = userEvent.setup(); const fetcher = installApi({ liveGpt: "configured", runner: "configured" }); @@ -1436,7 +1691,7 @@ describe("CounterLab judged flow", () => { it("accepts a notebook from the question-first landing without running it", async () => { const user = userEvent.setup(); - const fetcher = installApi(); + const fetcher = installApi({ liveGpt: "configured", runner: "configured" }); render(); await user.upload( @@ -1485,6 +1740,41 @@ describe("CounterLab judged flow", () => { expect(story.getByText(/sha-256/i)).toBeInTheDocument(); }); + it("installs the sample session route before exposing the Question stage", async () => { + const user = userEvent.setup(); + const originalPushState = window.history.pushState.bind(window.history); + const questionVisibilityAtSessionActivation: boolean[] = []; + const pushStateSpy = vi + .spyOn(window.history, "pushState") + .mockImplementation((data, unused, url) => { + if (String(url).startsWith("/session/")) { + questionVisibilityAtSessionActivation.push( + screen.queryByRole("heading", { + name: /what do you think the score means/i, + }) !== null, + ); + } + originalPushState(data, unused, url); + }); + + try { + render(); + await user.click( + screen.getByRole("button", { name: /try verified sample/i }), + ); + + expect( + await screen.findByRole("heading", { + name: /what do you think the score means/i, + }), + ).toBeInTheDocument(); + expect(window.location.pathname).toBe("/session/session_ui"); + expect(questionVisibilityAtSessionActivation).toEqual([false]); + } finally { + pushStateSpy.mockRestore(); + } + }); + it("opens one deterministic stage hint and records only its fixed identity", async () => { const user = userEvent.setup(); const fetcher = installApi(); @@ -1649,15 +1939,11 @@ describe("CounterLab judged flow", () => { }), ).toBeInTheDocument(); expect( - screen.getByText(/partly remembers customers it already saw/i), + screen.getByText(liveBeliefTest.competingHypothesis.statement), ).toBeInTheDocument(); - expect(document.body).not.toHaveTextContent(/live competing hypothesis/i); expect( - screen.getByText(/notebook schema selected as evidence/i), + screen.getByText(/the claim targets unseen customers/i), ).toBeInTheDocument(); - expect(document.body).not.toHaveTextContent( - /the claim targets unseen customers/i, - ); expect( screen.getByText(/customer_id identifies the evaluation boundary/i), ).toBeInTheDocument(); @@ -1751,7 +2037,7 @@ describe("CounterLab judged flow", () => { expect( await screen.findByRole("heading", { - name: /this session could not be restored/i, + name: /this session could not be loaded/i, }), ).toBeInTheDocument(); expect( @@ -1792,13 +2078,14 @@ describe("CounterLab judged flow", () => { it("restores focus when reviewing a completed imbalance stage", async () => { const user = userEvent.setup(); - const replay = replayFixture("class_imbalance"); installApi({ restoredSessionState: "EXPERIMENT_COMPLETED", restoredSessionExtra: { - beliefSpec: replay.beliefSpec, - prediction: replay.prediction, + beliefSpec: liveImbalanceBeliefSpec, + prediction: committedImbalancePrediction, verifiedResult: verifiedImbalanceResult, + evidenceVerdict: supportingImbalanceEvidenceVerdict, + epistemicReportHash, }, }); window.history.replaceState({}, "", "/session/session_ui"); @@ -2110,7 +2397,7 @@ describe("CounterLab judged flow", () => { ).not.toBeInTheDocument(); }); - it("renders only closed Subject Pack framing before Prediction", async () => { + it("renders the exact reviewed Belief Spec before Prediction", async () => { const modelAuthoredSentinel = "A bespoke analyst phrase for cohort omega."; const proposed = { ...liveBeliefSpec, @@ -2143,15 +2430,104 @@ describe("CounterLab judged flow", () => { name: /does your current explanation capture what you mean/i, }), ).toBeInTheDocument(); - expect(document.body).not.toHaveTextContent(/bespoke analyst/i); + expect(screen.getByText(modelAuthoredSentinel)).toBeInTheDocument(); expect( - screen.getByText( - /The model learned a useful pattern that will work for new customers/i, + screen.getByText("Bespoke analyst condition omega."), + ).toBeInTheDocument(); + expect( + screen.getAllByText("Bespoke analyst limitation omega."), + ).toHaveLength(2); + expect(screen.getAllByText("AI-suggested draft")).toHaveLength(2); + expect( + screen.getByText("Bespoke analyst alternative omega."), + ).toBeInTheDocument(); + expect(document.body).not.toHaveTextContent( + /The model learned a useful pattern that will work for new customers/i, + ); + }); + + it("keeps optional Director setup from surviving Belief Spec confirmation", async () => { + const user = userEvent.setup(); + installApi({ + restoredSessionState: "BELIEF_TEST_PROPOSED", + restoredSessionExtra: { + beliefSpec: liveBeliefSpec, + learningDirector: { + schemaVersion: "1", + beliefSpecHash: "a".repeat(64), + approvedPacketHash: "b".repeat(64), + subjectPackVersion: "2.1.0", + clarificationUsed: true, + decision: { + status: "CLARIFICATION_REQUIRED", + questionId: "learning-emphasis", + choices: ["controls-first", "boundary-first"], + }, + provenance: { + modelId: "gpt-5.6", + promptHash: "c".repeat(64), + turns: 1, + toolTrace: [ + { + toolName: "get_subject_pack_capabilities", + argsHash: "d".repeat(64), + outputHash: "e".repeat(64), + durationMs: 1, + }, + ], + }, + }, + }, + }); + window.history.replaceState({}, "", "/session/session_ui"); + + render(); + + expect( + await screen.findByRole("heading", { + name: /which part should the introduction foreground/i, + }), + ).toBeInTheDocument(); + await user.click( + screen.getByRole("button", { name: /yes, this captures my view/i }), + ); + + expect( + await screen.findByRole("heading", { + name: /seal what you expect before the result appears/i, + }), + ).toBeInTheDocument(); + expect( + screen.queryByRole("heading", { + name: /which part should the introduction foreground/i, + }), + ).not.toBeInTheDocument(); + }); + + it("shows a neutral question prompt before notebook concept routing", async () => { + const user = userEvent.setup(); + installApi({ liveGpt: "configured", runner: "configured" }); + render(); + + await openLiveSetup(user); + await user.upload( + await screen.findByLabelText(/attach a supported notebook/i), + new File(["{}"], uploadedArtifact.fileName, { + type: "application/json", + }), + ); + + expect( + await screen.findByText( + /write one bounded claim about what the reported result supports/i, ), ).toBeInTheDocument(); expect( - screen.getByText(/Pre-result explanation wording comes from/i), + screen.getByText(/because the notebook reported this result/i), ).toBeInTheDocument(); + expect(document.body).not.toHaveTextContent( + /completely new customers|rare fraud|minority class/i, + ); }); it.each([ @@ -2176,10 +2552,7 @@ describe("CounterLab judged flow", () => { }; window.localStorage.setItem("counterlab.sessionId", "session_ui"); window.localStorage.setItem("counterlab.claim", learnerClaim); - window.localStorage.setItem( - "counterlab.claimSessionId", - "session_ui", - ); + window.localStorage.setItem("counterlab.claimSessionId", "session_ui"); window.localStorage.setItem("counterlab.mode", "live"); window.history.replaceState({}, "", "/session/session_ui"); const fetcher = installApi({ @@ -2326,6 +2699,12 @@ describe("CounterLab judged flow", () => { ); render(); + const planningPhase = ( + await screen.findByText(/Generated planning · In progress/i) + ).closest("li"); + expect(planningPhase).toHaveAttribute("data-state", "active"); + expect(screen.getByText(/Fixed testing · Waiting/i)).toBeInTheDocument(); + expect(screen.getByText(/Verified result · Waiting/i)).toBeInTheDocument(); await user.click( await screen.findByRole("button", { name: /cancel this test/i }), ); @@ -2336,6 +2715,9 @@ describe("CounterLab judged flow", () => { expect( screen.getByText(/cancelled this test before it could release a result/i), ).toBeInTheDocument(); + expect( + screen.getByText(/Generated planning · Stopped safely/i), + ).toBeInTheDocument(); expect( fetcher.mock.calls.some( ([path, init]) => @@ -2392,7 +2774,7 @@ describe("CounterLab judged flow", () => { window.dispatchEvent(new PopStateEvent("popstate")); }); await screen.findByRole("heading", { - name: /see a verified belief break in ten seconds/i, + name: /inspect a verified belief break from question to proof/i, }); await act(async () => { pending.resolve(cancellationResponse); @@ -2505,7 +2887,7 @@ describe("CounterLab judged flow", () => { window.dispatchEvent(new PopStateEvent("popstate")); }); await screen.findByRole("heading", { - name: /see a verified belief break in ten seconds/i, + name: /inspect a verified belief break from question to proof/i, }); await act(async () => { pending.resolve(cancellationResponse); @@ -2515,7 +2897,7 @@ describe("CounterLab judged flow", () => { expect(window.location.pathname).toBe("/judge"); expect( screen.getByRole("heading", { - name: /see a verified belief break in ten seconds/i, + name: /inspect a verified belief break from question to proof/i, }), ).toBeInTheDocument(); }); @@ -2627,12 +3009,21 @@ describe("CounterLab judged flow", () => { ).toBeInTheDocument(); }); - it("opens a shareable session route without depending on local mode storage", async () => { + it("hydrates a private session route without flashing the landing page", async () => { installApi({ restoredSessionState: "INGESTED" }); window.history.pushState({}, "", "/session/session_ui"); render(); + expect( + screen.getByRole("heading", { name: /opening this investigation/i }), + ).toBeInTheDocument(); + expect( + screen.queryByRole("heading", { + name: /what result are you trying to understand/i, + }), + ).not.toBeInTheDocument(); + expect( await screen.findByRole("heading", { name: /what do you think the score means/i, @@ -2641,20 +3032,118 @@ describe("CounterLab judged flow", () => { expect(screen.getByText(/live notebook analysis/i)).toBeInTheDocument(); }); - it("hydrates restored session evidence from the canonical private endpoint", async () => { + it("clears private session state before opening a new analysis", async () => { const user = userEvent.setup(); - const storedEvents = [ - storedEvidenceEvent("session_ui", 1, "session.created"), - storedEvidenceEvent("session_ui", 2, "prediction.committed", "learner"), - storedEvidenceEvent("session_ui", 3, "experiment.completed", "kernel"), - ]; - const storedCompilerEvents = [ - { - schemaVersion: "1" as const, - eventId: "compiler_plan_1", - jobId: "job_compile_history", - cursor: 1, - at: "2026-07-19T10:01:00.000Z", + installApi({ restoredSessionState: "INGESTED" }); + window.history.replaceState({}, "", "/session/session_ui"); + window.localStorage.setItem("counterlab.sessionId", "session_ui"); + + render(); + await screen.findByRole("heading", { + name: /what do you think the score means/i, + }); + await user.click( + screen.getByRole("button", { name: /project & evidence/i }), + ); + await user.click(screen.getByRole("button", { name: /new analysis/i })); + + expect( + await screen.findByRole("heading", { name: /test my notebook/i }), + ).toBeInTheDocument(); + expect(window.location.pathname).toBe("/new"); + expect(window.localStorage.getItem("counterlab.sessionId")).toBeNull(); + expect(document.body).not.toHaveTextContent(uploadedArtifact.fileName); + }); + + it("continues to a new analysis after cancelling registered live work", async () => { + const user = userEvent.setup(); + installApi({ restoredSessionState: "INGESTED" }); + window.history.replaceState({}, "", "/session/session_ui"); + window.localStorage.setItem("counterlab.sessionId", "session_ui"); + window.localStorage.setItem( + "counterlab.activeRunnerJob.session_ui", + JSON.stringify({ + schemaVersion: "1", + sessionId: "session_ui", + jobId: liveRunnerJob.jobId, + kind: liveRunnerJob.kind, + }), + ); + + render(); + await screen.findByRole("heading", { + name: /what do you think the score means/i, + }); + await user.click( + screen.getByRole("button", { name: /project & evidence/i }), + ); + await user.click(screen.getByRole("button", { name: /new analysis/i })); + expect( + screen.getByRole("heading", { name: /stop live work and start over/i }), + ).toBeInTheDocument(); + await user.click( + screen.getByRole("button", { name: /stop jobs and start over/i }), + ); + + expect( + await screen.findByRole("heading", { name: /test my notebook/i }), + ).toBeInTheDocument(); + expect(window.location.pathname).toBe("/new"); + }); + + it("detaches private UI but retains active-job recovery when history returns home", async () => { + installApi({ + liveGpt: "configured", + runner: "configured", + stallRunner: true, + }); + window.history.replaceState({}, "", "/session/session_ui"); + window.localStorage.setItem("counterlab.sessionId", "session_ui"); + window.localStorage.setItem( + "counterlab.activeRunnerJob.session_ui", + JSON.stringify({ + schemaVersion: "1", + sessionId: "session_ui", + jobId: liveRunnerJob.jobId, + kind: liveRunnerJob.kind, + }), + ); + + render(); + await screen.findByRole("button", { name: /cancel this test/i }); + act(() => { + window.history.pushState({}, "", "/"); + window.dispatchEvent(new PopStateEvent("popstate")); + }); + + expect( + await screen.findByRole("heading", { + name: /what result are you trying to understand/i, + }), + ).toBeInTheDocument(); + expect(screen.getByRole("alert")).toHaveTextContent( + /live test is still registered/i, + ); + expect(document.body).not.toHaveTextContent(uploadedArtifact.fileName); + expect( + window.localStorage.getItem("counterlab.activeRunnerJob.session_ui"), + ).not.toBeNull(); + }); + + it("hydrates restored session evidence from the canonical private endpoint", async () => { + const user = userEvent.setup(); + const storedEvents = [ + storedEvidenceEvent("session_ui", 1, "session.created"), + storedEvidenceEvent("session_ui", 2, "prediction.committed", "learner"), + storedEvidenceEvent("session_ui", 3, "experiment.completed", "kernel"), + ]; + const storedCompilerEvents = [ + { + schemaVersion: "1" as const, + eventId: "compiler_plan_1", + jobId: "job_compile_history", + cursor: 1, + at: "2026-07-19T10:01:00.000Z", kind: "plan.summary" as const, title: "Stored bounded compile plan", steps: ["Read approved evidence", "Emit typed experiment IR"], @@ -2676,6 +3165,8 @@ describe("CounterLab judged flow", () => { beliefSpec: liveBeliefSpec, prediction: committedPrediction, verifiedResult: liveResult, + evidenceVerdict: supportingEvidenceVerdict, + epistemicReportHash, }, eventsHandler: () => response({ @@ -2869,6 +3360,8 @@ describe("CounterLab judged flow", () => { beliefSpec: liveBeliefSpec, prediction: committedPrediction, verifiedResult: liveResult, + evidenceVerdict: supportingEvidenceVerdict, + epistemicReportHash, }, }); window.history.replaceState({}, "", "/proof/session_ui"); @@ -2902,7 +3395,7 @@ describe("CounterLab judged flow", () => { ).toBe(false); }); - it("restores exact claim evidence with closed framing after a live session refresh", async () => { + it("restores the exact reviewed Belief Spec and evidence after refresh", async () => { installApi({ restoredSessionState: "BELIEF_TEST_PROPOSED", restoredSessionExtra: { beliefSpec: liveBeliefSpec }, @@ -2914,31 +3407,95 @@ describe("CounterLab judged flow", () => { render(); expect( - await screen.findByText( - /The model learned a useful pattern that will work for new customers/i, - ), + await screen.findByText(liveBeliefSpec.hypotheses[0].statement), ).toBeInTheDocument(); expect( - screen.getByText(/The model partly remembers customers it already saw/i), + screen.getByText(liveBeliefSpec.hypotheses[1].statement), ).toBeInTheDocument(); - expect(document.body).not.toHaveTextContent( - liveBeliefSpec.hypotheses[0].statement, - ); - expect(document.body).not.toHaveTextContent( - liveBeliefSpec.hypotheses[1].statement, - ); expect( - screen.getByText(/notebook schema selected as evidence/i), + screen.getByText(liveBeliefSpec.evidenceRefs[0]!.relevance), + ).toBeInTheDocument(); + expect( + screen.getByText(liveBeliefSpec.evidenceRefs[0]!.excerpt), ).toBeInTheDocument(); - expect(document.body).not.toHaveTextContent( - liveBeliefSpec.evidenceRefs[0]!.relevance, - ); expect(screen.getByText(liveBeliefSpec.claim)).toBeInTheDocument(); expect(document.body).not.toHaveTextContent( /customer_id encoded|98\.5% accuracy/i, ); }); + it("does not rerun a restored epistemically rejected test", async () => { + const fetcher = installApi({ + restoredSessionState: "LAB_VERIFIED", + restoredSessionExtra: { + beliefSpec: liveBeliefSpec, + prediction: committedPrediction, + evidenceVerdict: rejectedEvidenceVerdict, + epistemicReportHash, + }, + }); + window.history.replaceState({}, "", "/session/session_ui"); + + render(); + + expect( + await screen.findByRole("heading", { + name: /experimental result was withheld/i, + }), + ).toBeInTheDocument(); + expect(screen.getByText(/result binding mismatch/i)).toBeInTheDocument(); + expect(document.body).not.toHaveTextContent(/98\.5%|59\.4%/i); + expect( + fetcher.mock.calls.some(([path]) => String(path).endsWith("/lab/run")), + ).toBe(false); + }); + + it("shows an inconclusive result without unlocking repair", async () => { + installApi({ + restoredSessionState: "TRANSFER_PASSED", + restoredSessionExtra: { + beliefSpec: liveBeliefSpec, + prediction: committedPrediction, + verifiedResult: liveResult, + evidenceVerdict: inconclusiveEvidenceVerdict, + epistemicReportHash, + revision: + "Evaluation must match deployment timing and use exact code evidence.", + transferResult: { + ...failedLeakageTransfer, + id: "transfer_passed_ui", + outcome: "PASSED", + selectedStrategy: "time_ordered_holdout", + identifiedRisks: ["centered_window_reads_future"], + evidenceChoices: [ + "center_true_uses_later_targets", + "random_split_mixes_dates", + ], + checks: failedLeakageTransfer.checks.map((check) => ({ + ...check, + passed: true, + })), + }, + }, + }); + window.history.replaceState({}, "", "/session/session_ui"); + + render(); + + expect( + await screen.findByRole("heading", { + name: /fixed forecasting transfer passed/i, + }), + ).toBeInTheDocument(); + expect( + screen.getByText(/evidence verdict · inconclusive/i), + ).toBeInTheDocument(); + expect(screen.getByText(/repair remains locked/i)).toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: /verify notebook patch/i }), + ).not.toBeInTheDocument(); + }); + it("restores the immutable Prediction Seal after refresh", async () => { installApi({ restoredSessionState: "LAB_VERIFIED", @@ -2946,6 +3503,8 @@ describe("CounterLab judged flow", () => { beliefSpec: liveBeliefSpec, prediction: committedPrediction, verifiedResult: liveResult, + evidenceVerdict: supportingEvidenceVerdict, + epistemicReportHash, }, }); window.localStorage.setItem("counterlab.sessionId", "session_ui"); @@ -2973,6 +3532,8 @@ describe("CounterLab judged flow", () => { beliefSpec: liveBeliefSpec, prediction: committedPrediction, verifiedResult: liveResult, + evidenceVerdict: supportingEvidenceVerdict, + epistemicReportHash, revision: "Evaluation must match deployment timing and use exact code evidence.", transferResult: failedLeakageTransfer, @@ -3044,6 +3605,8 @@ describe("CounterLab judged flow", () => { beliefSpec: liveImbalanceBeliefSpec, prediction: committedImbalancePrediction, verifiedResult: verifiedImbalanceResult, + evidenceVerdict: supportingImbalanceEvidenceVerdict, + epistemicReportHash, revision: "Overall accuracy needs minority-sensitive metrics and direct rare-class evidence.", transferResult: failedImbalanceTransfer, @@ -3147,12 +3710,11 @@ describe("CounterLab judged flow", () => { ); expect( - ( - await screen.findAllByText( - /overall accuracy obscures rare-case behavior/i, - ) - ).length, - ).toBeGreaterThan(0); + await screen.findByText(imbalanceBeliefTest.currentHypothesis.statement), + ).toBeInTheDocument(); + expect( + screen.getByText(imbalanceBeliefTest.competingHypothesis.statement), + ).toBeInTheDocument(); expect( screen.getByText(/same predictions with overall and class-specific/i), ).toBeInTheDocument(); @@ -3215,7 +3777,7 @@ describe("CounterLab judged flow", () => { screen.getByRole("heading", { name: /the fair test is ready/i }), ).toBeInTheDocument(); expect( - screen.getByRole("button", { name: /run the fair test/i }), + screen.getByRole("button", { name: /reveal verified sample result/i }), ).toBeEnabled(); expect(screen.queryByText(/new customers 59\.4%/i)).not.toBeInTheDocument(); }); @@ -3251,7 +3813,7 @@ describe("CounterLab judged flow", () => { expect(document.body).not.toHaveTextContent(/new customers 59\.4%/i); await user.click( - screen.getByRole("button", { name: /run the fair test/i }), + screen.getByRole("button", { name: /reveal verified sample result/i }), ); expect( @@ -3288,8 +3850,15 @@ describe("CounterLab judged flow", () => { ).toHaveValue(""); expect(document.body).not.toHaveTextContent(/98\.5% became 59\.4%/i); + const interpretation = screen.getByRole("textbox", { + name: /what do you notice/i, + }); + await user.type(interpretation, "xxxxxxxxxxxxxxxxxxxx"); + expect(exploreTab).toBeDisabled(); + expect(boundaryTab).toBeDisabled(); + await user.clear(interpretation); await user.type( - screen.getByRole("textbox", { name: /what do you notice/i }), + interpretation, "The result changes when the evaluation boundary changes.", ); expect(document.body).toHaveTextContent( @@ -3327,14 +3896,44 @@ describe("CounterLab judged flow", () => { }), ).toBeInTheDocument(); expect(screen.getByText(/integrity-hashed/i)).toBeInTheDocument(); + expect(applyTab).toBeDisabled(); + expect( + window.localStorage.getItem( + "counterlab.sampleBoundaryClassifiedSessionId", + ), + ).toBeNull(); + + await user.click( + screen.getByRole("radio", { + name: /test fraction 10%.*observations per customer 2 observations/i, + }), + ); + await user.click( + screen.getByRole("button", { name: /check this condition/i }), + ); expect(applyTab).toBeEnabled(); expect( - window.localStorage.getItem("counterlab.sampleBoundarySessionId"), + window.localStorage.getItem( + "counterlab.sampleBoundaryClassifiedSessionId", + ), ).toBe("session_ui"); await user.click(applyTab); - expect(screen.getByLabelText(/your revised mental model/i)).toHaveValue( - "The result changes when the evaluation boundary changes.", + expect(screen.getByLabelText(/your revised mental model/i)).toHaveValue(""); + expect( + screen.getByRole("button", { name: /try the rule on a new problem/i }), + ).toBeDisabled(); + await user.selectOptions( + screen.getByLabelText(/choose the condition/i), + "repeated-entity", + ); + await user.selectOptions( + screen.getByLabelText(/choose the action/i), + "whole-entities", + ); + await user.selectOptions( + screen.getByLabelText(/choose the evidence-based reason/i), + "identity-overlap", ); expect( screen.getByRole("button", { name: /try the rule on a new problem/i }), @@ -3352,6 +3951,9 @@ describe("CounterLab judged flow", () => { expect( (await screen.findAllByText(/verified replay/i)).length, ).toBeGreaterThan(0); + expect( + screen.queryByRole("button", { name: /start|run.*test|new analysis/i }), + ).not.toBeInTheDocument(); await user.click(screen.getByRole("button", { name: /continue replay/i })); expect( @@ -3374,6 +3976,12 @@ describe("CounterLab judged flow", () => { expect( screen.queryByRole("button", { name: /check my answer/i }), ).not.toBeInTheDocument(); + expect( + screen.getByRole("button", { name: /return to counterlab home/i }), + ).toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: /start|run.*test|new analysis/i }), + ).not.toBeInTheDocument(); expect( fetcher.mock.calls.every(([, init]) => [undefined, "GET"].includes(init?.method), @@ -3413,6 +4021,59 @@ describe("CounterLab judged flow", () => { ).toBe(false); }); + it("returns to an honest chooser when a replay cannot be loaded", async () => { + const user = userEvent.setup(); + installApi({ replayFailure: true }); + render(); + + await user.click( + screen.getByRole("button", { name: /watch verified replay/i }), + ); + + expect(await screen.findByRole("alert")).toHaveTextContent( + /could not reach the api/i, + ); + expect( + screen.getByRole("heading", { + name: /what result are you trying to understand/i, + }), + ).toBeInTheDocument(); + expect( + screen.queryByRole("heading", { name: /opening this replay/i }), + ).not.toBeInTheDocument(); + }); + + it("canonicalizes a stored replay session route to its read-only replay URL", async () => { + const replayId = "replay:stored.session"; + window.history.replaceState({}, "", "/session/session_ui"); + const fetcher = installApi({ + restoredSessionState: "INGESTED", + restoredSessionExtra: { + mode: { kind: "verified_replay", replayId }, + }, + }); + + render(); + + expect( + (await screen.findAllByText(replayId, { exact: true })).length, + ).toBeGreaterThan(0); + expect(window.location.pathname).toBe( + `/replay/${encodeURIComponent(replayId)}`, + ); + expect( + fetcher.mock.calls.some( + ([path]) => String(path) === "/api/sessions/session_ui/artifact", + ), + ).toBe(false); + expect( + fetcher.mock.calls.some( + ([path]) => + String(path) === `/api/replays/${encodeURIComponent(replayId)}`, + ), + ).toBe(true); + }); + it("lets an explicit session URL override stale replay storage", async () => { window.localStorage.setItem("counterlab.mode", "replay"); window.localStorage.setItem("counterlab.replayId", "replay_retention_913"); @@ -3786,7 +4447,7 @@ describe("CounterLab judged flow", () => { ).toBe(false); await user.click( - screen.getByRole("button", { name: /start new analysis/i }), + screen.getByRole("button", { name: /return to counterlab home/i }), ); expect(window.location.pathname).toBe("/"); expect( @@ -3819,6 +4480,151 @@ describe("CounterLab judged flow", () => { expect(document.body).not.toHaveTextContent(/verified replay/i); }); + it("ignores a direct replay load after the learner returns home", async () => { + window.history.replaceState({}, "", "/replay/leakage-01"); + const normalFetch = installApi(); + const replayResponse = await normalFetch("/api/replays/leakage-01"); + const pending = deferredResponse(); + vi.stubGlobal( + "fetch", + vi.fn((input, init) => + String(input) === "/api/replays/leakage-01" + ? pending.promise + : normalFetch(input, init), + ), + ); + const user = userEvent.setup(); + + render(); + await user.click( + await screen.findByRole("button", { + name: /return to counterlab home/i, + }), + ); + await act(async () => { + pending.resolve(replayResponse); + await pending.promise; + }); + + expect(window.location.pathname).toBe("/"); + expect( + screen.getByRole("heading", { + name: /what result are you trying to understand/i, + }), + ).toBeInTheDocument(); + expect(screen.queryByLabelText("Replay status")).not.toBeInTheDocument(); + }); + + it("treats a 404 session as inaccessible private work and offers durable recovery actions", async () => { + window.history.replaceState({}, "", "/session/missing-session"); + vi.stubGlobal( + "fetch", + vi.fn(async () => + errorResponse("SESSION_NOT_FOUND", "Session was not found", 404), + ), + ); + + render(); + + expect( + await screen.findByRole("heading", { + name: /this private session could not be opened/i, + }), + ).toBeInTheDocument(); + expect(screen.getByRole("alert")).toHaveTextContent( + /does not reveal whether an inaccessible address exists/i, + ); + expect(screen.getByRole("button", { name: "Back" })).toBeEnabled(); + expect( + screen.getByRole("button", { name: /go to counterlab home/i }), + ).toBeEnabled(); + }); + + it("keeps browser recovery metadata when private-session access is denied", async () => { + window.history.replaceState({}, "", "/session/private-session"); + window.localStorage.setItem( + "counterlab.recentWork.v1", + JSON.stringify({ + schemaVersion: "1", + records: [ + { + id: "private-session", + mode: "live", + status: "BELIEF_TEST_PROPOSED", + updatedAt: "2026-07-19T01:00:00.000Z", + }, + ], + }), + ); + vi.stubGlobal( + "fetch", + vi.fn(async () => + errorResponse( + "SESSION_ACCESS_DENIED", + "Private session access was not provided", + 404, + ), + ), + ); + + render(); + + const recentWork = ( + await screen.findByRole("heading", { + name: /recent work from this browser/i, + }) + ).closest("section"); + expect(recentWork).not.toBeNull(); + const privateSession = within(recentWork!).getByRole("button", { + name: /live notebook session/i, + }); + expect(privateSession).toBeEnabled(); + expect(privateSession).toHaveTextContent(/explanation ready to review/i); + expect(window.localStorage.getItem("counterlab.recentWork.v1")).toContain( + "private-session", + ); + }); + + it("does not misreport a non-404 proof failure as missing", async () => { + window.history.replaceState({}, "", "/proof/session-unavailable"); + vi.stubGlobal( + "fetch", + vi.fn(async () => + errorResponse( + "SESSION_STORE_UNAVAILABLE", + "Session storage is unavailable", + 503, + ), + ), + ); + + render(); + + expect( + await screen.findByRole("heading", { + name: /this proof could not be loaded/i, + }), + ).toBeInTheDocument(); + expect(screen.getByRole("alert")).not.toHaveTextContent( + /proof was not found/i, + ); + expect(screen.getByRole("button", { name: "Back" })).toBeEnabled(); + }); + + it("gives an unknown route a distinct not-found recovery surface", async () => { + window.history.replaceState({}, "", "/not-a-counterlab-route"); + + render(); + + expect( + await screen.findByRole("heading", { + name: /we couldn't find that counterlab page/i, + }), + ).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Back" })).toBeEnabled(); + expect(window.location.pathname).toBe("/not-a-counterlab-route"); + }); + it("does not misreport a replay integrity failure as a missing link", async () => { window.history.replaceState({}, "", "/replay/replay-integrity-failed"); vi.stubGlobal( diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 4b736c9d..8a8ceca6 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -19,6 +19,7 @@ import { ApiClientError, counterLabApi, getSessionBeliefAuthority, + isExactLiveAuthorityReady, type ArtifactView, type BeliefAnalysisPreview, type BeliefTest, @@ -47,7 +48,11 @@ import { LearnerCompletion } from "./components/learner/LearnerCompletion"; import { LearnerCoach } from "./components/learner/LearnerCoach"; import { NeedAHint } from "./components/learner/NeedAHint"; import { LearnerProgress } from "./components/learner/LearnerProgress"; -import { ReflectionBuilder } from "./components/learner/ReflectionBuilder"; +import { LearningDirectorPanel } from "./components/learner/LearningDirectorPanel"; +import { + isMeaningfulLearnerText, + ReflectionBuilder, +} from "./components/learner/ReflectionBuilder"; import { RepairPreview } from "./components/learner/RepairPreview"; import { RouteRecovery, @@ -70,6 +75,7 @@ import { type FairTestRepairStory, type PublicTechnicalDetail, } from "./components/learner/FairTestBuilder"; +import { CompileAuthorityPhase } from "./components/learner/CompileAuthorityPhase"; import { ModelDuel, type DuelModel } from "./components/learner/ModelDuel"; import { NotebookEvidenceStory, @@ -82,7 +88,9 @@ import { } from "./components/learner/PredictionSeal"; import { PrivacyPacketSummary } from "./components/learner/PrivacyPacketSummary"; import { QuestionComposer } from "./components/learner/QuestionComposer"; -import { VerifiedBeliefBreakMechanism } from "./components/learner/VerifiedBeliefBreakTheater"; +import { LockedBeliefBreakPreview } from "./components/learner/LockedBeliefBreakPreview"; +import { LearnerThesisStrip } from "./components/learner/LearnerThesisStrip"; +import { LiveAuthorityStrip } from "./components/learner/LiveAuthorityStrip"; import { currentLearnerStage, learnerSessionStatusLabel, @@ -393,6 +401,16 @@ function sessionProofReady(session: SessionView | null): boolean { if (session.mode.kind === "live_notebook") { return ( session.state === "PROOF_CAPSULE_ISSUED" && + session.beliefSpec !== undefined && + session.prediction !== undefined && + session.verifiedResult !== undefined && + session.evidenceVerdict?.kind === "SUPPORTS" && + session.epistemicReportHash !== undefined && + session.boundaryMapAuthority !== undefined && + session.revision !== undefined && + session.transferResult?.outcome === "PASSED" && + session.patchResult?.status === "VERIFIED" && + session.reasoningDiffV2 !== undefined && session.proofCapsule !== undefined ); } @@ -401,6 +419,18 @@ function sessionProofReady(session: SessionView | null): boolean { ); } +function sessionEvidencePermitsRepair(session: SessionView | null): boolean { + return ( + session === null || + session.mode.kind !== "live_notebook" || + session.evidenceVerdict?.kind === "SUPPORTS" + ); +} + +function humanizeAuthorityToken(value: string): string { + return value.replaceAll("_", " ").toLowerCase(); +} + function beliefResponseClosed(session: SessionView | null): boolean { return ( session?.state === "INSUFFICIENT_EVIDENCE" || @@ -539,6 +569,35 @@ function presentationMode(mode: SessionView["mode"]): Mode { return "live"; } +function patchSourceCopy(mode: SessionView["mode"] | undefined) { + if (mode?.kind === "live_notebook") { + return { + activity: "The uploaded notebook remains read-only.", + source: "Original upload is never overwritten", + unlock: "The original upload will not be overwritten.", + } as const; + } + if (mode?.kind === "sample_lesson") { + return { + activity: "The bundled sample artifact remains read-only.", + source: "Bundled sample source is never overwritten", + unlock: "The bundled sample source will not be overwritten.", + } as const; + } + if (mode?.kind === "verified_replay") { + return { + activity: "The stored replay artifact remains read-only.", + source: "Stored replay source is never overwritten", + unlock: "The stored replay source will not be overwritten.", + } as const; + } + return { + activity: "The source artifact remains read-only.", + source: "Source artifact is never overwritten", + unlock: "The source artifact will not be overwritten.", + } as const; +} + const storageKeys = { sessionId: "counterlab.sessionId", mode: "counterlab.mode", @@ -549,11 +608,21 @@ const storageKeys = { replayIntro: "counterlab.replayIntro", replayTransferState: "counterlab.replayTransferState", replayRevision: "counterlab.replayRevision", - sampleBoundarySessionId: "counterlab.sampleBoundarySessionId", + sampleOriginQuestion: "counterlab.sampleOriginQuestion", + sampleOriginSessionId: "counterlab.sampleOriginSessionId", + sampleBoundaryClassifiedSessionId: + "counterlab.sampleBoundaryClassifiedSessionId", activeRunnerJobId: "counterlab.activeRunnerJobId", activeRunnerJobKind: "counterlab.activeRunnerJobKind", } as const; +function activateSessionRoute(sessionId: string): void { + const path = `/session/${encodeURIComponent(sessionId)}`; + if (window.location.pathname !== path) { + window.history.pushState({}, "", path); + } +} + function SkipLink() { return ( @@ -701,7 +770,7 @@ function notebookScoreDisplay( }; } -function notebookEvidenceReferences( +export function notebookEvidenceReferences( artifact: ArtifactView | null, ): readonly NotebookEvidenceReference[] { if (artifact?.fileSha256 === sampleArtifact.fileSha256) { @@ -726,9 +795,7 @@ function notebookEvidenceReferences( if (artifact === null) return []; const metricReferences = artifact.cells.flatMap((cell) => cell.metricCandidates.map((metric) => ({ - id: - cell.outputHashes[metric.outputIndex] ?? - `${cell.sourceSha256}-${metric.outputIndex}-${metric.name}`, + id: `${cell.outputHashes[metric.outputIndex] ?? cell.sourceSha256}-${metric.outputIndex}-${metric.name}`, reference: `Cell ${cell.index} · output ${metric.outputIndex}`, relevance: `${metric.name} is displayed notebook evidence attached to this question.`, excerpt: `${metric.name}: ${metric.value.toLocaleString()}`, @@ -1033,7 +1100,7 @@ function Header({ ); @@ -1076,7 +1143,6 @@ function Landing({ }, [railOpen]); const startSample = () => { setRailOpen(false); - updateClaim(""); chooseMode("instant"); }; const openReplay = () => { @@ -1098,17 +1164,26 @@ function Landing({

What result are you trying to understand?

-

- For learners testing whether a notebook result means what they - think it means. -

-

- Evidence-first learning: seal a Prediction, change one - condition, and let fixed evidence—not AI prose—release the - bounded result. +

+ CounterLab is a belief debugger—not a tutor or notebook linter. + Ask a question or attach a supported notebook; it reads the + evidence and never runs its cells. Seal a Prediction, then let + one controlled test answer—not AI prose.

+
+ - + +

- No account needed Supported - Python/scikit-learn notebooks - We read the evidence and never run the cells + No account needed Sample, live, + and replay stay clearly labelled

Question · Your idea

What do you think the score means?

-

Write one sentence about who you think this model will work for.

+

+ {isSample + ? "Write one sentence about who you think this model will work for." + : "Write one bounded claim about what the reported result supports."} +

{requiresFreshSession && ( @@ -1456,7 +1519,7 @@ function ClaimScreen({

{isSample ? SAMPLE_LEAKAGE_QUESTION - : "“Because the notebook scored highly, I think the model…”"} + : "“Because the notebook reported this result, I think…”"}

{isSample ? ( @@ -1477,7 +1540,7 @@ function ClaimScreen({ type="button" onClick={() => updateClaim( - "I think the high score means the model will work for completely new customers.", + "I think the reported result means the model will work on new data that matches its intended use.", ) } > @@ -1487,7 +1550,7 @@ function ClaimScreen({ id="learner-claim" value={claim} onChange={(event) => updateClaim(event.target.value)} - placeholder="I think this score means the model will work for…" + placeholder="I think this reported result supports…" rows={7} /> @@ -1538,7 +1601,12 @@ function ClaimScreen({ {analysisPreview.requiresSensitiveApproval && (