From 6645614a7714d1f2e30cb2a99be5ced92004d66b Mon Sep 17 00:00:00 2001 From: Harshith Pasupuleti Date: Sat, 24 Oct 2026 11:30:31 +0530 Subject: [PATCH 01/10] docs: add inline widget design guide, tool specs and browser service Draft the design rules agents load before rendering inline widgets, mapped from Fluid Functionalism tokens; the widget_guide, preview_widget, show_widget and update_widget tool descriptions; and a spec for a headless browser service hosted in the main process over per-session MCP endpoints, backing widget previews and general agent browsing. Co-Authored-By: Claude Opus 5.5 --- docs/inline-widgets/browser-service.md | 208 +++++++++++++++++++ docs/inline-widgets/design-guide.md | 244 +++++++++++++++++++++++ docs/inline-widgets/tool-descriptions.md | 176 ++++++++++++++++ 3 files changed, 628 insertions(+) create mode 100644 docs/inline-widgets/browser-service.md create mode 100644 docs/inline-widgets/design-guide.md create mode 100644 docs/inline-widgets/tool-descriptions.md diff --git a/docs/inline-widgets/browser-service.md b/docs/inline-widgets/browser-service.md new file mode 100644 index 0000000..c3c95c7 --- /dev/null +++ b/docs/inline-widgets/browser-service.md @@ -0,0 +1,208 @@ +# Browser service + +A headless browser that Pipper hosts in its main process and exposes to agents +over MCP. It backs `preview_widget` and also gives any ACP agent a general +browser: open a page, read it, click, type, screenshot, inspect console and +network. + +Status: design draft. Nothing here is implemented yet. + +## Goals + +- One engine for widget previews and agent browsing, so a preview renders with + the same Chromium, fonts and sandbox rules as the real widget frame. +- No extra install: use Electron's bundled Chromium. No separate Chrome, Rust + binary or long-lived daemon. +- Every page belongs to one ACP session and is torn down with it. +- Compact, text-first output that keeps agent context small. +- Bounded resource use under load. + +## Non-goals + +- Driving the user's own browser or reusing their sign-ins. +- A remote debugging port. The service never calls + `app.commandLine.appendSwitch("remote-debugging-port")`; an open port would let + any local process control every Pipper window. +- Visible browsing UI. A browser pane can come later and reuse this service. + +## Where it lives + +The subagent tool already solves the hard part: a client-hosted MCP endpoint +per ACP session. Follow the same pattern rather than adding a second server. + +| Piece | Existing precedent | Browser service | +| ------------------------------------------ | ------------------------------------------ | ------------------------------------------------------ | +| HTTP MCP server on `127.0.0.1:0` | `electron/subagents/mcp-http-server.ts` | Same server class, generalized (see below) | +| Per-session token at `/mcp/` | `SubagentManager.attachMcpServers` | `BrowserService.attachMcpServers` | +| stdio fallback for agents without HTTP MCP | `electron/subagents/stdio-proxy.ts` | Same shim, different URL | +| Bind token to session id | `bindSession` | Same | +| Release on session close | `releaseSessionMcp` | Same, plus closing every page the session owns | +| Wiring into `session/new` | `AgentConnectionManager.sessionMcpServers` | Chain a second `attachMcpServers` call after subagents | + +Proposed files: + +- `electron/mcp/mcp-http-server.ts`: move the existing server here and make it + shared. `SERVER_INFO` becomes a per-endpoint value so `pipper-subagents` and + `pipper-browser` identify separately, and `McpToolResult.content` gains + `{ type: "image", data, mimeType }` for screenshots. +- `electron/browser/browser-service.ts`: page pool, sessions, limits, lifecycle. +- `electron/browser/browser-page.ts`: one hidden page and its CDP wrapper. +- `electron/browser/snapshot.ts`: accessibility tree → compact text with refs. +- `electron/browser/widget-preview.ts`: `preview_widget` on top of the pool. +- `electron/browser/browser-tools.ts`: MCP tool definitions and argument + validation. + +Run one HTTP server for both endpoint types if the generalization stays small; +otherwise one instance each. Either way, tokens and endpoints stay per session. + +## Engine + +Each page is a hidden `BrowserWindow`: + +```ts +new BrowserWindow({ + show: false, + width: 1280, + height: 800, + webPreferences: { + partition: `browser:${sessionId}`, // not persist:, so nothing touches disk + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + javascript: true, + backgroundThrottling: false, + offscreen: false, + }, +}); +``` + +- **CDP without a port:** `webContents.debugger.attach("1.3")`, then + `sendCommand` for `Accessibility.getFullAXTree`, `Input.dispatchMouseEvent`, + `Runtime.evaluate`, `Network.*`, `Emulation.setEmulatedMedia`, etc. +- **Screenshots:** `webContents.capturePage()`, PNG, downscaled to at most + 1280px wide before returning. +- **Console:** the `console-message` event, ring-buffered per page (200 entries). +- **Network:** `Network.requestWillBeSent` / `responseReceived` / + `loadingFailed`, ring-buffered (500 entries). Bodies are fetched only on + request with `Network.getResponseBody`. +- **Isolation:** each ACP session gets its own in-memory partition. Agents never + share cookies with each other or with Pipper's UI windows. + +## Tools + +Tool results are text unless noted. Every page-scoped tool takes an optional +`page_id`; without it the session's most recent page is used. + +| Tool | Does | Returns | +| ----------------------------------------------------- | ----------------------------------------------------- | ------------------------------------------ | +| `browser_open` | Opens a URL in a new or existing page | `page_id`, final URL, title, HTTP status | +| `browser_snapshot` | Accessibility tree with refs; `interactive_only` flag | Text tree, e.g. `- button "Save" [ref=e4]` | +| `browser_click` | Clicks a ref | Short result plus any navigation | +| `browser_type` | Types into a ref; optional `submit` | Short result | +| `browser_press` | Presses a key or chord | Short result | +| `browser_scroll` | Scrolls page or ref | New scroll position | +| `browser_screenshot` | Viewport, full page or one ref | Image content | +| `browser_console` | Console entries since last call; `level` filter | Text lines | +| `browser_network` | Requests since last call; `failed_only` filter | Text lines | +| `browser_eval` | Runs a JS expression; returns JSON | JSON, truncated to 8 KB | +| `browser_back` / `browser_forward` / `browser_reload` | History | URL and title | +| `browser_close` | Closes a page | — | +| `preview_widget` | See `tool-descriptions.md` | Screenshots, size, console, blocked, lint | + +Refs are valid until the next navigation or snapshot. A stale ref returns an +error telling the agent to snapshot again, never a click on the wrong element. + +Snapshot output targets 200–400 tokens for a typical page: roles and names +only, collapsed generic containers, text runs truncated at 80 characters, +hidden nodes dropped. + +## Widget preview path + +`preview_widget` doesn't go through the general tools. It keeps one or two warm +pages that already have the widget frame loaded (`widget-frame.html` with +`widget-frame.css`, fonts and icons), so a preview only swaps content: + +1. Post the widget code into the warm frame. +2. Wait for fonts, scripts, then two animation frames. +3. Read the measured size from the frame's `ResizeObserver`. +4. Capture with `data-workspace-theme="light"`, flip to `"dark"`, capture again. +5. Collect console errors, failed requests and lint results. +6. Reset the frame for the next call. + +Target: under 300 ms per preview with a warm page. The page is recreated after +50 previews or any crash. + +## Limits + +Defaults, configurable in settings: + +| Limit | Default | +| ------------------------- | ----------------------------------------------- | +| Pages per session | 4 | +| Pages across all sessions | 12 (excluding warm preview pages) | +| Idle page timeout | 5 min | +| Navigation timeout | 30 s | +| Tool call timeout | 30 s (matches the MCP server's request timeout) | +| Renderer memory per page | Close and report an error over 1 GB | +| Screenshot size | 1280 px wide, PNG under 1.5 MB | + +When a limit is hit, the tool returns an error naming the limit and what to do +(for example "Close a page with browser_close"), never a silent failure. + +## Safety + +- **Blocked schemes:** `file:`, `chrome:`, `devtools:`, `about:` other than + `about:blank`, and Pipper's own origins. Enforced in `will-navigate`, + `setWindowOpenHandler` and `session.webRequest.onBeforeRequest`. +- **Local addresses:** `localhost` and private ranges are allowed, since agents + need to test dev servers, but each is shown in the thread with its port. +- **Downloads:** cancelled by default; an agent asks through Pipper's existing + permission coordinator (`electron/permission-coordinator.ts`) to allow one. +- **Permissions:** camera, mic, location, notifications and clipboard are denied + via `setPermissionRequestHandler`. +- **Popups:** `setWindowOpenHandler` opens them as new hidden pages in the same + session, counted against the page limit. +- **Credentials:** the service never fills saved passwords. Typing into a + password field is allowed only when the agent's own text is the value. +- **Audit:** every navigation and tool call is recorded on the thread, so the + user can see what an agent did. + +## Lifecycle invariants + +- **Trusted state:** `BrowserService.sessions: Map`. +- **Writers:** `attachMcpServers` (create), `bindSession` (assign session id), + tool calls (add and remove pages), `releaseSessionMcp` and `dispose` (destroy). +- **Session close or agent crash:** `releaseSessionMcp` closes every page, + detaches the debugger and clears the partition before unregistering the + token. +- **Pipper quit:** `dispose` runs inside `bounded-shutdown.ts` with a 2 s + budget; pages are destroyed, not closed gracefully. +- **Renderer crash (`render-process-gone`):** the page is removed and the next + call returns "page crashed; open it again". The session stays usable. +- **Reconnect and resume:** a resumed ACP session gets a new token and starts + with no pages. Pages are never carried across a reconnect. +- **Unbound tokens:** a token never bound to a session is released after 60 s. + +## Tests + +Behaviour-level, in line with `AGENT.md`: + +- A session's pages close when the session closes or its agent crashes. +- Two sessions can't see each other's cookies or storage. +- Blocked schemes and Pipper origins can't be opened, including via redirects + and popups. +- Hitting the page limit returns an actionable error and leaves existing pages + working. +- A stale ref errors instead of clicking another element. +- `preview_widget` reports a console error, a blocked script and a hardcoded + colour from a deliberately broken widget. +- Agents without HTTP MCP reach the same tools through the stdio shim. + +## Open questions + +1. Should `browser_*` tools be on by default for every agent, or enabled per + project? +2. Should a browser pane in the UI mirror an agent's page live, or only show + screenshots? +3. Should sessions be allowed to keep sign-in state across turns (a persistent + partition per project), and if so, how does the user clear it? diff --git a/docs/inline-widgets/design-guide.md b/docs/inline-widgets/design-guide.md new file mode 100644 index 0000000..c5784d5 --- /dev/null +++ b/docs/inline-widgets/design-guide.md @@ -0,0 +1,244 @@ +# Inline widget design guide + +This is the text the `widget_guide` tool returns to the agent before it draws a +widget. It translates Pipper's Fluid Functionalism system (see `design.md` and +`src/index.css`) into rules that work inside a sandboxed frame, where the agent +writes plain HTML/SVG and cannot import React components. + +The guide is split into modules so the agent loads only what it needs: +`core` (always), `diagram`, `chart`, `table`, `interactive`. Everything below the +"Agent-facing text" line is returned verbatim. + +## Host contract (not sent to the agent) + +The frame must provide, before any widget code runs: + +1. **Tokens.** The `:root` / `[data-workspace-theme]` blocks from `src/index.css`, + generated at build time into `widget-frame.css`. Same variable names as the + app, so a token change ships to widgets automatically. +2. **Theme scope.** `data-workspace-theme="light|dark"` on the frame root, updated + via `postMessage` when the app theme changes. Widgets never read + `prefers-color-scheme` themselves. +3. **Shape mode.** `data-shape="pill|rounded"` on the root, following the app's + **R** toggle. `--ff-radius` resolves from it. +4. **Motion.** `--ff-ease-fast|moderate|slow` and `--ff-dur-fast|moderate|slow`, + approximating `src/lib/springs.ts` (0.08s / 0.16s / 0.24s). Durations collapse + to `0s` under `prefers-reduced-motion`. +5. **Fonts.** Inter Variable and JetBrains Mono Variable, self-hosted. +6. **Icons.** Phosphor regular icon font, exposed as ``. +7. **Component kit.** The `.ff-*` classes listed below, implemented in + `widget-frame.css` on top of the tokens. +8. **Runtime API.** `pipper.sendPrompt(text)`, `pipper.openLink(url)`. Height is + measured by the host with a `ResizeObserver`; widgets never set it. +9. **CSP.** Scripts only from the frame origin and the CDN allowlist below. + +--- + +## Agent-facing text + +### core + +You are drawing a widget that renders inline in a Pipper thread. It must look +like part of Pipper, not like an embedded web page. The user should not be able +to tell where the app ends and the widget begins. + +**Output shape** + +- Write an HTML fragment or a single ``. No ``, ``, + `` or ``. +- Order: one short ` + +
+`; + return cachedHtml; +} diff --git a/electron/widgets/widget-guide.ts b/electron/widgets/widget-guide.ts new file mode 100644 index 0000000..da1c6a3 --- /dev/null +++ b/electron/widgets/widget-guide.ts @@ -0,0 +1,37 @@ +import guideMarkdown from "../../docs/inline-widgets/design-guide.md?raw"; + +/** + * Serves the agent-facing part of `docs/inline-widgets/design-guide.md` + * module by module, so the doc stays the single source for widget rules. + */ + +export const WIDGET_GUIDE_MODULES = ["core", "chart", "diagram", "table", "interactive"] as const; +export type WidgetGuideModule = (typeof WIDGET_GUIDE_MODULES)[number]; + +export function parseGuideModules(markdown: string): Map { + const start = markdown.indexOf("## Agent-facing text"); + const body = start >= 0 ? markdown.slice(start) : markdown; + const modules = new Map(); + const parts = body.split(/^### /m).slice(1); + for (const part of parts) { + const newline = part.indexOf("\n"); + const name = part.slice(0, newline).trim(); + modules.set(name, part.slice(newline + 1).trim()); + } + return modules; +} + +let cached: Map | null = null; + +export function widgetGuide(requested: readonly string[]): string { + cached ??= parseGuideModules(guideMarkdown); + const names = ["core", ...requested.filter((name) => name !== "core")]; + const sections: string[] = []; + for (const name of names) { + const text = cached.get(name); + if (!text) + throw new Error(`Unknown guide module "${name}"; use ${WIDGET_GUIDE_MODULES.join(", ")}`); + sections.push(`# ${name}\n\n${text}`); + } + return sections.join("\n\n"); +} diff --git a/electron/widgets/widget-lint.ts b/electron/widgets/widget-lint.ts new file mode 100644 index 0000000..4d99d11 --- /dev/null +++ b/electron/widgets/widget-lint.ts @@ -0,0 +1,83 @@ +import { WIDGET_SCRIPT_ORIGINS } from "./widget-frame-origins.ts"; + +/** + * Checks on widget source text, before it runs. They mirror the "never" list + * in the design guide (`docs/inline-widgets/design-guide.md`); the in-frame + * checks in `widget-frame.ts` cover what only rendering can show (overflow, + * text size, contrast). + */ + +export interface LintFinding { + rule: string; + message: string; + line?: number; +} + +const HEX_COLOR = /#(?:[0-9a-f]{8}|[0-9a-f]{6}|[0-9a-f]{3,4})\b/gi; +const COLOR_FUNCTION = /\b(?:rgba?|hsla?|oklch|oklab|lab|lch|hwb)\(\s*[\d.]/gi; +const SCRIPT_SRC = /]*\bsrc\s*=\s*["']([^"']+)["']/gi; + +export function lintWidgetSource(code: string): LintFinding[] { + const findings: LintFinding[] = []; + const lines = code.split("\n"); + const add = (rule: string, message: string, line?: number) => { + if (findings.length < 30) findings.push({ rule, message, ...(line ? { line } : {}) }); + }; + + if (/]|]|]/i.test(code)) { + add("document-wrapper", "Send a fragment: no , , or ."); + } + + lines.forEach((text, index) => { + const line = index + 1; + for (const match of text.matchAll(HEX_COLOR)) { + const before = text.slice(0, match.index); + if (/(?:href|xlink:href)\s*=\s*["']$/i.test(before) || /url\(\s*["']?$/i.test(before)) + continue; + add( + "hardcoded-color", + `${match[0]} on line ${line}; use a token such as var(--foreground).`, + line, + ); + } + for (const match of text.matchAll(COLOR_FUNCTION)) { + add("hardcoded-color", `${match[0]}…) on line ${line}; use a token instead.`, line); + } + if (/position\s*:\s*fixed/i.test(text)) { + add( + "fixed-position", + `position: fixed on line ${line}; the frame sizes to in-flow content.`, + line, + ); + } + if (/\b(?:request|exit)Fullscreen\b|fullscreenElement/.test(text)) { + add("fullscreen", `Fullscreen API on line ${line}; toggle an expanded layout instead.`, line); + } + if (/\bwindow\.open\s*\(|\btop\.location\b|\bparent\.location\b/.test(text)) { + add("navigation", `Line ${line} opens or navigates windows; use pipper.openLink(url).`, line); + } + if (/\b(?:localStorage|sessionStorage)\b/.test(text)) { + add("storage", `Line ${line} uses browser storage, which widgets can't rely on.`, line); + } + if (/font-weight\s*:\s*(?:[6-9]00|bold(?:er)?)\b/i.test(text)) { + add("font-weight", `Line ${line} uses a heavy weight; use 400 or 500.`, line); + } + }); + + for (const match of code.matchAll(SCRIPT_SRC)) { + const src = match[1]; + let origin: string | null = null; + try { + origin = new URL(src).origin; + } catch { + origin = null; + } + if (!origin || !WIDGET_SCRIPT_ORIGINS.includes(origin)) { + add("script-origin", `${src} isn't on an allowed CDN (${WIDGET_SCRIPT_ORIGINS.join(", ")}).`); + } else if (!/@\d+\.\d+\.\d+/.test(src) && !/\/\d+\.\d+\.\d+\//.test(src)) { + add("unpinned-script", `${src} has no exact version; pin one, e.g. chart.js@4.4.1.`); + } + } + + return findings; +} diff --git a/electron/widgets/widget-preview.ts b/electron/widgets/widget-preview.ts new file mode 100644 index 0000000..c3aeaf6 --- /dev/null +++ b/electron/widgets/widget-preview.ts @@ -0,0 +1,193 @@ +import type { McpContent, McpToolResult } from "../mcp/mcp-http-server.ts"; +import type { BrowserPage, PageHost } from "../browser/browser-page.ts"; +import type { UrlDecision } from "../browser/url-policy.ts"; +import { buildWidgetFrameHtml } from "./widget-frame.ts"; +import { lintWidgetSource, type LintFinding } from "./widget-lint.ts"; + +/** + * Renders widget code in the real widget frame on a headless page and + * reports what the agent needs to fix before showing it: screenshots in both + * themes, measured size, script errors, blocked resources and design checks. + * + * One page is shared by all sessions (previews are short and serialized), + * kept in its own partition and recycled after a number of uses. + */ + +export interface WidgetPreviewOptions { + host: PageHost; + checkUrl: (url: string) => UrlDecision; + downloadDir: string; + navigationTimeoutMs: number; + screenshotMaxBytes: number; +} + +interface FrameState { + size: { width: number; height: number }; + errors: string[]; + blocked: Array<{ url: string; reason: string }>; + actions: Array<{ type: string }>; + lint: LintFinding[]; +} + +const PARTITION = "pipper-widget-preview"; +const RECYCLE_AFTER_USES = 50; +const MAX_CAPTURE_HEIGHT = 4000; +export const DEFAULT_WIDGET_WIDTH = 680; +const MIN_WIDGET_WIDTH = 280; +const MAX_WIDGET_WIDTH = 1200; + +export class WidgetPreviewer { + private readonly options: WidgetPreviewOptions; + private page: BrowserPage | null = null; + private uses = 0; + private queue: Promise = Promise.resolve(); + private disposed = false; + + constructor(options: WidgetPreviewOptions) { + this.options = options; + } + + preview(code: string, width = DEFAULT_WIDGET_WIDTH): Promise { + const run = this.queue.then(() => this.render(code, width)); + this.queue = run.catch(() => undefined); + return run; + } + + dispose(): void { + this.disposed = true; + this.page?.close(); + this.page = null; + } + + private async render(code: string, requestedWidth: number): Promise { + if (this.disposed) throw new Error("Widget previews are shutting down."); + const width = Math.round( + Math.min( + MAX_WIDGET_WIDTH, + Math.max(MIN_WIDGET_WIDTH, requestedWidth || DEFAULT_WIDGET_WIDTH), + ), + ); + const sourceFindings = lintWidgetSource(code); + const page = await this.freshPage(); + const html = buildWidgetFrameHtml(); + await page.navigate( + `data:text/html;base64,${Buffer.from(html).toString("base64")}`, + this.options.navigationTimeoutMs, + ); + + const light = (await page.evaluate( + `window.__pipperRender(${JSON.stringify(code)}, ${JSON.stringify({ width, theme: "light" })})`, + )) as FrameState; + const height = Math.max(1, Math.min(MAX_CAPTURE_HEIGHT, light.size.height)); + const clip = { x: 0, y: 0, width, height }; + const lightShot = await page.screenshot({ + clip, + maxWidth: width, + maxBytes: this.options.screenshotMaxBytes, + }); + const dark = (await page.evaluate(`window.__pipperSetTheme("dark")`)) as FrameState; + const darkShot = await page.screenshot({ + clip, + maxWidth: width, + maxBytes: this.options.screenshotMaxBytes, + }); + + const frameFindings = mergeFindings(light.lint, dark.lint); + if (light.size.height > MAX_CAPTURE_HEIGHT) { + frameFindings.push({ + rule: "too-tall", + message: `Content is ${light.size.height}px tall; screenshots stop at ${MAX_CAPTURE_HEIGHT}px. Keep widgets compact.`, + }); + } + const findings = [...sourceFindings, ...frameFindings]; + const errors = unique([...light.errors, ...dark.errors]); + const blocked = uniqueBy([...light.blocked, ...dark.blocked], (item) => item.url); + + const lines = [`Rendered at ${width}×${light.size.height}px.`]; + lines.push( + errors.length === 0 ? "Script errors: none" : `Script errors:\n${bullets(errors)}`, + blocked.length === 0 + ? "Blocked or missing resources: none" + : `Blocked or missing resources:\n${bullets(blocked.map((b) => `${b.url} (${b.reason})`))}`, + findings.length === 0 + ? "Design checks: all passed" + : `Design checks:\n${bullets(findings.map((f) => `[${f.rule}] ${f.message}`))}`, + ); + const clean = errors.length === 0 && blocked.length === 0 && findings.length === 0; + lines.push( + clean + ? "Clean. Show it with the same code." + : "Fix these and preview again before showing the widget.", + ); + + const content: McpContent[] = [ + { type: "text", text: lines.join("\n\n") }, + { type: "text", text: "Light theme:" }, + { type: "image", data: lightShot.data, mimeType: lightShot.mimeType }, + { type: "text", text: "Dark theme:" }, + { type: "image", data: darkShot.data, mimeType: darkShot.mimeType }, + ]; + return { content }; + } + + private async freshPage(): Promise { + if (this.page && (this.page.closed || this.uses >= RECYCLE_AFTER_USES)) { + this.page.close(); + this.page = null; + } + if (!this.page) { + this.uses = 0; + const page = await this.options.host.createPage({ + partition: PARTITION, + checkUrl: this.options.checkUrl, + downloadDir: this.options.downloadDir, + events: { + onCrash: () => { + if (this.page === page) this.page = null; + }, + onPopup: () => undefined, + onClosed: () => { + if (this.page === page) this.page = null; + }, + onDownload: async () => false, + onDownloadFinished: () => undefined, + }, + }); + if (this.disposed) { + page.close(); + throw new Error("Widget previews are shutting down."); + } + this.page = page; + } + this.uses++; + return this.page; + } +} + +function mergeFindings(light: LintFinding[], dark: LintFinding[]): LintFinding[] { + const out = light.map((finding) => ({ ...finding })); + const seen = new Set(light.map((finding) => finding.message)); + for (const finding of dark) { + if (seen.has(finding.message)) continue; + out.push({ ...finding, message: `In dark theme: ${finding.message}` }); + } + return out; +} + +function unique(items: string[]): string[] { + return [...new Set(items)]; +} + +function uniqueBy(items: T[], key: (item: T) => string): T[] { + const seen = new Set(); + return items.filter((item) => { + const k = key(item); + if (seen.has(k)) return false; + seen.add(k); + return true; + }); +} + +function bullets(items: string[]): string { + return items.map((item) => `- ${item}`).join("\n"); +} diff --git a/electron/widgets/widget-tokens.ts b/electron/widgets/widget-tokens.ts new file mode 100644 index 0000000..5ffbfbd --- /dev/null +++ b/electron/widgets/widget-tokens.ts @@ -0,0 +1,33 @@ +/** + * Pulls the light and dark token blocks out of the app stylesheet, so widgets + * use exactly the same variables as Pipper's UI. A token change in + * `src/index.css` reaches widgets with no second copy to update. + */ + +const LIGHT_SELECTOR = ':root,\n[data-workspace-theme="light"] {'; +const DARK_SELECTOR = '.dark,\n[data-workspace-theme="dark"] {'; + +export function extractThemeTokens(css: string): string { + const light = extractBlock(css, LIGHT_SELECTOR); + const dark = extractBlock(css, DARK_SELECTOR); + if (!light || !dark) { + throw new Error("Couldn't find the light and dark token blocks in index.css"); + } + return `${light}\n${dark}`; +} + +/** The rule starting at `selector`, through its matching closing brace. */ +function extractBlock(css: string, selector: string): string | null { + const start = css.indexOf(selector); + if (start < 0) return null; + let depth = 0; + for (let i = start + selector.length - 1; i < css.length; i++) { + const char = css[i]; + if (char === "{") depth++; + else if (char === "}") { + depth--; + if (depth === 0) return css.slice(start, i + 1); + } + } + return null; +} diff --git a/electron/widgets/widgets.test.ts b/electron/widgets/widgets.test.ts new file mode 100644 index 0000000..ffed672 --- /dev/null +++ b/electron/widgets/widgets.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, test } from "vitest"; +import phosphorFont from "@phosphor-icons/web/regular/Phosphor.woff2?inline"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { BrowserPage, PageEvents, PageHost } from "../browser/browser-page.ts"; +import { buildWidgetFrameHtml } from "./widget-frame.ts"; +import { parseGuideModules, widgetGuide } from "./widget-guide.ts"; +import { lintWidgetSource } from "./widget-lint.ts"; +import { WidgetPreviewer } from "./widget-preview.ts"; +import { extractThemeTokens } from "./widget-tokens.ts"; + +const rules = (code: string) => lintWidgetSource(code).map((finding) => finding.rule); + +describe("widget source checks", () => { + test("token-only widgets built from the kit pass", () => { + const code = `

Summary

+
+412Details
+`; + expect(lintWidgetSource(code)).toEqual([]); + }); + + test("hardcoded colours are reported with their line", () => { + const findings = lintWidgetSource( + `
\n

x

\n`, + ); + expect(findings.map((f) => [f.rule, f.line])).toEqual([ + ["hardcoded-color", 2], + ["hardcoded-color", 3], + ]); + }); + + test("banned patterns from the guide are caught", () => { + expect(rules("x")).toContain("document-wrapper"); + expect(rules('
')).toContain("fixed-position"); + expect(rules("")).toContain("fullscreen"); + expect(rules("")).toContain("navigation"); + expect(rules("")).toContain("storage"); + expect(rules('x')).toContain("font-weight"); + }); + + test("scripts must come from an allowed CDN at an exact version", () => { + expect(rules('')).toEqual(["script-origin"]); + expect(rules('')).toEqual(["unpinned-script"]); + }); +}); + +describe("widget frame", () => { + test("uses the app's own light and dark tokens", () => { + const css = readFileSync(join(import.meta.dirname, "../../src/index.css"), "utf8"); + const tokens = extractThemeTokens(css); + expect(tokens).toContain('[data-workspace-theme="light"]'); + expect(tokens).toContain('[data-workspace-theme="dark"]'); + expect(tokens).toContain("--workspace-fill-ready"); + + const html = buildWidgetFrameHtml(); + expect(html).toContain(tokens); + expect(html).toContain(".ff-badge"); + expect(html).toContain("window.pipper"); + }); + + test("icons and fonts are self-contained and scripts are limited to the allowlist", () => { + const html = buildWidgetFrameHtml(); + // Production builds inline this as a data: URL; Vitest serves a path. + expect(html).toContain(`src: url("${phosphorFont}") format("woff2");`); + expect(html).not.toContain('url("./Phosphor'); + expect(html).toMatch(/script-src 'unsafe-inline' https:\/\/cdnjs\.cloudflare\.com/); + expect(html).not.toMatch(/]+https?:/); + }); + + test("a stylesheet without the token blocks is rejected loudly", () => { + expect(() => extractThemeTokens(":root { --x: 1 }")).toThrow(/token blocks/); + }); +}); + +describe("widget guide", () => { + test("splits the agent-facing doc into modules", () => { + const modules = parseGuideModules( + "# Title\n## Host contract\n### not-this\nsecret\n## Agent-facing text\n### core\nRules\n### chart\nCharts", + ); + expect([...modules.keys()]).toEqual(["core", "chart"]); + expect(modules.get("core")).toBe("Rules"); + }); + + test("unknown modules are an error the agent can act on", () => { + expect(() => widgetGuide(["maps"])).toThrow(/core, chart/); + }); +}); + +describe("widget preview", () => { + function fakeHost(state: Record) { + const evaluated: string[] = []; + let created = 0; + const host: PageHost = { + async createPage(options: { events: PageEvents }) { + created++; + const page: Partial = { + closed: false, + navigate: async (url) => ({ url, title: "", status: null }), + evaluate: async (expression) => { + evaluated.push(expression); + return state; + }, + screenshot: async () => ({ data: "cG5n", mimeType: "image/png" }), + close: () => options.events.onClosed(), + }; + return page as BrowserPage; + }, + clearPartition: async () => undefined, + }; + return { host, evaluated, created: () => created }; + } + + const preview = (state: Record) => { + const fake = fakeHost(state); + const previewer = new WidgetPreviewer({ + host: fake.host, + checkUrl: () => ({ allowed: true }), + downloadDir: "/tmp/unused", + navigationTimeoutMs: 1000, + screenshotMaxBytes: 1_000_000, + }); + return { previewer, fake }; + }; + + test("a clean widget gets both themes and a go-ahead", async () => { + const { previewer, fake } = preview({ + size: { width: 680, height: 120 }, + errors: [], + blocked: [], + actions: [], + lint: [], + }); + const result = await previewer.preview('
ok
', 360); + const text = result.content.filter((c) => c.type === "text").map((c) => c.text); + expect(text[0]).toContain("Rendered at 360×120px"); + expect(text[0]).toContain("Design checks: all passed"); + expect(text[0]).toContain("Clean."); + expect(text).toContain("Light theme:"); + expect(text).toContain("Dark theme:"); + expect(result.content.filter((c) => c.type === "image")).toHaveLength(2); + expect(fake.evaluated[0]).toContain('"theme":"light"'); + expect(fake.evaluated[1]).toContain('__pipperSetTheme("dark")'); + previewer.dispose(); + }); + + test("errors, blocked resources and design problems are all reported", async () => { + const { previewer } = preview({ + size: { width: 680, height: 5000 }, + errors: ["Uncaught ReferenceError: x is not defined"], + blocked: [{ url: "https://evil.example/x.js", reason: "blocked by script-src-elem" }], + actions: [], + lint: [{ rule: "small-text", message: 'Text "a" is 9px; the minimum is 11px.' }], + }); + const result = await previewer.preview('

a

'); + const summary = result.content[0].type === "text" ? result.content[0].text : ""; + expect(summary).toContain("ReferenceError"); + expect(summary).toContain("https://evil.example/x.js"); + expect(summary).toContain("[hardcoded-color]"); + expect(summary).toContain("[small-text]"); + expect(summary).toContain("[too-tall]"); + expect(summary).toContain("Fix these"); + previewer.dispose(); + }); + + test("one page is reused across previews", async () => { + const { previewer, fake } = preview({ + size: { width: 680, height: 10 }, + errors: [], + blocked: [], + actions: [], + lint: [], + }); + await previewer.preview("

a

"); + await previewer.preview("

b

"); + expect(fake.created()).toBe(1); + previewer.dispose(); + }); +}); diff --git a/package.json b/package.json index b5f2b28..310c87f 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@hugeicons/core-free-icons": "^4.3.0", "@hugeicons/react": "^1.1.10", "@phosphor-icons/react": "^2.1.10", + "@phosphor-icons/web": "2.1.1", "@pierre/diffs": "^1.3.6", "@pierre/trees": "^1.0.0-beta.6", "@radix-ui/react-accordion": "^1.2.20", diff --git a/src/electron.d.ts b/src/electron.d.ts index c45812e..bbb0457 100644 --- a/src/electron.d.ts +++ b/src/electron.d.ts @@ -30,6 +30,7 @@ import type { McpServerInput, McpServerRecord, SessionConfigOption, + BrowserConfig, SubagentConfig, SubagentRunSnapshot, } from "../../contracts/acp.ts"; @@ -325,6 +326,10 @@ declare global { setConfig: (partial: Partial) => Promise; listRuns: () => Promise; }; + browser: { + getConfig: () => Promise; + setConfig: (partial: Partial) => Promise; + }; mcp: { list: () => Promise; create: (input: McpServerInput) => Promise; diff --git a/src/settings/app.tsx b/src/settings/app.tsx index bffa03c..f7ceb84 100644 --- a/src/settings/app.tsx +++ b/src/settings/app.tsx @@ -16,7 +16,7 @@ import { Elevated } from "@/lib/elevated"; import type { IconComponent } from "@/lib/icon-context"; import { useAgentRegistryStore } from "@/store/agent-registry-store"; import { useTabGestureStore } from "@/store/tab-gesture-store"; -import type { AcpAgentDescriptor } from "../../contracts/acp.ts"; +import type { AcpAgentDescriptor, BrowserConfig } from "../../contracts/acp.ts"; import type { WorkspaceDiskUsageReport } from "../../contracts/storage.ts"; import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; @@ -33,6 +33,7 @@ import { SidebarMenuItem, SidebarProvider, } from "@/components/ui/sidebar"; +import { SliderComfortable } from "@/components/ui/slider"; import { Switch } from "@/components/ui/switch"; import { Accordion, @@ -52,6 +53,7 @@ type SectionId = | "design" | "appearance" | "agents" + | "browser" | "accounts" | "keyboard" | "power" @@ -62,6 +64,7 @@ const NAV_ITEMS: Array<{ id: SectionId; label: string }> = [ { id: "appearance", label: "Appearance" }, { id: "design", label: "Design system" }, { id: "agents", label: "Agents" }, + { id: "browser", label: "Browser" }, { id: "accounts", label: "Account" }, { id: "keyboard", label: "Keyboard & trackpad" }, { id: "power", label: "Power" }, @@ -354,6 +357,10 @@ const SECTION_META: Record = { title: "Agents", blurb: "Choose which coding agents Pipper can use.", }, + browser: { + title: "Agent browser", + blurb: "Headless pages your agents can open to read docs and test web apps.", + }, accounts: { title: "Account", blurb: "", @@ -683,6 +690,140 @@ function StorageView() { ); } +const IDLE_MINUTE_CHOICES = [1, 2, 5, 10, 15, 30, 60]; +const SAVE_DEBOUNCE_MS = 300; + +function closestIndex(choices: number[], value: number): number { + let best = 0; + for (let i = 1; i < choices.length; i++) { + if (Math.abs(choices[i] - value) < Math.abs(choices[best] - value)) best = i; + } + return best; +} + +function BrowserView() { + const [config, setConfig] = useState(null); + const [error, setError] = useState(null); + const pending = useRef>({}); + const timer = useRef | null>(null); + + useEffect(() => { + let disposed = false; + window.omni.browser + .getConfig() + .then((value) => { + if (!disposed) setConfig(value); + }) + .catch(() => { + if (!disposed) setError("Could not load browser settings."); + }); + return () => { + disposed = true; + if (timer.current) clearTimeout(timer.current); + }; + }, []); + + // Sliders report every step while dragging; save once they settle. + const update = (partial: Partial, immediate = false) => { + setConfig((current) => (current ? { ...current, ...partial } : current)); + pending.current = { ...pending.current, ...partial }; + if (timer.current) clearTimeout(timer.current); + const save = () => { + const changes = pending.current; + pending.current = {}; + setError(null); + window.omni.browser + .setConfig(changes) + .then(setConfig) + .catch(() => setError("Could not save browser settings.")); + }; + if (immediate) save(); + else timer.current = setTimeout(save, SAVE_DEBOUNCE_MS); + }; + + const disabled = config === null; + return ( + <> + +
+ + config && update({ enabled: !config.enabled }, true)} + className="gap-0 px-0 py-0 [&>span:last-of-type]:sr-only" + data-pipper-id="settings-browser-enabled" + /> + + + + config && update({ downloads: config.downloads === "ask" ? "block" : "ask" }, true) + } + className="gap-0 px-0 py-0 [&>span:last-of-type]:sr-only" + data-pipper-id="settings-browser-downloads" + /> + +
+ update({ pagesPerSession: value })} + min={1} + max={8} + step={1} + variant="pips" + label="Pages per session" + formatValue={(value) => String(value)} + disabled={disabled || !config?.enabled} + data-pipper-id="settings-browser-pages" + /> +
+
+ update({ idleMinutes: IDLE_MINUTE_CHOICES[index] ?? 5 })} + min={0} + max={IDLE_MINUTE_CHOICES.length - 1} + step={1} + variant="pips" + label="Close idle pages after" + formatValue={(index) => `${IDLE_MINUTE_CHOICES[index] ?? 5} min`} + disabled={disabled || !config?.enabled} + data-pipper-id="settings-browser-idle" + /> +
+
+ + {error && ( +

+ {error} +

+ )} + + + What agents can and can't open + + Each session browses in its own private storage, so agents never share cookies with each + other or with you. Websites and local dev servers work. Local files, browser settings + pages and Pipper itself are blocked, and camera, microphone and location requests are + denied. + + + + + ); +} + function RemoteView() { return ( <> @@ -760,6 +901,7 @@ export function SettingsApp() { {section === "design" && } {section === "appearance" && } {section === "agents" && } + {section === "browser" && } {section === "accounts" && } {section === "keyboard" && } {section === "power" && } diff --git a/vitest.config.ts b/vitest.config.ts index 180d156..a7e412b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -36,6 +36,9 @@ export default defineConfig({ "scripts/**/*.test.ts", ...(expoInstalled ? ["native/pipper-remote-expo/src/**/*.test.ts"] : []), ], + // Vitest blanks CSS by default, which would make `?raw` stylesheet + // imports empty; the widget frame reads its tokens and icon styles that way. + css: { include: [/src\/index\.css/, /@phosphor-icons\/web/] }, clearMocks: true, restoreMocks: true, unstubGlobals: true, From 272ae5836057651679e5f7ee42ecb42ae28421a8 Mon Sep 17 00:00:00 2001 From: Harshith Pasupuleti Date: Sat, 24 Oct 2026 15:36:37 +0530 Subject: [PATCH 03/10] feat: show agent widgets inline in the thread Agents could preview widgets but had no way to show them. Add show_widget and update_widget to the pipper-browser MCP server and render shown widgets in the conversation. - show_widget validates the code (size, document wrapper, script origins), stores it in a new inline_widgets table and returns `pipper-widget:` as the first line of its result. That line reaches the tool call's output preview for any agent, so the renderer finds widgets there and shows them between the turn's text, in order. - update_widget replaces a widget's code (same session or thread only) and emits widget-updated so the frame reloads in place. - The thread frame is an iframe with sandbox="allow-scripts" loading the same document preview_widget uses. It sizes to its content, follows the app theme and shape over postMessage, opens https links after a confirm, and sends prompts only from a click or keypress in the widget. - The clean-preview message now points at show_widget. Co-Authored-By: Claude Opus 5.5 --- contracts/acp.ts | 28 +++ docs/inline-widgets/browser-service.md | 16 +- docs/inline-widgets/tool-descriptions.md | 12 +- electron/agent-connection-manager.ts | 5 + .../browser/browser-service.behavior.test.ts | 97 ++++++++++ electron/browser/browser-service.ts | 106 +++++++++++ electron/browser/browser-tools.ts | 45 +++++ electron/db.ts | 15 ++ electron/main.ts | 21 ++- electron/preload.ts | 5 + electron/widgets/widget-frame.ts | 71 +++++++- electron/widgets/widget-preview.ts | 2 +- electron/widgets/widget-store.ts | 97 ++++++++++ electron/widgets/widgets.test.ts | 16 +- src/components/agent-panel.tsx | 47 +++-- src/components/inline-widget.tsx | 167 ++++++++++++++++++ src/electron.d.ts | 4 + src/lib/inline-widgets.test.ts | 50 ++++++ src/lib/inline-widgets.ts | 46 +++++ 19 files changed, 819 insertions(+), 31 deletions(-) create mode 100644 electron/widgets/widget-store.ts create mode 100644 src/components/inline-widget.tsx create mode 100644 src/lib/inline-widgets.test.ts create mode 100644 src/lib/inline-widgets.ts diff --git a/contracts/acp.ts b/contracts/acp.ts index 1019fa6..e402ab5 100644 --- a/contracts/acp.ts +++ b/contracts/acp.ts @@ -382,6 +382,8 @@ export type AcpBridgeEvent = | { type: "stop"; sessionId: string; threadId: string | null; stopReason: string } /** Thread IDs whose agent is currently streaming, across all open threads. */ | { type: "running-threads"; threadIds: string[] } + /** A widget's code changed (update_widget); frames showing it reload. */ + | { type: "widget-updated"; widgetId: string; threadId: string | null } /** Snapshot of all subagent runs whenever any run changes state. */ | { type: "subagent-runs"; runs: SubagentRunSnapshot[] } /** Provider model catalogs that changed (cached in main; see getModelCatalogs). */ @@ -390,6 +392,32 @@ export type AcpBridgeEvent = catalogs: Record>; }; +/** A visual an agent showed in a thread with the `show_widget` tool. */ +export interface InlineWidgetRecord { + /** `wdg_…`; referenced from the tool result as `pipper-widget:`. */ + id: string; + threadId: string | null; + sessionId: string | null; + /** snake_case name chosen by the agent; also the download name. */ + title: string; + /** Plain-text fallback for screen readers, exports and failed renders. */ + summary: string; + /** HTML fragment or single . */ + code: string; + createdAt: number; + updatedAt: number; +} + +/** What the renderer loads to display a widget: its sandboxed document. */ +export interface InlineWidgetDocument { + id: string; + title: string; + summary: string; + updatedAt: number; + /** Complete HTML document for the widget's sandboxed frame. */ + document: string; +} + /** * User preferences for the agent browser (headless pages agents drive over * MCP). Persisted as `browser.json` in the app's userData directory. diff --git a/docs/inline-widgets/browser-service.md b/docs/inline-widgets/browser-service.md index 4e9530d..827cac5 100644 --- a/docs/inline-widgets/browser-service.md +++ b/docs/inline-widgets/browser-service.md @@ -217,8 +217,20 @@ Where the code differs from the design above: endpoints with the stdio fallback; limits; idle and memory sweeps; blocked schemes, Pipper's dev renderer origin, permissions and popups; crash handling; per-session call serialization; stale-ref errors. -- **Not built yet:** `show_widget` / `update_widget` and the thread's widget - timeline item, and a browser pane. +- **Shown widgets:** `show_widget` stores the code in the `inline_widgets` table + and returns `pipper-widget:` as the first line of its result. That line + survives into the tool call's output preview for any agent, so the renderer + finds widgets there (`src/lib/inline-widgets.ts`) and shows them between the + turn's text, in order. `update_widget` replaces the code and emits a + `widget-updated` event so the frame reloads in place. +- **Thread frame:** `src/components/inline-widget.tsx` loads the document from + `widgets:getDocument` into an `
412 passed
', + summary: "412 tests passed and 3 failed.", + }); + expect(shown.isError).toBe(false); + const id = /^pipper-widget:(wdg_[a-f0-9]{32})\n/.exec(shown.text)?.[1]; + expect(id).toBeDefined(); + expect(widgetStore.get(id!)).toMatchObject({ + threadId: "thread-1", + sessionId: "s1", + title: "test_results", + code: '
412 passed
', + }); + }); + + test("update_widget replaces code in place and tells the renderer", async () => { + const widgetStore = memoryWidgetStore(); + const updates: Array<{ widgetId: string; threadId: string | null }> = []; + const service = new BrowserService({ + host: new FakeHost(), + widgetStore, + baseDir: await mkdtemp(join(tmpdir(), "pipper-browser-test-")), + }); + services.push(service); + service.setWidgetHooks({ + resolveThread: (sessionId) => `thread-of-${sessionId}`, + onWidgetUpdated: (update) => updates.push(update), + }); + const mine = await openSession(service, "s1"); + const other = await openSession(service, "s2"); + const shown = await mine.call("show_widget", { + title: "chart", + widget_code: "

v1

", + summary: "First version.", + }); + const id = /pipper-widget:(wdg_[a-f0-9]{32})/.exec(shown.text)![1]; + + const updated = await mine.call("update_widget", { widget_id: id, widget_code: "

v2

" }); + expect(updated.isError).toBe(false); + expect(updated.text.startsWith(`pipper-widget-updated:${id}`)).toBe(true); + expect(widgetStore.get(id)?.code).toBe("

v2

"); + expect(widgetStore.get(id)?.summary).toBe("First version."); + expect(updates).toEqual([{ widgetId: id, threadId: "thread-of-s1" }]); + + const foreign = await other.call("update_widget", { widget_id: id, widget_code: "

x

" }); + expect(foreign.isError).toBe(true); + expect(widgetStore.get(id)?.code).toBe("

v2

"); + }); + + test("widgets that can't run are refused with what to fix", async () => { + const service = new BrowserService({ + host: new FakeHost(), + widgetStore: memoryWidgetStore(), + baseDir: await mkdtemp(join(tmpdir(), "pipper-browser-test-")), + }); + services.push(service); + const { call } = await openSession(service, "s1"); + const bad = await call("show_widget", { + title: "bad", + widget_code: '', + summary: "Broken.", + }); + expect(bad.isError).toBe(true); + expect(bad.text).toContain("preview_widget"); + const badTitle = await call("show_widget", { + title: "Not Snake", + widget_code: "

x

", + summary: "x", + }); + expect(badTitle.isError).toBe(true); + }); + + test("without a widget store, show_widget says widgets aren't supported", async () => { + const { service } = await setup(); + const { call } = await openSession(service, "s1"); + const result = await call("show_widget", { + title: "abc", + widget_code: "

x

", + summary: "x", + }); + expect(result.isError).toBe(true); + expect(result.text).toContain("can't be shown"); + }); }); diff --git a/electron/browser/browser-service.ts b/electron/browser/browser-service.ts index 798478c..852ab8a 100644 --- a/electron/browser/browser-service.ts +++ b/electron/browser/browser-service.ts @@ -25,6 +25,8 @@ import { } from "./browser-tools.ts"; import { buildSnapshot } from "./snapshot.ts"; import { widgetGuide } from "../widgets/widget-guide.ts"; +import { lintWidgetSource } from "../widgets/widget-lint.ts"; +import type { WidgetStore } from "../widgets/widget-store.ts"; import { WidgetPreviewer } from "../widgets/widget-preview.ts"; import { withTimeout } from "./timing.ts"; import { checkUrl, normalizeUserUrl, type UrlDecision } from "./url-policy.ts"; @@ -86,10 +88,23 @@ export interface BrowserServiceOptions { baseDir?: string; /** Parent folder for per-session download folders; defaults to temp. */ downloadsDir?: string; + /** Where show_widget saves widgets; without one, widgets can't be shown. */ + widgetStore?: WidgetStore; now?: () => number; } /** Asks the user in the session's thread; resolves true only on approval. */ +/** Lets the service attribute widgets to threads and tell the renderer. */ +export interface WidgetHooks { + resolveThread(sessionId: string): string | null; + onWidgetUpdated(widget: { widgetId: string; threadId: string | null }): void; +} + +/** Hard limit on widget source; a widget this large is a page, not a reply. */ +export const MAX_WIDGET_CODE_CHARS = 512 * 1024; +/** Lint rules that make a widget unshowable, not just imperfect. */ +const BLOCKING_WIDGET_RULES = new Set(["document-wrapper", "script-origin"]); + export type BrowserPermissionRequester = (request: { sessionId: string; title: string; @@ -125,12 +140,18 @@ const CONSOLE_RANK: Record = { error: 3, }; +/** Prefix of show_widget results; `src/lib/inline-widgets.ts` matches it. */ +export const WIDGET_MARKER = "pipper-widget:"; +export const WIDGET_UPDATED_MARKER = "pipper-widget-updated:"; + export class BrowserService { private readonly host: PageHost; private readonly limits: BrowserLimits; private config: BrowserConfig = { ...DEFAULT_BROWSER_CONFIG }; private requestPermission: BrowserPermissionRequester | null = null; private readonly downloadsDir: string | undefined; + private readonly widgetStore: WidgetStore | null; + private widgetHooks: WidgetHooks | null = null; private readonly appOrigins: () => string[]; private readonly isEnabled: () => boolean; private readonly baseDir: string | undefined; @@ -149,6 +170,7 @@ export class BrowserService { this.isEnabled = options.isEnabled ?? (() => this.config.enabled); this.baseDir = options.baseDir; this.downloadsDir = options.downloadsDir; + this.widgetStore = options.widgetStore ?? null; this.now = options.now ?? Date.now; } @@ -166,6 +188,10 @@ export class BrowserService { return this.getConfig(); } + setWidgetHooks(hooks: WidgetHooks | null): void { + this.widgetHooks = hooks; + } + /** Wire download approvals to the thread's permission prompt. */ setPermissionRequester(requester: BrowserPermissionRequester | null): void { this.requestPermission = requester; @@ -280,6 +306,82 @@ export class BrowserService { } } + private requireWidgetStore(): WidgetStore { + if (!this.widgetStore) throw new Error("Widgets can't be shown in this client."); + return this.widgetStore; + } + + private widgetCode(args: Record): string { + const code = stringArg(args, "widget_code"); + if (code.length > MAX_WIDGET_CODE_CHARS) { + throw new ToolArgumentError( + `widget_code is ${Math.round(code.length / 1024)} KB; keep widgets under ${MAX_WIDGET_CODE_CHARS / 1024} KB.`, + ); + } + const blocking = lintWidgetSource(code).filter((finding) => + BLOCKING_WIDGET_RULES.has(finding.rule), + ); + if (blocking.length > 0) { + throw new ToolArgumentError( + `This widget can't be shown:\n${blocking.map((finding) => `- ${finding.message}`).join("\n")}\nFix it and check with preview_widget.`, + ); + } + return code; + } + + private showWidget(record: SessionRecord, args: Record): McpToolResult { + const store = this.requireWidgetStore(); + const title = stringArg(args, "title"); + if (!/^[a-z0-9_]{3,60}$/.test(title)) { + throw new ToolArgumentError('"title" must be 3–60 characters of a-z, 0-9 and _.'); + } + const summary = stringArg(args, "summary").trim().slice(0, 280); + const code = this.widgetCode(args); + const now = this.now(); + const id = `wdg_${randomUUID().replace(/-/g, "")}`; + store.insert({ + id, + threadId: record.sessionId + ? (this.widgetHooks?.resolveThread(record.sessionId) ?? null) + : null, + sessionId: record.sessionId, + title, + summary, + code, + createdAt: now, + updatedAt: now, + }); + // The marker leads the result so it survives into the tool call's + // output preview, which is how the thread finds the widget to render. + return textResult( + `${WIDGET_MARKER}${id}\nThe widget is now visible in the thread. Don't describe or repeat ` + + `its contents in text. To change it, call update_widget with widget_id "${id}".`, + ); + } + + private updateWidget(record: SessionRecord, args: Record): McpToolResult { + const store = this.requireWidgetStore(); + const id = stringArg(args, "widget_id").trim(); + const existing = store.get(id); + const threadId = record.sessionId + ? (this.widgetHooks?.resolveThread(record.sessionId) ?? null) + : null; + const sameOwner = + existing && + ((existing.sessionId !== null && existing.sessionId === record.sessionId) || + (existing.threadId !== null && existing.threadId === threadId)); + if (!existing || !sameOwner) { + throw new ToolArgumentError(`No widget ${id} in this thread. Use show_widget to create one.`); + } + const code = this.widgetCode(args); + const summary = optionalString(args, "summary")?.trim().slice(0, 280) || undefined; + store.update(id, { code, summary, updatedAt: this.now() }); + this.widgetHooks?.onWidgetUpdated({ widgetId: id, threadId: existing.threadId }); + return textResult( + `${WIDGET_UPDATED_MARKER}${id}\nUpdated in place; the user sees the new version.`, + ); + } + private widgetPreviewer(): WidgetPreviewer { this.previewer ??= new WidgetPreviewer({ host: this.host, @@ -594,6 +696,10 @@ export class BrowserService { const code = stringArg(args, "widget_code"); return this.widgetPreviewer().preview(code, optionalNumber(args, "width")); } + case "show_widget": + return this.showWidget(record, args); + case "update_widget": + return this.updateWidget(record, args); default: throw new ToolArgumentError(`Unknown browser tool: ${name}`); } diff --git a/electron/browser/browser-tools.ts b/electron/browser/browser-tools.ts index 89d313c..2acef17 100644 --- a/electron/browser/browser-tools.ts +++ b/electron/browser/browser-tools.ts @@ -172,6 +172,51 @@ export const WIDGET_TOOLS: McpToolDefinition[] = [ ), ]; +WIDGET_TOOLS.push( + tool( + "show_widget", + "Shows a visual widget inline in the thread: charts, diagrams, data tables, comparisons, " + + "stat summaries, and small interactive explainers. Use it when a picture is clearer than " + + "text. Don't use it for short answers, plain lists, prose, or code the user will copy. " + + "Pass an HTML fragment or a single written to widget_guide's rules, and preview it " + + "with preview_widget first. Don't set a height or width; the frame sizes itself. Put " + + "explanations in your reply text, never inside the widget. Returns a widget id; the user " + + "already sees the widget, so don't describe its contents in text. To change it later, " + + "call update_widget instead of showing a new one.", + { + title: { + type: "string", + pattern: "^[a-z0-9_]{3,60}$", + description: "Specific snake_case name, e.g. q3_latency_by_region.", + }, + widget_code: { + type: "string", + description: "HTML fragment or a single . No doctype, html, head or body.", + }, + summary: { + type: "string", + maxLength: 280, + description: + "One or two plain sentences saying what the widget shows, for screen readers and " + + "places that can't render it.", + }, + }, + ["title", "widget_code", "summary"], + ), + tool( + "update_widget", + "Replaces the code of a widget you already showed in this thread, keeping its place in " + + "the conversation. Use it to fix or extend a widget instead of showing a near-duplicate. " + + "Preview the new code first.", + { + widget_id: { type: "string", description: "Id returned by show_widget." }, + widget_code: { type: "string" }, + summary: { type: "string", maxLength: 280 }, + }, + ["widget_id", "widget_code"], + ), +); + export class ToolArgumentError extends Error {} export function stringArg(args: Record, key: string): string { diff --git a/electron/db.ts b/electron/db.ts index 8d77674..1e56fb9 100644 --- a/electron/db.ts +++ b/electron/db.ts @@ -277,6 +277,21 @@ export function getDb(): DatabaseSync { ); `); + db.exec(` + CREATE TABLE IF NOT EXISTS inline_widgets ( + id TEXT PRIMARY KEY, + thread_id TEXT, + session_id TEXT, + title TEXT NOT NULL, + summary TEXT NOT NULL, + code TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_inline_widgets_thread + ON inline_widgets(thread_id); + `); + db.exec(` CREATE TABLE IF NOT EXISTS auth_users ( provider TEXT NOT NULL, diff --git a/electron/main.ts b/electron/main.ts index 4f112c4..a186ed4 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -78,7 +78,7 @@ import { } from "./db"; import { getThread, listThreads, listThreadsByIds, listProjectThreads } from "./threads"; import type { OpenTabsState } from "../contracts/threads.ts"; -import type { BrowserConfig } from "../contracts/acp.ts"; +import type { BrowserConfig, InlineWidgetDocument } from "../contracts/acp.ts"; import { listMcpServers, createMcpServer, updateMcpServer, deleteMcpServer } from "./mcp-servers"; import { installAgentInstanceProvider, @@ -96,6 +96,8 @@ import { import { AgentManager } from "./agent"; import { BrowserService } from "./browser/browser-service.ts"; import { electronPageHost, isAgentBrowserWindow } from "./browser/electron-page.ts"; +import { buildWidgetDocument } from "./widgets/widget-frame.ts"; +import { sqliteWidgetStore } from "./widgets/widget-store.ts"; import { RemoteTaskError, getRemoteRequests } from "./remote-requests.ts"; import { prepareIsolatedAgentTask } from "./isolated-agent-task.ts"; import { createElectronOsNotifier } from "./os-notifications"; @@ -2749,6 +2751,22 @@ function registerIpc(): void { ); ipcMain.handle("subagents:listRuns", () => requireAgentManager().getSubagentRuns()); + ipcMain.handle( + "widgets:getDocument", + (_event, id: unknown, theme: unknown): InlineWidgetDocument | null => { + if (typeof id !== "string" || !/^wdg_[a-f0-9]{32}$/.test(id)) return null; + const widget = sqliteWidgetStore.get(id); + if (!widget) return null; + return { + id: widget.id, + title: widget.title, + summary: widget.summary, + updatedAt: widget.updatedAt, + document: buildWidgetDocument(widget.code, theme === "dark" ? "dark" : "light"), + }; + }, + ); + ipcMain.handle("browser:getConfig", () => { if (!browserService) throw new Error("Browser service is not ready"); return browserService.getConfig(); @@ -3156,6 +3174,7 @@ app.whenReady().then(async () => { } browserService = new BrowserService({ host: electronPageHost, + widgetStore: sqliteWidgetStore, // Agents may browse localhost dev servers, but not Pipper's own renderer. appOrigins: () => { const rendererUrl = process.env["ELECTRON_RENDERER_URL"]; diff --git a/electron/preload.ts b/electron/preload.ts index cebb5ed..8d346cb 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -24,6 +24,7 @@ import type { AgentAccountSchema, AgentProbeResult, BrowserConfig, + InlineWidgetDocument, SubagentConfig, SubagentRunSnapshot, } from "../contracts/acp.ts"; @@ -521,6 +522,10 @@ const api = { ipcRenderer.invoke("subagents:setConfig", partial), listRuns: (): Promise => ipcRenderer.invoke("subagents:listRuns"), }, + widgets: { + getDocument: (id: string, theme: "light" | "dark"): Promise => + ipcRenderer.invoke("widgets:getDocument", id, theme), + }, browser: { getConfig: (): Promise => ipcRenderer.invoke("browser:getConfig"), setConfig: (partial: Partial): Promise => diff --git a/electron/widgets/widget-frame.ts b/electron/widgets/widget-frame.ts index 4d0f9ee..983c851 100644 --- a/electron/widgets/widget-frame.ts +++ b/electron/widgets/widget-frame.ts @@ -60,7 +60,7 @@ body { font-family: var(--font-sans); font-size: 13px; line-height: 1.6; color: var(--foreground); -webkit-font-smoothing: antialiased; } -#widget-root { width: var(--frame-width, 680px); overflow: hidden; } +#widget-root { width: var(--frame-width, 100%); overflow: hidden; } h1, h2, h3 { font-weight: 500; margin: 0 0 8px; color: var(--foreground); } h1 { font-size: 15px; } h2 { font-size: 14px; } h3 { font-size: 13px; } a { color: var(--workspace-tone-stale); } @@ -116,6 +116,7 @@ ${["neutral", "action", "ready", "merged", "stale"] transition: transform var(--ff-dur-fast) var(--ff-ease-fast), opacity var(--ff-dur-fast) var(--ff-ease-fast); } .ff-btn:active { transform: scale(0.97); } +.ff-stack > .ff-btn, .ff-stat > .ff-btn { align-self: flex-start; } .ff-btn[data-variant="secondary"] { background: var(--accent); color: var(--accent-foreground); } .ff-btn:focus-visible, .ff-input:focus-visible, .ff-select:focus-visible { outline: 2px solid var(--ring); outline-offset: 2px; @@ -164,12 +165,24 @@ const RUNTIME = String.raw` originalError(...args); }; + const toHost = (message) => { + if (window.parent !== window) window.parent.postMessage({ source: "pipper-widget", ...message }, "*"); + }; const post = (message) => { actions.push(message); - if (window.parent !== window) window.parent.postMessage({ source: "pipper-widget", ...message }, "*"); + toHost(message); }; + // Only a click or keypress inside the widget may send a prompt, so a + // widget can't post messages to the agent on its own. + const userActive = () => !navigator.userActivation || navigator.userActivation.isActive; window.pipper = { - sendPrompt: (text) => post({ type: "sendPrompt", text: String(text) }), + sendPrompt: (text) => { + if (!userActive()) { + console.warn("pipper.sendPrompt ignored: call it from a click or keypress handler."); + return; + } + post({ type: "sendPrompt", text: String(text) }); + }, openLink: (url) => post({ type: "openLink", url: String(url) }), }; document.addEventListener("click", (event) => { @@ -195,7 +208,7 @@ const RUNTIME = String.raw` }); window.__pipperRender = async (code, options) => { - document.documentElement.style.setProperty("--frame-width", options.width + "px"); + if (options.width) document.documentElement.style.setProperty("--frame-width", options.width + "px"); document.documentElement.dataset.workspaceTheme = options.theme; const template = document.createElement("template"); template.innerHTML = code; @@ -309,6 +322,35 @@ const RUNTIME = String.raw` lint, }; }; + + // Display mode: the thread embeds the widget code; report height to the + // host and follow its theme. + const embedded = document.getElementById("pipper-widget-code"); + if (embedded) { + let lastHeight = -1; + const reportSize = () => { + const height = Math.ceil(root.getBoundingClientRect().height); + if (height === lastHeight) return; + lastHeight = height; + toHost({ type: "resize", height }); + }; + new ResizeObserver(reportSize).observe(root); + window.addEventListener("message", (event) => { + if (event.source !== window.parent) return; + const data = event.data || {}; + if (data.type === "theme" && (data.theme === "light" || data.theme === "dark")) { + document.documentElement.dataset.workspaceTheme = data.theme; + } + if (data.type === "shape" && (data.shape === "pill" || data.shape === "rounded")) { + document.documentElement.dataset.shape = data.shape; + } + }); + const theme = document.documentElement.dataset.workspaceTheme || "light"; + window.__pipperRender(JSON.parse(embedded.textContent || '""'), { theme }).then(() => { + reportSize(); + toHost({ type: "ready" }); + }); + } })(); `; @@ -333,3 +375,24 @@ ${KIT_CSS} `; return cachedHtml; } + +/** + * A complete document for showing one widget in the thread: the frame plus + * the widget's code, rendered on load in the given theme. + */ +export function buildWidgetDocument(code: string, theme: "light" | "dark" = "light"): string { + // JSON inside
', + // A function replacer, so "$&" and friends in widget code stay literal. + () => + `
`, + ); +} diff --git a/electron/widgets/widget-preview.ts b/electron/widgets/widget-preview.ts index c3aeaf6..2d10b10 100644 --- a/electron/widgets/widget-preview.ts +++ b/electron/widgets/widget-preview.ts @@ -116,7 +116,7 @@ export class WidgetPreviewer { const clean = errors.length === 0 && blocked.length === 0 && findings.length === 0; lines.push( clean - ? "Clean. Show it with the same code." + ? "Clean. Show it with show_widget using the same code." : "Fix these and preview again before showing the widget.", ); diff --git a/electron/widgets/widget-store.ts b/electron/widgets/widget-store.ts new file mode 100644 index 0000000..2e983e1 --- /dev/null +++ b/electron/widgets/widget-store.ts @@ -0,0 +1,97 @@ +import { getDb } from "../db.ts"; +import type { InlineWidgetRecord } from "../../contracts/acp.ts"; + +/** + * Widgets an agent showed in a thread. The thread's timeline keeps only the + * `show_widget` tool call; the widget code lives here, keyed by id, so the + * transcript never carries the markup and widgets survive reloads. + */ +export interface WidgetStore { + insert(widget: InlineWidgetRecord): void; + update(id: string, changes: { code: string; summary?: string; updatedAt: number }): boolean; + get(id: string): InlineWidgetRecord | null; +} + +interface WidgetRow { + id: string; + thread_id: string | null; + session_id: string | null; + title: string; + summary: string; + code: string; + created_at: number; + updated_at: number; +} + +function fromRow(row: WidgetRow): InlineWidgetRecord { + return { + id: row.id, + threadId: row.thread_id, + sessionId: row.session_id, + title: row.title, + summary: row.summary, + code: row.code, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + +export const sqliteWidgetStore: WidgetStore = { + insert(widget) { + getDb() + .prepare( + `INSERT INTO inline_widgets (id, thread_id, session_id, title, summary, code, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + widget.id, + widget.threadId, + widget.sessionId, + widget.title, + widget.summary, + widget.code, + widget.createdAt, + widget.updatedAt, + ); + }, + update(id, changes) { + const result = getDb() + .prepare( + `UPDATE inline_widgets SET code = ?, summary = COALESCE(?, summary), updated_at = ? + WHERE id = ?`, + ) + .run(changes.code, changes.summary ?? null, changes.updatedAt, id); + return Number(result.changes) > 0; + }, + get(id) { + const row = getDb().prepare("SELECT * FROM inline_widgets WHERE id = ?").get(id) as + | WidgetRow + | undefined; + return row ? fromRow(row) : null; + }, +}; + +/** In-memory store for tests. */ +export function memoryWidgetStore(): WidgetStore { + const rows = new Map(); + return { + insert(widget) { + rows.set(widget.id, { ...widget }); + }, + update(id, changes) { + const row = rows.get(id); + if (!row) return false; + rows.set(id, { + ...row, + code: changes.code, + summary: changes.summary ?? row.summary, + updatedAt: changes.updatedAt, + }); + return true; + }, + get(id) { + const row = rows.get(id); + return row ? { ...row } : null; + }, + }; +} diff --git a/electron/widgets/widgets.test.ts b/electron/widgets/widgets.test.ts index ffed672..e29734a 100644 --- a/electron/widgets/widgets.test.ts +++ b/electron/widgets/widgets.test.ts @@ -3,7 +3,7 @@ import phosphorFont from "@phosphor-icons/web/regular/Phosphor.woff2?inline"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import type { BrowserPage, PageEvents, PageHost } from "../browser/browser-page.ts"; -import { buildWidgetFrameHtml } from "./widget-frame.ts"; +import { buildWidgetDocument, buildWidgetFrameHtml } from "./widget-frame.ts"; import { parseGuideModules, widgetGuide } from "./widget-guide.ts"; import { lintWidgetSource } from "./widget-lint.ts"; import { WidgetPreviewer } from "./widget-preview.ts"; @@ -177,3 +177,17 @@ describe("widget preview", () => { previewer.dispose(); }); }); + +describe("widget document for the thread", () => { + test("embeds the code so it can't break out of its script tag", () => { + const code = '

a

x";'; + const doc = buildWidgetDocument(code, "dark"); + const match = /"); + const next = previewer.preview('
ok
'); + await expect(stuck).rejects.toThrow(/did not finish rendering/); + const result = await next; + expect(result.content[0].type === "text" && result.content[0].text).toContain("Rendered"); + expect(created).toBe(2); + previewer.dispose(); + }); + test("one page is reused across previews", async () => { const { previewer, fake } = preview({ size: { width: 680, height: 10 }, diff --git a/src/components/file-preview/file-preview.tsx b/src/components/file-preview/file-preview.tsx index 443e5fc..2a1e7d6 100644 --- a/src/components/file-preview/file-preview.tsx +++ b/src/components/file-preview/file-preview.tsx @@ -85,7 +85,13 @@ export function FilePreview({ workspaceKey }: { workspaceKey: string }) { setSending(true); try { await sendFileComments(preview, comments, { - onUndelivered: (error) => reportError("The agent didn't get your comments", error), + onTurnFailed: (error, markUnsent) => + toast({ + icon: , + title: "The agent's turn ended with an error", + description: `${error instanceof Error ? error.message : String(error)} Your comments stay marked as sent.`, + action: { label: "Mark unsent", onClick: markUnsent }, + }), }); } catch (error) { reportError("Could not send comments", error); diff --git a/src/components/inline-widget.tsx b/src/components/inline-widget.tsx index 2bacb41..bb08b9f 100644 --- a/src/components/inline-widget.tsx +++ b/src/components/inline-widget.tsx @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react"; import { WarningCircle } from "@phosphor-icons/react"; import { shapeMap, useShape } from "@/lib/shape-context"; import { useAgentStore } from "@/store/agent-store"; +import { createWidgetPromptGate } from "@/lib/inline-widgets"; import type { InlineWidgetDocument, InlineWidgetRef } from "../../contracts/acp.ts"; /** @@ -43,6 +44,7 @@ export function InlineWidget({ threadId: string | null; }) { const frameRef = useRef(null); + const allowPrompt = useRef(createWidgetPromptGate()); const [widget, setWidget] = useState(null); const [status, setStatus] = useState("loading"); const [height, setHeight] = useState(LOADING_HEIGHT); @@ -108,6 +110,9 @@ export function InlineWidget({ } else if (data.type === "sendPrompt" && typeof data.text === "string") { const text = data.text.trim().slice(0, 4000); if (!text) return; + // The frame's own check runs in code the agent wrote; enforce it here. + const interacting = navigator.userActivation?.isActive ?? false; + if (!allowPrompt.current(interacting)) return; const store = useAgentStore.getState(); void store.sendPrompt({ threadId, diff --git a/src/lib/file-preview-actions.test.ts b/src/lib/file-preview-actions.test.ts index 207d1d3..1ebbe23 100644 --- a/src/lib/file-preview-actions.test.ts +++ b/src/lib/file-preview-actions.test.ts @@ -134,7 +134,7 @@ describe("sendFileComments", () => { expect(status()).toBe("draft"); }); - test("a turn that fails before the agent took the comment puts it back and says so", async () => { + test("a failed turn keeps the comment sent and lets the user mark it unsent", async () => { const { sendFileComments, comment, status } = await setup(); let failTurn: (error: Error) => void = () => {}; sendWorkspaceAgentPrompt.mockResolvedValueOnce({ @@ -143,12 +143,17 @@ describe("sendFileComments", () => { failTurn = reject; }), }); - const onUndelivered = vi.fn(); - await sendFileComments({ projectId: "p1", worktreePath: null }, [comment], { onUndelivered }); + const onTurnFailed = vi.fn(); + await sendFileComments({ projectId: "p1", worktreePath: null }, [comment], { onTurnFailed }); + + // The turn can fail after the agent already read the comments: keep them + // sent so nothing goes twice on its own. + failTurn(new Error("timed out")); + await vi.waitFor(() => expect(onTurnFailed).toHaveBeenCalledTimes(1)); expect(status()).toBe("sent"); - failTurn(new Error("Sign in to continue")); - await vi.waitFor(() => expect(onUndelivered).toHaveBeenCalledTimes(1)); + const markUnsent = onTurnFailed.mock.calls[0][1] as () => void; + markUnsent(); expect(status()).toBe("draft"); }); }); diff --git a/src/lib/file-preview-actions.ts b/src/lib/file-preview-actions.ts index 92d6cdd..8062ca8 100644 --- a/src/lib/file-preview-actions.ts +++ b/src/lib/file-preview-actions.ts @@ -104,14 +104,17 @@ async function findProject(projectId: string): Promise { * Only comments still unsent are sent, and they are marked sent before the * prompt goes out, so a second click (or "Send now" on a comment a batch is * already sending) never sends anything twice. If the prompt can't be - * dispatched, or the agent's turn fails before it took the comments (it can't - * start, it needs sign-in), they go back to unsent and `onUndelivered` hears - * about a turn failure; a dispatch failure is thrown. + * dispatched they go back to unsent and the error is thrown. + * + * A failed turn is different: its promise covers the whole turn, so it can + * reject after the agent already took the comments (a timeout, a stop). + * Restoring them then could send them twice, so they stay sent and + * `onTurnFailed` gets a `markUnsent` the user can choose to run. */ export async function sendFileComments( workspace: { projectId: string; worktreePath: string | null }, comments: readonly FileComment[], - options: { onUndelivered?: (error: unknown) => void } = {}, + options: { onTurnFailed?: (error: unknown, markUnsent: () => void) => void } = {}, ): Promise { const key = previewWorkspaceKey(workspace.projectId, workspace.worktreePath); const store = usePreviewStore.getState(); @@ -132,10 +135,7 @@ export async function sendFileComments( message: buildFileCommentsPrompt(pending), title: "File review comments", }); - turn.catch((error: unknown) => { - restore(); - options.onUndelivered?.(error); - }); + turn.catch((error: unknown) => options.onTurnFailed?.(error, restore)); } catch (error) { restore(); throw error; diff --git a/src/lib/inline-widgets.test.ts b/src/lib/inline-widgets.test.ts index 64f36a7..83ca16b 100644 --- a/src/lib/inline-widgets.test.ts +++ b/src/lib/inline-widgets.test.ts @@ -100,3 +100,19 @@ describe("inline widgets in the timeline", () => { ]); }); }); + +describe("widget prompts", () => { + test("need a user interaction, and one interaction sends one prompt", async () => { + const { createWidgetPromptGate } = await import("./inline-widgets"); + let now = 0; + const allow = createWidgetPromptGate(() => now, 1000); + // A widget posting on load, with nobody touching it. + expect(allow(false)).toBe(false); + expect(allow(true)).toBe(true); + // The same click can't be reused for a burst. + now += 200; + expect(allow(true)).toBe(false); + now += 1000; + expect(allow(true)).toBe(true); + }); +}); diff --git a/src/lib/inline-widgets.ts b/src/lib/inline-widgets.ts index 63b85df..33ce243 100644 --- a/src/lib/inline-widgets.ts +++ b/src/lib/inline-widgets.ts @@ -98,3 +98,27 @@ export function buildMessageBodyBlocks( } return blocks; } + +/** Shortest gap between two prompts a widget may send. */ +export const WIDGET_PROMPT_INTERVAL_MS = 1000; + +/** + * Decides whether a widget's `sendPrompt` message may reach the agent. The + * widget's code comes from the agent, so it can post the message itself + * without the user doing anything. Accept it only during a real user + * interaction (a click or keypress in the frame also activates the host + * page), and at most once per interval, so one click can't fire a burst. + */ +export function createWidgetPromptGate( + now: () => number = Date.now, + intervalMs = WIDGET_PROMPT_INTERVAL_MS, +): (userIsInteracting: boolean) => boolean { + let lastSentAt = Number.NEGATIVE_INFINITY; + return (userIsInteracting) => { + if (!userIsInteracting) return false; + const time = now(); + if (time - lastSentAt < intervalMs) return false; + lastSentAt = time; + return true; + }; +} diff --git a/src/lib/settings-saver.test.ts b/src/lib/settings-saver.test.ts new file mode 100644 index 0000000..fe34996 --- /dev/null +++ b/src/lib/settings-saver.test.ts @@ -0,0 +1,68 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { createSettingsSaver } from "./settings-saver"; + +type Config = { enabled: boolean; pages: number }; + +function harness() { + const calls: Array<{ changes: Partial; resolve: (saved: Config) => void }> = []; + const saved: Config[] = []; + const saver = createSettingsSaver({ + save: (changes) => new Promise((resolve) => calls.push({ changes, resolve })), + onSaved: (value) => saved.push(value), + onError: () => undefined, + debounceMs: 300, + }); + return { saver, calls, saved }; +} + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("settings saver", () => { + test("saves run in order, and a late reply doesn't undo a newer change", async () => { + const { saver, calls, saved } = harness(); + saver.update({ enabled: false }, true); + saver.update({ enabled: true }, true); + await vi.waitFor(() => expect(calls).toHaveLength(1)); + // The second save waits for the first. + expect(calls).toHaveLength(1); + + calls[0].resolve({ enabled: false, pages: 4 }); + await vi.waitFor(() => expect(calls).toHaveLength(2)); + expect(calls[1].changes).toEqual({ enabled: true }); + // The first reply is older than the second change, so it isn't applied. + expect(saved).toEqual([]); + + calls[1].resolve({ enabled: true, pages: 4 }); + await saver.idle(); + expect(saved).toEqual([{ enabled: true, pages: 4 }]); + }); + + test("slider steps are saved once they settle", async () => { + const { saver, calls } = harness(); + saver.update({ pages: 2 }); + saver.update({ pages: 3 }); + vi.advanceTimersByTime(299); + expect(calls).toHaveLength(0); + vi.advanceTimersByTime(1); + await vi.waitFor(() => expect(calls).toHaveLength(1)); + expect(calls[0].changes).toEqual({ pages: 3 }); + }); + + test("leaving the page within the debounce window still saves the change", async () => { + const { saver, calls, saved } = harness(); + saver.update({ pages: 6 }); + saver.dispose(); + await vi.waitFor(() => expect(calls).toHaveLength(1)); + expect(calls[0].changes).toEqual({ pages: 6 }); + calls[0].resolve({ enabled: true, pages: 6 }); + await saver.idle(); + // The page is gone; nothing is reported back to it. + expect(saved).toEqual([]); + }); +}); diff --git a/src/lib/settings-saver.ts b/src/lib/settings-saver.ts new file mode 100644 index 0000000..56e9abb --- /dev/null +++ b/src/lib/settings-saver.ts @@ -0,0 +1,64 @@ +/** + * Saves partial settings changes the way a settings page needs: + * + * - Debounced: sliders report every step while dragging; one save once they + * settle (or at once for toggles). + * - In order: saves run one at a time, in the order the user made them. Each + * save writes the whole config on the other side, so an older one must + * never land last. + * - Newest wins: a save's response is applied only if nothing changed since + * it was sent, so a late reply can't undo a newer value. + * - Nothing lost: `dispose` (the page closing) still sends a pending change. + */ +export function createSettingsSaver(options: { + save: (changes: Partial) => Promise; + onSaved: (saved: T) => void; + onError: (error: unknown) => void; + debounceMs: number; +}) { + let pending: Partial = {}; + let timer: ReturnType | null = null; + let chain: Promise = Promise.resolve(); + let version = 0; + let disposed = false; + + const flush = () => { + if (timer) clearTimeout(timer); + timer = null; + if (Object.keys(pending).length === 0) return; + const changes = pending; + pending = {}; + const sentAt = version; + chain = chain + .catch(() => undefined) + .then(() => options.save(changes)) + .then((saved) => { + if (!disposed && sentAt === version) options.onSaved(saved); + }) + .catch((error: unknown) => { + if (!disposed) options.onError(error); + }); + }; + + return { + update(partial: Partial, immediate = false) { + version += 1; + pending = { ...pending, ...partial }; + if (timer) clearTimeout(timer); + if (immediate) flush(); + else timer = setTimeout(flush, options.debounceMs); + }, + /** Send any pending change now, then stop reporting results. */ + dispose() { + flush(); + disposed = true; + }, + /** Resolves once every save sent so far has finished. */ + idle(): Promise { + return chain.then( + () => undefined, + () => undefined, + ); + }, + }; +} diff --git a/src/settings/app.tsx b/src/settings/app.tsx index f7ceb84..49ca4e0 100644 --- a/src/settings/app.tsx +++ b/src/settings/app.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useRef, useState } from "react"; +import { createSettingsSaver } from "@/lib/settings-saver"; import { ArrowsClockwiseIcon, CheckCircleIcon, @@ -704,41 +705,37 @@ function closestIndex(choices: number[], value: number): number { function BrowserView() { const [config, setConfig] = useState(null); const [error, setError] = useState(null); - const pending = useRef>({}); - const timer = useRef | null>(null); + const saver = useRef> | null>(null); useEffect(() => { - let disposed = false; + let mounted = true; + const current = createSettingsSaver({ + save: (changes) => window.omni.browser.setConfig(changes), + onSaved: setConfig, + onError: () => setError("Could not save browser settings."), + debounceMs: SAVE_DEBOUNCE_MS, + }); + saver.current = current; window.omni.browser .getConfig() .then((value) => { - if (!disposed) setConfig(value); + if (mounted) setConfig(value); }) .catch(() => { - if (!disposed) setError("Could not load browser settings."); + if (mounted) setError("Could not load browser settings."); }); return () => { - disposed = true; - if (timer.current) clearTimeout(timer.current); + mounted = false; + // Leaving the page within the debounce window still saves the change. + current.dispose(); }; }, []); // Sliders report every step while dragging; save once they settle. const update = (partial: Partial, immediate = false) => { setConfig((current) => (current ? { ...current, ...partial } : current)); - pending.current = { ...pending.current, ...partial }; - if (timer.current) clearTimeout(timer.current); - const save = () => { - const changes = pending.current; - pending.current = {}; - setError(null); - window.omni.browser - .setConfig(changes) - .then(setConfig) - .catch(() => setError("Could not save browser settings.")); - }; - if (immediate) save(); - else timer.current = setTimeout(save, SAVE_DEBOUNCE_MS); + setError(null); + saver.current?.update(partial, immediate); }; const disabled = config === null; diff --git a/src/store/agent-store.behavior.test.ts b/src/store/agent-store.behavior.test.ts index 44badba..9a73ac4 100644 --- a/src/store/agent-store.behavior.test.ts +++ b/src/store/agent-store.behavior.test.ts @@ -588,6 +588,29 @@ describe("agent store ACP bridge behavior", () => { resolveSwitch?.(); }); + test("a failed fallback read after a switch releases the pending target", async () => { + let calls = 0; + const agentApi = { + onEvent: vi.fn((_handler: (payload: AcpBridgeEvent) => void) => vi.fn()), + // Answers the connect-time read, then fails like a dropped bridge. + getState: vi.fn(async () => { + calls += 1; + if (calls > 1) throw new Error("bridge closed"); + return sessionState("thread-a"); + }), + getCapabilities: vi.fn(async () => null), + // An older bridge: the switch resolves but no session-state arrives. + switchThread: vi.fn(async () => {}), + }; + (globalThis as any).window = { omni: { agent: agentApi } }; + + const store = await loadStore(); + await store.getState().connect(); + await expect(store.getState().switchThread("thread-b")).rejects.toThrow("bridge closed"); + expect(store.getState().pendingThreadTarget).toBeNull(); + expect(store.getState().error).toBe("bridge closed"); + }); + test("surfaces switch-thread failures", async () => { const agentApi = { onEvent: vi.fn((_handler: (payload: AcpBridgeEvent) => void) => vi.fn()), diff --git a/src/store/agent-store.ts b/src/store/agent-store.ts index 79fde8c..5360898 100644 --- a/src/store/agent-store.ts +++ b/src/store/agent-store.ts @@ -1175,6 +1175,17 @@ export const useAgentStore = create((set, get) => ({ // Send every click to main at once. Main supersedes the older activation // when a newer one arrives (see electron/activation.ts), so a slow thread // never holds later clicks back; superseded calls settle quietly here. + // A failed switch must release the pending target: while it is set, most + // bridge events are ignored. Returns false for a superseded switch. + const failed = (err: unknown): boolean => { + if (switchId !== latestThreadSwitchId) return false; + if (pendingThreadTarget === threadId) pendingThreadTarget = null; + set({ + pendingThreadTarget: null, + error: err instanceof Error ? err.message : "Failed to switch thread", + }); + return true; + }; try { await withTimeout( window.omni.agent.switchThread(threadId), @@ -1182,13 +1193,8 @@ export const useAgentStore = create((set, get) => ({ `Switch to ${threadId}`, ); } catch (err) { - if (switchId !== latestThreadSwitchId) return; - if (pendingThreadTarget === threadId) pendingThreadTarget = null; - set({ - pendingThreadTarget: null, - error: err instanceof Error ? err.message : "Failed to switch thread", - }); - throw err; + if (failed(err)) throw err; + return; } // A new click can arrive after this thread's snapshot paints but before // its IPC promise settles. Only the latest switch may settle the pending @@ -1198,7 +1204,14 @@ export const useAgentStore = create((set, get) => ({ // promise resolves. Avoid a second renderer→main→renderer trip. Keep // fallback for older ACP bridges that do not emit that snapshot. if (get().state?.threadId !== threadId) { - const state = preserveOptimisticState(get().state, await window.omni.agent.getState()); + let fetched: Awaited>; + try { + fetched = await window.omni.agent.getState(); + } catch (err) { + if (failed(err)) throw err; + return; + } + const state = preserveOptimisticState(get().state, fetched); set(() => { if (switchId !== latestThreadSwitchId || state.threadId !== threadId) return {}; discardStreamDeltas();