From 558b54f8fa0cb3679852c6ece2b07e23a3b10ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Val=20Alexander=20=F0=9F=91=91?= Date: Thu, 12 Nov 2026 09:12:35 +0000 Subject: [PATCH 1/8] test(local-models): preserve drafts and selection across metadata races (#503) * test(local-models): preserve drafts and selection across metadata races * docs(local-models): record preservation qualification --- .../e2e/fixtures/local-model-preservation.mjs | 29 +++++ .../e2e/local-model-preservation.spec.ts | 121 +++++++++++++++++ .../e2e/local-model-profile-revision.spec.ts | 123 ++++++++++++++++++ ...-11-12-local-model-refresh-preservation.md | 38 ++++++ docs/reviews/2026-11-12-model-search.md | 6 + docs/status.md | 9 +- ...-11-12-local-model-refresh-preservation.md | 27 ++++ .../plans/2026-11-12-model-search.md | 2 +- 8 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 apps/desktop/e2e/fixtures/local-model-preservation.mjs create mode 100644 apps/desktop/e2e/local-model-preservation.spec.ts create mode 100644 apps/desktop/e2e/local-model-profile-revision.spec.ts create mode 100644 docs/reviews/2026-11-12-local-model-refresh-preservation.md create mode 100644 docs/superpowers/plans/2026-11-12-local-model-refresh-preservation.md diff --git a/apps/desktop/e2e/fixtures/local-model-preservation.mjs b/apps/desktop/e2e/fixtures/local-model-preservation.mjs new file mode 100644 index 00000000..d519b352 --- /dev/null +++ b/apps/desktop/e2e/fixtures/local-model-preservation.mjs @@ -0,0 +1,29 @@ +// Observer-only entry point for disposable, scripted preservation tests. Never packaged. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +if (process.env.MODEX_E2E !== "1" || process.env.MODEX_LM_NATIVE_E2E === "1") throw new Error("Preservation fixture requires scripted E2E mode."); +const home = fs.realpathSync(process.env.MODEX_HOME ?? ""); +if (!home.startsWith(fs.realpathSync(os.tmpdir()) + path.sep) || !path.basename(home).startsWith("modex-e2e-home-") || JSON.parse(fs.readFileSync(path.join(home, "app/state.json"), "utf8")).settings.default_backend !== "mock") throw new Error("Preservation fixture requires a disposable mock home."); +import { LocalModelDiscovery } from "../../dist/src/main/engine/local-models/discovery.js"; +globalThis.__lm4DiscoveryResults = []; +globalThis.__lm4DiscoveryStarts = []; +const discover = LocalModelDiscovery.prototype.discover; +LocalModelDiscovery.prototype.discover = async function (ref, requestId) { + globalThis.__lm4DiscoveryStarts.push({ connection: { ...ref }, requestId }); + const result = await discover.call(this, ref, requestId); + globalThis.__lm4DiscoveryResults.push({ connection: { ...ref }, requestId, state: result.state }); + return result; +}; + + +import { LocalExecution } from "../../dist/src/main/engine/local-models/execution.js"; +import { ThreadRunner } from "../../dist/src/main/engine/runner.js"; +globalThis.__lm4Calls = { localStarts: 0, sends: 0, durableSends: 0, retries: 0 }; +const start = LocalExecution.prototype.start; +LocalExecution.prototype.start = function (...args) { globalThis.__lm4Calls.localStarts++; return start.apply(this, args); }; +for (const [method, key] of [["send", "sends"], ["sendDurably", "durableSends"], ["retry", "retries"]]) { + const original = ThreadRunner.prototype[method]; + ThreadRunner.prototype[method] = function (...args) { globalThis.__lm4Calls[key]++; return original.apply(this, args); }; +} +await import("./local-model-workflow.mjs"); diff --git a/apps/desktop/e2e/local-model-preservation.spec.ts b/apps/desktop/e2e/local-model-preservation.spec.ts new file mode 100644 index 00000000..10c47fb8 --- /dev/null +++ b/apps/desktop/e2e/local-model-preservation.spec.ts @@ -0,0 +1,121 @@ +// #381 metadata races through production IPC/store/renderer with the existing scripted native fixture. +// These cases request no coding turns and do not qualify ordinary product admission or real models. +import { test, expect, _electron as electron, type ElectronApplication } from "@playwright/test"; +import fs from "node:fs"; +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; +import path from "node:path"; +import http from "node:http"; +import crypto from "node:crypto"; +import type { AddressInfo } from "node:net"; +import { appDir, seedHome, tid } from "./support.js"; + +const PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; +const sha = (bytes: Buffer) => crypto.createHash("sha256").update(bytes).digest("hex"); +async function runtime(model: string) { + let current = model, hold = false, release: (() => void) | undefined; + let arrived!: () => void; + const barrier = new Promise(resolve => { arrived = resolve; }); + const hits: Array<{ method?: string; url?: string; auth: boolean }> = []; + const server = http.createServer((req, res) => { + hits.push({ method: req.method, url: req.url, auth: Boolean(req.headers.authorization) }); + req.resume(); res.setHeader("content-type", "application/json"); + const tags = () => res.end(JSON.stringify({ models: [{ name: current, details: { family: "qwen3moe" } }] })); + if (req.url === "/api/tags") { + if (hold) { hold = false; release = () => { current = "Late/Alpha:Q4_0"; if (!res.destroyed) tags(); }; arrived(); } + else tags(); + } else if (req.url === "/api/ps") res.end(JSON.stringify({ models: [] })); + else if (req.url === "/api/show") res.end(JSON.stringify({ capabilities: ["completion", "tools"], model_info: { "qwen3moe.context_length": 65536 } })); + else { res.writeHead(404); res.end(); } + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + return { hits, url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, hold: () => { hold = true; }, barrier, release: () => release?.(), stop: async () => { release?.(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } }; +} + +test("delayed metadata preserves two thread drafts, image bytes, target and search focus", async ({}, testInfo) => { + const receipt = testInfo.outputPath("local-model-preservation.json"); + expect(fs.existsSync(receipt)).toBe(false); + const out: Record = { startedAt: new Date().toISOString(), gitHeadAtRun: execFileSync("git", ["rev-parse", "HEAD"], { cwd: appDir, encoding: "utf8" }).trim(), scope: "existing temporary-home scripted LocalExecution bootstrap; production IPC/store/renderer; no real model or product admission", state: "started" }; + const persist = () => fs.writeFileSync(receipt, JSON.stringify(out, null, 2) + "\n"); persist(); + let app: ElectronApplication | undefined; + let alpha: Awaited> | undefined, beta: Awaited> | undefined; + try { + alpha = await runtime("Fixture/Alpha:Q4_0"); beta = await runtime("Fixture/Beta:Q4_0"); + const { home } = seedHome(); + app = await electron.launch({ args: [path.join(appDir, "e2e/fixtures/local-model-preservation.mjs")], cwd: appDir, env: { ...process.env, MODEX_HOME: home, MODEX_E2E: "1", MODEX_E2E_NODE: process.execPath, MODEX_LM_NATIVE_E2E: "", MODEX_LM3_LIVE: "", MODEX_LM3_WIRE: "", MODEX_NO_LOGIN_PATH: "1", TYPESAFE_API_KEY: "", JEV_API_KEY: "" } }); + out.mainPid = app.process().pid; persist(); + const page = await app.firstWindow(); await page.waitForLoadState("domcontentloaded"); + await expect(tid(page, "composer-input")).toBeVisible(); + out.document = await page.evaluate(() => location.href); expect(out.document).toBe(pathToFileURL(path.join(appDir, "dist/renderer/index.html")).href); + for (const [label, url] of [["Alpha lab", alpha.url], ["Beta lab", beta.url]]) await page.evaluate(({ label, url }) => window.modex!.invoke("local-models:command", { op: "create", input: { label, runtime: "ollama", url } }), { label, url }); + await tid(page, "model-picker").click(); + await tid(page, "model-local-option").filter({ hasText: "Fixture/Beta:Q4_0" }).click(); + await tid(page, "model-local-context").getByRole("menuitemradio", { name: /65,536/ }).click(); + await expect(tid(page, "model-local-badge")).toBeVisible(); + const local = await page.evaluate(async () => (await window.modex!.invoke("state:get", undefined)).threads.find(t => t.localTarget)!); + expect(local.localTarget?.model.id).toBe("Fixture/Beta:Q4_0"); + out.targetBefore = local.localTarget; + const cloud = await page.evaluate(() => window.modex!.invoke("thread:create", { projectId: "p1", backend: "mock", mode: "chat" })); + // Direct thread:create returns to its caller; unlike routine creation it does not broadcast a window refresh. + await page.reload(); await expect(tid(page, "composer-input")).toBeVisible(); + out.setup = "Reload after direct IPC thread creation before the race; production source unchanged"; + const select = async (id: string) => page.locator(`[data-testid="thread-row"][data-thread-id="${id}"] button.thread-main`).click(); + await select(cloud.id); + await tid(page, "composer-input").fill(" cloud draft with image\nkept exactly "); + await page.evaluate(b64 => { + const dt = new DataTransfer(); dt.items.add(new File([Uint8Array.from(atob(b64), c => c.charCodeAt(0))], "dot.png", { type: "image/png" })); + document.querySelector('[data-testid="composer-input"]')!.dispatchEvent(new ClipboardEvent("paste", { clipboardData: dt, bubbles: true, cancelable: true })); + }, PNG_B64); + await expect(tid(tid(page, "composer-attachments"), "attachment-chip")).toContainText("dot.png"); + const staging = path.join(home, "attachments", "staging"); + expect(fs.readdirSync(staging)).toHaveLength(1); const staged = path.join(staging, fs.readdirSync(staging)[0]!); + out.imageBefore = sha(fs.readFileSync(staged)); expect(out.imageBefore).toBe(sha(Buffer.from(PNG_B64, "base64"))); + await tid(page, "open-settings").click(); await page.getByRole("button", { name: "Local models", exact: true }).click(); + const alphaCard = tid(page, "local-connection").filter({ hasText: "Alpha lab" }); + await expect(alphaCard).toHaveAttribute("data-state", "ready"); + alpha.hold(); + await alphaCard.getByTestId("local-refresh").click(); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Alpha refresh did not reach its barrier within 10s")), 10000); + alpha!.barrier.then(() => { clearTimeout(timer); resolve(); }, error => { clearTimeout(timer); reject(error); }); + }); + await page.keyboard.press("Escape"); + await select(local.id); + await tid(page, "composer-input").fill(" local draft\nkept separately "); + await expect(tid(page, "local-inference-host")).toContainText(beta.url); + await tid(page, "composer-input").evaluate((el: HTMLTextAreaElement) => el.setSelectionRange(1, 5)); + await tid(page, "model-picker").click(); + const search = page.getByRole("searchbox", { name: "Search models" }); await search.fill("late/ALPHA"); + await search.press("Home"); for (let key = 0; key < 4; key++) await search.press("Shift+ArrowRight"); + expect(await search.evaluate((el: HTMLInputElement) => ({ start: el.selectionStart, end: el.selectionEnd }))).toEqual({ start: 0, end: 4 }); + alpha.release(); + await expect(tid(page, "model-local-title")).toHaveText(["Late/Alpha:Q4_0"]); + await expect(search).toHaveValue("late/ALPHA"); await expect(search).toBeFocused(); + out.searchSelectionAfter = await search.evaluate((el: HTMLInputElement) => ({ start: el.selectionStart, end: el.selectionEnd })); + expect(out.searchSelectionAfter).toEqual({ start: 0, end: 4 }); + const after = await page.evaluate(() => window.modex!.invoke("state:get", undefined)); + out.targetAfter = after.threads.find(t => t.id === local.id)!.localTarget; expect(out.targetAfter).toEqual(out.targetBefore); + expect(after.threads.find(t => t.id === cloud.id)!.localTarget).toBeUndefined(); + await page.keyboard.press("Escape"); await expect(tid(page, "composer-input")).toHaveValue(" local draft\nkept separately "); + out.localDraftAfter = await tid(page, "composer-input").inputValue(); + await tid(page, "composer-input").focus(); + out.localSelectionAfter = await tid(page, "composer-input").evaluate((el: HTMLTextAreaElement) => ({ start: el.selectionStart, end: el.selectionEnd })); + expect(out.localSelectionAfter).toEqual({ start: 1, end: 5 }); + await select(cloud.id); await expect(tid(page, "composer-input")).toHaveValue(" cloud draft with image\nkept exactly "); + await expect(tid(tid(page, "composer-attachments"), "attachment-chip")).toContainText("dot.png"); + out.cloudDraftAfter = await tid(page, "composer-input").inputValue(); + out.callCounts = await app.evaluate(() => (globalThis as any).__lm4Calls); + expect(out.callCounts).toEqual({ localStarts: 0, sends: 0, durableSends: 0, retries: 0 }); + for (const id of [local.id, cloud.id]) expect(await page.evaluate(threadId => window.modex!.invoke("thread:items", { threadId }), id)).toEqual([]); + out.imageAfter = sha(fs.readFileSync(staged)); expect(out.imageAfter).toBe(out.imageBefore); + for (const r of [alpha, beta]) expect(r.hits.every(h => !h.auth && ((h.method === "GET" && ["/api/tags", "/api/ps"].includes(h.url!)) || (h.method === "POST" && h.url === "/api/show")))).toBe(true); + out.state = "passed"; + } catch (error) { out.state = "failed"; out.failure = String(error); throw error; } + finally { + alpha?.release(); + out.hits = { alpha: alpha?.hits ?? [], beta: beta?.hits ?? [] }; + try { if (app) { await app.close(); out.appClose = "returned"; } else out.appClose = "no returned handle"; } + catch (error) { out.state = "close-failed"; out.closeError = String(error); throw error; } + finally { await Promise.all([alpha, beta].filter((r): r is Awaited> => Boolean(r)).map(r => r.stop())); out.finishedAt = new Date().toISOString(); persist(); await testInfo.attach("local-model-preservation.json", { path: receipt, contentType: "application/json" }); } + } +}); diff --git a/apps/desktop/e2e/local-model-profile-revision.spec.ts b/apps/desktop/e2e/local-model-profile-revision.spec.ts new file mode 100644 index 00000000..d3f96b0f --- /dev/null +++ b/apps/desktop/e2e/local-model-profile-revision.spec.ts @@ -0,0 +1,123 @@ +// #381 metadata races through production IPC/store/renderer with the existing scripted native fixture. +// These cases request no coding turns and do not qualify ordinary product admission or real models. +import { test, expect, _electron as electron, type ElectronApplication } from "@playwright/test"; +import fs from "node:fs"; +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; +import path from "node:path"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { appDir, seedHome, tid } from "./support.js"; + +async function runtime(initial: string) { + let model = initial; + let arrival: (() => void) | undefined; + let held: http.ServerResponse | undefined; + const hits: Array<{ method?: string; url?: string; auth: boolean }> = []; + const heldReleases: Array<{ model: string; destroyedBeforeRelease: boolean }> = []; + const server = http.createServer((req, res) => { + hits.push({ method: req.method, url: req.url, auth: Boolean(req.headers.authorization) }); req.resume(); + res.setHeader("content-type", "application/json"); + if (req.url === "/api/tags") { + if (arrival) { held = res; const resolve = arrival; arrival = undefined; resolve(); } + else res.end(JSON.stringify({ models: [{ name: model, details: { family: "qwen3moe" } }] })); + } else if (req.url === "/api/ps") res.end(JSON.stringify({ models: [] })); + else if (req.url === "/api/show") res.end(JSON.stringify({ capabilities: ["completion", "tools"], model_info: { "qwen3moe.context_length": 65536 } })); + else { res.writeHead(404); res.end(); } + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const release = (next = model) => { + model = next; + if (held) { + heldReleases.push({ model: next, destroyedBeforeRelease: held.destroyed }); + if (!held.destroyed) held.end(JSON.stringify({ models: [{ name: model, details: { family: "qwen3moe" } }] })); + held = undefined; + } + }; + return { hits, heldReleases, url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + hold: () => new Promise(resolve => { arrival = resolve; }), release, + stop: async () => { release(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } }; +} +async function arrived(barrier: Promise) { + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Metadata barrier did not arrive within 10s")), 10000); + barrier.then(() => { clearTimeout(timer); resolve(); }, error => { clearTimeout(timer); reject(error); }); + }); +} + +test("profile form survives metadata and endpoint revision never silently rebinds its thread", async ({}, testInfo) => { + const receipt = testInfo.outputPath("local-model-profile-revision.json"); expect(fs.existsSync(receipt)).toBe(false); + const out: Record = { startedAt: new Date().toISOString(), gitHeadAtRun: execFileSync("git", ["rev-parse", "HEAD"], { cwd: appDir, encoding: "utf8" }).trim(), state: "started", scope: "scripted temporary-home existing workflow bootstrap; no provider inference or product activation" }; + const persist = () => fs.writeFileSync(receipt, JSON.stringify(out, null, 2) + "\n"); persist(); + let old: Awaited> | undefined, replacement: Awaited> | undefined; + let app: ElectronApplication | undefined; + try { + old = await runtime("Fixture/Beta:Q4_0"); replacement = await runtime("Replacement/Coder:Q8"); + const { home } = seedHome(); + app = await electron.launch({ args: [path.join(appDir, "e2e/fixtures/local-model-preservation.mjs")], cwd: appDir, + env: { ...process.env, MODEX_HOME: home, MODEX_E2E: "1", MODEX_E2E_NODE: process.execPath, MODEX_LM_NATIVE_E2E: "", MODEX_LM3_LIVE: "", MODEX_LM3_WIRE: "", MODEX_NO_LOGIN_PATH: "1", TYPESAFE_API_KEY: "", JEV_API_KEY: "" } }); + out.mainPid = app.process().pid; persist(); + const page = await app.firstWindow(); await page.waitForLoadState("domcontentloaded"); await expect(tid(page, "composer-input")).toBeVisible(); + out.document = await page.evaluate(() => location.href); expect(out.document).toBe(pathToFileURL(path.join(appDir, "dist/renderer/index.html")).href); + const created = await page.evaluate(url => window.modex!.invoke("local-models:command", { op: "create", input: { label: "Beta lab", runtime: "ollama", url } }), old.url); + expect(created.op).toBe("create"); + await tid(page, "model-picker").click(); await tid(page, "model-local-option").click(); + await tid(page, "model-local-context").getByRole("menuitemradio", { name: /65,536/ }).click(); + await expect(tid(page, "model-local-badge")).toBeVisible(); + const thread = await page.evaluate(async () => (await window.modex!.invoke("state:get", undefined)).threads.find(t => t.localTarget)!); + out.targetBefore = thread.localTarget; + await tid(page, "composer-input").fill(" pinned draft\nkeep every byte "); + await tid(page, "open-settings").click(); await page.getByRole("button", { name: "Local models", exact: true }).click(); + const card = tid(page, "local-connection"); await expect(card).toHaveAttribute("data-state", "ready"); + const first = old.hold(); await tid(card, "local-refresh").click(); await arrived(first); + await tid(card, "local-edit").click(); const form = tid(card, "local-edit-form"); + const name = form.getByLabel("Name", { exact: true }); + await name.fill(" Edited Beta "); await form.getByLabel("Extra model IDs (one per line)").fill("Custom/Exact ID:Q4_0"); await name.focus(); + // Establish the range directly: macOS Home does not move this input caret. + await name.evaluate((el: HTMLInputElement) => el.setSelectionRange(0, 3)); + out.editSelectionBefore = await name.evaluate((el: HTMLInputElement) => ({ start: el.selectionStart, end: el.selectionEnd })); + expect(out.editSelectionBefore).toEqual({ start: 0, end: 3 }); + old.release("Late/Beta:Q4_0"); + await expect(tid(card, "local-model")).toContainText("Late/Beta:Q4_0"); + await expect(name).toHaveValue(" Edited Beta "); await expect(name).toBeFocused(); + out.editSelectionAfter = await name.evaluate((el: HTMLInputElement) => ({ start: el.selectionStart, end: el.selectionEnd })); + expect(out.editSelectionAfter).toEqual({ start: 0, end: 3 }); + await expect(form.getByLabel("Extra model IDs (one per line)")).toHaveValue("Custom/Exact ID:Q4_0"); + out.editDraftBeforeSave = await name.inputValue(); + const beforeSecond = await app.evaluate(() => (globalThis as any).__lm4DiscoveryStarts.length); + const second = old.hold(); await tid(card, "local-refresh").click(); await arrived(second); + const secondStarts = await app.evaluate(({}, arg) => (globalThis as any).__lm4DiscoveryStarts.slice(arg.after).filter((r: any) => r.connection.id === arg.ref.id && r.connection.revision === arg.ref.revision), { after: beforeSecond, ref: out.targetBefore.model.connection }); + expect(secondStarts).toHaveLength(1); out.secondHeldRequestId = secondStarts[0].requestId; + await form.getByLabel("Address", { exact: true }).fill(replacement.url); + await form.getByRole("button", { name: "Save connection", exact: true }).click(); + await expect(card).toHaveAttribute("data-state", "ready"); + await expect(tid(card, "local-model")).toContainText(["Replacement/Coder:Q8", "Custom/Exact ID:Q4_0"]); + old.release("Stale/MustNotAppear"); + await expect.poll(() => app!.evaluate(({}, requestId) => (globalThis as any).__lm4DiscoveryResults.some((r: any) => r.requestId === requestId && r.state === "stale"), out.secondHeldRequestId)).toBe(true); + out.discoveryResults = await app.evaluate(() => (globalThis as any).__lm4DiscoveryResults); + out.staleIdAbsentAtCheckedPoint = !(await card.innerText()).includes("Stale/MustNotAppear"); + expect(out.staleIdAbsentAtCheckedPoint).toBe(true); + const profiles = await page.evaluate(() => window.modex!.invoke("local-models:command", { op: "list" })); + if (profiles.op !== "list") throw new Error("Expected profile list"); + out.currentConnection = profiles.connections[0]; expect(out.currentConnection.revision).toBe(out.targetBefore.model.connection.revision + 1); + expect(out.currentConnection.origin).toBe(replacement.url); expect(out.currentConnection.manualModels).toEqual(["Custom/Exact ID:Q4_0"]); + const state = await page.evaluate(() => window.modex!.invoke("state:get", undefined)); + out.targetAfter = state.threads.find(t => t.id === thread.id)!.localTarget; expect(out.targetAfter).toEqual(out.targetBefore); + await page.keyboard.press("Escape"); + await expect(tid(page, "composer-input")).toHaveValue(" pinned draft\nkeep every byte "); + out.rawDraftAfter = await tid(page, "composer-input").inputValue(); + await expect(tid(page, "local-inference-host")).toContainText("an unavailable local server"); + await expect(tid(page, "local-inference-host")).not.toContainText(replacement.url); + out.provenanceText = await tid(page, "local-inference-host").innerText(); + out.callCounts = await app.evaluate(() => (globalThis as any).__lm4Calls); expect(out.callCounts).toEqual({ localStarts: 0, sends: 0, durableSends: 0, retries: 0 }); + expect(await page.evaluate(threadId => window.modex!.invoke("thread:items", { threadId }), thread.id)).toEqual([]); + for (const r of [old, replacement]) expect(r.hits.every(h => !h.auth && ((h.method === "GET" && ["/api/tags", "/api/ps"].includes(h.url!)) || (h.method === "POST" && h.url === "/api/show")))).toBe(true); + out.state = "passed"; + } catch (error) { out.state = "failed"; out.failure = String(error); throw error; } + finally { + old?.release(); out.runtime = { old: { hits: old?.hits ?? [], releases: old?.heldReleases ?? [] }, replacement: { hits: replacement?.hits ?? [] } }; + try { if (app) { await app.close(); out.appClose = "returned"; } else out.appClose = "no returned app handle"; } + catch (error) { out.state = "close-failed"; out.closeError = String(error); throw error; } + finally { await Promise.all([old, replacement].filter((r): r is Awaited> => Boolean(r)).map(r => r.stop())); out.finishedAt = new Date().toISOString(); persist(); await testInfo.attach("local-model-profile-revision.json", { path: receipt, contentType: "application/json" }); } + } +}); diff --git a/docs/reviews/2026-11-12-local-model-refresh-preservation.md b/docs/reviews/2026-11-12-local-model-refresh-preservation.md new file mode 100644 index 00000000..e0fdd2da --- /dev/null +++ b/docs/reviews/2026-11-12-local-model-refresh-preservation.md @@ -0,0 +1,38 @@ +# Local-model refresh preservation + +## Candidate and scope + +The `model-refresh-preservation` candidate starts from protected main `0e4e2126737874bb6604e9dece6334aa789f7c34`, including #502's model search and #501's Code Review decision record. This change adds two canonical Electron cases and an observer fixture; production application source is unchanged. It covers delayed metadata preservation under #381, without claiming the complete S2 or AC3 clauses, real-model qualification, ordinary product admission, paired-client provenance or human acceptance. + +The fixture requires a disposable `modex-e2e-home-*` directory under the system temporary directory, scripted E2E mode and a mock default backend. It reuses the existing local-model workflow bootstrap. Discovery and coding observers call the original methods with their original receivers and return their results. The tests use bounded local metadata servers and request no coding turns. + +## Reproducible cases + +- `local-model-preservation.spec.ts` holds Alpha's metadata response while switching between a cloud thread with a staged image and a separately pinned Beta thread. It verifies exact multiline drafts, staged PNG bytes, unchanged target/revision, search query/focus/selection and composer selection after metadata delivery. Both transcripts remain empty and sampled coding dispatch counters remain zero. +- `local-model-profile-revision.spec.ts` holds metadata while editing an unsaved connection form, then checks the form's raw values, focus and selected text after delivery. A second held request is bound to its exact main-owned request ID. Saving a replacement endpoint increments the connection revision; the old request completes as stale and the old thread target remains pinned to its original revision, with unavailable-server provenance and its draft retained. + +The second case explicitly records whether the old HTTP response was already destroyed at release. Cancellation followed by a stale main completion is distinct from rejection of a delivered late body. The absence of the stale model ID is asserted at a checked point, not as an exhaustive future guarantee. Metadata logs contain method/path/auth-presence observations, not account credentials or file contents. + +## First failures and setup correction + +The first canonical run passed the cross-thread case but failed the profile field's post-refresh selection assertion: the caret was at `{start:15,end:15}` instead of `{start:0,end:3}`. A diagnostic run added the same assertion immediately before releasing metadata and failed there with the identical range. The keyboard setup had not established the intended selection, so these failures do not demonstrate refresh-driven product selection loss. + +The correction establishes the profile field's range with `setSelectionRange(0, 3)` after focusing it. The pre-release and post-delivery assertions remain unchanged. The search case separately retains real keyboard selection and navigation coverage. No production repair or relaxed deadline was introduced. Both failed runs, source snapshots, traces and receipts are retained outside the worktree before subsequent Playwright runs can replace their output. + +## Verification and delivery + +The corrected targeted command `pnpm exec playwright test e2e/local-model-preservation.spec.ts e2e/local-model-profile-revision.spec.ts` passed both cases in 3.8 seconds with no skips or observed retries. Both receipts report passed state and returned application close. Five generated artifact files were copied outside the worktree and byte-verified. Source hashes and original process handles are retained in the external closeout checkpoint. + +Independent read-only review found no Critical, Important or Minor findings in the three test/fixture files and independently verified all 17 archived files across the first canonical failure, pre-release control failure and corrected passing run. Programmatic profile selection does not qualify native Home-key behavior. Root build and typechecks passed. Full root units passed with 2,478 passes, 36 skips and zero failures or cancellations (2,514 total); full Electron/site E2E then passed with 519 desktop passes, 11 skips and seven site passes, with no failures or observed retries. The original sequential validation command exited zero. All 178 final artifact files were archived and byte-verified, and both last-run records report passed with no failed tests. Independent terminal review verified all archive and log hashes and the signed source blobs. The two new receipts bind to signed source `16d4fa877b3ad09a5b2107ed2d237499e364a362`; committing during E2E changed no working source bytes, and production source is unchanged from run-start `0e4e212`. The qualification below binds to implementation source `16d4fa8` and its exact checkout tree. The documentation-only follow-up needs its own hosted checks; protected delivery is tracked in PR #503. No issue checkbox advances on these scoped results alone. #381 still needs the full grouping/qualification/reachability/state-preservation requirements, native compatibility-test controls and evidence states, supported paired provenance, and applicable keyboard/minimum-window/human evidence. + +#502 itself is already delivered as protected main `bf497f72b07b2c283a90b5aaa7dfa84f57e3cf39`; its exact reviewed tree and queue results are recorded in the [model-search review](2026-11-12-model-search.md). This candidate's later tests are separate evidence, not retroactive source-head qualification for #502. + +## Hosted platform results and visual observation + +Linux and Windows jobs `131046081211` and `131046080897` both succeeded on attempt one in run `43177772321`. Their actual checkout `156e6cd4a5ed93693731fb060bdb7b56fdd973b6` has the exact signed candidate tree `afc170eec41b712bec2130a39eac6e017c905739`. Independent audit verified both original log hashes, checkout parents and fresh terminal statuses. Both receipts explicitly retain installer acceptance as pending: Linux still needs KWallet6/full graphical login and complete CLI/account acceptance; Windows still needs complete CLI/account acceptance. These hosted jobs are not full personal installed-platform acceptance. + +Direct inspection of the final ready and minimum-window Graphite model-search captures found visible retained query/focus, exact opaque identifiers, connection health/reason and the management action within the window. These are existing model-search fixture captures, not captures of the two new preservation cases or complete human/theme/state acceptance. + +Hosted iPhone job `131046081135` in run `43177772414` succeeded on attempt one against the same `156e6cd` checkout/tree. The log contains 44 distinct UI start/pass pairs: 43 ordinary passes and one deliberate expected screenshot-failure selftest, with zero unexpected failures or skips. The 88,346,257-byte test-results ZIP matches GitHub's SHA-256 digest `4106d48ce6b5dafea9fb3c9f2346e563689554655b2144398f3060943b4b33d1` and passes CRC across 1,604 members. This is simulator evidence, not physical-device or human acceptance, and does not establish a TLS/shutdown cause or repair. + +Hosted macOS job `131046080786` in run `43177772414` succeeded on attempt one against checkout `156e6cd4a5ed93693731fb060bdb7b56fdd973b6`, tree `afc170eec41b712bec2130a39eac6e017c905739`, matching implementation source `16d4fa8`. Its original log reports 2,478 unit passes with 36 skips, 522 desktop E2E passes with eight skips and seven site passes, with zero failures, cancellations or observed retries. The E2E, runner-memory and coverage ZIPs match fresh GitHub artifact digests and source provenance and pass CRC. The E2E ZIP contains 176 members; both new case receipts report passed state and returned app close at the actual checkout. Hidden last-run files are absent from the hosted ZIP, so no hosted last-run JSON result is inferred. diff --git a/docs/reviews/2026-11-12-model-search.md b/docs/reviews/2026-11-12-model-search.md index 3cc0ea39..93bc76d9 100644 --- a/docs/reviews/2026-11-12-model-search.md +++ b/docs/reviews/2026-11-12-model-search.md @@ -19,3 +19,9 @@ Independent review found no remaining Critical, Important or Minor findings acro First failures, original tool handles, logs and captures are retained under `/Users/buns/Documents/Codex/2026-11-11/modex-closeout/`, including `model-search-first-red.log`, `model-search-first-red-artifacts/`, `model-search-return-to-filter-first-red.log`, `model-search-return-to-filter-first-red-artifacts/` and `model-search-keyboard-focused-first.log`. The latter run returned zero with seven passes and no skips or retries. Private receipt paths are not portable public evidence. The candidate was exercised as uncommitted source on base `4f1d80c5b25115a1debcf37a276d749a245e1589`, with all eight file hashes frozen before final validation. The terminal controller reverified those hashes before and after every stage. Final qualification is recorded in `model-search-final-validation.json`, `model-search-final-local-validation.json` and `model-search-full-unit-first-terminal.json`; these manifests distinguish the base Git head from the tested changed source. Later edits to this receipt, status and the plan reconcile evidence only; production and test bytes remain frozen. + +## Protected delivery + +PR #502 delivered signed main `bf497f72b07b2c283a90b5aaa7dfa84f57e3cf39`, with tree `5dc4896008f5b0070f00c1bf18f55f548dff9b08` exactly matching reviewed source `1ed3cd07c86440a46718128e20cbc65e6d933bfc` and its hosted test merge. Protected queue [43169825746](https://github.com/TypeSafeAI/modex/actions/runs/43169825746), required macOS job 131020191299, passed 2,478 unit cases with 36 explicit skips, 520 desktop E2E cases with eight skips and seven site cases, with zero failures, cancellations or observed E2E retries. Both queue ZIPs matched fresh GitHub SHA-256 digests and passed full CRC verification; the E2E ZIP contains 172 members and excludes hidden last-run files. Queue iPhone was intentionally skipped. The separate source-head simulator suite passed 43 ordinary cases plus one deliberate expected-failure selftest, with no unexpected failures, skips or observed retries. Linux/Windows hosted exercises passed while retaining installer acceptance as pending. + +Independent review verified actual merge, signature, exact tree, original queue logs and artifacts. This supersedes the earlier pending-delivery checkpoint. #381 remains open at two of 13 boxes; this search delivery does not establish the full attachment/thread/profile matrix, compatibility testing, paired provenance, production admission or human acceptance. The owned worktree was released after preserving all known diagnostic evidence and restoring its exact signed tip from a complete-history bundle; no worktree deletion was requested. diff --git a/docs/status.md b/docs/status.md index c5c6c94a..78c74ee5 100644 --- a/docs/status.md +++ b/docs/status.md @@ -63,6 +63,8 @@ and [landing verification](reviews/2026-10-11-landing-clarity.md). ## On main, unreleased +- **Model-picker search** (#381, #502). Protected main `bf497f7` has connection/model filtering with exact IDs, retained query/focus through metadata completion, and keyboard return to the filter. Its signed merge exactly matches the reviewed/tested tree; the protected queue passed 2,478 units with 36 skips, 520 desktop E2E cases with eight skips and seven site cases. The separate source iPhone suite passed 43 ordinary cases and one expected-failure selftest; installer/human/admission limits remain. The full #381 preservation and qualification matrix stays open. [Delivery and limits](reviews/2026-11-12-model-search.md). + The published application source is `6a97d23aaea315b29008890f25dde2d7ea07f1d0` (v0.4.1). v0.4.2 is in preparation with Code Review hidden ([review](reviews/2026-11-03-v0.4.2-release-review.md)). Later source deliveries remain unreleased: #462 records local-model context provenance; #461 enforces local-only auxiliary dispatch; #465 refuses known cloud-routed and non-generative models; #463 (`32f1cb4`) defers Markdown far from the view so long threads open fast. Main still does not construct `LocalExecution`; native qualification and product admission remain separate gates. #459, #460 and #464 (`3d72c6e`, the CR-3 acceptance re-run) are evidence/documentation deliveries after the release source. Also unreleased, since that summary: #467 (`8a28df3`) adds the Claude account pool with OrcaRouter keys; #477 (`49aa449`) keeps Code Review links right after a repository rename; #487 (`b785054`) adds CF-3 draft and YOLO regressions; #488 (`68fe543`) adds owner-authorized worktree retirement; #490 (`a4f2f70`) adds the Code Review setting and its gates, and turned Code Review on by default. #492 (`4297235`) turned that default off, because Val chose v0.4.2 with Code Review hidden on 2026-11-03. #468, #470, #472, #473, #474, #476 and #483 are evidence or documentation only. @@ -116,11 +118,11 @@ before relying on a tick. | [#325](https://github.com/TypeSafeAI/modex/issues/325) CR-6 | 13/14 | Authorized live native smoke (AC6) | Run the smoke once on a small open TypeSafeAI/modex PR (authorized 2026-11-04) | | [#326](https://github.com/TypeSafeAI/modex/issues/326) CR-7 | 13/14 | Live canary on a disposable repository (AC6) | Val names a disposable repository | | [#327](https://github.com/TypeSafeAI/modex/issues/327) CR-8 | 5/17 | Reference comparison (retired 2026-11-04, boxes still name it), VoiceOver and the live gates. Budgets and C76 landed in #490; re-check item 18 and AC5 once Val confirms the budget numbers | Val's [one session](reviews/2026-11-02-v0.5.0-val-session.md) | -| [#376](https://github.com/TypeSafeAI/modex/issues/376) LM tracker | 1/14 | LM-2 to LM-8 acceptance; `LocalExecution` is not constructed, so the product refuses local turns | Val answers the five questions in `docs/reviews/2026-11-03-lm5-val-decision.md` on branch `lm-ready` | +| [#376](https://github.com/TypeSafeAI/modex/issues/376) LM tracker | 1/14 | LM-2 to LM-7 core acceptance; LM-8 is optional and does not block the core release; `LocalExecution` is not constructed, so the product refuses local turns | Val answers the five questions in `docs/reviews/2026-11-03-lm5-val-decision.md` on branch `lm-ready` | | [#378](https://github.com/TypeSafeAI/modex/issues/378) LM-2 | 12/13 | #498 delivered the cold-load explanation and token/export regressions; #499 delivered manual-ID visibility and the complete metadata IPC fixture matrix. Packaged secure-storage acceptance remains open | Complete secure-storage acceptance and re-audit every criterion; existing ticks do not replace current evidence. [Receipt](reviews/2026-11-12-lm2-manual-visibility.md) | | [#379](https://github.com/TypeSafeAI/modex/issues/379) LM-3 | 12/13 | AC1's real native read/edit/check and AC4's regression/configuration receipts now bind to actual main. AC6 still has native redirect and managed-policy gaps; simultaneous isolation claims retain their tested fixture scope | Resolve AC6 without weakening destination binding; product admission stays separate. [Receipt](https://github.com/TypeSafeAI/modex/issues/379#issuecomment-6811858808) | | [#380](https://github.com/TypeSafeAI/modex/issues/380) LM-5 | 6/13 | S5 shared capacity/cancellable waiting/explicit status and AC3/AC5 are reconciled on protected main. Redirect privacy, action-safe compaction, genuine OOM and older-client presentation remain open | Complete remaining engineering/native gates; preserve #475's owner disposition. [Delivered S5 evidence](https://github.com/TypeSafeAI/modex/issues/380#issuecomment-6815410455) | -| [#381](https://github.com/TypeSafeAI/modex/issues/381) LM-4 | 2/13 | S4 separates inference/workspace/harness labels and AC6 preserves existing cloud/workspace behavior on main. Search, native compatibility-test/qualification UI and phone inference provenance are not implemented | Complete missing implementation, delayed-refresh/state/keyboard coverage and human/device gates; product admission remains separate. [Criterion evidence](https://github.com/TypeSafeAI/modex/issues/381#issuecomment-6813640283) | +| [#381](https://github.com/TypeSafeAI/modex/issues/381) LM-4 | 2/13 | S4 separates inference/workspace/harness labels and AC6 preserves existing cloud/workspace behavior on main. Search is delivered by #502; native compatibility-test/qualification UI and phone inference provenance are not implemented | Complete missing implementation, delayed-refresh/state/keyboard coverage and human/device gates; product admission remains separate. [Criterion evidence](https://github.com/TypeSafeAI/modex/issues/381#issuecomment-6813640283) | | [#382](https://github.com/TypeSafeAI/modex/issues/382) LM-6 | 0/11 | Bounded native bundled llama-server read/edit/check, Stop and two-worker restart receipts exist; the full runtime matrix is incomplete | Qualify remaining runtime/resource cases without generalizing one named combination | | [#383](https://github.com/TypeSafeAI/modex/issues/383) LM-7 | 0/15 | [Evidence map](reviews/2026-11-03-lm7-evidence-map.md) reconciles actual #498/#499 protected delivery and separates fixtures, native models, network-denied and human evidence; privacy/compaction/resource gaps remain | Complete producer criteria, integrated golden/adversarial workflow and runtime/human gates | | [#384](https://github.com/TypeSafeAI/modex/issues/384) LM-8 | 0/11 | Optional; an outside contributor's scope question is unanswered | A maintainer answers the [scope question](https://github.com/TypeSafeAI/modex/issues/384#issuecomment-6390714922) | @@ -252,7 +254,8 @@ that fixed each. ## Next -- **Model-picker search** (#381), owned `model-search` candidate: exact-ID/connection filtering, delayed-listing query/focus and draft preservation, plus keyboard return to the retained filter. Seven focused Electron cases pass; final build/typecheck, 2,478 units with 36 skips, 517 desktop E2E cases with 11 skips and seven site cases passed. Hosted checks and protected delivery remain pending. The full issue, attachment/thread/profile matrix, native qualification and paired-client provenance stay open. [Evidence](reviews/2026-11-12-model-search.md). +- **Local-model refresh preservation** (#381), owned `model-refresh-preservation` candidate: promote two reviewed scripted-fixture characterizations into canonical coverage for cross-thread image/raw-draft preservation, edit-form focus, exact-request stale completion and unchanged old revision-bound targets. Both targeted cases pass with before/after selection checks. Build/typecheck, 2,478 units with 36 skips, 519 desktop E2E cases with 11 skips and seven site cases pass; final artifacts and terminal results are independently verified. Draft #503 has signed source `16d4fa8`; source-head hosted macOS, iPhone and native-platform jobs pass. Final-head checks and protected delivery are tracked in #503. [Plan](superpowers/plans/2026-11-12-local-model-refresh-preservation.md); [evidence and limits](reviews/2026-11-12-local-model-refresh-preservation.md). + Roughly in the order they earn their place; none is scheduled. diff --git a/docs/superpowers/plans/2026-11-12-local-model-refresh-preservation.md b/docs/superpowers/plans/2026-11-12-local-model-refresh-preservation.md new file mode 100644 index 00000000..edf8022d --- /dev/null +++ b/docs/superpowers/plans/2026-11-12-local-model-refresh-preservation.md @@ -0,0 +1,27 @@ +# Local-model refresh preservation implementation plan + +> Execute inline with the existing verification and independent-review workflow. + +**Goal:** Make #381's cross-thread and connection-revision preservation evidence reproducible in the canonical Electron suite, including text selection. + +**Spec:** `docs/design/local-models-settings.md` and #381 S2/AC3. Product admission, compatibility testing, paired provenance, real models and human acceptance retain their independent requirements. + +**Architecture:** Keep production code unchanged unless a meaningful regression demonstrates a defect. Promote two reviewed temporary-home characterizations into canonical E2E files and a guarded observer entry point that reuses the existing scripted LocalExecution workflow. Observers call original methods and retain their results. Metadata servers permit only the existing Ollama metadata paths; tests request no coding turns. + +## Task 1: Reproducible preservation coverage + +- [x] Start an owned helper worktree from protected main `bf497f72b07b2c283a90b5aaa7dfa84f57e3cf39`; retain the finished worktree's recovery bundle and evidence before releasing it. +- [x] Promote the reviewed cross-thread and connection-revision cases into `apps/desktop/e2e/local-model-preservation.spec.ts` and `local-model-profile-revision.spec.ts`; make receipts portable through Playwright output and attachments. +- [x] Add `e2e/fixtures/local-model-preservation.mjs`, guarded to disposable scripted homes, with observer-only coding/discovery counters and exact-request stale-completion binding. +- [x] Run targeted cases with literal selection-range assertions for query, composer and unsaved profile fields. Preserve first failures; repair production only if evidence establishes a defect. +- [x] Reconcile #502 delivery, dated receipts and status. Clarify existing optional LM-8 scope without changing requirements or approval gates. +- [x] Run root build/typecheck/units and full E2E, inspect relevant captures, and independently review the final source and qualification evidence. +- [ ] Sign, push, open a PR, satisfy hosted checks and use the protected queue. Verify actual merged main before advancing any complete criterion. + +## Review focus + +- Separate direct-IPC setup from ordinary caller refresh; retain the first setup failure without claiming a product regression repair. +- Bind stale completion to the exact second held request, excluding earlier cancellation results. +- Record cancellation versus delivered late bodies explicitly. +- Preserve raw multiline text, image bytes, exact opaque IDs, model target/revision and text selection. +- Keep fixture admission, test encryption, sampled counters and human/native acceptance limits explicit. diff --git a/docs/superpowers/plans/2026-11-12-model-search.md b/docs/superpowers/plans/2026-11-12-model-search.md index 20df62f6..a6377f68 100644 --- a/docs/superpowers/plans/2026-11-12-model-search.md +++ b/docs/superpowers/plans/2026-11-12-model-search.md @@ -24,5 +24,5 @@ - [x] Run the cases against unchanged production source and retain the first missing-search failures. - [x] Extend `ModelMenu.tsx` with local query state, input and filtered cloud/local groups; clear query on closure. Keep management and cloud-return controls available. - [x] Extend `components/ui/Menu.tsx` with optional initial input focus and text-editing-aware key handling. Add theme-token styling in `styles.css`. -- [ ] Run focused search and existing local-model/menu suites, inspect actual Electron captures, then root build/typecheck/units and full E2E. Review before signed commit/push and protected delivery. +- [x] Run focused search and existing local-model/menu suites, inspect actual Electron captures, then root build/typecheck/units and full E2E. Review before signed commit/push and protected delivery. - [x] Reconcile the delivered #500 queue evidence and #380 S5 count in `docs/status.md` and dated receipts; record search's actual verification and limits separately. From 8059acc767edebc26a8c152a2522e607436b1e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Val=20Alexander=20=F0=9F=91=91?= Date: Thu, 12 Nov 2026 10:11:49 +0000 Subject: [PATCH 2/8] Conserve low coding credits across Auto routes and retries (#505) * feat: conserve low coding allowance and credits in Auto * fix: enforce quota admission on retries and selected routes * docs: record credit conservation qualification and limits * fix: admit every automatic credit replacement and preserve rotation * docs: record integrated credit conservation qualification --- apps/desktop/e2e/credit-conservation.spec.ts | 36 ++++ .../src/main/engine/backends/account-codex.ts | 3 +- .../src/main/engine/backends/claude.ts | 16 +- .../desktop/src/main/engine/backends/codex.ts | 9 +- .../desktop/src/main/engine/backends/types.ts | 6 +- apps/desktop/src/main/engine/routing/fit.ts | 6 +- .../desktop/src/main/engine/routing/policy.ts | 107 ++++++++++- apps/desktop/src/main/engine/routing/quota.ts | 77 ++++++++ .../desktop/src/main/engine/routing/router.ts | 100 +++++++--- apps/desktop/src/main/engine/runner.ts | 142 ++++++++++---- apps/desktop/src/main/engine/store.ts | 3 + .../renderer/components/SettingsDialog.tsx | 16 ++ apps/desktop/src/shared/types.ts | 11 +- apps/desktop/test/backend-quota.test.ts | 96 ++++++++++ .../test/conservation-settings.test.ts | 18 ++ apps/desktop/test/credit-fallback.test.ts | 5 +- .../desktop/test/routing-conservation.test.ts | 174 ++++++++++++++++++ apps/desktop/test/routing-quota.test.ts | 76 ++++++++ .../test/runner-automatic-quota.test.ts | 148 +++++++++++++++ apps/desktop/test/runner-local-models.test.ts | 11 +- apps/desktop/test/runner-quota.test.ts | 82 +++++++++ docs/auto-routing.md | 16 ++ ...2026-11-12-credit-conservation-delivery.md | 11 ++ .../reviews/2026-11-12-credit-conservation.md | 22 +++ docs/status.md | 2 + .../plans/2026-11-12-credit-routing.md | 56 ++++++ .../specs/2026-11-12-credit-routing-design.md | 23 +++ 27 files changed, 1192 insertions(+), 80 deletions(-) create mode 100644 apps/desktop/e2e/credit-conservation.spec.ts create mode 100644 apps/desktop/src/main/engine/routing/quota.ts create mode 100644 apps/desktop/test/backend-quota.test.ts create mode 100644 apps/desktop/test/conservation-settings.test.ts create mode 100644 apps/desktop/test/routing-conservation.test.ts create mode 100644 apps/desktop/test/routing-quota.test.ts create mode 100644 apps/desktop/test/runner-automatic-quota.test.ts create mode 100644 apps/desktop/test/runner-quota.test.ts create mode 100644 docs/reviews/2026-11-12-credit-conservation-delivery.md create mode 100644 docs/reviews/2026-11-12-credit-conservation.md create mode 100644 docs/superpowers/plans/2026-11-12-credit-routing.md create mode 100644 docs/superpowers/specs/2026-11-12-credit-routing-design.md diff --git a/apps/desktop/e2e/credit-conservation.spec.ts b/apps/desktop/e2e/credit-conservation.spec.ts new file mode 100644 index 00000000..da51de0c --- /dev/null +++ b/apps/desktop/e2e/credit-conservation.spec.ts @@ -0,0 +1,36 @@ +import { test, expect } from "@playwright/test"; +import fs from "node:fs"; +import path from "node:path"; +import { appDir, launch, seedHome, tid } from "./support"; + +test("automatic credit conservation defaults on, thresholds save, and disabling persists", async () => { + const { home } = seedHome(); + const { app, page } = await launch(home); + try { + const open = async () => { + await tid(page, "open-settings").click(); + await tid(page, "settings-nav").getByRole("button", { name: "Auto routing", exact: true }).click(); + }; + await open(); + const enabled = tid(page, "conserve-credits"), percent = tid(page, "low-quota-percent"), balance = tid(page, "low-credit-balance"); + await expect(enabled).toBeChecked(); + await expect(percent).toHaveValue("20"); + await expect(balance).toHaveValue("10"); + await percent.fill("25"); + await balance.fill("5"); + await page.getByText("Conservation chooses economical models", { exact: false }).scrollIntoViewIfNeeded(); + await page.screenshot({ path: path.join(appDir, "test-results", "credit-conservation-settings.png") }); + await tid(page, "settings").getByRole("button", { name: "Save", exact: true }).click(); + await open(); + await expect(percent).toHaveValue("25"); + await expect(balance).toHaveValue("5"); + await enabled.uncheck(); + await tid(page, "settings").getByRole("button", { name: "Save", exact: true }).click(); + await open(); + await expect(enabled).not.toBeChecked(); + await expect(percent).toBeDisabled(); + await expect(balance).toBeDisabled(); + const saved = JSON.parse(fs.readFileSync(path.join(home, "app", "state.json"), "utf8")).settings.routing; + expect([saved.conserve_credits, saved.low_quota_percent, saved.low_credit_balance]).toEqual([false, 25, 5]); + } finally { await app.close(); } +}); diff --git a/apps/desktop/src/main/engine/backends/account-codex.ts b/apps/desktop/src/main/engine/backends/account-codex.ts index f14bc566..835b4cfd 100644 --- a/apps/desktop/src/main/engine/backends/account-codex.ts +++ b/apps/desktop/src/main/engine/backends/account-codex.ts @@ -88,7 +88,8 @@ export class AccountCodexBackend implements Backend { const id = opts.account ?? this.identity(); return this.use(id, (backend) => generateTitle(backend, opts, text, signal)); } - listModels(): Promise { return this.use(this.identity(), (backend) => backend.listModels()); } + listModels(account = this.identity()): Promise { return this.use(account, (backend) => backend.listModels()); } + quota(account = this.identity()) { return this.use(account, (backend) => backend.quota()); } async health() { const id = this.identity(); if (id === "cli") return this.cli.health(); const installed = await installation(this.bin, this.spawnImpl); diff --git a/apps/desktop/src/main/engine/backends/claude.ts b/apps/desktop/src/main/engine/backends/claude.ts index 032f0ec9..4f116e0b 100644 --- a/apps/desktop/src/main/engine/backends/claude.ts +++ b/apps/desktop/src/main/engine/backends/claude.ts @@ -1,3 +1,4 @@ +import { claudeQuota, type QuotaSnapshot } from "../routing/quota.js"; import { actionDigest } from "../../../shared/human-requests.js"; import { AgentTracker } from "./agents.js"; import type { CliExecutable } from "../cli-path.js"; @@ -67,6 +68,9 @@ export function confinedEnvironment(bin: string, env: NodeJS.ProcessEnv = proces */ export class ClaudeBackend implements Backend { readonly id = "claude" as const; + private readonly quotas = new Map(); + + async quota(account = "cli"): Promise { return this.quotas.get(account); } protected get bin(): string { return typeof this.executable === "function" ? this.executable() : this.executable; } constructor( @@ -134,7 +138,7 @@ export class ClaudeBackend implements Backend { if (opts.model) args.push("--model", opts.model); if (opts.effort) args.push("--effort", opts.effort); // Fast mode is a settings key in Claude Code; `--settings` accepts inline JSON for this session only. - if (opts.fast) args.push("--settings", JSON.stringify({ fastMode: true })); + if (opts.fast || opts.conserveCredits) args.push("--settings", JSON.stringify({ fastMode: opts.conserveCredits ? false : true })); if (opts.resume) args.push("--resume", opts.resume); for (const d of opts.addDirs ?? []) args.push("--add-dir", d); if (opts.knowledge || opts.pages || opts.images) args.push("--mcp-config", JSON.stringify({ mcpServers: { @@ -378,6 +382,14 @@ export class ClaudeBackend implements Backend { } catch { return; } + if (msg.type === "rate_limit_event") { + const next = claudeQuota(msg.rate_limit_info); + const account = opts.account ?? "cli"; + const previous = this.quotas.get(account); + const replaced = new Set(next.windows.map(w => w.key)); + this.quotas.set(account, { ...next, windows: [...(previous?.windows ?? []).filter(w => !replaced.has(w.key)), ...next.windows] }); + return; + } // A heartbeat (the CLI sends one while a control request runs long) is no sign of the turn going on. if (msg.type === "keep_alive") return; // The CLI no longer needs an answer to one of its prompts (Stop, or it settled the prompt itself). @@ -705,6 +717,8 @@ interface ClaudeMessage { tasks?: { task_id?: string }[]; session_id?: string; message?: { id?: string; model?: string; content?: ContentBlock[]; usage?: ClaudeUsage }; + /** Metadata-only rate_limit_event from the CLI. */ + rate_limit_info?: unknown; /** A synthetic assistant reply's API error class ("rate_limit", "billing_error", …). */ error?: unknown; /** `result`: the turn's token totals. */ diff --git a/apps/desktop/src/main/engine/backends/codex.ts b/apps/desktop/src/main/engine/backends/codex.ts index ba3d7c02..348cfcbd 100644 --- a/apps/desktop/src/main/engine/backends/codex.ts +++ b/apps/desktop/src/main/engine/backends/codex.ts @@ -1,3 +1,4 @@ +import { codexQuota, type QuotaSnapshot } from "../routing/quota.js"; import { requestedPermissions } from "./codex-permissions.js"; import { AgentTracker } from "./agents.js"; import { PAGES_TOOLS } from "../../../shared/pages.js"; @@ -178,6 +179,12 @@ export class CodexBackend implements Backend { this.send({ id, result }); } + async quota(): Promise { + if (this.configuration?.local) return undefined; + await this.ensure(); + return codexQuota(await this.request("account/rateLimits/read", {}, 3000)); + } + async listModels(): Promise { await this.ensure(); const res = await this.request<{ data: CodexModel[] }>("model/list", {}); @@ -703,7 +710,7 @@ export class CodexBackend implements Backend { model: opts.model || null, effort: opts.effort ?? null, // "fast" is the Codex fast-mode service tier; only this turn, the thread's tier is untouched. - serviceTierForTurn: opts.fast ? "fast" : null, + serviceTierForTurn: opts.conserveCredits ? "default" : opts.fast ? "fast" : null, ...(mode ? { collaborationMode: mode } : {}), }); turnStart(input, collaborationMode) diff --git a/apps/desktop/src/main/engine/backends/types.ts b/apps/desktop/src/main/engine/backends/types.ts index e53bd92a..758bdc35 100644 --- a/apps/desktop/src/main/engine/backends/types.ts +++ b/apps/desktop/src/main/engine/backends/types.ts @@ -129,6 +129,8 @@ export interface TurnOptions { effort?: string; /** Fast mode for this turn (Codex "fast" service tier, Claude fastMode setting). */ fast?: boolean; + /** Auto conservation must override a saved fast setting, not inherit it. */ + conserveCredits?: boolean; /** Resumable handle from a previous turn on this thread. */ resume?: string; account?: string; @@ -209,7 +211,9 @@ export interface Backend { */ forceStop?(account?: string): Promise; /** Current model catalogue for this backend. */ - listModels(): Promise; + listModels(account?: string): Promise; + /** Metadata-only CLI quota read, scoped to the admitted account. Never starts inference. */ + quota?(account?: string): Promise; health?(): Promise; identity?(): string; dispose(): Promise; diff --git a/apps/desktop/src/main/engine/routing/fit.ts b/apps/desktop/src/main/engine/routing/fit.ts index 68d7a701..cce6d7f7 100644 --- a/apps/desktop/src/main/engine/routing/fit.ts +++ b/apps/desktop/src/main/engine/routing/fit.ts @@ -136,7 +136,7 @@ export class Fit { this.write(); } - recordOutcome(threadId: string, outcome: "completed" | "failed" | "interrupted", usage?: TurnUsage): void { + recordOutcome(threadId: string, outcome: "completed" | "failed" | "interrupted", usage?: TurnUsage, learn = true): void { const r = this.open(threadId); if (!r) return; r.outcome = outcome; @@ -145,10 +145,10 @@ export class Fit { if (usage.outputTokens !== undefined) r.outputTokens = usage.outputTokens; } const t = (this.state.tasks[r.task] ??= empty()); - if (outcome === "failed") { + if (learn && outcome === "failed") { t.failures += 1; t.offset = Math.min(MAX_OFFSET, t.offset + FAILURE_STEP); - } else if (outcome === "completed") t.completions += 1; + } else if (learn && outcome === "completed") t.completions += 1; this.write(); } diff --git a/apps/desktop/src/main/engine/routing/policy.ts b/apps/desktop/src/main/engine/routing/policy.ts index aab53698..374b46be 100644 --- a/apps/desktop/src/main/engine/routing/policy.ts +++ b/apps/desktop/src/main/engine/routing/policy.ts @@ -1,3 +1,4 @@ +import { quotaState, type QuotaSnapshot } from "./quota.js"; import type { BackendId, EffortLevel, LastTurn, RoutingPolicy } from "../../../shared/types.js"; import type { Candidate, Tier } from "./catalog.js"; import { pick } from "./catalog.js"; @@ -42,6 +43,11 @@ export interface DecisionInput { /** Learned per-task tier offset (see fit.ts). */ fitOffset: number; premiumExhausted: boolean; + quotas?: Partial>; + /** Retry retains the previous attempt's conservation admission even when telemetry is now unknown. */ + conserveCredits?: boolean; + /** An automatic account replacement retains the admitted task capability floor. */ + minimumTier?: Tier; /** * What the thread's prompt cache was built with, and how long ago. The router always passes it; * a caller that omits it gets a fresh choice every turn. @@ -59,6 +65,11 @@ export interface Decision { pinned: boolean; /** The request must not run because an effort or premium budget limit cannot be guaranteed. */ blocked?: true; + minimumTier?: Tier; + quotaBlocked?: true; + quotaResetAt?: number; + /** Requires an explicit standard-speed override in the CLI, even with saved fast settings. */ + conserving?: true; /** True when a warm prompt cache kept the thread's model, effort or fast mode. */ held?: true; /** The cache warmth the hold step saw; absent when no cache information was passed. */ @@ -244,7 +255,7 @@ function hold(input: DecisionInput, free: Decision, current: Candidate | undefin return { ...kept, warmth: w }; } -export function decide(input: DecisionInput): Decision { +function decideWithLimits(input: DecisionInput): Decision { const { judgments: j, policy, current } = input; const reasons: string[] = []; const currentLadder = input.ladders[current.backend] ?? []; @@ -317,3 +328,97 @@ export function decide(input: DecisionInput): Decision { // 7. Hold: a warm prompt cache keeps the thread where it is unless the jump is worth a rebuild. return hold(input, { backend, model: candidate.model, effort, fast, tier: candidate.tier, pinned: false, reasons }, candidateNow, reasons); } + + +/** Quota is an eligibility constraint; conservation changes discretionary spend, never session ownership. */ +export function decide(input: DecisionInput): Decision { return decideWithQuota(input, Boolean(input.conserveCredits)); } + +function decideWithQuota(input: DecisionInput, retainConservation: boolean): Decision { + const now = Date.now(); + const state = (backend: BackendId, model: string) => quotaState(input.quotas?.[backend], input.policy, model, now); + const current = currentCandidate(input); + const currentUsage = state(input.current.backend, current?.model ?? input.current.model); + const planned = decideWithLimits(input); + const plannedUsage = state(planned.backend, planned.model); + const reasons = [`${input.current.backend}: ${currentUsage.reason}`]; + if (planned.model !== current?.model && plannedUsage.state === "low") reasons.push(`Initial candidate ${planned.model}: ${plannedUsage.reason}`); + const ladders: DecisionInput["ladders"] = {}; + for (const [backend, candidates] of Object.entries(input.ladders) as [BackendId, Candidate[]][]) { + ladders[backend] = candidates.filter(c => { + const usage = state(backend, c.model); + if (usage.state !== "exhausted") return true; + reasons.push(`Excluded ${c.label}: ${usage.reason}`); + return false; + }); + } + const conserve = input.policy.conserve_credits && (retainConservation || ["low", "exhausted"].includes(currentUsage.state) || plannedUsage.state === "low"); + let minimumTier = input.minimumTier; + const stop = (detail: string, quotaBlocked = false): Decision => ({ ...blocked(input, reasons, detail), minimumTier, ...(quotaBlocked ? { quotaBlocked: true as const, quotaResetAt: currentUsage.resetsAt } : {}), ...(conserve ? { conserving: true as const } : {}) }); + let effective = { ...input, ladders }; + if (conserve) { + const j = input.judgments; + const light = j.complexity < 1.5 && j.needsDeepReasoning < 0.7 && j.blastRadius < 1.5; + effective = { ...effective, fitOffset: light ? Math.min(0, input.fitOffset) : input.fitOffset, + policy: { ...conservationPolicy(input.policy), + posture: light ? "economy" : input.policy.posture } }; + reasons.push("Conserving low allowance or credits: economical suitable model, at most high effort, paid fast mode off; capability tiers estimate cost, not dollar prices."); + } + // Compare with the same cache/effort rules before exclusions. Unrelated quota buckets must + // never invalidate a legitimate cache hold that already ran below the free-choice tier. + const baseline = decideWithLimits({ ...effective, ladders: input.ladders }); + const free = decideWithLimits({ ...effective, ladders: input.ladders, cache: undefined }); + const wantedTier = Math.max(input.minimumTier ?? 0, Math.min(baseline.tier, free.tier)) as Tier; + minimumTier = wantedTier; + const healthy = (b: BackendId) => ladders[b]?.filter(c => c.tier >= wantedTier && state(b, c.model).state === "healthy") ?? []; + const quotaDriven = conserve || state(baseline.backend, baseline.model).state === "exhausted" || currentUsage.state === "exhausted"; + if (quotaDriven && input.policy.allow_backend_switch && !input.current.hasSession) { + const allowed = input.policy.allow_backends.filter(b => b === input.current.backend || healthy(b).length > 0); + effective.policy = { ...effective.policy, allow_backends: allowed }; + // Restrict all quota-driven alternatives, including the ordinary capability/unavailable + // fallback path, to candidates with fresh healthy readings. + for (const b of allowed) if (b !== input.current.backend) effective.ladders = { ...effective.ladders, [b]: healthy(b) }; + const alternate = allowed.find(b => b !== input.current.backend); + if (alternate && (conserve || currentUsage.state === "exhausted")) { + reasons.push(`Selected ${alternate} on a fresh conversation: ${state(alternate, healthy(alternate)[0]!.model).reason}`); + effective = { ...effective, ladders: { ...effective.ladders, [input.current.backend]: [] }, policy: { ...effective.policy, allow_backends: [alternate] } }; + } + } + if (currentUsage.state === "exhausted" && input.source === "jev" && input.judgments.taskConfidence < input.policy.min_confidence) { + return stop("The pinned model is exhausted and the judge is unsure; the route was stopped.", true); + } + const decision = decideWithLimits(effective); + const selectedUsage = state(decision.backend, decision.model || current?.model || ""); + // A different candidate can become low only after exhaustion filtering or backend selection. + // One repeat applies conservation to that candidate; it cannot loop because the flag is sticky. + if (!conserve && input.policy.conserve_credits && selectedUsage.state === "low") return decideWithQuota(input, true); + if (selectedUsage.state === "exhausted") return stop("No permitted model has usable allowance or credits; the route was stopped. Wait for a reset or choose another account.", true); + if (!decision.blocked && decision.tier < wantedTier) { + return stop("The remaining models do not meet this task's capability target; the route was stopped instead of silently downgrading the work.", Object.values(input.ladders).some(cs => cs?.some(c => state(c.backend, c.model).state === "exhausted"))); + } + if (decision.model !== current?.model || decision.backend !== input.current.backend) reasons.push(`${decision.backend}/${decision.model}: ${selectedUsage.reason}`); + return { ...decision, minimumTier: wantedTier, ...(conserve ? { conserving: true as const } : {}), reasons: decision.blocked ? [...reasons, ...decision.reasons] : [decision.reasons[0]!, ...reasons, ...decision.reasons.slice(1)] }; +} + + +function conservationPolicy(policy: RoutingPolicy): RoutingPolicy { + const configured = effortIndex(policy.max_effort); + return { ...policy, allow_fast: false, ...(configured === undefined ? {} : { max_effort: EFFORTS[Math.min(configured, effortIndex("high")!)]! }) }; +} + +/** Revalidate the actual fixed replacement; never turn a provider retry into a weaker or unmetered route. */ +export function admitReplacement(input: DecisionInput, prior: Decision): Decision { + const candidate = currentCandidate(input); + const usage = quotaState(input.quotas?.[input.current.backend], input.policy, candidate?.model ?? input.current.model); + const minimumTier = prior.minimumTier ?? prior.tier; + const conserve = input.policy.conserve_credits && (Boolean(prior.conserving) || usage.state === "low" || usage.state === "unknown"); + const reasons = [`Replacement ${input.current.backend}/${input.current.model}: ${usage.reason}`]; + const stop = (detail: string): Decision => ({ ...blocked(input, reasons, detail), minimumTier, ...(conserve ? { conserving: true as const } : {}) }); + if (usage.state === "exhausted") return { ...stop("The replacement is exhausted; Auto stopped before retrying."), quotaBlocked: true, quotaResetAt: usage.resetsAt }; + if (!candidate) return stop("The replacement model is not in the account's catalogue; Auto stopped before retrying."); + if (candidate.tier < minimumTier) return stop("The replacement cannot meet this task's capability target; Auto stopped before retrying."); + const effective = conserve ? { ...input, policy: conservationPolicy(input.policy) } : input; + const decision = keepCurrent(effective, candidate, reasons); + if (conserve) reasons.push("Conserving replacement allowance or credits: supported effort at most high and paid fast mode explicitly off."); + return { ...decision, minimumTier, ...(conserve ? { conserving: true as const } : {}), + fast: !decision.blocked && !conserve && input.policy.allow_fast && prior.fast && Boolean(candidate.fastTier) }; +} diff --git a/apps/desktop/src/main/engine/routing/quota.ts b/apps/desktop/src/main/engine/routing/quota.ts new file mode 100644 index 00000000..4da4d5d9 --- /dev/null +++ b/apps/desktop/src/main/engine/routing/quota.ts @@ -0,0 +1,77 @@ +import type { RoutingPolicy } from "../../../shared/types.js"; + +/** Account-bound metadata only. Provider credit units are not dollars or included allowance. */ +export interface QuotaSnapshot { + at: number; + windows: { key: string; remainingPercent?: number; resetsAt?: number; model?: string; at?: number; state?: "low" | "exhausted" }[]; + credits?: { hasCredits?: boolean; unlimited?: boolean; balance?: number }; + blocked?: boolean; + /** Explicit backend statement that included usage is unavailable; missing windows alone cannot prove this. */ + includedUnavailable?: boolean; +} +export interface QuotaState { state: "unknown" | "healthy" | "low" | "exhausted"; reason: string; resetsAt?: number } +export const QUOTA_MAX_AGE_MS = 5 * 60_000; +const object = (v: unknown): Record => v && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; +const finite = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v); +const epoch = (v: unknown): number | undefined => finite(v) && v > 0 && v * 1000 <= 8.64e15 ? v * 1000 : undefined; +const fresh = (at: number, now: number) => finite(at) && now >= at && now - at <= QUOTA_MAX_AGE_MS; + +export function quotaState(snapshot: QuotaSnapshot | undefined, policy: RoutingPolicy, model = "", now = Date.now()): QuotaState { + const unknown = { state: "unknown", reason: "Remaining allowance and credits are unknown; no available balance was assumed." } as const; + if (!snapshot || !fresh(snapshot.at, now)) return unknown; + if (snapshot.blocked) return { state: "exhausted", reason: "The CLI reports a spend-control limit; this account is unavailable." }; + const windows = snapshot.windows.filter(w => (!w.model || model === w.model || model.startsWith(`claude-${w.model}-`)) + && fresh(w.at ?? snapshot.at, now) && (w.resetsAt === undefined || (finite(w.resetsAt) && w.resetsAt > now)) + && (w.state || (finite(w.remainingPercent) && w.remainingPercent >= 0 && w.remainingPercent <= 100))); + const worst = [...windows].sort((a, b) => (a.state === "exhausted" ? 0 : a.remainingPercent ?? policy.low_quota_percent) - (b.state === "exhausted" ? 0 : b.remainingPercent ?? policy.low_quota_percent))[0]; + const c = snapshot.credits; + const paid = c?.unlimited === true || (c?.hasCredits === true && c.balance !== 0) || (c?.hasCredits !== false && c?.balance !== undefined && c.balance > 0); + if (worst) { + const remaining = worst.state === "exhausted" ? 0 : worst.remainingPercent; + const reset = worst.resetsAt === undefined ? "" : `; resets ${new Date(worst.resetsAt).toISOString()}`; + const detail = remaining === undefined ? "The CLI warns that included allowance is low" : `${Math.round(remaining * 10) / 10}% included allowance remains`; + const paidDetail = c?.balance === undefined && !c?.unlimited + ? " Paid credits are available; amount is unknown; conserving credits." + : " Paid credits remain; conserving credits."; + if (remaining === 0) return { state: paid ? "low" : "exhausted", resetsAt: worst.resetsAt, + reason: `Included allowance is exhausted${reset}.${paid ? paidDetail : " No usable paid-credit balance is established."}` }; + return { state: worst.state === "low" || remaining! <= policy.low_quota_percent ? "low" : "healthy", reason: `${detail}${reset}.` }; + } + if (c?.unlimited) return { state: "healthy", reason: "The CLI reports unlimited credits." }; + if (snapshot.includedUnavailable && (c?.hasCredits === false || c?.balance === 0)) return { state: "exhausted", reason: "The CLI reports included usage unavailable and no paid credits remain." }; + if (c?.balance !== undefined && finite(c.balance) && c.balance > 0) return { state: c.balance <= policy.low_credit_balance ? "low" : "healthy", reason: `${c.balance} provider credit units remain; included allowance is unknown.` }; + return unknown; +} + +/** Codex app-server account/rateLimits/read. Never copies credentials or unrecognized payload text. */ +export function codexQuota(value: unknown, at = Date.now()): QuotaSnapshot { + const response = object(value), primary = object(response.rateLimits); + const windows: QuotaSnapshot["windows"] = []; + const buckets = [["account", primary], ...Object.entries(object(response.rateLimitsByLimitId)).map(([key, v]) => [key, object(v)] as const)] as const; + for (const [key, bucket] of buckets) { + // The single-bucket view is authoritative for general usage. Other buckets need an explicit model scope. + const model = typeof bucket.normalModelSlug === "string" ? bucket.normalModelSlug : undefined; + if (key !== "account" && !model) continue; + for (const kind of ["primary", "secondary"]) { + const w = object(bucket[kind]); + if (!finite(w.usedPercent) || w.usedPercent < 0 || w.usedPercent > 100) continue; + windows.push({ key: `${key}:${kind}`, remainingPercent: 100 - w.usedPercent, resetsAt: epoch(w.resetsAt), ...(model ? { model } : {}) }); + } + } + const raw = object(primary.credits); + const balance = (typeof raw.balance === "string" && /^\d+(?:\.\d+)?$/.test(raw.balance.trim())) ? Number(raw.balance) : undefined; + const credits = Object.keys(raw).length ? { hasCredits: typeof raw.hasCredits === "boolean" ? raw.hasCredits : undefined, + unlimited: typeof raw.unlimited === "boolean" ? raw.unlimited : undefined, balance: finite(balance) && balance >= 0 ? balance : undefined } : undefined; + return { at, windows, ...(response.ordinaryUsageAllowed === false ? { includedUnavailable: true } : {}), ...(credits ? { credits } : {}), ...(primary.spendControlReached === true ? { blocked: true } : {}) }; +} + +/** Claude stream-json rate_limit_event; utilization is a fraction, resetsAt is epoch seconds. */ +export function claudeQuota(value: unknown, at = Date.now()): QuotaSnapshot { + const info = object(value); + const kind = typeof info.rateLimitType === "string" ? info.rateLimitType : "account"; + const model = /^(?:seven_day|five_hour)_(opus|sonnet|haiku|fable)$/.exec(kind)?.[1]; + if (!["account", "five_hour", "seven_day"].includes(kind) && !model) return { at, windows: [] }; + const remainingPercent = finite(info.utilization) && info.utilization >= 0 && info.utilization <= 1 ? 100 * (1 - info.utilization) : undefined; + const state = info.status === "rejected" ? "exhausted" : info.status === "allowed_warning" ? "low" : undefined; + return { at, windows: [{ key: kind, at, remainingPercent, resetsAt: epoch(info.resetsAt), ...(model ? { model } : {}), ...(state ? { state } : {}) }] }; +} diff --git a/apps/desktop/src/main/engine/routing/router.ts b/apps/desktop/src/main/engine/routing/router.ts index 2ebab199..ab37abf1 100644 --- a/apps/desktop/src/main/engine/routing/router.ts +++ b/apps/desktop/src/main/engine/routing/router.ts @@ -1,24 +1,31 @@ +import type { QuotaSnapshot } from "./quota.js"; import path from "node:path"; import type { BackendId, FollowUp, ModelInfo, RoutingPolicy, RoutingStatus, RoutingTest, Thread, ThreadItem } from "../../../shared/types.js"; import type { SecretStore } from "../secrets.js"; -import { ladder, tierOf, type Candidate } from "./catalog.js"; +import { ladder, tierOf, type Candidate, type Tier } from "./catalog.js"; import { Decisions, type DecisionsOptions } from "./decisions.js"; import { Fit } from "./fit.js"; import { JevError } from "./jev.js"; import { judgeHeuristically, judgeWithJev, stateFor, QUESTION_SET_VERSION, type JudgeSource, type Judgments } from "./judge.js"; -import { decide, type Decision } from "./policy.js"; +import { admitReplacement, decide, type Decision } from "./policy.js"; import { fallbackFollowUp, FOLLOW_UP_QUESTION, FOLLOW_UPS, followUpState } from "./followup.js"; import type { TurnUsage } from "../backends/types.js"; +type DecisionInputQuotas = Partial>; + /** The Jev options (transport, key, timeout, test seams) build the shared decision service. */ export interface RouterOptions extends Omit { home: string; policy: () => RoutingPolicy; - listModels: (backend: BackendId) => Promise<{ models: ModelInfo[]; error?: string }>; + listModels: (backend: BackendId, account?: string) => Promise<{ models: ModelInfo[]; error?: string }>; + quota?: (backend: BackendId, account?: string) => Promise; } export interface RouteInput { thread: Thread; + accounts?: Partial>; + conserveCredits?: boolean; + minimumTier?: Tier; text: string; items: ThreadItem[]; project: { name: string; branch?: string | null; changedFiles?: number }; @@ -39,12 +46,12 @@ export interface RouteReceipt { export class Router { readonly fit: Fit; readonly decisions: Decisions; - private readonly ladders = new Map(); + private readonly ladders = new Map(); private readonly blockedThreads = new Set(); constructor(private readonly o: RouterOptions) { this.fit = new Fit(path.join(o.home, "app", "routing-fit.json")); - const { listModels: _listModels, ...decisions } = o; + const { listModels: _listModels, quota: _quota, ...decisions } = o; this.decisions = new Decisions(decisions); } @@ -72,14 +79,15 @@ export class Router { } /** Model ladders are cached for a minute; the CLIs' lists rarely change mid-session. */ - private async ladderFor(backend: BackendId): Promise { - const hit = this.ladders.get(backend); + private async ladderFor(backend: BackendId, account?: string): Promise { + const key = JSON.stringify([backend, account ?? "cli"]); + const hit = this.ladders.get(key); if (hit && Date.now() - hit.at < 60_000) return hit.rungs; - const r = await this.o.listModels(backend); + const r = await this.o.listModels(backend, account); const rungs = ladder(backend, r.models); if (r.error) throw new Error(`${backend} model discovery failed: ${r.error}`); - if (rungs.length) this.ladders.set(backend, { at: Date.now(), rungs }); - else this.ladders.delete(backend); + if (rungs.length) this.ladders.set(key, { at: Date.now(), rungs }); + else this.ladders.delete(key); return rungs; } @@ -102,7 +110,7 @@ export class Router { }; } - async route(input: RouteInput, signal?: AbortSignal): Promise { + async route(input: RouteInput, signal?: AbortSignal, reuse?: RouteReceipt): Promise { const started = Date.now(); const policy = this.o.policy(); const { thread } = input; @@ -113,30 +121,41 @@ export class Router { let judgments: Judgments; let source: JudgeSource = "heuristic"; let fallback: string | undefined; - const jev = await this.decisions.connect("routing", { threadId: thread.id }); - if (jev.transport) { - try { - judgments = await judgeWithJev(state, jev.transport, jev.model, signal); - source = "jev"; - } catch (err) { - const why = err instanceof JevError ? `${err.message} (${err.code})` : (err as Error).message; - fallback = `Jev unavailable — ${why}; used the built-in heuristic.`; + if (reuse) { judgments = reuse.judgments; source = reuse.source; } + else { + const jev = await this.decisions.connect("routing", { threadId: thread.id }); + if (jev.transport) { + try { + judgments = await judgeWithJev(state, jev.transport, jev.model, signal); + source = "jev"; + } catch (err) { + const why = err instanceof JevError ? `${err.message} (${err.code})` : (err as Error).message; + fallback = `Jev unavailable — ${why}; used the built-in heuristic.`; + judgments = judgeHeuristically(state); + } + } else if (jev.disabled) { + fallback = `${jev.disabled} Used the built-in heuristic.`; + judgments = judgeHeuristically(state); + } else { + fallback = `${(jev.unavailable ?? "No Jev transport available").replace(/[.!;]+$/, "")}; used the built-in heuristic.`; judgments = judgeHeuristically(state); } - } else if (jev.disabled) { - fallback = `${jev.disabled} Used the built-in heuristic.`; - judgments = judgeHeuristically(state); - } else { - fallback = `${(jev.unavailable ?? "No Jev transport available").replace(/[.!;]+$/, "")}; used the built-in heuristic.`; - judgments = judgeHeuristically(state); } const backends = new Set([thread.backend, ...(policy.allow_backend_switch ? policy.allow_backends : [])]); const ladders: Partial> = {}; const discoveryErrors: string[] = []; + const quotas: DecisionInputQuotas = {}; await Promise.all([...backends].map(async (b) => { - try { ladders[b] = await this.ladderFor(b); } - catch (err) { ladders[b] = []; discoveryErrors.push((err as Error).message); } + const account = input.accounts?.[b] ?? (b === "codex" ? thread.codexAccount : b === "claude" ? thread.claudeAccount : undefined) ?? undefined; + await Promise.all([ + this.ladderFor(b, account).then(rungs => { ladders[b] = rungs; }).catch(err => { + ladders[b] = []; discoveryErrors.push((err as Error).message); + }), + Promise.resolve().then(() => this.o.quota?.(b, account)).then(snapshot => { quotas[b] = snapshot; }).catch(() => { + // Policy reports unknown usage. Raw errors may contain account credentials. + }), + ]); })); const last = thread.lastTurn; @@ -144,7 +163,7 @@ export class Router { const idleMs = Number.isFinite(lastAt) ? Math.max(0, Date.now() - lastAt) : undefined; const premiumExhausted = policy.premium_turns_per_day != null && this.fit.premiumToday() >= policy.premium_turns_per_day; const decision = decide({ - judgments, source, policy, ladders, premiumExhausted, + judgments, source, policy, ladders, premiumExhausted, quotas, conserveCredits: input.conserveCredits, minimumTier: input.minimumTier, current: { backend: thread.backend, model: thread.model, effort: thread.effort, hasSession: Boolean(thread.sessionHandle) }, fitOffset: this.fit.offsetFor(judgments.task), cache: { lastTurn: last, idleMs }, @@ -179,6 +198,7 @@ export class Router { complexity: Math.round(judgments.complexity * 10) / 10, pinned: decision.pinned, blocked: decision.blocked, + ...(decision.conserving ? { conserving: true as const } : {}), ...(decision.held ? { held: true as const } : {}), reasons: fallback ? [fallback, ...decision.reasons] : decision.reasons, durationMs: Date.now() - started, @@ -187,6 +207,26 @@ export class Router { return { item, judgments, decision, source, questionSetVersion: QUESTION_SET_VERSION }; } + /** Metadata-only admission for a provider-selected model/account retry; Jev is not asked again. */ + async admitReplacement(thread: Thread, previous: RouteReceipt, account?: string): Promise { + const policy = this.o.policy(); + let rungs: Candidate[] = [], snapshot: QuotaSnapshot | undefined; + await Promise.all([ + this.ladderFor(thread.backend, account).then(value => { rungs = value; }).catch(() => {}), + Promise.resolve().then(() => this.o.quota?.(thread.backend, account)).then(value => { snapshot = value; }).catch(() => {}), + ]); + // The failed provider attempt and its replacement share the original admitted premium turn. + const recorded = this.fit.latest(thread.id); + const used = this.fit.premiumToday() - Number(Boolean(recorded?.premium && recorded.at.slice(0, 10) === new Date().toISOString().slice(0, 10))); + const decision = admitReplacement({ judgments: previous.judgments, source: previous.source, policy, + current: { backend: thread.backend, model: thread.model, effort: thread.effort, hasSession: Boolean(thread.sessionHandle) }, + ladders: { [thread.backend]: rungs }, quotas: { [thread.backend]: snapshot }, fitOffset: 0, + premiumExhausted: policy.premium_turns_per_day != null && used >= policy.premium_turns_per_day, + }, previous.decision); + if (decision.blocked) this.blockedThreads.add(thread.id); + return decision; + } + /** A provider-selected fallback changes what ran, never the user's learned preference. */ noteFallback(thread: Thread, fast: boolean): void { const tier = tierOf({ id: thread.model, label: thread.model }); @@ -196,8 +236,8 @@ export class Router { } catch { /* Learning persistence must not prevent a safe provider fallback. */ } } - noteOutcome(threadId: string, outcome: "completed" | "failed" | "interrupted", usage?: TurnUsage): void { - try { this.fit.recordOutcome(threadId, outcome, usage); } catch { /* Learning persistence cannot change a coding turn's outcome. */ } + noteOutcome(threadId: string, outcome: "completed" | "failed" | "interrupted", usage?: TurnUsage, learn = true): void { + try { this.fit.recordOutcome(threadId, outcome, usage, learn); } catch { /* Learning persistence cannot change a coding turn's outcome. */ } } /** diff --git a/apps/desktop/src/main/engine/runner.ts b/apps/desktop/src/main/engine/runner.ts index 517face3..f7ee6507 100644 --- a/apps/desktop/src/main/engine/runner.ts +++ b/apps/desktop/src/main/engine/runner.ts @@ -14,7 +14,8 @@ import { ClaudeBackend } from "./backends/claude.js"; import { CodexBackend } from "./backends/codex.js"; import { AcpBackend } from "./backends/acp.js"; import { MockBackend } from "./backends/mock.js"; -import { Router } from "./routing/router.js"; +import { Router, type RouteReceipt } from "./routing/router.js"; +import type { Decision } from "./routing/policy.js"; import { answerFor, decide as decideApproval, gateApplies, receiptFor } from "./approvals/gate.js"; import { policyAnswer, policyReceipt } from "./approvals/policy.js"; import type { SecretStore } from "./secrets.js"; @@ -118,7 +119,7 @@ export interface RunnerOptions { /** Whether a process still exists; tests replace the signal-0 probe. */ isProcessAlive?: (pid: number) => boolean; /** Modex's Claude account pool: usage-limit cooldowns and rotation (engine/claude-accounts.ts). */ - claudeAccounts?: Pick; + claudeAccounts?: Pick; } /** The thread's account binding for its current backend: Codex and Claude Code keep separate ones. */ @@ -170,7 +171,7 @@ export class ThreadRunner { mimo: o.backends?.mimo ?? new AcpBackend("mimo", () => resolveCli("mimo", cliSettings.mimo_bin)), mock: o.backends?.mock ?? new MockBackend(() => s().mock_script, o.home), }; - this.router = o.router ?? new Router({ home: o.home, policy: () => s().routing, listModels: (b) => this.listModels(b), secrets: o.secrets }); + this.router = o.router ?? new Router({ home: o.home, policy: () => s().routing, listModels: (b, account) => this.listModels(b, account), quota: (b, account) => this.background.run(async () => this.backends[b].quota?.(account)), secrets: o.secrets }); // Jev is never asked about a local thread, whichever caller asks (#380). this.router.decisions.guardThreads(id => localOnly(o.store.thread(id))); } @@ -189,11 +190,11 @@ export class ThreadRunner { && this.o.store.thread(threadId)?.localTarget?.model.connection.id === connectionId); } - async listModels(id: BackendId): Promise<{ models: ModelInfo[]; error?: string }> { + async listModels(id: BackendId, account?: string): Promise<{ models: ModelInfo[]; error?: string }> { if (this.disposing) return { models: [], error: "Modex is shutting down." }; if (this.restartHold) return { models: [], error: "Modex is preparing to restart." }; try { - return { models: await this.background.run(() => this.backends[id].listModels()) }; + return { models: await this.background.run(() => this.backends[id].listModels(account)) }; } catch (err) { return { models: [], error: (err as Error).message }; } @@ -679,7 +680,7 @@ export class ThreadRunner { /** * Runs the thread's last message again, in place: the transcript keeps its one user bubble and the - * new attempt follows the failure card. Auto routing is not re-asked; the pick already applied. + * new attempt follows the failure card. Auto obtains a fresh quota admission before each attempt. */ async retry(threadId: string, by = "desktop"): Promise { this.assertLocalSender(threadId, by); @@ -748,6 +749,12 @@ export class ThreadRunner { // auxiliary setup can observe a different active account. const prospectiveCodexAccount = thread.localTarget === undefined && (thread.backend === "codex" || thread.auto) ? this.backends.codex.identity?.() ?? null : null; + const prospectiveClaudeAccount = thread.localTarget === undefined && (thread.backend === "claude" || thread.auto) + ? this.backends.claude.identity?.() ?? "cli" : "cli"; + const routeAccounts = { + codex: thread.codexAccount ?? (thread.backend === "codex" && thread.sessionHandle ? "cli" : prospectiveCodexAccount ?? "cli"), + claude: thread.claudeAccount ?? (thread.backend === "claude" && thread.sessionHandle ? "cli" : prospectiveClaudeAccount), + }; const admittedExecution = executionBinding(thread, prospectiveCodexAccount); // Staged files move into the thread's folder now, so a Retry (or a restart) still finds them. let draft: AttachmentDraft | undefined; @@ -793,7 +800,9 @@ export class ThreadRunner { const sink = this.sinkFor(threadId, l); this.setStatus(threadId, "running"); let fast = false; + let conserveCredits = false; let routeId: string | undefined; + let routeAdmission: RouteReceipt | undefined; let routingBlocked: string | undefined; // A local thread runs only on its exact target: never Auto, never a cloud backend, never a cloud account. const local = thread.localTarget !== undefined; @@ -806,12 +815,38 @@ export class ThreadRunner { if (thread.plan || thread.imageJobId || files.some(file => file.kind === "image")) throw new Error("This local target runs text and tools only. Turn off Plan and remove images."); } catch (err) { routingBlocked = (err as Error).message; } } - if (auto && !retry) { + if (auto) { // Auto: judge the request, then re-read the thread — the pick is applied as a thread update // so the composer, header, and persisted state all show what this turn runs on. try { const project = this.o.store.project(thread.projectId); - const receipt = await this.router.route({ thread, text, items: l.items.slice(0, -1), project: { name: project?.name ?? "", branch: thread.worktree?.branch ?? null } }, abort.signal); + const previousRoute = retry ? [...l.items].reverse().find(i => i.kind === "route") : undefined; + const retainConservation = previousRoute?.kind === "route" && previousRoute.conserving === true; + let receipt = await this.router.route({ thread, accounts: routeAccounts, conserveCredits: retainConservation, text, items: l.items.slice(0, -1), project: { name: project?.name ?? "", branch: thread.worktree?.branch ?? null } }, abort.signal); + const pool = this.o.claudeAccounts; + if (receipt.decision.quotaBlocked && thread.backend === "claude" && !thread.sessionHandle && pool?.status().rotate) { + const seen = new Set(); + let limited = routeAccounts.claude; + while (receipt.decision.quotaBlocked && !seen.has(limited) && seen.size <= 20 && !abort.signal.aborted) { + seen.add(limited); + pool.noteLimit(limited, receipt.decision.quotaResetAt); + const next = pool.pick(limited); + if (seen.has(next)) break; + const proposed = { ...thread, claudeAccount: next }; + const candidate = await this.router.route({ thread: proposed, accounts: { ...routeAccounts, claude: next }, conserveCredits: true, + minimumTier: receipt.decision.minimumTier, text, items: l.items.slice(0, -1), project: { name: project?.name ?? "", branch: thread.worktree?.branch ?? null } }, abort.signal, receipt); + receipt = candidate; + if (!candidate.decision.blocked) { + this.o.store.updateThread(threadId, { claudeAccount: next }); + thread = this.o.store.thread(threadId) ?? proposed; + routeAccounts.claude = next; + this.addItem(threadId, { id: newId(), kind: "notice", level: "warn", text: "The selected Claude account is exhausted; enabled account rotation selected another account before coding.", at: new Date().toISOString() }); + break; + } + limited = next; + } + } + routeAdmission = receipt; this.addItem(threadId, receipt.item); routeId = receipt.item.id; const d = receipt.decision; @@ -822,6 +857,7 @@ export class ThreadRunner { if (!d.blocked && d.backend !== thread.backend) this.updateThread(threadId, { backend: d.backend, model: d.model, effort: d.effort }, { fromRouter: true }); else if (!d.blocked && (d.model !== thread.model || d.effort !== thread.effort)) this.updateThread(threadId, { model: d.model, effort: d.effort }, { fromRouter: true }); fast = d.fast; + conserveCredits = d.conserving === true; thread = this.o.store.thread(threadId) ?? thread; } catch (err) { if (thread.backend === "mock") { @@ -880,7 +916,7 @@ export class ThreadRunner { const account = !local && (thread.backend === "codex" || thread.backend === "claude") && backend.identity ? thread.backend === "codex" ? admittedExecution.backend === "codex" ? admittedExecution.account ?? undefined : prospectiveCodexAccount ?? undefined - : bound ?? (thread.sessionHandle ? "cli" : backend.identity()) : undefined; + : bound ?? (thread.sessionHandle ? "cli" : prospectiveClaudeAccount) : undefined; if (account && !bound) this.o.store.updateThread(threadId, thread.backend === "codex" ? { codexAccount: account } : { claudeAccount: account }); const accountFor = (ran: Thread) => threadAccount(ran) ?? account; const freshConversation = !thread.sessionHandle; @@ -896,7 +932,7 @@ export class ThreadRunner { const result = await backend.runTurn( describeAttachments(text, turnFiles), { - cwd: ran.cwd, mode: ran.mode, plan: ran.plan, model: ran.model, effort: ran.effort, fast, resume: ran.sessionHandle, account: accountFor(ran), + cwd: ran.cwd, mode: ran.mode, plan: ran.plan, model: ran.model, effort: ran.effort, fast, ...(conserveCredits ? { conserveCredits: true } : {}), resume: ran.sessionHandle, account: accountFor(ran), // Files the CLI reads by path sit outside cwd; --add-dir keeps that from prompting. addDirs: turnFiles.some((f) => f.kind === "file") ? [path.join(this.attachments.root, threadId)] : [], ...(turnFiles.length ? { attachments: turnFiles } : {}), ...(knowledge ? { knowledge: knowledge.connection } : {}), ...(pages ? { pages: pages.connection } : {}), ...(images ? { images: images.connection } : {}), @@ -911,25 +947,50 @@ export class ThreadRunner { throw err; } }; - // A model refused for usage credits recently is refused again: start on its fallback instead. - const skip = local ? undefined : this.recentCreditFallback(thread); - if (skip) thread = this.fallBack(threadId, thread, skip, routeId, false); + let replacementBlocked = false; + let replacementFailure: TurnResult = { status: "failed", error: "Auto stopped before an unsafe replacement attempt." }; + const validateReplacement = async (proposed: Thread): Promise<{ decision?: Decision } | undefined> => { + if (!auto || !routeAdmission) return {}; + const decision = await this.router.admitReplacement(proposed, routeAdmission, accountFor(proposed)); + if (abort.signal.aborted) { replacementBlocked = true; replacementFailure = { status: "interrupted" }; return undefined; } + if (decision.blocked) { + replacementBlocked = true; + replacementFailure = { status: "failed", error: decision.reasons.at(-1) ?? "Auto rejected the replacement." }; + if (decision.quotaBlocked && proposed.backend === "claude" && accountFor(proposed)) this.o.claudeAccounts?.noteLimit(accountFor(proposed)!, decision.quotaResetAt); + this.addItem(threadId, { id: newId(), kind: "notice", level: "warn", text: `Auto replacement stopped: ${replacementFailure.error}`, at: new Date().toISOString() }); + if (routeId) this.patchItem(threadId, routeId, { blocked: true, reasons: [...routeAdmission.item.reasons, ...decision.reasons] }); + return undefined; + } + fast = decision.fast; + conserveCredits = decision.conserving === true; + routeAdmission = { ...routeAdmission, decision }; + return { decision }; + }; + // A recent refusal is scoped to this account, and its replacement is admitted before persistence. + const skip = local ? undefined : this.recentCreditFallback(thread, accountFor(thread)); + if (skip) { + const checked = await validateReplacement({ ...thread, model: skip }); + if (checked) thread = this.fallBack(threadId, thread, skip, routeId, false, checked.decision); + } let replyStart = l.items.length; - let result = await runOn(thread); + let result = replacementBlocked ? replacementFailure : await runOn(thread); // A failed local turn never moves to a cloud model. - const fallback = !local && result.status === "failed" && !abort.signal.aborted ? this.creditFallback(thread, result.error, l.items.slice(turnStart)) : undefined; + const fallback = !local && !replacementBlocked && result.status === "failed" && !abort.signal.aborted ? this.creditFallback(thread, result.error, l.items.slice(turnStart)) : undefined; if (fallback) { const from = thread.model; - this.outOfCredits.set(`${thread.backend}:${from}`, Date.now()); - thread = this.fallBack(threadId, thread, fallback, routeId, true); - replyStart = l.items.length; - result = await runOn(thread); - if (result.status === "failed") result = { ...result, recovery: [`${modelFamilyName(from)} was out of usage credits; retried on ${modelFamilyName(fallback)}.`, ...(result.recovery ?? [])] }; + this.outOfCredits.set(JSON.stringify([thread.backend, accountFor(thread) ?? "cli", from]), Date.now()); + const checked = await validateReplacement({ ...thread, model: fallback }); + if (checked) { + thread = this.fallBack(threadId, thread, fallback, routeId, true, checked.decision); + replyStart = l.items.length; + result = await runOn(thread); + if (result.status === "failed") result = { ...result, recovery: [`${modelFamilyName(from)} was out of usage credits; retried on ${modelFamilyName(fallback)}.`, ...(result.recovery ?? [])] }; + } else result = replacementFailure; } // A Claude account that reports a usage limit cools down; with rotation on, a turn that had not started work moves on. const pool = this.o.claudeAccounts; const limited = accountFor(thread); - if (!local && pool && thread.backend === "claude" && limited && result.status === "failed" && !abort.signal.aborted && classifyFailure(result.error ?? "") === "rate_limited") { + if (!local && !replacementBlocked && pool && thread.backend === "claude" && limited && result.status === "failed" && !abort.signal.aborted && classifyFailure(result.error ?? "") === "rate_limited") { const { until } = pool.noteLimit(limited, limitResetAt(result.error ?? "")); const next = pool.pick(limited); const name = (id: string) => id === "cli" ? "Claude Code's own login" : `"${pool.account(id)?.label ?? "removed account"}"`; @@ -938,13 +999,22 @@ export class ThreadRunner { this.addItem(threadId, { id: newId(), kind: "notice", level: "warn", text: `Claude account ${name(limited)} hit its usage limit; Modex holds it out of rotation until ${new Date(until).toLocaleTimeString([], { hour: "numeric", minute: "2-digit" })}.${movable ? ` Retrying on ${name(next)}.` : next !== limited ? ` New Claude conversations start on ${name(next)}.` : ""}`, at: new Date().toISOString() }); if (movable) { // A conversation that had no session before this turn starts fresh: the failed attempt's session may live in the old profile. - this.o.store.updateThread(threadId, { claudeAccount: next, ...(freshConversation ? { sessionHandle: undefined } : {}) }); - thread = this.o.store.thread(threadId) ?? thread; - replyStart = l.items.length; - result = await runOn(thread); - // A second limit still updates selection for the next conversation; never retry a third time. - if (result.status === "failed" && !abort.signal.aborted && classifyFailure(result.error ?? "") === "rate_limited") pool.noteLimit(next, limitResetAt(result.error ?? "")); - if (result.status === "failed") result = { ...result, recovery: [`Claude account ${name(limited)} hit its usage limit; retried on ${name(next)}.`, ...(result.recovery ?? [])] }; + const proposed = { ...thread, claudeAccount: next, ...(freshConversation ? { sessionHandle: undefined } : {}) }; + const checked = await validateReplacement(proposed); + if (checked) { + this.o.store.updateThread(threadId, { claudeAccount: next, ...(freshConversation ? { sessionHandle: undefined } : {}), ...(checked.decision ? { effort: checked.decision.effort } : {}) }); + thread = this.o.store.thread(threadId) ?? proposed; + if (auto && checked.decision) { + this.router.noteFallback(thread, fast); + if (routeId) this.patchItem(threadId, routeId, { effort: checked.decision.effort, fast, conserving: checked.decision.conserving, + reasons: [...(routeAdmission?.item.reasons ?? []), ...checked.decision.reasons] }); + } + replyStart = l.items.length; + result = await runOn(thread); + // A second limit still updates selection for the next conversation; never retry a third time. + if (result.status === "failed" && !abort.signal.aborted && classifyFailure(result.error ?? "") === "rate_limited") pool.noteLimit(next, limitResetAt(result.error ?? "")); + if (result.status === "failed") result = { ...result, recovery: [`Claude account ${name(limited)} hit its usage limit; retried on ${name(next)}.`, ...(result.recovery ?? [])] }; + } else result = replacementFailure; } } if (pool && thread.backend === "claude" && result.status === "completed" && accountFor(thread)) pool.clearLimit(accountFor(thread)!); @@ -954,11 +1024,11 @@ export class ThreadRunner { if (fast && !ranFast && routeId) this.noteFastOff(threadId, routeId, result.fastMode?.why); this.noteLastTurn(threadId, thread, ranFast, result); // Naming is a separate model conversation; a local thread keeps its opening-message title. - if (!local && result.status === "completed" && shouldName && !abort.signal.aborted) { + if (!local && !conserveCredits && result.status === "completed" && shouldName && !abort.signal.aborted) { const reply = l.items.slice(turnStart).reverse().find((item) => item.kind === "assistant" && item.text.trim()); this.nameThread(threadId, text, fallbackTitle, backend, thread.model, reply?.kind === "assistant" ? reply.text : undefined); } - if (auto) this.router.noteOutcome(threadId, result.status, result.usage); + if (auto) this.router.noteOutcome(threadId, result.status, result.usage, !replacementBlocked); this.recordUsage(thread, accountFor(thread), result); // A failure can precede Stop; its outcome does not establish process exit. if (result.stop?.outcome === "unconfirmed" && result.stop.pid) (l.unconfirmed ??= new Set()).add(result.stop.pid); @@ -1305,9 +1375,9 @@ export class ThreadRunner { } /** The fallback for a model that was refused for usage credits within CREDIT_SKIP_MS, or undefined. */ - private recentCreditFallback(thread: Thread): string | undefined { + private recentCreditFallback(thread: Thread, account = threadAccount(thread) ?? "cli"): string | undefined { if (localOnly(thread) || !this.o.store.settings.model_fallback) return undefined; - const key = `${thread.backend}:${thread.model}`; + const key = JSON.stringify([thread.backend, account, thread.model]); const at = this.outOfCredits.get(key); if (at === undefined) return undefined; if (Date.now() - at >= CREDIT_SKIP_MS) { this.outOfCredits.delete(key); return undefined; } @@ -1315,17 +1385,17 @@ export class ThreadRunner { } /** Moves the thread to `to` for this turn and onward, and says so in the thread and on Auto's receipt. */ - private fallBack(threadId: string, thread: Thread, to: string, routeId: string | undefined, refusedNow: boolean): Thread { + private fallBack(threadId: string, thread: Thread, to: string, routeId: string | undefined, refusedNow: boolean, admission?: Decision): Thread { const from = modelFamilyName(thread.model), next = modelFamilyName(to); // fromRouter: Modex made this switch, so Auto must not learn it as the user's preference. - this.updateThread(threadId, { model: to }, { fromRouter: true }); + this.updateThread(threadId, { model: to, ...(admission ? { effort: admission.effort } : {}) }, { fromRouter: true }); const why = refusedNow ? `${from} is out of usage credits, so this turn moved to ${next}.` : `${from} ran out of usage credits within the last hour, so this turn runs on ${next}.`; this.addItem(threadId, { id: newId(), kind: "notice", level: "warn", text: `${why} Add usage credits to use ${from} again, or turn off the fallback in Settings.`, at: new Date().toISOString() }); const route = routeId ? this.slot(threadId).items.find((i) => i.id === routeId) : undefined; if (route?.kind === "route") { const actual = this.o.store.thread(threadId) ?? { ...thread, model: to }; - this.router.noteFallback(actual, route.fast); - this.patchItem(threadId, route.id, { model: to, held: undefined, reasons: [...route.reasons, why] }); + this.router.noteFallback(actual, admission?.fast ?? route.fast); + this.patchItem(threadId, route.id, { model: to, held: undefined, ...(admission ? { effort: admission.effort, fast: admission.fast, conserving: admission.conserving } : {}), reasons: [...route.reasons, why, ...(admission?.reasons ?? [])] }); } return this.o.store.thread(threadId) ?? { ...thread, model: to }; } diff --git a/apps/desktop/src/main/engine/store.ts b/apps/desktop/src/main/engine/store.ts index 91c5c4cc..86ff0421 100644 --- a/apps/desktop/src/main/engine/store.ts +++ b/apps/desktop/src/main/engine/store.ts @@ -72,6 +72,9 @@ export function migrateRouting(raw: unknown): RoutingPolicy { allow_fast: typeof r.allow_fast === "boolean" ? r.allow_fast : d.allow_fast, min_confidence: num(r.min_confidence, 0, 1, d.min_confidence), premium_turns_per_day: r.premium_turns_per_day === null ? null : typeof r.premium_turns_per_day === "number" && r.premium_turns_per_day >= 0 ? Math.floor(r.premium_turns_per_day) : d.premium_turns_per_day, + conserve_credits: typeof r.conserve_credits === "boolean" ? r.conserve_credits : d.conserve_credits, + low_quota_percent: num(r.low_quota_percent, 0, 100, d.low_quota_percent), + low_credit_balance: num(r.low_credit_balance, 0, Number.MAX_SAFE_INTEGER, d.low_credit_balance), jev_model: typeof r.jev_model === "string" && r.jev_model.trim() ? r.jev_model.trim() : d.jev_model, jev_transport: r.jev_transport === "auto" || r.jev_transport === "cli" || r.jev_transport === "http" ? r.jev_transport : d.jev_transport, jev_bin: typeof r.jev_bin === "string" && r.jev_bin.trim() ? r.jev_bin.trim() : d.jev_bin, diff --git a/apps/desktop/src/renderer/components/SettingsDialog.tsx b/apps/desktop/src/renderer/components/SettingsDialog.tsx index 4e049f85..a972cb56 100644 --- a/apps/desktop/src/renderer/components/SettingsDialog.tsx +++ b/apps/desktop/src/renderer/components/SettingsDialog.tsx @@ -561,6 +561,22 @@ export function SettingsDialog({ settings, projects, currentProjectId, onSave, o setR("premium_turns_per_day", e.target.value === "" ? null : Math.max(0, Math.floor(Number(e.target.value) || 0)))} /> + +
+ + +
+

Conservation chooses economical models for light work, limits reasoning effort and turns paid fast mode off. Complex work keeps its capability target; warm conversations keep their context. Unavailable balance readings appear as unknown in each route receipt.