diff --git a/src/persist.ts b/src/persist.ts index 78714b0910..2098d989b9 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -36,6 +36,7 @@ import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js"; import { sha256Hex, type CheckpointRow } from "./storage/store.js"; import type { WireProtocol } from "./util.js"; import { currentContextObservation, CALIBRATION_CLAMP_MAX, CALIBRATION_CLAMP_MIN, CALIBRATION_SAMPLE_MAX, CALIBRATION_SAMPLE_MIN, CALIBRATION_SAMPLE_WINDOW } from "./cache-ledger.js"; +import { isValidSystemSurface, SYSTEM_SURFACE_SUFFIX, type SystemSurfaceRecord } from "./system-surface.js"; /** * On-disk persistence for proxy sessions. @@ -324,6 +325,13 @@ export function contentStoreRelPathFor(id: string, protocol?: string, upstreamOr return base.slice(0, -".json".length) + ".content-store.json"; } +/** #3201: system-surface sidecar path — same sibling discipline as the + * content store above. */ +function systemSurfaceRelPathFor(id: string, protocol?: string, upstreamOrigin?: string): string { + const base = relPathFor(id, protocol, upstreamOrigin); + return base.slice(0, -".json".length) + SYSTEM_SURFACE_SUFFIX; +} + /** #1937: true when a walked FILENAME is the canonical (or legacy flat / * .fb fallback) name for record `id` — mirrors the kernel StateStore.loadAll * owner check so independent directory walkers agree with loadSync about @@ -553,12 +561,16 @@ export class SessionStore { legacy: (parsed) => (isValidRecord(parsed) ? { id: parsed.id, payload: parsed, version: parsed.version, savedAt: parsed.savedAt } : null), validate: (envelope) => isValidRecord(envelope.payload), // #2298: CCR content-store envelopes (#1097) co-reside in the SAME - // namespace by design (/_.content-store.json) but + // namespace by design (/_.content-store.json), as + // do #3201 system-surface sidecars (.system-surface.json), but // are not session records — declaring them foreign keeps the boot // scan from parsing each one and warning "skipping invalid record" // per file (hundreds of warns per start at multi-relay scale). Same // suffix predicate the GC (#1180) and web walkers already use. - foreignFile: (file) => path.basename(file).endsWith(".content-store.json"), + foreignFile: (file) => { + const base = path.basename(file); + return base.endsWith(".content-store.json") || base.endsWith(SYSTEM_SURFACE_SUFFIX); + }, }); } @@ -573,6 +585,45 @@ export class SessionStore { ]); } + /** #3201: load the session's system-surface sidecar (same probe order as + * loadContentStore). It is a DERIVED artifact, unlike the content store: + * a corrupt/unreadable file degrades to "not captured" (warned) and must + * never refuse boot or block a save. */ + loadSystemSurface(session: Session): SystemSurfaceRecord | null { + return this.readSystemSurfaceRels(session.id, [ + systemSurfaceRelPathFor(session.id, session.meta.protocol, session.meta.upstreamOrigin), + systemSurfaceRelPathFor(session.id), + ]); + } + + private readSystemSurfaceRels(id: string, rels: string[]): SystemSurfaceRecord | null { + if (!this.enabled) return null; + const source = this.sourceFiles.get(id); + const paths = [...rels.map(rel => path.join(this.sessionsDir, rel)), + ...(source ? [source.replace(/\.json$/, SYSTEM_SURFACE_SUFFIX)] : []), + ...this.diskFiles.filter(file => fileNameMatchesId(path.basename(file), id)) + .map(file => file.replace(/\.json$/, SYSTEM_SURFACE_SUFFIX))]; + for (const file of new Set(paths)) { + let buf: Buffer; + try { + buf = readFileSync(file); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") continue; + this.log("warn", `[persist] system surface for ${id} unreadable, ignoring: ${String(err)}`); + return null; + } + try { + const parsed: unknown = JSON.parse(this.readCodec.decode(buf)); + if (isValidSystemSurface(parsed)) return parsed; + throw new Error(`system surface for ${id} is malformed`); + } catch (err) { + this.log("warn", `[persist] system surface for ${id} unreadable, ignoring: ${err instanceof Error ? err.message : String(err)}`); + return null; + } + } + return null; + } + /** Id+meta variant for replica freeze/restore (#2671 review 6): the * checkpoint path has no live Session object, only the frozen record's * meta — same rel-path candidates, same parse-and-degrade semantics. */ @@ -689,6 +740,34 @@ export class SessionStore { } } + /** #3201: persist the system-surface sidecar when dirty; an absent + * surface deletes the file. Same atomic tmp+rename discipline as + * saveContentStore. Unlike that one, a failure here only re-arms dirty — + * it never blocks the session state write (derived artifact, see + * readSystemSurfaceRels). */ + private saveSystemSurface(session: Session): void { + if (!this.enabled || !session.systemSurfaceDirty) return; + const rel = systemSurfaceRelPathFor(session.id, session.meta.protocol, session.meta.upstreamOrigin); + const abs = path.join(this.sessionsDir, rel); + const tmp = path.join(path.dirname(abs), `.tmp-${path.basename(abs, ".json")}-${process.pid}-${this.tmpSeq++}`); + try { + if (!session.systemSurface) { + rmSync(abs, { force: true }); + session.systemSurfaceDirty = false; + return; + } + mkdirSync(path.dirname(abs), { recursive: true }); + const text = JSON.stringify(session.systemSurface); + writeFileSync(tmp, this.codec ? this.codec.encode(text) : text, { mode: 0o600 }); + renameSync(tmp, abs); + session.systemSurfaceDirty = false; + } catch (err) { + try { rmSync(tmp, { force: true }); } catch { /* best-effort: a stuck temp is swept at next boot */ } + session.systemSurfaceDirty = true; + this.log("warn", `[persist] system-surface write failed for ${session.id}: ${err instanceof Error ? err.message : String(err)}`); + } + } + /** Bulk-load every persisted session from disk into a map keyed by the * REAL session id (read from the file body, not the filename). Called once * at startup before the server accepts traffic. Corrupt individual files @@ -790,7 +869,8 @@ export class SessionStore { for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) { const files = existsSync(root) ? await walkJsonFiles(root) : []; for (const file of files) { - if (path.basename(file).endsWith(".content-store.json")) continue; + const base = path.basename(file); + if (base.endsWith(".content-store.json") || base.endsWith(SYSTEM_SURFACE_SUFFIX)) continue; this.diskFiles.push(file); filesByRoot.set(root, [...(filesByRoot.get(root) ?? []), file]); } @@ -2289,6 +2369,7 @@ export class SessionStore { * internal debounce could not do: every per-turn save then proved its * bytes on disk and the ownership fast path stayed live. */ scheduleSave(session: Session): void { + this.saveSystemSurface(session); if (!this.saveContentStore(session)) return; if (this.writerOffActive()) { // Phase 3 (#2960): index-only persistence — the builder closure @@ -2383,6 +2464,7 @@ export class SessionStore { * check and the build), so concurrent callers there cannot interleave * either. Do NOT regress this by eager-snapshotting the record here. */ async writeNow(session: Session): Promise { + this.saveSystemSurface(session); if (!this.saveContentStore(session)) throw new Error(`Originals were not saved for ${session.id}`); if (this.writerOffActive()) { // #3133: drop the armed microtask (token mismatch below) — this @@ -2463,6 +2545,7 @@ export class SessionStore { * Returns true on success, false on failure (caller must NOT evict on * failure for a never-persisted session or it is lost permanently). */ flushSync(session: Session): boolean { + this.saveSystemSurface(session); if (!this.saveContentStore(session)) return false; if (this.writerOffActive()) { // #3133: same subsume rule as writeNow. @@ -2512,7 +2595,10 @@ export class SessionStore { // a stray microtask would build against a closed store and burn a // 3-strike disable for no reason. this.drainWriterOffPending(); - for (const session of sessions) this.saveContentStore(session); + for (const session of sessions) { + this.saveSystemSurface(session); + this.saveContentStore(session); + } await this.flushScheduledSaves(); await this.store.flushAll(); this.shadow?.close(); diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index aafb3fb290..33b58eacc0 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -27,6 +27,7 @@ import { estimateCoreMessages, estimateCoreMessagesUpper, extractBillingAttribut import { dshNativeCompactionWitness, recordConflict } from "../conflict-watch.js"; import { anthropicToolsCarryCacheControl, computeAnthropicMessageMarks, stampAnthropicSystemCacheControl } from "../loop/cache-control.js"; import { reconcileSystemAnchor } from "../system-anchor.js"; +import { captureSystemSurface } from "../system-surface.js"; import { isStrictReasoningEcho, modelIdOf } from "../strict-echo.js"; import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker/engine.js"; import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js"; @@ -513,6 +514,9 @@ export async function prepareAnthropic( // SysPrompt row — the kernel breakdown classifies messages only, and on // this wire the system rides the top-level `system` field outside the fold. session.metadata.systemPromptTokens = countSystemAndToolsTokens(extractSystem(systemOut), toolsOut); + // #3201: persist the system+tools surface text (web trajectory view) at + // the same point the overhead is measured, so both stay in lockstep. + captureSystemSurface(session, extractSystem(systemOut), toolsOut); snapshotMessages(session, originalMessages); markDirty(session); diff --git a/src/server/prepare-google.ts b/src/server/prepare-google.ts index 0cd9ed03c9..ed35cd3d63 100644 --- a/src/server/prepare-google.ts +++ b/src/server/prepare-google.ts @@ -27,6 +27,7 @@ import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js"; import { applyImageRescalePass } from "../image-rescale.js"; import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed } from "./budget.js"; +import { captureSystemSurface } from "../system-surface.js"; import { estimateCoreMessages } from "../preflight.js"; import { effectiveAbsorbBlock } from "./prepare-responses.js"; import { injectGoogleTool, injectTool } from "./inject.js"; @@ -356,6 +357,10 @@ export async function prepareGoogle( // #532: title-gen side requests carry their own tiny system — skip them. if (!isTitleGen && googleOutboundSystem !== undefined) { session.metadata.systemPromptTokens = countSystemAndToolsTokens(googleOutboundSystem, toolsOut); + // #3201: persist the system+tools surface text (web trajectory view); + // same skip guard as the measurement above — title-gen side requests + // carry their own tiny system and must not clobber it. + captureSystemSurface(session, googleOutboundSystem, toolsOut); } // #1933 F1 + #2407: billed-caliber denominator of the k̂ learning pair. // The Google lane was the only one of the four never recording it, so diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 6b825e22e9..a38b085852 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -26,6 +26,7 @@ import { attachSubagentSessions } from "../subagent-sessions.js"; import { estimateCoreMessages, estimateCoreMessagesUpper } from "../preflight.js"; import { dshNativeCompactionWitness, recordConflict } from "../conflict-watch.js"; import { isStrictReasoningEcho, modelIdOf, normalizeStrictEchoReasoning } from "../strict-echo.js"; +import { captureSystemSurface } from "../system-surface.js"; import { reconcileSystemAnchor } from "../system-anchor.js"; import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker/engine.js"; import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js"; @@ -453,6 +454,10 @@ export async function prepareOpenai( // clobber the conversation's measured overhead — skip them. if (!isTitleGen && openaiOutboundSystem !== undefined) { session.metadata.systemPromptTokens = countSystemAndToolsTokens(openaiOutboundSystem, toolsOut); + // #3201: persist the system+tools surface text (web trajectory view); + // same skip guard as the measurement above — title-gen side requests + // carry their own tiny system and must not clobber it. + captureSystemSurface(session, openaiOutboundSystem, toolsOut); } // #728: record this turn's outbound payload upper bound as the fallback // token source for upstreams that never report usage (see effectiveTokenCount). diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 1bbd183c6e..b95e98b5bb 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -39,6 +39,7 @@ import { applyImageRescaleItems } from "../image-rescale.js"; import { clampOutgoingOutput, countLoadedToolTokens, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed, modelVisibleTools, responsesSysDevItemText } from "./budget.js"; import { FORCE_TEXT_PROTOCOL, injectResponsesTool, injectTool } from "./inject.js"; import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js"; +import { captureSystemSurface } from "../system-surface.js"; export async function prepareResponses( parsed: ResponsesRequestBody, @@ -642,6 +643,10 @@ export async function prepareResponses( const loadedToolTokens = countLoadedToolTokens(rebuilt); if (transformOk) { session.metadata.systemPromptTokens = countSystemAndToolsTokens(responsesDevContent ?? "", toolsOut) + loadedToolTokens + inplaceSysDevTokens; + // #3201: persist the system+tools surface text (web trajectory view); + // inside transformOk like the measurement — a failed kernel transform + // forwards unchanged and must not record a stale surface. + captureSystemSurface(session, responsesDevContent ?? "", toolsOut); const catalogTokens = defaultCountTokens(JSON.stringify(toolsOut ?? [])); const visibleTokens = defaultCountTokens(JSON.stringify(modelVisibleTools(toolsOut))); if (catalogTokens >= 50_000 || catalogTokens !== visibleTokens || loadedToolTokens > 0) { diff --git a/src/session-gc.ts b/src/session-gc.ts index 6df3fb915c..65cbf6fda4 100644 --- a/src/session-gc.ts +++ b/src/session-gc.ts @@ -9,6 +9,7 @@ import { dropSessionForGc, peekSession } from "./session.js"; import { stateDir } from "./paths.js"; import { restoreExportDir, restoreExportDirName } from "./image-restore.js"; import { gcSettings as knobGcSettings } from "./knobs.js"; +import { isValidSystemSurface, SYSTEM_SURFACE_SUFFIX } from "./system-surface.js"; /** * Session-file garbage collection (#1082). OPT-IN: disabled unless @@ -46,8 +47,10 @@ import { gcSettings as knobGcSettings } from "./knobs.js"; * once #1083 lands — works via the store's format-agnostic reader). * Corrupt/unreadable files are left in place, never guessed at. * - * #1180: the per-session CCR content store (#1097) lives next to its session - * file as `.content-store.json` and shares the session's lifecycle — + * #1180: the per-session CCR content store (#1097) lives next to its session + * file as `.content-store.json` (the deterministic sibling computed by + * contentStoreRelPathFor in persist.ts — the suffix pairing below must stay + * consistent with it) and shares the session's lifecycle — * it holds lossless payload bytes (originals of oversized tool results), so * deleting the session loses exactly those bytes, the class of file this * sweep exists to clean: @@ -62,10 +65,14 @@ import { gcSettings as knobGcSettings } from "./knobs.js"; * - its token footprint (unique-content via the kernel's CJK-aware * defaultCountTokens, same estimator as rawInputTokens) counts toward the * size gate, so a tiny session with a huge store does not slip under it; - * - attached to a kept session → kept untouched; an unreadable/foreign - * companion next to an otherwise-eligible session → both kept (never - * guess at unreadable files). - */ + * - attached to a kept session → kept untouched; an unreadable/foreign + * companion next to an otherwise-eligible session → both kept (never + * guess at unreadable files). + * + * #3201: the system-surface sidecar (.system-surface.json) follows the same + * co-delete / twin-fallback / orphan-sweep rules; being a bounded derived + * artifact its size does NOT count toward the token gate. + */ interface GcConfig { enabled: boolean; @@ -148,6 +155,31 @@ export function contentStoreTokens(parsed: unknown): number | null { return tokens; } +/** #1180/#3201: true when a surviving TWIN of `file`'s session id still reads + * its companion through this stale duplicate's sibling — meta fill-once can + * migrate _unknown/ -> ns/ without cleaning the old path, and loaders probe + * the namespaced rel first then fall back, so co-deleting here would lose + * live companion bytes. `readable` probes the twin's own sibling and must + * check SHAPE, not mere existence (an unreadable own-sibling means the stale + * one is still loadable). Never guesses: any probe failure keeps it. */ +async function survivingTwinNeedsCompanion(file: string, suffix: string, sessionFiles: string[], deletedSessions: Set, readable: (absPath: string) => Promise): Promise { + const stem = path.basename(file, ".json"); + for (const sf of sessionFiles) { + if (sf === file || deletedSessions.has(sf)) continue; + if (!path.basename(sf).endsWith(stem + ".json")) continue; + let ownReadable: boolean; + try { + ownReadable = await readable(sf.slice(0, -".json".length) + suffix); + } catch { + // Probe failure (twin vanished mid-sweep, EACCES, …): keep — + // never guess at bytes we cannot inspect. + return true; + } + if (!ownReadable) return true; + } + return false; +} + export function isGcEligible(view: FileView, now: number, cfg: Pick): boolean { if (now - view.savedAt < cfg.maxAgeMs) return false; if (view.everCompressed) return false; @@ -176,7 +208,8 @@ interface GcResult { kept: number; unreadable: number; bytesFreed: number; - /** #1180: content-store companion files removed (co-deletions + orphan sweeps) */ + /** #1180/#3201: companion files removed — content stores + system-surface + * sidecars (co-deletions + orphan sweeps) */ companionsRemoved: number; /** #1995: image-spill directories removed (co-deletions + stale-mtime sweep) */ imgDirsRemoved: number; @@ -196,13 +229,15 @@ export async function gcSessionFiles(opts?: { dir?: string; store?: SessionStore } catch { return result; } - // #1180: split sessions from CCR companions (#1097); companion path is the - // deterministic sibling per contentStoreRelPathFor — no decode to pair them. + // #1180: split sessions from CCR companions (#1097), plus #3201 + // system-surface sidecars; companion paths are deterministic siblings — + // no decode to pair them. const companionSuffix = ".content-store.json"; + const companionSuffixes = [companionSuffix, SYSTEM_SURFACE_SUFFIX]; const sessionFiles: string[] = []; const companions: string[] = []; for (const f of files) { - if (f.endsWith(companionSuffix)) companions.push(f); + if (companionSuffixes.some((s) => f.endsWith(s))) companions.push(f); else sessionFiles.push(f); } const deletedSessions = new Set(); @@ -245,6 +280,14 @@ export async function gcSessionFiles(opts?: { dir?: string; store?: SessionStore } view.storedTokens = stored; } + // #3201: system-surface sidecar — bounded derived artifact (no token + // gate), co-deleted with the same twin-fallback rule below. + const surfaceCompanion = file.slice(0, -".json".length) + SYSTEM_SURFACE_SUFFIX; + let surfSt: Stats | undefined; + try { + const s = await stat(surfaceCompanion); + if (s.isFile()) surfSt = s; + } catch { /* no surface sidecar */ } if (!isGcEligible(view, now, cfg)) { result.kept++; continue; @@ -294,36 +337,13 @@ export async function gcSessionFiles(opts?: { dir?: string; store?: SessionStore if (compSt) { // Co-delete the companion so the sweep never leaves orphans. On // failure it surfaces as an orphan on the very next pass below. - // Cross-namespace exception: meta fill-once can migrate a session - // _unknown/.json -> ns/host_.json without cleaning the old - // path (kernel writes never delete the previous relPath), and - // saveContentStore only rewrites when dirty — so a moved session - // whose store was not re-saved since the move still reads its LIVE - // store from this stale duplicate's sibling (loadContentStore - // probes _unknown/ when the namespaced store is absent, and stops - // at the first EXISTING candidate). Co-deleting it here would lose - // retrievable originals. Skip when a surviving twin shares the - // hash stem and has no store of its own; the bytes stay referenced - // until the twin is swept, then the orphan pass collects them. - const stem = path.basename(file, ".json"); - let keepCompanion = false; - for (const sf of sessionFiles) { - if (sf === file || deletedSessions.has(sf)) continue; - if (!path.basename(sf).endsWith(stem + ".json")) continue; - let twinHasOwnStore = false; - try { - const s2 = await stat(sf.slice(0, -".json".length) + companionSuffix); - // Existence alone is not adoption: loadContentStore falls - // through to _unknown/ when the namespaced store exists but - // is UNREADABLE — the stale sibling stays the twin's live - // store. Probe the shape, not the inode. - twinHasOwnStore = s2.isFile() && contentStoreTokens(await store.readRawFile(sf.slice(0, -".json".length) + companionSuffix)) !== null; - } catch { /* no sibling, or unreadable: keep — never guess */ } - if (!twinHasOwnStore) { - keepCompanion = true; - break; - } - } + // Cross-namespace exception: see survivingTwinNeedsCompanion — a + // stale duplicate's sibling may still be a surviving twin's live + // fallback store; skip it then, the orphan pass collects it later. + const keepCompanion = await survivingTwinNeedsCompanion(file, companionSuffix, sessionFiles, deletedSessions, async (p) => { + const s2 = await stat(p); + return s2.isFile() && contentStoreTokens(await store.readRawFile(p)) !== null; + }); if (keepCompanion) { loggerLog("info", `[gc] kept content store ${path.relative(dir, companion)} (${compSt.size} B): live fallback store of a surviving twin`); } else { @@ -336,6 +356,25 @@ export async function gcSessionFiles(opts?: { dir?: string; store?: SessionStore } catch { /* orphan pass retries */ } } } + if (surfSt) { + // #3201: system-surface sidecar — same lifecycle rules as the + // content store above (twin-fallback keep, orphan pass retry). + const keepSurface = await survivingTwinNeedsCompanion(file, SYSTEM_SURFACE_SUFFIX, sessionFiles, deletedSessions, async (p) => { + const s2 = await stat(p); + return s2.isFile() && isValidSystemSurface(await store.readRawFile(p)); + }); + if (keepSurface) { + loggerLog("info", `[gc] kept system surface ${path.relative(dir, surfaceCompanion)} (${surfSt.size} B): live fallback surface of a surviving twin`); + } else { + try { + await rm(surfaceCompanion, { force: true }); + loggerLog("info", `[gc] removed system surface ${path.relative(dir, surfaceCompanion)} (${surfSt.size} B)`); + result.bytesFreed += surfSt.size; + result.companionsRemoved++; + coDeleted.add(surfaceCompanion); + } catch { /* orphan pass retries */ } + } + } const parent = path.dirname(file); if (parent !== dir) await rmdir(parent).catch(() => {}); } @@ -358,8 +397,9 @@ export async function gcSessionFiles(opts?: { dir?: string; store?: SessionStore // namespaced `host_.json` and flat `_unknown/.json` both end // with `.json`, and meta fill-once can migrate a session between // those namespaces without cleaning the old path: sibling-only matching - // would false-orphan (delete live store bytes) across that move. - const stem = path.basename(companion).slice(0, -companionSuffix.length); + // would false-orphan (delete live companion bytes) across that move. + const suffix = companionSuffixes.find((s) => companion.endsWith(s)) ?? ""; + const stem = suffix ? path.basename(companion).slice(0, -suffix.length) : path.basename(companion, ".json"); let referenced = false; for (const sf of sessionFiles) { if (deletedSessions.has(sf)) continue; @@ -378,7 +418,7 @@ export async function gcSessionFiles(opts?: { dir?: string; store?: SessionStore result.kept++; continue; } - loggerLog("info", `[gc] removed orphaned content store ${path.relative(dir, companion)} (${st.size} B)`); + loggerLog("info", `[gc] removed orphaned companion file ${path.relative(dir, companion)} (${st.size} B)`); // Companions count in their own field — `removed` stays "session files". result.companionsRemoved++; result.bytesFreed += st.size; @@ -414,7 +454,7 @@ export async function gcSessionFiles(opts?: { dir?: string; store?: SessionStore } catch { /* no spill tree yet — nothing to sweep */ } if (result.removed > 0 || result.unreadable > 0 || result.companionsRemoved > 0 || result.imgDirsRemoved > 0) { loggerLog(result.unreadable > 0 ? "warn" : "info", - `[gc] removed ${result.removed} stale session file(s)${result.companionsRemoved > 0 ? ` + ${result.companionsRemoved} content store file(s)` : ""}${result.imgDirsRemoved > 0 ? ` + ${result.imgDirsRemoved} image spill dir(s)` : ""} (age>${Math.round(cfg.maxAgeMs / DAY_MS)}d, ≤${cfg.maxTokens}tok), freed ${(result.bytesFreed / 1024).toFixed(1)} KB${result.unreadable > 0 ? `; left ${result.unreadable} unreadable file(s) in place` : ""}`); + `[gc] removed ${result.removed} stale session file(s)${result.companionsRemoved > 0 ? ` + ${result.companionsRemoved} companion file(s)` : ""}${result.imgDirsRemoved > 0 ? ` + ${result.imgDirsRemoved} image spill dir(s)` : ""} (age>${Math.round(cfg.maxAgeMs / DAY_MS)}d, ≤${cfg.maxTokens}tok), freed ${(result.bytesFreed / 1024).toFixed(1)} KB${result.unreadable > 0 ? `; left ${result.unreadable} unreadable file(s) in place` : ""}`); } return result; } diff --git a/src/session.ts b/src/session.ts index 8a0b5d1e89..2d76d46ade 100644 --- a/src/session.ts +++ b/src/session.ts @@ -10,6 +10,7 @@ import { durableMessageToolPaths, withDurableProtection } from "./durable-messag import type { AnonymousProvenance } from "./anon-provenance.js"; import type { WireProtocol } from "./util.js"; import type { InboundSlot, SeamSlot } from "./cache-ledger.js"; +import type { SystemSurfaceRecord } from "./system-surface.js"; export type BlockView = { text: string; count: number }; @@ -539,8 +540,20 @@ export type Session = { * full-state rebase). In-memory only here; buildRecord omits it. */ contentStore?: MessageContentStore; /** In-memory only (NOT persisted): content-store.json is rewritten only - * while this is set (adopt grew entries / reset cleared the store). */ + * while this is set (adopt grew entries / reset cleared the store). */ contentStoreDirty?: boolean; + /** #3201 In-memory only (NOT persisted — buildRecord omits it): last + * outbound system surface, captured per request by captureSystemSurface; + * persisted as the .system-surface.json sidecar (same pattern as + * contentStore). Lazily loaded from disk on first web-view need. */ + systemSurface?: SystemSurfaceRecord; + /** #3201 In-memory only (NOT persisted): true once the sidecar probe has + * run for this process lifetime (found or absent) so repeated web views + * do not re-stat the file per render. */ + systemSurfaceLoaded?: boolean; + /** In-memory only (NOT persisted): the system-surface sidecar is + * rewritten only while this is set (capture saw a changed hash). */ + systemSurfaceDirty?: boolean; /** #1336 in-memory only (NOT persisted — buildRecord omits it): acp_retrieve * hit count per ref, used by plan-aware search steering to flag refs the * model keeps re-fetching. Bounded by trimRetrieveCounts; lost on restart diff --git a/src/system-surface.ts b/src/system-surface.ts new file mode 100644 index 0000000000..5db527c099 --- /dev/null +++ b/src/system-surface.ts @@ -0,0 +1,108 @@ +import { createHash } from "node:crypto"; +import type { Session } from "./session.js"; + +/** #3201: the OUTBOUND system surface of a session — the system prompt text + * plus the tool catalog exactly as last sent to the upstream. The web UI's + * trajectory view renders it so a user can see the full context head without + * capturing the raw wire. Captured per request on the four prepare-* lanes, + * at the same sites (and under the same title-gen guards) where + * metadata.systemPromptTokens is stamped, so the two always describe the + * same outbound payload. + * + * Persisted as a separate sidecar file next to the session JSON (same + * pattern as the kernel CCR content store, #1097): it is payload bytes with + * its own lifecycle, NOT part of the session record — buildRecord omits it, + * which preserves whole-file encryption of the record and .json discovery. + * + * Bounded by design: both fields are capped (see MAX_*_CHARS below); the + * hash covers the UNTRUNCATED bytes so a change only inside the truncated + * tail still invalidates the cache and triggers a rewrite. */ +export interface SystemSurfaceRecord { + v: 1; + /** sha256 over fullText + "\u0000" + fullToolsJson (untruncated). */ + hash: string; + systemText: string; + systemTruncated: boolean; + toolsJson: string; + toolsTruncated: boolean; + /** Best-effort extraction across all four wire shapes (display aid only — + * toolsJson stays the source of truth). */ + toolNames: string[]; + updatedAt: number; +} + +/** Companion filename suffix — sibling of the session JSON, mirroring + * `.content-store.json` (#1097). Every directory walker / GC predicate that + * special-cases the content-store suffix must special-case this one too. */ +export const SYSTEM_SURFACE_SUFFIX = ".system-surface.json"; + +const MAX_SYSTEM_CHARS = 400_000; +const MAX_TOOLS_CHARS = 400_000; + +export function isValidSystemSurface(value: unknown): value is SystemSurfaceRecord { + if (!value || typeof value !== "object") return false; + const r = value as Record; + return r.v === 1 + && typeof r.hash === "string" + && typeof r.systemText === "string" + && typeof r.systemTruncated === "boolean" + && typeof r.toolsJson === "string" + && typeof r.toolsTruncated === "boolean" + && Array.isArray(r.toolNames) + && r.toolNames.every((n) => typeof n === "string") + && typeof r.updatedAt === "number"; +} + +/** Best-effort tool-name extraction across the four wire shapes: + * OpenAI `{type:"function",function:{name}}`, Anthropic `{name}`, + * Responses `{type:"function",name}`, Google `{functionDeclarations:[{name}]}`. */ +function extractToolNames(tools: unknown[]): string[] { + const names: string[] = []; + for (const t of tools) { + if (!t || typeof t !== "object") continue; + const o = t as Record; + const fn = o.function; + if (typeof o.name === "string") { + names.push(o.name); + continue; + } + if (fn && typeof fn === "object" && typeof (fn as Record).name === "string") { + names.push((fn as Record).name as string); + continue; + } + const decls = o.functionDeclarations; + if (Array.isArray(decls)) { + for (const d of decls) { + if (d && typeof d === "object" && typeof (d as Record).name === "string") { + names.push((d as Record).name as string); + } + } + } + } + return names; +} + +/** Capture the outbound system surface for this request. No-op when the + * surface is unchanged (hash compare) — steady-state turns cost one sha256 + * over strings already in memory, no allocation beyond the digest, and never + * arm a sidecar rewrite. */ +export function captureSystemSurface(session: Session, systemText: string | null | undefined, tools: unknown): void { + const text = typeof systemText === "string" ? systemText : ""; + const list = Array.isArray(tools) ? tools : []; + if (text.length === 0 && list.length === 0) return; + const toolsJson = JSON.stringify(list); + const hash = createHash("sha256").update(text, "utf8").update("\u0000", "utf8").update(toolsJson, "utf8").digest("hex"); + const prev = session.systemSurface; + if (prev && prev.hash === hash) return; + session.systemSurface = { + v: 1, + hash, + systemText: text.length > MAX_SYSTEM_CHARS ? text.slice(0, MAX_SYSTEM_CHARS) : text, + systemTruncated: text.length > MAX_SYSTEM_CHARS, + toolsJson: toolsJson.length > MAX_TOOLS_CHARS ? toolsJson.slice(0, MAX_TOOLS_CHARS) : toolsJson, + toolsTruncated: toolsJson.length > MAX_TOOLS_CHARS, + toolNames: extractToolNames(list), + updatedAt: Date.now(), + }; + session.systemSurfaceDirty = true; +} diff --git a/src/web/client.ts b/src/web/client.ts index f94bb80bc1..b1793eed20 100644 --- a/src/web/client.ts +++ b/src/web/client.ts @@ -1790,6 +1790,12 @@ export const WEB_CLIENT = `(function () { return html.join(""); } window.bili_renderHandoffMd = renderHandoffMd; + const KIND_LABEL = { user: t("kind.user"), assistant: t("kind.assistant"), system: t("kind.system"), tool: t("kind.tool"), "tool-call": t("kind.tool-call"), "tool-result": t("kind.tool-result"), reasoning: t("kind.reasoning"), compacted: t("kind.compacted") }; + function trajPreview(text) { + const s = String(text || ""); + const nl = s.indexOf("\\n"); + return nl >= 0 ? s.slice(0, nl) : s.slice(0, 160); + } function buildDetailHtml(d) { const parts = []; parts.push('' + t("common.back") + ""); @@ -2125,13 +2131,44 @@ export const WEB_CLIENT = `(function () { parts.push('
"); parts.push(""); } - parts.push('
' + t("det.handoff") + '' + t("det.handoff_hint") + '
'); - // #1426: copy / download actions over the rendered handoff document - parts.push('
"); + // #3201: trajectory view replaces the default-rendered handoff document — the same + // last-outbound snapshot as a per-record ledger; handoff md stays available via buttons. + parts.push('
' + t("det.traj_title") + '' + t("det.traj_hint") + '
'); + parts.push('
"); if (d.handoffTruncated) parts.push('"); - if (d.handoffMd) parts.push('
' + renderHandoffMd(d.handoffMd) + "
"); - else if (d.handoffHtml) parts.push('
' + d.handoffHtml + "
"); - else parts.push('
' + t("common.empty") + "
"); + const traj = d.trajectory; + if (!traj || !traj.records.length) { + parts.push('
' + t("det.traj_empty") + "
"); + } else { + if (traj.folded) parts.push('"); + if (traj.truncatedTotal) parts.push('"); + if (traj.recordsCapped) parts.push('"); + if (Array.isArray(traj.unanchoredBlocks) && traj.unanchoredBlocks.length > 0) parts.push('
' + escapeHtml(t("det.traj_unanchored", { ids: traj.unanchoredBlocks.join(", ") })) + "
"); + if (traj.system) { + const sys = traj.system; + parts.push('
' + t("det.traj_system") + (typeof sys.tokens === "number" ? ' · ~' + fmtW(sys.tokens) + " tok" : "") + '' + timeAgo(sys.updatedAt) + ""); + parts.push('
' + renderHandoffMd(sys.text || "") + (sys.truncated ? '
' + t("det.traj_sys_truncated") + "
" : "") + "
"); + parts.push('
' + t("det.traj_tools", { n: (sys.toolNames || []).length }) + "
"); + for (const name of sys.toolNames || []) parts.push('' + escapeHtml(name) + ""); + parts.push("
" + escapeHtml(sys.toolsJson || "") + (sys.toolsTruncated ? "\\n…(truncated)" : "") + "
"); + parts.push("
"); + } + for (const r of traj.records) { + const metaBits = []; + if (r.ref) metaBits.push(escapeHtml(r.ref)); + metaBits.push(escapeHtml(t("det.traj_turn", { n: r.turn }))); + if (typeof r.tokens === "number") metaBits.push("~" + fmtW(r.tokens)); + if (r.thinkingTokens) metaBits.push(escapeHtml(t("det.thinking")) + " ~" + fmtW(r.thinkingTokens)); + if (r.toolName) metaBits.push("" + escapeHtml(r.toolName) + ""); + let extra = ""; + if (r.kind === "compacted") { + extra = " · T" + String(typeof r.tier === "number" ? r.tier : "?"); + if (r.compressedTokens) extra += " · ~" + fmtW(r.compressedTokens); + if (r.blockId) metaBits.push(escapeHtml(r.blockId)); + } + parts.push('
' + String(r.n) + '' + escapeHtml(KIND_LABEL[r.kind] || r.kind) + '' + metaBits.join(" · ") + extra + '' + escapeHtml(trajPreview(r.text)) + (r.truncated ? "…" : "") + "
" + escapeHtml(r.text) + (r.truncated ? "\\n\\n…(truncated)" : "") + "
"); + } + } parts.push("
"); return parts.join(""); } @@ -2150,6 +2187,7 @@ export const WEB_CLIENT = `(function () { try { host.innerHTML = buildDetailHtml(d); bindHandoffActions(d); + bindTrajectoryActions(d); bindBlocksActions(d); bindCacheReportActions(d); bindSessionReportActions(d); @@ -2371,6 +2409,11 @@ export const WEB_CLIENT = `(function () { } catch (e) { toast(t("toast.failed", { msg: e.message }), "err"); } finally { busy(dlB, false); } }); } + function bindTrajectoryActions(d) { + const btn = $("traj-copy-json"); + if (!btn || !d.trajectory || !d.trajectory.records.length) { if (btn) btn.hidden = true; return; } + btn.addEventListener("click", () => copyText(JSON.stringify(d.trajectory, null, 2), btn)); + } function bindHandoffActions(d) { const copyBtn = $("handoff-copy-md"); const dlBtn = $("handoff-dl"); diff --git a/src/web/i18n.ts b/src/web/i18n.ts index 296ee1bbf8..5282f79ffc 100644 --- a/src/web/i18n.ts +++ b/src/web/i18n.ts @@ -264,7 +264,7 @@ export const MESSAGES: Record> = { "det.ccr": "CCR 内容检索", "det.ccr_detail": "{calls} 次调用 · {hits} 命中 · {misses} 未中", "det.store": "内容存储", - "det.trajectory": "上下文轨迹", + "det.trajectory": "上下文曲线", "det.trajectory_sub": "每次请求的输入 token 与缓存命中;红色竖线为压缩折叠点", "det.trajectory_empty": "该会话还没有用量采样", "det.hitrate": "缓存命中率走势", @@ -373,12 +373,30 @@ export const MESSAGES: Record> = { "det.rpt_missed": "未命中", "det.rpt_hit": "命中率", "det.blocks_copy_md": "复制 Markdown", - "det.handoff": "会话交接文档", - "det.handoff_hint": "与 bili export 相同的内容 —— 复制粘贴到新会话即可继续", "det.thinking": "思考", "det.handoff_truncated": "⚠️ 内容过长已截断", "det.handoff_copy_md": "复制 Markdown", "det.handoff_download": "下载 .md", + "det.traj_title": "上下文轨迹", + "det.traj_hint": "最近一次出站请求实际携带的完整上下文 —— 系统提示词、每条消息、折叠摘要,无需抓包", + "det.traj_system": "系统提示词与工具", + "det.traj_tools": "工具定义({n} 个)", + "det.traj_sys_truncated": "(超长已截断)", + "det.traj_copy_json": "复制 JSON", + "det.traj_empty": "该会话还没有可重建的出站快照(旧会话或尚未收到请求)", + "det.traj_folded": "视图来自有界折叠快照,部分引用号可能缺失", + "det.traj_trunc_total": "内容总量超出展示预算,已截断", + "det.traj_records_capped": "记录数超过 {n} 条上限,仅显示前部", + "det.traj_unanchored": "以下折叠块无法定位到可见消息:{ids}", + "det.traj_turn": "轮 {n}", + "kind.user": "用户", + "kind.assistant": "助手", + "kind.system": "系统", + "kind.tool": "工具", + "kind.tool-call": "工具调用", + "kind.tool-result": "工具结果", + "kind.reasoning": "思考", + "kind.compacted": "压缩摘要", "det.not_found": "找不到该会话", "det.not_found_hint": "它可能已被删除,或不属于本机数据目录", "cfg.title": "配置", @@ -776,7 +794,7 @@ export const MESSAGES: Record> = { "det.ccr": "CCR content retrieval", "det.ccr_detail": "{calls} calls · {hits} hits · {misses} misses", "det.store": "Content store", - "det.trajectory": "Context trajectory", + "det.trajectory": "Context curve", "det.trajectory_sub": "Per-request input tokens and cache hits; red marks are compression folds", "det.trajectory_empty": "No usage samples for this session yet", "det.hitrate": "Cache hit rate", @@ -885,12 +903,30 @@ export const MESSAGES: Record> = { "det.rpt_missed": "missed", "det.rpt_hit": "hit %", "det.blocks_copy_md": "copy Markdown", - "det.handoff": "Handoff document", - "det.handoff_hint": "Same content as bili export — paste into a new session to continue", "det.thinking": "thinking", "det.handoff_truncated": "⚠️ Content truncated for display", "det.handoff_copy_md": "Copy as Markdown", "det.handoff_download": "Download .md", + "det.traj_title": "Context trajectory", + "det.traj_hint": "The full context the LAST outbound request actually carried — system prompt, every message, fold summaries; no packet capture needed", + "det.traj_system": "System prompt & tools", + "det.traj_tools": "Tool definitions ({n})", + "det.traj_sys_truncated": "(truncated)", + "det.traj_copy_json": "Copy JSON", + "det.traj_empty": "No outbound snapshot to reconstruct yet (older session or no requests received)", + "det.traj_folded": "View built from a bounded folded snapshot — some ref ids may be missing", + "det.traj_trunc_total": "Content exceeded the display budget and was truncated", + "det.traj_records_capped": "Record count exceeded the {n} cap; only the head is shown", + "det.traj_unanchored": "Fold blocks that could not be anchored to a visible message: {ids}", + "det.traj_turn": "turn {n}", + "kind.user": "user", + "kind.assistant": "assistant", + "kind.system": "system", + "kind.tool": "tool", + "kind.tool-call": "tool call", + "kind.tool-result": "tool result", + "kind.reasoning": "reasoning", + "kind.compacted": "compacted", "det.not_found": "Session not found", "det.not_found_hint": "It may have been deleted or lives outside this data dir", "cfg.title": "Configuration", diff --git a/src/web/sessions-data.ts b/src/web/sessions-data.ts index 94a9f2c791..8b297ebec4 100644 --- a/src/web/sessions-data.ts +++ b/src/web/sessions-data.ts @@ -1,4 +1,4 @@ -import { orderedRefPair } from "acp-kernel"; +import { defaultCountTokens, orderedRefPair } from "acp-kernel"; import { listSessions, displayContextBest, type ContextBest, type Session } from "../session.js"; import { conflictClientOf, conflictEventsOf } from "../conflict-watch.js"; import { isDisplayOnlyConflictDetail, isSiblingConflictDetail } from "../thirdparty-scan.js"; @@ -12,6 +12,7 @@ import { markdownToHtml } from "./markdown.js"; import { log } from "../logger.js"; import { dataDir } from "../paths.js"; import { isExternalSummaryBlock } from "../external-summary-marker.js"; +import { SYSTEM_SURFACE_SUFFIX } from "../system-surface.js"; import { readdirSync, statSync } from "node:fs"; import { readdir, stat } from "node:fs/promises"; import * as path from "node:path"; @@ -212,6 +213,12 @@ interface WebSessionDetail extends WebSessionSummary { * summary chain — the web block badge mirrors /acp's "⚡ext". */ external?: true; }>; + /** #3201: dsh-style trajectory ledger of the LAST outbound context — the + * persisted system+tools surface plus a per-message view of the client's + * re-sent history with fold markers anchored at active-block endRefs. + * Absent when the session holds no outbound snapshot (older sessions or + * none received yet). */ + trajectory?: WebTrajectory; } // --------------------------------------------------------------------------- @@ -248,11 +255,11 @@ function yieldToGc(): Promise { /** Recursive *.json walk under the sessions dir, stat-based (metadata only, * no decoding). Excludes dotfiles (.bili-migration-* markers), temp files - * (.tmp-* kernel temps, *.tmp-enc-* codec temps) and CCR content-store - * envelopes (*.content-store.json — payload bytes, not session records; they - * would fail validation anyway and can be huge). A TOP-LEVEL failure (missing - * dir / EACCES) throws so callers can distinguish "no data" from "unreadable"; - * subdirectory failures skip that subtree. */ + * (.tmp-* kernel temps, *.tmp-enc-* codec temps) and sidecar envelopes + * (*.content-store.json CCR payload bytes #1097, *.system-surface.json #3201 + * — not session records; they would fail validation anyway). A TOP-LEVEL + * failure (missing dir / EACCES) throws so callers can distinguish "no data" + * from "unreadable"; subdirectory failures skip that subtree. */ async function walkSessionFiles(dir: string): Promise> { await readdir(dir); const out: Array<{ abs: string; mtimeMs: number; size: number }> = []; @@ -274,7 +281,7 @@ async function walkSessionFiles(dir: string): Promise !(e.kind === "third-party-plugin" && (isSiblingConflictDetail(e.detail) || isDisplayOnlyConflictDetail(e.detail)))); const conflictClient = conflicts.length > 0 ? conflictClientOf(session) : undefined; + // #3201: disk sessions never saw captureSystemSurface in this process — + // probe the sidecar once per process; an in-memory surface always wins + // (a live session may hold not-yet-flushed new bytes). + if (!session.systemSurfaceLoaded) { + session.systemSurfaceLoaded = true; + if (!session.systemSurface) session.systemSurface = getDiskStore().loadSystemSurface(session) ?? undefined; + } + const trajectory = buildTrajectory(session); return { ...summaryOf(session, live), @@ -1089,5 +1104,174 @@ function renderDetail(session: Session, live: boolean): WebSessionDetail { ...(isExternalSummaryBlock(b) ? { external: true } : {}), }; }), + ...(trajectory !== undefined ? { trajectory } : {}), + }; +} + +// --------------------------------------------------------------------------- +// #3201: trajectory ledger — reconstructs what the LAST outbound request +// carried, without packet capture. Two sources, both already persisted: +// - the system+tools surface sidecar (captureSystemSurface at every +// systemPromptTokens stamp site — same guard as the measurement); +// - session.lastMessages: the client's full history as re-sent this turn +// (the PRE-compression view; fold markers below show where summaries ride). +// Protocol-neutral by design: the four wire shapes differ (top-level system vs +// developer message vs systemInstruction), so this ledger shows normalized +// content, never byte-exact wire bodies. +// --------------------------------------------------------------------------- + +interface WebTrajectoryRecord { + /** 1-based position in the outbound context order. */ + n: number; + kind: "user" | "assistant" | "system" | "tool" | "tool-call" | "tool-result" | "reasoning" | "compacted"; + /** mNNNNN ref when resolvable from state.messageRefs.byRaw. */ + ref?: string; + toolName?: string; + toolCallId?: string; + thinkingTokens?: number; + /** Token estimate over the FULL (untruncated) text. */ + tokens?: number; + /** Display text after per-record / total caps. */ + text: string; + truncated: boolean; + turn: number; + // compacted records only: + blockId?: string; + tier?: number; + topic?: string; + compressedTokens?: number; +} + +interface WebTrajectory { + records: WebTrajectoryRecord[]; + turns: number; + /** Chars actually included across record texts (post-cap). */ + totalChars: number; + truncatedTotal: boolean; + recordsCapped?: boolean; + /** Built from a bounded folded snapshot — ref ids may be missing. */ + folded?: boolean; + /** Active blocks whose endRef could not be anchored to a visible record. */ + unanchoredBlocks?: string[]; + system?: { + text: string; + truncated: boolean; + toolNames: string[]; + toolsJson: string; + toolsTruncated: boolean; + updatedAt: number; + /** Measured sys+tools overhead tokens (metadata.systemPromptTokens). */ + tokens?: number; + }; +} + +export const TRAJ_RECORD_CAP = 64_000; +export const TRAJ_TOTAL_CAP = 2_000_000; +export const TRAJ_RECORDS_CAP = 5_000; + +export function capText(text: string, budget: { totalChars: number; truncatedTotal: boolean }): { text: string; truncated: boolean } { + if (text.length <= TRAJ_RECORD_CAP && budget.totalChars + text.length <= TRAJ_TOTAL_CAP) { + budget.totalChars += text.length; + return { text, truncated: false }; + } + const room = Math.max(0, Math.min(TRAJ_RECORD_CAP, TRAJ_TOTAL_CAP - budget.totalChars)); + const out = text.slice(0, room); + // The consumed chars must count against the total even when capped — + // otherwise every oversized record gets the full room again and the total + // cap stops meaning anything past the first truncation. + budget.totalChars += out.length; + return { text: out, truncated: true }; +} + +export function buildTrajectory(session: Session): WebTrajectory | undefined { + const msgs = session.lastMessages; + if (!msgs || msgs.length === 0) return undefined; + const byRaw = session.state.messageRefs?.byRaw ?? {}; + const budget = { totalChars: 0, truncatedTotal: false }; + let turn = 0; + let recordsCapped = false; + const records: WebTrajectoryRecord[] = []; + for (const m of msgs) { + if (records.length >= TRAJ_RECORDS_CAP) { recordsCapped = true; break; } + if (m.role === "user" && m.contentType === "text") turn++; + const full = m.text ?? ""; + const capped = capText(full, budget); + if (capped.truncated) budget.truncatedTotal = true; + records.push({ + n: 0, + kind: m.contentType === "tool-call" ? "tool-call" + : m.contentType === "tool-result" ? "tool-result" + : m.contentType === "reasoning" ? "reasoning" + : m.role, + ...(byRaw[m.id] !== undefined ? { ref: byRaw[m.id] } : {}), + ...(m.toolName !== undefined ? { toolName: m.toolName } : {}), + ...(m.toolCallId !== undefined ? { toolCallId: m.toolCallId } : {}), + ...(typeof m.thinkingTokens === "number" ? { thinkingTokens: m.thinkingTokens } : {}), + ...(full.length > 0 ? { tokens: defaultCountTokens(full) } : {}), + text: capped.text, + truncated: capped.truncated, + turn, + }); + } + // Fold markers: each ACTIVE block's summary rides where its span ends in + // the outbound context (true for both compression modes — the proxy-mode + // acp_summary carrier and plugin-mode folds both replace the span), so a + // compacted record is inserted right after the anchor record carrying + // endRef. Blocks whose endRef resolves to nothing visible stay honest: + // they are listed separately instead of guessed into a position. + let unanchoredBlocks: string[] = []; + const activeBlocks = session.state.blocks.filter((b) => b.active && b.endRef !== undefined); + if (activeBlocks.length > 0) { + const refIndex = new Map(); + records.forEach((r, i) => { if (r.ref !== undefined && !refIndex.has(r.ref)) refIndex.set(r.ref, i); }); + const sorted = [...activeBlocks].sort((a, b) => (a.endRef as string) < (b.endRef as string) ? -1 : (a.endRef as string) > (b.endRef as string) ? 1 : a.tier - b.tier); + const inserts: Array<{ at: number; block: (typeof activeBlocks)[number] }> = []; + for (const b of sorted) { + const idx = refIndex.get(b.endRef as string); + if (idx === undefined) unanchoredBlocks.push(b.blockId); + else inserts.push({ at: idx, block: b }); + } + for (let k = inserts.length - 1; k >= 0; k--) { + const { at, block } = inserts[k]; + const full = block.summary ?? ""; + const capped = capText(full, budget); + if (capped.truncated) budget.truncatedTotal = true; + records.splice(at + 1, 0, { + n: 0, + kind: "compacted", + ...(block.endRef !== undefined ? { ref: block.endRef } : {}), + ...(full.length > 0 ? { tokens: defaultCountTokens(full) } : {}), + text: capped.text, + truncated: capped.truncated, + turn: records[at]?.turn ?? turn, + blockId: block.blockId, + tier: block.tier, + ...(block.topic !== undefined ? { topic: block.topic } : {}), + compressedTokens: block.compressedTokens, + }); + } + } + records.forEach((r, i) => { r.n = i + 1; }); + const surface = session.systemSurface; + const sysPromptTok = typeof session.metadata["systemPromptTokens"] === "number" ? (session.metadata["systemPromptTokens"] as number) : undefined; + return { + records, + turns: turn, + totalChars: budget.totalChars, + truncatedTotal: budget.truncatedTotal, + ...(recordsCapped ? { recordsCapped: true } : {}), + ...(session.lastMessagesFolded ? { folded: true } : {}), + ...(unanchoredBlocks.length > 0 ? { unanchoredBlocks } : {}), + ...(surface ? { + system: { + text: surface.systemText, + truncated: surface.systemTruncated, + toolNames: surface.toolNames, + toolsJson: surface.toolsJson, + toolsTruncated: surface.toolsTruncated, + updatedAt: surface.updatedAt, + ...(sysPromptTok !== undefined ? { tokens: sysPromptTok } : {}), + }, + } : {}), }; } diff --git a/src/web/styles.ts b/src/web/styles.ts index 13b37ad43c..48f59d805c 100644 --- a/src/web/styles.ts +++ b/src/web/styles.ts @@ -249,6 +249,37 @@ td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; font-fam .block-item .body.md hr { border: none; border-top: 1px solid var(--border); margin: 10px 0; } .block-item .body.md strong { color: var(--text); } +/* #3201 trajectory view — mirrors the .block-item row pattern */ +.traj-sys { border-bottom: 1px solid var(--border-soft); } +.traj-sys:last-child { border-bottom: none; } +.traj-sys > summary { list-style: none; cursor: pointer; padding: 9px 16px; display: flex; gap: 10px; align-items: baseline; } +.traj-sys > summary::-webkit-details-marker { display: none; } +.traj-sys > summary:hover { background: var(--bg-muted); } +.traj-sys > summary .topic { font-weight: 600; font-size: 13px; flex: 1; } +.traj-sys > summary .meta { font-size: 11.5px; color: var(--text-muted); white-space: nowrap; } +.traj-sys .body { padding: 0 16px 12px 16px; font-size: 12.5px; color: var(--text-muted); } +.traj-tools { margin-top: 8px; border: 1px solid var(--border-soft); border-radius: 8px; } +.traj-tools > summary { cursor: pointer; padding: 7px 12px; font-size: 12px; color: var(--text-muted); } +.tool-chips { display: flex; flex-wrap: wrap; gap: 6px; padding: 0 12px 8px; } +.chip { font-family: var(--mono); font-size: 11px; background: var(--bg-muted); border: 1px solid var(--border); border-radius: 10px; padding: 1px 8px; color: var(--text-muted); } +.traj-json { margin: 0 12px 10px; background: var(--bg-muted); border: 1px solid var(--border-soft); border-radius: 8px; padding: 8px 10px; overflow-x: auto; font-family: var(--mono); font-size: 11px; white-space: pre; max-height: 320px; } +.traj-row { border-bottom: 1px solid var(--border-soft); } +.traj-row:last-child { border-bottom: none; } +.traj-row > summary { list-style: none; cursor: pointer; padding: 7px 16px; display: flex; gap: 8px; align-items: baseline; } +.traj-row > summary::-webkit-details-marker { display: none; } +.traj-row > summary:hover { background: var(--bg-muted); } +.traj-n { font-family: var(--mono); font-size: 11px; color: var(--text-faint); min-width: 34px; text-align: right; } +.badge.k-user { background: var(--accent-soft); color: var(--accent); border-color: var(--accent); } +.badge.k-assistant { background: var(--green-soft); color: var(--green); border-color: var(--green); } +.badge.k-system { background: var(--accent-soft); color: var(--accent); border-color: var(--accent); font-family: var(--mono); font-weight: 500; } +.badge.k-tool, .badge.k-tool-call, .badge.k-tool-result { background: var(--amber-soft); color: var(--amber); border-color: var(--amber); } +.badge.k-reasoning { background: var(--bg-muted); color: var(--text-muted); border-color: var(--border); } +.badge.k-compacted { background: var(--accent-soft); color: var(--accent); border-color: var(--accent); font-family: var(--mono); font-weight: 500; } +.traj-preview { flex: 1; font-size: 12px; color: var(--text-faint); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.traj-text { margin: 0; padding: 4px 16px 12px 58px; font-family: var(--mono); font-size: 11.5px; line-height: 1.55; color: var(--text-muted); white-space: pre-wrap; word-break: break-word; max-height: 480px; overflow-y: auto; } +.traj-row.traj-fold > summary { background: var(--accent-soft); } +.traj-fold .traj-text { color: var(--text); } + .handoff { font-size: 13.5px; line-height: 1.65; } .handoff h1, .handoff h2, .handoff h3, .handoff h4 { margin: 18px 0 8px; line-height: 1.3; } .handoff h1 { font-size: 17px; } .handoff h2 { font-size: 15.5px; } .handoff h3 { font-size: 14px; } diff --git a/tests/issue2129-restart-measurement-state.test.ts b/tests/issue2129-restart-measurement-state.test.ts index b147fbedaa..2202289556 100644 --- a/tests/issue2129-restart-measurement-state.test.ts +++ b/tests/issue2129-restart-measurement-state.test.ts @@ -45,7 +45,7 @@ function makeSession(id: string): Session { type StatsBlob = Record; function findRecordFile(dir: string): string { - const files = readdirSync(dir, { recursive: true }).map(String).filter((f) => f.endsWith(".json")); + const files = readdirSync(dir, { recursive: true }).map(String).filter((f) => f.endsWith(".json") && !f.endsWith(".content-store.json") && !f.endsWith(".system-surface.json")); assert.equal(files.length, 1, `expected exactly one session record, found ${files.length}`); return join(dir, files[0] as string); } diff --git a/tests/proxy-persist.test.ts b/tests/proxy-persist.test.ts index 19456628a2..40fecbee14 100644 --- a/tests/proxy-persist.test.ts +++ b/tests/proxy-persist.test.ts @@ -27,6 +27,13 @@ function jsonFilesUnder(dir: string): string[] { return out; } +/** #2671: with no storage tier configured the unified store defaults to + * shadow mode and opens /index.db (+ -wal/-shm) beside the namespace + * dirs — legitimate top-level entries, not stray files. */ +function nonSqliteTopLevel(dir: string): string[] { + return readdirSync(dir).filter((n) => !/^index\.db(-wal|-shm)?$/.test(n)); +} + function makeSession(id: string): Session { return { id, @@ -131,7 +138,7 @@ await withTempStore("scheduleSave debounces and eventually writes", async (store // land well past the 5ms debounce. Deadline covers the retry ladder // (~1.5s) with headroom. const loaded = await waitFor(() => store.loadSync("debounce-1"), 5000, "debounced write to land"); - assert.equal(readdirSync(dir).length, 1, "exactly one top-level entry (the _unknown subdir) happened"); + assert.equal(nonSqliteTopLevel(dir).length, 1, "exactly one top-level entry (the _unknown subdir) happened"); assert.equal(loaded!.stats.requests, 3, "latest value persisted"); }); @@ -148,7 +155,7 @@ await withTempStore("flushSync writes immediately (for eviction)", async (store, const s = makeSession("evict-1"); s.stats.tokensSaved = 555; store.flushSync(s); - assert.equal(readdirSync(dir).length, 1); + assert.equal(nonSqliteTopLevel(dir).length, 1); assert.equal(store.loadSync("evict-1")!.stats.tokensSaved, 555); }); diff --git a/tests/session-gc.test.ts b/tests/session-gc.test.ts index da91a4ba36..741a8b2710 100644 --- a/tests/session-gc.test.ts +++ b/tests/session-gc.test.ts @@ -78,6 +78,11 @@ function contentStoreFile(dir: string, rel: string, chars: number, ageDays: numb return writeFile(dir, rel, JSON.stringify(store), ageDays); } +function systemSurfaceFile(dir: string, rel: string, ageDays: number): string { + const rec = { v: 1, hash: "ab".repeat(32), systemText: "sys", systemTruncated: false, toolsJson: "[]", toolsTruncated: false, toolNames: [], updatedAt: Date.now() }; + return writeFile(dir, rel, JSON.stringify(rec), ageDays); +} + function writeFile(dir: string, rel: string, content: string | Buffer, ageDays: number): string { const full = path.join(dir, rel); mkdirSync(path.dirname(full), { recursive: true }); @@ -481,6 +486,77 @@ test("gcSessionFiles: orphaned content stores are swept once old, fresh ones kep }); }); +test("gcSessionFiles: stale session co-deletes its system-surface companion (#3201)", async () => { + const dir = tmpDir("bili-gc-surf-"); + await withEnv({ BILI_SESSION_GC: "1" }, async () => { + const store = new SessionStore({ dir, debounceMs: 500 }); + const oldSavedAt = Date.now() - 10 * DAY; + const fSess = writeFile(dir, "anthropic/host_surf.json", JSON.stringify(envelope("gc-surf", oldSavedAt, { metadata: { rawInputTokens: 5000 } })), 10); + const fSurf = systemSurfaceFile(dir, "anthropic/host_surf.system-surface.json", 10); + const res = await gcSessionFiles({ dir, store, now: Date.now() }); + assert.equal(res.removed, 1, JSON.stringify(res)); + assert.equal(res.companionsRemoved, 1, JSON.stringify(res)); + assert.ok(!existsSync(fSess)); + assert.ok(!existsSync(fSurf), "surface co-deleted — no orphan left behind"); + }); +}); + +test("gcSessionFiles: stale duplicate keeps its surface while a namespaced twin still relies on it (#3201)", async () => { + const dir = tmpDir("bili-gc-surftwin-"); + await withEnv({ BILI_SESSION_GC: "1" }, async () => { + const store = new SessionStore({ dir, debounceMs: 500 }); + // Same meta fill-once migration story as the content-store twin tests: + // the _unknown/ sibling is still the LIVE surface — loadSystemSurface + // falls back to it when the namespaced one is absent. + const stem = "surfbeefcafe1234"; + const fDup = writeFile(dir, `_unknown/${stem}.json`, JSON.stringify(envelope(`gc-surftwin-${stem}`, Date.now() - 10 * DAY, { metadata: { rawInputTokens: 5000 } })), 10); + const fTwin = writeFile(dir, `anthropic/api.example.com_${stem}.json`, JSON.stringify(envelope(`gc-surftwin-${stem}`, Date.now() - DAY, { metadata: { rawInputTokens: 5000 } })), 1); + const fSurf = systemSurfaceFile(dir, `_unknown/${stem}.system-surface.json`, 10); + const res = await gcSessionFiles({ dir, store, now: Date.now() }); + assert.equal(res.removed, 1, JSON.stringify(res)); + assert.equal(res.companionsRemoved, 0, JSON.stringify(res)); + assert.ok(!existsSync(fDup), "stale duplicate deleted"); + assert.ok(existsSync(fTwin), "live twin untouched"); + assert.ok(existsSync(fSurf), "live fallback surface of the surviving twin is never co-deleted"); + }); +}); + +test("gcSessionFiles: stale duplicate's surface IS co-deleted once the twin owns its own (#3201)", async () => { + const dir = tmpDir("bili-gc-surftwinown-"); + await withEnv({ BILI_SESSION_GC: "1" }, async () => { + const store = new SessionStore({ dir, debounceMs: 500 }); + // The session was re-saved after the namespace move: the namespaced + // surface shadows the flat one, so the flat sibling is dead bytes and + // co-deletion is safe. + const stem = "surftwincafe1234"; + const fDup = writeFile(dir, `_unknown/${stem}.json`, JSON.stringify(envelope(`gc-surftwown-${stem}`, Date.now() - 10 * DAY, { metadata: { rawInputTokens: 5000 } })), 10); + const fTwin = writeFile(dir, `anthropic/api.example.com_${stem}.json`, JSON.stringify(envelope(`gc-surftwown-${stem}`, Date.now() - DAY, { metadata: { rawInputTokens: 5000 } })), 1); + const fDead = systemSurfaceFile(dir, `_unknown/${stem}.system-surface.json`, 10); + const fLive = systemSurfaceFile(dir, `anthropic/api.example.com_${stem}.system-surface.json`, 1); + const res = await gcSessionFiles({ dir, store, now: Date.now() }); + assert.equal(res.removed, 1, JSON.stringify(res)); + assert.equal(res.companionsRemoved, 1, JSON.stringify(res)); + assert.ok(!existsSync(fDup)); + assert.ok(existsSync(fTwin)); + assert.ok(!existsSync(fDead), "shadowed dead surface co-deleted with its duplicate"); + assert.ok(existsSync(fLive), "the live namespaced surface stays referenced"); + }); +}); + +test("gcSessionFiles: orphaned system surfaces are swept once old, fresh ones kept (#3201)", async () => { + const dir = tmpDir("bili-gc-surforph-"); + await withEnv({ BILI_SESSION_GC: "1" }, async () => { + const store = new SessionStore({ dir, debounceMs: 500 }); + const fOld = systemSurfaceFile(dir, "anthropic/host_sorph_old.system-surface.json", 10); + const fFresh = systemSurfaceFile(dir, "anthropic/host_sorph_fresh.system-surface.json", 1); + const res = await gcSessionFiles({ dir, store, now: Date.now() }); + assert.equal(res.removed, 0, "no session files were removed"); + assert.equal(res.companionsRemoved, 1, JSON.stringify(res)); + assert.ok(!existsSync(fOld), "aged orphan swept"); + assert.ok(existsSync(fFresh), "fresh orphan kept (still inside any resume window)"); + }); +}); + test("gcSessionFiles: tiny session with a huge store does not slip under the token gate (#1180)", async () => { const dir = tmpDir("bili-gc-hugestore-"); await withEnv({ BILI_SESSION_GC: "1", BILI_SESSION_GC_MAX_TOKENS: "1000" }, async () => { diff --git a/tests/session-report.test.ts b/tests/session-report.test.ts index 8f2e639a1c..1bac57afd0 100644 --- a/tests/session-report.test.ts +++ b/tests/session-report.test.ts @@ -52,7 +52,7 @@ test("markdown: mask=false reveals the raw title", () => { test("markdown: renders every section heading + key metrics (zh default)", () => { const md = buildSessionReportMd(D); - for (const h of ["# billion-context session report", "基本信息", "用量统计", "上下文轨迹", "缓存经济学"]) { + for (const h of ["# billion-context session report", "基本信息", "用量统计", "上下文曲线", "缓存经济学"]) { assert.ok(md.includes(h), "missing section: " + h); } assert.ok(md.includes("900K"), "input tokens formatted"); diff --git a/tests/system-surface-persist.test.ts b/tests/system-surface-persist.test.ts new file mode 100644 index 0000000000..90ef5ff495 --- /dev/null +++ b/tests/system-surface-persist.test.ts @@ -0,0 +1,172 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { tmpdir } from "node:os"; +import { mkdtempSync, readFileSync, writeFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { createInitialState } from "acp-kernel"; +import { SessionStore, relPathFor } from "../src/persist.ts"; +import { captureSystemSurface, isValidSystemSurface, SYSTEM_SURFACE_SUFFIX } from "../src/system-surface.ts"; +import type { SystemSurfaceRecord } from "../src/system-surface.ts"; +import type { Session } from "../src/session.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// #3201: the system-surface sidecar (outbound system prompt + tool catalog) +// must round-trip through SessionStore like the CCR content store does, and +// must degrade gracefully when corrupt — it is a derived artifact and may +// never block boot or a save. + +function makeSession(id: string): Session { + return { + id, + meta: {}, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +function sidecarPath(dir: string, id: string, protocol?: string): string { + const base = relPathFor(id, protocol); + return join(dir, base.slice(0, -".json".length) + SYSTEM_SURFACE_SUFFIX); +} + +test("captureSystemSurface records once per distinct surface", () => { + const s = makeSession("ss-dedup"); + captureSystemSurface(s, "you are helpful", [{ name: "bash" }]); + assert.ok(s.systemSurface); + assert.equal(s.systemSurfaceDirty, true); + assert.equal(s.systemSurface!.systemText, "you are helpful"); + assert.deepEqual(s.systemSurface!.toolNames, ["bash"]); + const first = s.systemSurface; + + s.systemSurfaceDirty = false; + captureSystemSurface(s, "you are helpful", [{ name: "bash" }]); + assert.equal(s.systemSurfaceDirty, false, "identical surface must not re-arm dirty"); + assert.equal(s.systemSurface, first, "record object stays stable when unchanged"); + + captureSystemSurface(s, "you are helpful v2", [{ name: "bash" }]); + assert.equal(s.systemSurfaceDirty, true); + assert.notEqual(s.systemSurface!.hash, first.hash); +}); + +test("captureSystemSurface truncates oversized payloads but hashes full content", () => { + const s = makeSession("ss-trunc"); + const big = "s".repeat(500_000); + captureSystemSurface(s, big, null); + assert.equal(s.systemSurface!.systemTruncated, true); + assert.equal(s.systemSurface!.systemText.length, 400_000); + + // Hash covers the UNTRUNCATED bytes: same head, different tail → new hash. + const s2 = makeSession("ss-trunc2"); + captureSystemSurface(s2, big.slice(0, 499_999) + "T", null); + assert.notEqual(s2.systemSurface!.hash, s.systemSurface!.hash); + + const s3 = makeSession("ss-empty"); + captureSystemSurface(s3, "", undefined); + assert.equal(s3.systemSurface, undefined, "empty surface is not captured"); +}); + +test("extractToolNames handles all four wire shapes", () => { + const cases: Array<[string, unknown[], string[]]> = [ + ["openai", [{ type: "function", function: { name: "read_file" } }], ["read_file"]], + ["anthropic", [{ name: "str_replace_editor" }], ["str_replace_editor"]], + ["responses", [{ type: "function", name: "apply_patch" }], ["apply_patch"]], + ["google", [{ functionDeclarations: [{ name: "g1" }, { name: "g2" }] }], ["g1", "g2"]], + ["mixed-junk", [null, 42, { name: "ok" }, {}], ["ok"]], + ]; + for (const [label, tools, expected] of cases) { + const s = makeSession(`ss-tools-${label}`); + captureSystemSurface(s, `sys ${label}`, tools); + assert.deepEqual(s.systemSurface!.toolNames, expected, label); + } +}); + +test("isValidSystemSurface rejects malformed records", () => { + const good: SystemSurfaceRecord = { + v: 1, hash: "h", systemText: "t", systemTruncated: false, + toolsJson: "[]", toolsTruncated: false, toolNames: [], updatedAt: 1, + }; + assert.equal(isValidSystemSurface(good), true); + assert.equal(isValidSystemSurface(null), false); + assert.equal(isValidSystemSurface({}), false); + assert.equal(isValidSystemSurface({ ...good, v: 2 }), false); + assert.equal(isValidSystemSurface({ ...good, toolNames: "nope" }), false); + assert.equal(isValidSystemSurface({ ...good, toolNames: ["ok", 5] }), false); + assert.equal(isValidSystemSurface({ ...good, updatedAt: "yesterday" }), false); +}); + +test("system surface sidecar survives a save/load round-trip", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-sys-srf-")); + try { + const store = new SessionStore({ dir, debounceMs: 5, enabled: true }); + const s = makeSession("ss-rt"); + s.meta.protocol = "openai"; + captureSystemSurface(s, "sys text", [{ name: "bash" }]); + await store.writeNow(s); + await store.flushAll([]); + assert.ok(existsSync(sidecarPath(dir, "ss-rt", "openai")), "sidecar written next to the session record"); + + const reloaded = new SessionStore({ dir, debounceMs: 5, enabled: true }); + // Namespaced sessions need their meta for the direct file probe — + // production callers pass it from the index row (same as loadAll). + const back = reloaded.loadSync("ss-rt", { protocol: "openai" }); + assert.ok(back, "session reloaded"); + assert.equal(back.systemSurface, undefined, "loadSync does not auto-hydrate the derived sidecar"); + const loaded = reloaded.loadSystemSurface(back); + assert.ok(loaded, "explicit probe finds the sidecar"); + assert.equal(loaded.systemText, "sys text"); + assert.deepEqual(loaded.toolNames, ["bash"]); + await reloaded.flushAll([]); + } finally { + rmrf(dir); + } +}); + +test("malformed system surface degrades to null without throwing", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-sys-srf-bad-")); + try { + const store = new SessionStore({ dir, debounceMs: 5, enabled: true }); + const s = makeSession("ss-bad"); + s.meta.protocol = "openai"; + captureSystemSurface(s, "sys", [{ name: "bash" }]); + await store.writeNow(s); + await store.flushAll([]); + + writeFileSync(sidecarPath(dir, "ss-bad", "openai"), "{not json"); + const reloaded = new SessionStore({ dir, debounceMs: 5, enabled: true }); + const back = reloaded.loadSync("ss-bad", { protocol: "openai" }); + assert.ok(back, "boot/load still succeeds with a corrupt sidecar"); + assert.equal(reloaded.loadSystemSurface(back), null); + await reloaded.flushAll([]); + } finally { + rmrf(dir); + } +}); + +test("re-capturing after clear deletes the stale sidecar on next save", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-sys-srf-del-")); + try { + const store = new SessionStore({ dir, debounceMs: 5, enabled: true }); + const s = makeSession("ss-del"); + s.meta.protocol = "openai"; + captureSystemSurface(s, "sys", [{ name: "bash" }]); + await store.writeNow(s); + await store.flushAll([]); + const p = sidecarPath(dir, "ss-del", "openai"); + assert.ok(existsSync(p)); + + s.systemSurface = undefined; + s.systemSurfaceDirty = true; + await store.writeNow(s); + await store.flushAll([]); + assert.ok(!existsSync(p), "absent surface removes the sidecar"); + } finally { + rmrf(dir); + } +}); diff --git a/tests/web-i18n.test.ts b/tests/web-i18n.test.ts index cc7d448f8b..ae6cffc765 100644 --- a/tests/web-i18n.test.ts +++ b/tests/web-i18n.test.ts @@ -43,11 +43,11 @@ function collectRefs(): Set { const clientSrc = readFileSync(join(here, "..", "src", "web", "client.ts"), "utf8"); const reportSrc = readFileSync(join(here, "..", "src", "web", "report.ts"), "utf8"); // #3099 server-side report renderer const refs = new Set(); - for (const m of pageSrc.matchAll(/data-i18n(?:-ph|-title)?="([\w.]+)"/g)) refs.add(m[1]); - for (const m of pageSrc.matchAll(/zh\("([\w.]+)"\)/g)) refs.add(m[1]); - for (const m of clientSrc.matchAll(/\bt\("([\w.]+)"/g)) refs.add(m[1]); - for (const m of reportSrc.matchAll(/\bt\("([\w.]+)"/g)) refs.add(m[1]); - for (const m of reportSrc.matchAll(/\btr\(\s*\w+,\s*"([\w.]+)"/g)) refs.add(m[1]); + for (const m of pageSrc.matchAll(/data-i18n(?:-ph|-title)?="([\w.-]+)"/g)) refs.add(m[1]); + for (const m of pageSrc.matchAll(/zh\("([\w.-]+)"\)/g)) refs.add(m[1]); + for (const m of clientSrc.matchAll(/\bt\("([\w.-]+)"/g)) refs.add(m[1]); + for (const m of reportSrc.matchAll(/\bt\("([\w.-]+)"/g)) refs.add(m[1]); + for (const m of reportSrc.matchAll(/\btr\(\s*\w+,\s*"([\w.-]+)"/g)) refs.add(m[1]); return refs; } diff --git a/tests/web-trajectory.test.ts b/tests/web-trajectory.test.ts new file mode 100644 index 0000000000..ecb23bbec7 --- /dev/null +++ b/tests/web-trajectory.test.ts @@ -0,0 +1,183 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createInitialState } from "acp-kernel"; +import type { CompressionBlock, CoreMessage } from "acp-kernel"; +import { buildTrajectory, capText, TRAJ_RECORD_CAP, TRAJ_TOTAL_CAP, TRAJ_RECORDS_CAP } from "../src/web/sessions-data.ts"; +import { SYSTEM_SURFACE_SUFFIX } from "../src/system-surface.ts"; +import type { SystemSurfaceRecord } from "../src/system-surface.ts"; +import type { Session } from "../src/session.ts"; + +// #3201: the web UI trajectory view — a protocol-neutral ledger of what the +// last outbound context actually contained (messages + fold markers + system +// surface). buildTrajectory is pure over the in-memory Session, so these tests +// pin its ordering, anchoring and capping rules without any disk or server. + +function makeSession(partial?: Partial): Session { + return { + id: "traj-1", + meta: {}, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + ...partial, + }; +} + +function mkBlock(over: Partial & Pick): CompressionBlock { + return { + runId: "r1", + tier: 1, + summary: "a summary", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + compressedTokens: 100, + createdAt: 1, + survivedCount: 0, + generation: "young", + active: true, + ...over, + }; +} + +test("buildTrajectory returns undefined without lastMessages", () => { + assert.equal(buildTrajectory(makeSession()), undefined); + assert.equal(buildTrajectory(makeSession({ lastMessages: [] })), undefined); +}); + +test("buildTrajectory maps kinds, counts turns, resolves refs", () => { + const msgs: CoreMessage[] = [ + { id: "u1", role: "user", contentType: "text", text: "hello" }, + { id: "a1", role: "assistant", contentType: "reasoning", text: "thinking" }, + { id: "a2", role: "assistant", contentType: "tool-call", text: "{\"x\":1}", toolName: "bash", toolCallId: "tc1" }, + { id: "t1", role: "tool", contentType: "tool-result", text: "ok" }, + { id: "u2", role: "user", contentType: "text", text: "again" }, + ]; + const s = makeSession({ lastMessages: msgs }); + s.state.messageRefs.byRaw = { u1: "m00001", a1: "m00002", a2: "m00003", t1: "m00004", u2: "m00005" }; + const traj = buildTrajectory(s)!; + assert.deepEqual(traj.records.map((r) => r.kind), ["user", "reasoning", "tool-call", "tool-result", "user"]); + assert.equal(traj.turns, 2); + assert.deepEqual(traj.records.map((r) => r.turn), [1, 1, 1, 1, 2]); + assert.deepEqual(traj.records.map((r) => r.ref), ["m00001", "m00002", "m00003", "m00004", "m00005"]); + assert.deepEqual(traj.records.map((r) => r.n), [1, 2, 3, 4, 5]); + assert.equal(traj.records[2].toolName, "bash"); + assert.equal(traj.records[2].toolCallId, "tc1"); + assert.equal(traj.truncatedTotal, false); +}); + +test("buildTrajectory carries thinkingTokens and token estimates", () => { + const s = makeSession({ + lastMessages: [ + { id: "a1", role: "assistant", contentType: "reasoning", text: "abc", thinkingTokens: 42 }, + { id: "e1", role: "user", contentType: "text" }, + ], + }); + const traj = buildTrajectory(s)!; + assert.equal(traj.records[0].thinkingTokens, 42); + const est = traj.records[0].tokens; + assert.equal(typeof est, "number"); + assert.ok((est ?? 0) > 0); + assert.ok(!("tokens" in traj.records[1]), "empty text must not carry a token estimate"); +}); + +test("fold markers insert after the anchor record; unanchored blocks are listed", () => { + const msgs: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "one" }, + { id: "m2", role: "assistant", contentType: "text", text: "two" }, + { id: "m3", role: "user", contentType: "text", text: "three" }, + ]; + const s = makeSession({ lastMessages: msgs }); + s.state.messageRefs.byRaw = { m1: "m00001", m2: "m00002", m3: "m00003" }; + // Two blocks end at the same ref (tiers ascend away from the anchor); one + // ends at a ref that is not visible (folded snapshot) and must stay honest; + // one inactive / one without endRef must be ignored entirely. + s.state.blocks.push( + mkBlock({ blockId: "bA", tier: 1, endRef: "m00002", topic: "topic A" }), + mkBlock({ blockId: "bB", tier: 2, endRef: "m00002" }), + mkBlock({ blockId: "bC", endRef: "m99999" }), + mkBlock({ blockId: "bD", active: false, endRef: "m00002" }), + mkBlock({ blockId: "bE" }), + ); + const traj = buildTrajectory(s)!; + assert.deepEqual(traj.records.map((r) => r.kind), ["user", "assistant", "compacted", "compacted", "user"]); + const folded = traj.records.filter((r) => r.kind === "compacted"); + assert.deepEqual(folded.map((r) => r.blockId), ["bA", "bB"], "same-anchor blocks order by tier"); + assert.equal(folded[0].turn, 1, "fold marker inherits the anchor's turn"); + assert.equal(folded[0].topic, "topic A"); + assert.equal(folded[0].compressedTokens, 100); + assert.deepEqual(traj.unanchoredBlocks, ["bC"]); + assert.deepEqual(traj.records.map((r) => r.n), [1, 2, 3, 4, 5], "n renumbered after inserts"); +}); + +test("capText enforces per-record and total budgets", () => { + const budget = { totalChars: 0, truncatedTotal: false }; + const big = "x".repeat(TRAJ_RECORD_CAP + 5_000); + const r1 = capText(big, budget); + assert.equal(r1.truncated, true); + assert.equal(r1.text.length, TRAJ_RECORD_CAP); + assert.equal(budget.totalChars, TRAJ_RECORD_CAP); + + const near = { totalChars: TRAJ_TOTAL_CAP - 100, truncatedTotal: false }; + const r2 = capText("y".repeat(500), near); + assert.equal(r2.text.length, 100, "only the remaining total room is allowed"); + assert.equal(r2.truncated, true); + + const r3 = capText("z".repeat(10), { totalChars: TRAJ_TOTAL_CAP, truncatedTotal: false }); + assert.equal(r3.text.length, 0); + assert.equal(r3.truncated, true); + + const ok = { totalChars: 0, truncatedTotal: false }; + const r4 = capText("hello", ok); + assert.equal(r4.text, "hello"); + assert.equal(r4.truncated, false); + assert.equal(ok.totalChars, 5); +}); + +test("buildTrajectory caps the record count", () => { + const msgs: CoreMessage[] = []; + for (let i = 0; i < TRAJ_RECORDS_CAP + 10; i++) { + msgs.push({ id: `m${i}`, role: "user", contentType: "text", text: "x" }); + } + const traj = buildTrajectory(makeSession({ lastMessages: msgs }))!; + assert.equal(traj.records.length, TRAJ_RECORDS_CAP); + assert.equal(traj.recordsCapped, true); +}); + +test("system surface and prompt tokens map onto trajectory.system", () => { + const surface: SystemSurfaceRecord = { + v: 1, hash: "h", systemText: "sys", systemTruncated: false, + toolsJson: "[{\"name\":\"bash\"}]", toolsTruncated: false, toolNames: ["bash"], updatedAt: 123, + }; + const withSurface = makeSession({ + lastMessages: [{ id: "u1", role: "user", contentType: "text", text: "hi" }], + systemSurface: surface, + metadata: { systemPromptTokens: 777 }, + }); + const traj = buildTrajectory(withSurface)!; + assert.equal(traj.system?.text, "sys"); + assert.equal(traj.system?.tokens, 777); + assert.equal(traj.system?.updatedAt, 123); + assert.deepEqual(traj.system?.toolNames, ["bash"]); + + const bare = buildTrajectory(makeSession({ lastMessages: [{ id: "u1", role: "user", contentType: "text", text: "hi" }] }))!; + assert.ok(!("system" in bare), "no sidecar → no system section"); +}); + +test("folded snapshot flag propagates", () => { + const traj = buildTrajectory(makeSession({ + lastMessages: [{ id: "u1", role: "user", contentType: "text", text: "hi" }], + lastMessagesFolded: true, + }))!; + assert.equal(traj.folded, true); +}); + +test("sidecar suffix is stable for walkers to exclude", () => { + assert.equal(SYSTEM_SURFACE_SUFFIX, ".system-surface.json"); +});