From f56854ba46a1bab2e889b8ab0e288ee6f8768bfa Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:19:05 +0000 Subject: [PATCH 01/30] fix(ui): chord hints follow a rebind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tooltips, menu chords and dialog hints spelled the shipped chords by hand (⌘E, ⌘[, ⌘], ⇧⌘F, ⌘,, ⌘N, ⇧⌘N, F2, ⇧F10, ⌘⌫, ⌘⏎, ⌃Tab), so they went stale the moment a user rebound one. They now come out of displayKeys, and the shell's static hints live in hints.ts and are repainted by setOverrides (the shell and the live editor's bar are the two surfaces render() doesn't rebuild, so #open-chat's title used to stay on the boot-time chord). hints.test.ts pins the hints under a rebind and walks every UI string literal through TypeScript's parser, failing on a chord spelled by hand. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/bindings.ts | 8 ++- ui/src/hints.test.ts | 150 +++++++++++++++++++++++++++++++++++++++++++ ui/src/hints.ts | 51 +++++++++++++++ ui/src/main.ts | 41 ++++++++---- ui/src/render.ts | 67 ++++++++++++------- ui/src/shortcuts.ts | 16 ++++- 6 files changed, 291 insertions(+), 42 deletions(-) create mode 100644 ui/src/hints.test.ts create mode 100644 ui/src/hints.ts diff --git a/ui/src/bindings.ts b/ui/src/bindings.ts index 21247e9..898bcf1 100644 --- a/ui/src/bindings.ts +++ b/ui/src/bindings.ts @@ -796,8 +796,10 @@ export function displayChord(spec: string): string { /** The chords for a row of the sheet, printed as one cell: "⌘G / ⇧⌘G". * * Distinct renderings only, so a row covering two commands that share a chord — ⏎ - * commits the link dialog *and* the delete confirm — reads "⏎" rather than "⏎ / ⏎". */ -export function displayKeys(ids: readonly string[]): string { + * commits the link dialog *and* the delete confirm — reads "⏎" rather than "⏎ / ⏎". + * `sep` is for the tight spots — a tooltip's "↑/↓ move" — where the sheet's spaced + * slash would read as two separate hints. */ +export function displayKeys(ids: readonly string[], sep = " / "): string { const out: string[] = []; for (const id of ids) { for (const spec of binding(id).keys) { @@ -805,5 +807,5 @@ export function displayKeys(ids: readonly string[]): string { if (!out.includes(text)) out.push(text); } } - return out.join(" / "); + return out.join(sep); } diff --git a/ui/src/hints.test.ts b/ui/src/hints.test.ts new file mode 100644 index 0000000..a56c40f --- /dev/null +++ b/ui/src/hints.test.ts @@ -0,0 +1,150 @@ +// Chord hints follow the live keyboard (hints.ts, and render.ts's own hints). +// +// Two halves. The first rebinds and reads the hints back: a tooltip that still names the +// shipped chord after a rebind is the bug this module exists to prevent. The second is +// the guard that keeps a new one from being written by hand: it walks every string +// literal in the UI's sources — through TypeScript's own parser, so comments are never +// mistaken for strings — and fails on anything spelled like a chord. The fix for a +// failure is `displayKeys([...])`, never an entry in the exemption list; that list is for +// text that names a key the user *can't* rebind, and each entry says why. + +import { strict as assert } from "node:assert"; +import { readFileSync, readdirSync } from "node:fs"; +import test from "node:test"; +import ts from "typescript"; +import { DEFAULT_BINDINGS, setActiveBindings } from "./bindings.ts"; +import { editDoneTitle, shellHints } from "./hints.ts"; +import { applyOverrides } from "./keymap.ts"; +import { contextMenuHtml, treePaneHtml } from "./render.ts"; +import { shortcuts } from "./shortcuts.ts"; +import { state, type AppState } from "./state.ts"; + +/** Run `fn` with a rebound keyboard installed, and always put the defaults back. */ +function rebound(overrides: Record, fn: () => void): void { + setActiveBindings(applyOverrides(DEFAULT_BINDINGS, overrides)); + try { + fn(); + } finally { + setActiveBindings(DEFAULT_BINDINGS); + } +} + +test("the shell's hints name the shipped chords by default", () => { + const h = shellHints(); + assert.equal(h["nav-back"].title, "Back (⌘[)"); + assert.equal(h["nav-forward"].title, "Forward (⌘])"); + assert.equal(h["search-input"].placeholder, "Search the vault… ⇧⌘F"); + assert.equal(h["search-input"].title, "Search the vault — ⇧⌘F (⌘F finds inside the open note)"); + assert.equal(h["open-chat"].title, "Ask your notes (⌘J)"); + assert.equal(h["open-settings"].title, "Settings (⌘,)"); + assert.equal(h["find-prev"].title, "Previous match (⇧⌘G / ⇧⏎)"); + assert.equal(h["find-next"].title, "Next match (⌘G / ⏎)"); + assert.equal(h["find-close"].title, "Close (Esc)"); + assert.equal(editDoneTitle(), "Save and return to reading — ⌘E (⌘S flushes anytime)"); +}); + +test("a rebind moves the shell's hints with it", () => { + rebound( + { + "nav.back": ["Mod-Alt-["], + "settings.toggle": ["Mod-Alt-,"], + "search.focus": ["Mod-Alt-f"], + "edit.toggle": ["Mod-Alt-e"], + }, + () => { + const h = shellHints(); + assert.equal(h["nav-back"].title, "Back (⌥⌘[)"); + assert.equal(h["open-settings"].title, "Settings (⌥⌘,)"); + assert.equal(h["search-input"].placeholder, "Search the vault… ⌥⌘F"); + assert.ok(editDoneTitle().includes("⌥⌘E"), editDoneTitle()); + }, + ); +}); + +test("a rebind moves the tree's and the menu's chords", () => { + const s: AppState = { + ...state, + vaultRoot: "/vault", + notes: [{ path: "a.md", title: "a" }] as AppState["notes"], + contextMenu: { kind: "tree", x: 0, y: 0, dir: "", node: { path: "a.md", nodeKind: "note" } }, + } as AppState; + rebound({ "tree.rename": ["F3"], "tree.new-note": ["Mod-Alt-n"] }, () => { + const menu = contextMenuHtml(s); + assert.ok(menu.includes(">F3<"), "Rename names the rebound chord"); + assert.ok(!menu.includes(">F2<"), "and not the shipped one"); + assert.ok(menu.includes(">⌥⌘N<"), "New note names the rebound chord"); + const tree = treePaneHtml(s); + assert.ok(tree.includes("F3 rename"), "the tree's crib follows the rebind"); + assert.ok(tree.includes("(⌥⌘N)"), "and so does the New note icon"); + }); +}); + +test("the sheet's Settings heading follows the rebind", () => { + const title = () => shortcuts().find((g) => g.title.startsWith("Settings"))?.title; + assert.equal(title(), "Settings (⌘,)"); + rebound({ "settings.toggle": ["Mod-Alt-,"] }, () => assert.equal(title(), "Settings (⌥⌘,)")); +}); + +// --- the guard ------------------------------------------------------------------------ + +/** A chord as a person would type it into a string: modifier glyphs then a key, or a bare + * function key. A lone glyph ("hold ⌘", "⇧ for a bigger step") is prose about a + * modifier, not a chord, and doesn't match. */ +const CHORD = + /[⌃⌥⇧⌘]+(?:F\d{1,2}|Enter|Tab|Esc|[A-Za-z0-9,.[\]/;'`=\-⏎⌫⌦←→↑↓])|(? { + if ( + ts.isStringLiteral(n) || + ts.isNoSubstitutionTemplateLiteral(n) || + ts.isTemplateHead(n) || + ts.isTemplateMiddle(n) || + ts.isTemplateTail(n) + ) { + out.push({ line: sf.getLineAndCharacterOfPosition(n.getStart()).line + 1, text: n.text }); + } + ts.forEachChild(n, visit); + }; + visit(sf); + return out; +} + +test("no UI string spells a chord by hand", () => { + const dir = new URL(".", import.meta.url); + const found: string[] = []; + for (const file of readdirSync(dir)) { + if (!file.endsWith(".ts") || file.endsWith(".test.ts") || OWNERS.has(file)) continue; + const src = readFileSync(new URL(file, dir), "utf8"); + for (const { line, text } of literalTexts(file, src)) { + // An HTML comment inside a template is markup nobody sees. + let t = text.replace(//g, ""); + for (const e of EXEMPT) t = t.split(e).join(""); + const m = t.match(CHORD); + if (m) found.push(`${file}:${line} spells "${m[0]}" — derive it with displayKeys([...])`); + } + } + assert.deepEqual(found, []); +}); + +test("the guard's pattern catches what it is for, and not prose about a modifier", () => { + for (const s of ["(⌘N)", "⇧⌘N", "⌘⏎ saves", "F2 rename", "⇧F10", "⌃Tab cycles", "⌘,", "⌘["]) + assert.ok(CHORD.test(s), s); + for (const s of ["Hold ⌘ on its own", "⇧ for a bigger step", "⌘", "F2F", "⏎ open"]) + assert.ok(!CHORD.test(s), s); +}); diff --git a/ui/src/hints.ts b/ui/src/hints.ts new file mode 100644 index 0000000..8f783ef --- /dev/null +++ b/ui/src/hints.ts @@ -0,0 +1,51 @@ +// The chord hints on the shell's static chrome — the top bar, the find bar, the editor's +// Done button — as data, derived from the live keyboard. +// +// Why a module. Chords are rebindable (#121), so a tooltip that spells "⌘[" is wrong for +// exactly the user who moved Back somewhere else — and the shell is painted once at boot, +// so even a hint that *was* derived went stale on the first rebind. Everything the shell +// says about a chord is listed here, keyed by element id, and main.ts writes it on to the +// DOM twice: when the shell is built, and again whenever the keyboard changes +// (`setOverrides`). The pane surfaces need no such help — render.ts derives their hints +// from `displayKeys` on every paint. +// +// Pure: it reads the registry and returns strings, so node tests it off the source, and +// hints.test.ts also scans the UI's string literals for a chord spelled by hand. + +import { displayKeys } from "./bindings.ts"; + +/** What one element says about its chord. `title` is its tooltip; `placeholder` is the + * search box's in-field hint, the one control that advertises its chord that way. */ +export interface ChordHint { + readonly title?: string; + readonly placeholder?: string; +} + +/** The Done button's tooltip while editing. */ +export function editDoneTitle(): string { + return `Save and return to reading — ${displayKeys(["edit.toggle"])} (${displayKeys([ + "editor.save", + ])} flushes anytime)`; +} + +/** Every static shell element's chord hint, by element id. An element that isn't on + * screen (Done, outside the editor) is simply skipped by the painter. */ +export function shellHints(): Record { + const search = displayKeys(["search.focus"]); + return { + "nav-back": { title: `Back (${displayKeys(["nav.back"])})` }, + "nav-forward": { title: `Forward (${displayKeys(["nav.forward"])})` }, + "search-input": { + placeholder: `Search the vault… ${search}`, + title: `Search the vault — ${search} (${displayKeys(["find.open"])} finds inside the open note)`, + }, + "open-chat": { title: `Ask your notes (${displayKeys(["chat.toggle"])})` }, + "open-settings": { title: `Settings (${displayKeys(["settings.toggle"])})` }, + // The buttons answer to the find-scope chords (rebindable) as well as to ⏎ / ⇧⏎ in + // the field (the text field's reflex), so both are named. + "find-prev": { title: `Previous match (${displayKeys(["find.prev", "find.input.prev"])})` }, + "find-next": { title: `Next match (${displayKeys(["find.next", "find.input.next"])})` }, + "find-close": { title: `Close (${displayKeys(["dismiss"])})` }, + "edit-done": { title: editDoneTitle() }, + }; +} diff --git a/ui/src/main.ts b/ui/src/main.ts index 7d02f21..383403c 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -112,6 +112,7 @@ import { menuDrift } from "./menukeys"; import { HOLD_MS, type HoldEvent, type HoldPhase, holdStep } from "./cmdhold"; import { markdownForPaste } from "./paste"; import { icon } from "./icons"; +import { editDoneTitle, shellHints } from "./hints"; import { activeAfter, countLabel, FIND_CAP, findMatches, locate, stepActive, type Match } from "./findbar"; import { BOUNDS, initPanes, visiblePanes } from "./panes"; import { @@ -2877,6 +2878,9 @@ function setOverrides(next: Overrides): void { saveOverrides(next); installKeymap(); render(); + // The two surfaces `render()` doesn't rebuild: the shell, and the live editor's bar. + paintShellHints(); + paintEditor(); } // The recorder. `state.recorder` is the whole of its state; these five actions are the @@ -3634,7 +3638,7 @@ function mountEditor(body: string): void { }" data-toggle-source aria-pressed="${state.sourceOpen}" title="${escapeHtml( editorSourceTitle(), )}"></> - +
- This block is yours — B2 changes nothing in it · ⌘⏎ saves · Esc cancels + This block is yours — B2 changes nothing in it · ${escapeHtml( + displayKeys(["fm.save"]), + )} saves · ${escapeHtml(displayKeys(["dismiss"]))} cancels
@@ -536,13 +552,7 @@ function noteBarHtml(state: AppState, note: NoteView): string { - + ${editToggleHtml(state.loading || editing, editing ? "Finish the frontmatter edit first" : undefined)} ${body} @@ -1372,6 +1382,16 @@ function graphToggleHtml(active: boolean): string { }">${icon("diagram-3")}`; } +/** The Edit chip, shared by the reading bar and the graph bar. Its chord comes out of the + * live registry for `graphToggleHtml`'s reason; `title` overrides the tooltip when the + * chip is disabled for a reason worth naming. */ +function editToggleHtml(disabled: boolean, title?: string): string { + const hint = title ?? `Edit this note — ${displayKeys(["edit.toggle"])} (autosaves as you type)`; + return ``; +} + /** Fixed-point SVG coordinate — keeps the markup compact and diff-stable. */ function px(v: number): string { return (Math.round(v * 10) / 10).toString(); @@ -1431,7 +1451,7 @@ function nodeTitle(n: GraphNode): string { // unavailable — an ungraded candidate was never measured at all. return `${n.full} — similar but not linked${ n.sub ? ` (${n.sub} above this note's other candidates)` : "" - }. Click or ⏎ to link it; right-click (or ⇧F10) for more.`; + }. Click or ⏎ to link it; right-click (or ${displayKeys(["menu.open"])}) for more.`; case "dangling": return `${n.full} resolves to no note or file — fix the link in the note.`; case "resource": @@ -1606,9 +1626,7 @@ function graphPaneHtml(state: AppState, n: NoteView): string {
${graphToggleHtml(true)} - + ${editToggleHtml(state.loading)}
@@ -2522,16 +2540,21 @@ function settingsScreenHtml(state: AppState): string {
- ↑↓ picks a section · ⌃Tab cycles · Esc closes + ${escapeHtml( + `${displayKeys(["settings.tab.prev", "settings.tab.next"], "/")} picks a section · ${displayKeys([ + "settings.section.next", + ])} cycles · ${displayKeys(["dismiss"])} closes`, + )}
`; } /** One menu row. `chord` names the direct shortcut where one exists — a menu is where - * a keyboard user *learns* the chord that lets them skip the menu next time. */ + * a keyboard user *learns* the chord that lets them skip the menu next time — so it + * comes out of the live registry (`displayKeys`), never spelled here. */ function contextItemHtml(attr: string, label: string, chord = "", danger = false): string { - const hint = chord ? `${chord}` : ""; + const hint = chord ? `${escapeHtml(chord)}` : ""; return ``; @@ -2561,11 +2584,11 @@ export function contextMenuHtml(state: AppState): string { // muted text, and the system path is nowhere in the UI at all. const node = m.node ? `
${escapeHtml(m.node.path)}
- ${contextItemHtml("data-ctx-rename", "Rename", "F2")} + ${contextItemHtml("data-ctx-rename", "Rename", displayKeys(["tree.rename"]))} ${contextItemHtml("data-ctx-move", "Move…")} ${contextItemHtml("data-ctx-copy-vault-path", "Copy vault path")} ${contextItemHtml("data-ctx-copy-system-path", "Copy system path")} - ${contextItemHtml("data-ctx-delete", "Delete", "⌘⌫", true)} + ${contextItemHtml("data-ctx-delete", "Delete", displayKeys(["delete.focused"]), true)} ` : `
${escapeHtml(m.dir ? `${m.dir}/` : "vault root")}
`; // Import files… is the drop gesture's keyboard half (K1): dragging a file in from @@ -2574,8 +2597,8 @@ export function contextMenuHtml(state: AppState): string { // folder context like the create pair, so it reads as the third way to put // something in this folder. items = `${node} - ${contextItemHtml("data-ctx-new-note", "New note", "⌘N")} - ${contextItemHtml("data-ctx-new-folder", "New folder", "⇧⌘N")} + ${contextItemHtml("data-ctx-new-note", "New note", displayKeys(["tree.new-note"]))} + ${contextItemHtml("data-ctx-new-folder", "New folder", displayKeys(["tree.new-folder"]))} ${contextItemHtml("data-ctx-import", "Import files…")}`; } else { // *Insert link at cursor* is the card drag's keyboard half (K1), the same shape diff --git a/ui/src/shortcuts.ts b/ui/src/shortcuts.ts index ffaa086..583125c 100644 --- a/ui/src/shortcuts.ts +++ b/ui/src/shortcuts.ts @@ -46,7 +46,13 @@ // owns. The host still declares the menu, because the two jobs that needed the // declaration are untouched: the conflict gate can see those keystrokes (menukeys.ts) and // the recorder refuses a chord spelled with one (keymap.ts). -import { type BindingId, activeBindings, displayChord, findBinding } from "./bindings.ts"; +import { + type BindingId, + activeBindings, + displayChord, + displayKeys, + findBinding, +} from "./bindings.ts"; /** One chord as the sheet paints it. */ export interface ShortcutKey { @@ -81,6 +87,9 @@ export type SheetRow = export interface SheetGroup { readonly title: string; + /** A chord the heading names after its title — "Settings (⌘,)" — resolved against the + * live keyboard like every row, so a rebind moves the heading too. */ + readonly titleIds?: readonly BindingId[]; readonly rows: readonly SheetRow[]; } @@ -214,7 +223,8 @@ const SHEET: readonly SheetGroup[] = [ // the kind of thing that ends up mouse-only if its moves aren't written down: the // sections are visibly *there*, so nobody thinks to look for a chord. { - title: "Settings (⌘,)", + title: "Settings", + titleIds: ["settings.toggle"], rows: [ { ids: ["settings.tab.prev", "settings.tab.next"], @@ -279,7 +289,7 @@ export function keyChips(ids: readonly BindingId[]): ShortcutKey[] { /** The sheet as render.ts paints it — every row's chords resolved to chips. */ export function shortcuts(): ShortcutGroup[] { return sheet().map((group) => ({ - title: group.title, + title: group.titleIds ? `${group.title} (${displayKeys(group.titleIds)})` : group.title, items: group.rows.map((row) => ({ keys: "ids" in row ? keyChips(row.ids) : [{ text: row.keys }], action: row.action, From ec7783390dfcdc2994e3c1638a64cfd485b8f1de Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:20:18 +0000 Subject: [PATCH 02/30] refactor(ui): drop dead code - api.ts: the ping and neighbors wrappers (nothing calls them; the host commands are going too). - bindings.ts: chordsOverlap, unused since menukeys.ts compares keystrokes. - keymap.ts: currentKeys, never called. - recorder.ts: reservedChords and its test. Its doc said the recorder names the menu's chords up front, but recorderHtml never rendered them. - style.css: .modal-wide and its stale anomaly-panel comment. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/api.ts | 6 ------ ui/src/bindings.ts | 6 ------ ui/src/keymap.ts | 6 ------ ui/src/recorder.test.ts | 14 +------------- ui/src/recorder.ts | 11 +---------- ui/style.css | 9 +-------- 6 files changed, 3 insertions(+), 49 deletions(-) diff --git a/ui/src/api.ts b/ui/src/api.ts index 74a573d..14590c0 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -23,7 +23,6 @@ import type { MenuChord, ModelChoice, MoveReport, - NeighborView, NoteSummary, NoteView, ProjectReport, @@ -85,9 +84,6 @@ export const VAULT_CHANGED_EVENT = "vault-changed"; export const MENU_COMMAND_EVENT = "menu-command"; export const api = { - /** Step 0's seam proof: round-trips a trivial command through the Rust host. */ - ping: (): Promise => invoke("ping"), - /** The active vault root + whether semantic ranking is live (real model). */ vaultInfo: (): Promise => invoke("vault_info"), @@ -259,8 +255,6 @@ export const api = { /** Delete a whole folder and everything inside it (unindexed files go too). */ deleteDir: (dir: string): Promise => invoke("delete_dir", { dir }), - /** A note's typed neighbors (both directions). */ - neighbors: (note: string): Promise => invoke("neighbors", { note }), /** A note's connections with their "why" (outbound + inbound). */ explain: (note: string): Promise => invoke("explain", { note }), diff --git a/ui/src/bindings.ts b/ui/src/bindings.ts index 898bcf1..aee8c9c 100644 --- a/ui/src/bindings.ts +++ b/ui/src/bindings.ts @@ -635,12 +635,6 @@ export function keystrokes(spec: string): string[] { return physicalForms(parseChord(spec)); } -/** Do two chords answer to any of the same keystrokes? */ -export function chordsOverlap(a: string, b: string): boolean { - const forms = new Set(keystrokes(a)); - return keystrokes(b).some((f) => forms.has(f)); -} - /** One chord, as one keystroke string — modifiers in Apple's order, then the key. */ function formOf(chord: Chord): string { return ( diff --git a/ui/src/keymap.ts b/ui/src/keymap.ts index c4693d2..2f77a2a 100644 --- a/ui/src/keymap.ts +++ b/ui/src/keymap.ts @@ -26,7 +26,6 @@ import { type Binding, DEFAULT_BINDINGS, - activeBindings, conflicts, displayChord, findBinding, @@ -306,8 +305,3 @@ export function saveOverrides(overrides: Overrides): void { // Non-fatal: the chords still hold for this session if they can't persist. } } - -/** The live table's chords for one command, for the panel's "currently" column. */ -export function currentKeys(id: string): readonly string[] { - return findBinding(activeBindings(), id)?.keys ?? []; -} diff --git a/ui/src/recorder.test.ts b/ui/src/recorder.test.ts index 361ae6a..7fb3ef3 100644 --- a/ui/src/recorder.test.ts +++ b/ui/src/recorder.test.ts @@ -14,8 +14,7 @@ // fine — which teaches them to ignore the recorder. So the tests below are as much about // when it stays quiet as when it speaks. import { type KeyEventLike, chordMatches, parseChord } from "./bindings.ts"; -import { MENU_CHORDS } from "./menukeys.ts"; -import { PROBE_AFTER_MS, capture, reservedChords, silenceHint } from "./recorder.ts"; +import { PROBE_AFTER_MS, capture, silenceHint } from "./recorder.ts"; let passed = 0; @@ -167,15 +166,4 @@ check("the probe under-reports rather than false-alarms, by construction", () => assertEq(silenceHint({ elapsedMs: -1, blurred: false }), null, "and never before it has cause"); }); -check("the reserved chords are the host's menu, rendered as the sheet renders chords", () => { - // What the recorder can name *up front*, and the only thing it can: these are the - // keystrokes guaranteed to arrive as silence, enumerable only because #119 made the - // host declare its own menu instead of inheriting Tauri's default. - const shown = reservedChords(); - assertEq(shown.length, MENU_CHORDS.length, "one per menu item with a chord"); - assert(shown.includes("⌘W"), "⌘W closes the window"); - assert(shown.includes("⌥⌘H"), "and modifiers print in Apple's order"); - assertEq(reservedChords([]), [], "an empty menu reserves nothing"); -}); - console.log(`recorder: ${passed} checks passed`); diff --git a/ui/src/recorder.ts b/ui/src/recorder.ts index 06b5b70..7d4e608 100644 --- a/ui/src/recorder.ts +++ b/ui/src/recorder.ts @@ -31,9 +31,7 @@ // The one signal stronger than silence is `blurred`. Spotlight, the app switcher and Hide // all take the key window away, and a window that lost focus mid-recording is positive // evidence that something outside B2 answered — not an inference from nothing happening. -import { type KeyEventLike, canonicalKey, displayChord, isBindableKey, shiftDistinguishes } from "./bindings.ts"; -import { MENU_CHORDS } from "./menukeys.ts"; -import type { MenuChord } from "./types.ts"; +import { type KeyEventLike, canonicalKey, isBindableKey, shiftDistinguishes } from "./bindings.ts"; /** Keys that are only ever *part* of a chord. A keydown for one of these is the user * starting a chord, not finishing one, so the recorder keeps waiting. */ @@ -115,10 +113,3 @@ export function silenceHint(s: Silence): string | null { } return null; } - -/** The menu bar's chords, as display text — the list the recorder names up front, since - * these are the ones guaranteed to arrive as silence (menukeys.ts says why AppKit takes - * them first). Enumerable only because #119 made the host declare its own menu. */ -export function reservedChords(menu: readonly MenuChord[] = MENU_CHORDS): string[] { - return menu.map((c) => displayChord(c.keys)); -} diff --git a/ui/style.css b/ui/style.css index 1d0477e..0550659 100644 --- a/ui/style.css +++ b/ui/style.css @@ -2165,14 +2165,7 @@ body.is-loading { color: var(--muted); } /* --- the keyboard reference (?) -------------------------------------------------- - The discoverable half of K1: every chord in one place. It lives in Settings' Keyboard - section now, but the two-column pack is the same, and `.modal-wide` is still what the - anomaly panel is built on. */ -.modal-wide { - max-width: 720px; - max-height: calc(100vh - 96px); - overflow-y: auto; -} + The discoverable half of K1: every chord in one place, in Settings' Keyboard section. */ /* CSS multi-column, not a grid: the groups are wildly different lengths, and a grid would leave a short column stranded beside a long one (rows align, so every group pays for the tallest in its row). Columns flow, so the page packs. */ From a423f593bbf6e1b18525e16021c784f0ce363d67 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:23:57 +0000 Subject: [PATCH 03/30] refactor(ui): one helper per repeated state transition in main.ts - beginIndexRun/endIndexRun for the three index runs' meter lifecycle, and refreshOpenNoteAfterIndex for the after-run re-read + discovery refresh (autoIndexOnOpen still skips it entirely while an editor is open). - enterDocument: the core loadNote and loadResource share. - clearDiscovery/clearConnections/adoptExplain, adoptResource, adoptCoverage (a VaultInfo's embedding coverage). - Chat: applyChatConfig is now really the shared tail of Save, the model picker and Remove key; adoptChatSetup; openChat; chatReady/chatStateOf in chat.ts replace three inline chatEmptyState(...) === "ready" reads; and LOCAL_CHAT_ENDPOINT moves to chat.ts, where the Endpoint placeholder uses it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/chat.test.ts | 11 ++ ui/src/chat.ts | 17 +++ ui/src/main.ts | 322 +++++++++++++++++++++++--------------------- ui/src/render.ts | 12 +- 4 files changed, 200 insertions(+), 162 deletions(-) diff --git a/ui/src/chat.test.ts b/ui/src/chat.test.ts index 8898465..07f3d95 100644 --- a/ui/src/chat.test.ts +++ b/ui/src/chat.test.ts @@ -5,6 +5,8 @@ import { STREAMING_ROW_KEY, answerMessage, chatEmptyState, + chatReady, + chatStateOf, chatHistory, chatRows, citationRowKey, @@ -147,6 +149,15 @@ test("the empty state is chosen once, from the vault and the probe", () => { assert.equal(chatEmptyState({ hasVault: true, setup: setup({ state: "fake" }) }), "ready"); }); +test("the app asks the same question off its own state", () => { + assert.equal(chatStateOf({ vaultRoot: null, chatSetup: setup() }), "no-vault"); + assert.equal(chatStateOf({ vaultRoot: "/v", chatSetup: null }), "loading"); + assert.equal(chatReady({ vaultRoot: "/v", chatSetup: setup() }), true); + assert.equal(chatReady({ vaultRoot: "/v", chatSetup: setup({ state: "fake" }) }), true); + assert.equal(chatReady({ vaultRoot: "/v", chatSetup: setup({ state: "unreachable" }) }), false); + assert.equal(chatReady({ vaultRoot: null, chatSetup: setup() }), false); +}); + test("an unembedded vault is a quiet note, never a blocker (M4)", () => { // Nothing indexed yet: the tree's own empty state covers it, so chat says nothing. assert.equal(retrievalNote({ semantic: true, notesEmbedded: 0, notesTotal: 0 }), ""); diff --git a/ui/src/chat.ts b/ui/src/chat.ts index d2bfea1..e81f8b3 100644 --- a/ui/src/chat.ts +++ b/ui/src/chat.ts @@ -252,6 +252,23 @@ export function chatEmptyState(s: { } } +/** The app's side of `chatEmptyState`: the vault and the probe, read off the app state. */ +export function chatStateOf(s: { vaultRoot: string | null; chatSetup: ChatSetup | null }): ChatEmptyState { + return chatEmptyState({ hasVault: s.vaultRoot !== null, setup: s.chatSetup }); +} + +/** Can a question be asked right now? What the composer's presence and *Why?* both wait on. */ +export function chatReady(s: { vaultRoot: string | null; chatSetup: ChatSetup | null }): boolean { + return chatStateOf(s) === "ready"; +} + +/** Ollama's OpenAI-compatible endpoint — the **Local** configuration's starting point, and + * the only place the frontend spells it: the Endpoint field's placeholder, and its seed + * when the user presses *Local* after typing a cloud URL. The host's + * `b2_llm::DEFAULT_BASE_URL` is the authority (it is what an unset endpoint resolves + * to) — change them together. */ +export const LOCAL_CHAT_ENDPOINT = "http://localhost:11434/v1"; + /** * The honest note about *retrieval* under the composer, or "" when there is nothing to * say — the search caveat (#26) applied to chat. diff --git a/ui/src/main.ts b/ui/src/main.ts index 383403c..9a49c4b 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -56,9 +56,10 @@ import { import { cardRowKey, sideArrowMove, sideNavFor, sideRowIndex, sideRows } from "./sidenav"; import { answerMessage, - chatEmptyState, chatHistory, + chatReady, errorMessage, + LOCAL_CHAT_ENDPOINT, toolCapInput, userMessage, whyQuestion, @@ -124,7 +125,14 @@ import { type Direction, } from "./zoom"; import { reconcileIndex } from "./reconcile"; -import type { AnswerView, ResourceExplainView, ResourceSummary } from "./types"; +import type { + AnswerView, + ChatSetup, + ExplainView, + ResourceExplainView, + ResourceSummary, + VaultInfo, +} from "./types"; import { cmdSheetHtml, contextMenuHtml, @@ -574,20 +582,10 @@ async function loadNote(ref: string, commit: (path: string) => void): Promise void): Promise void): void { + commit(path); + expandAncestors(path); + state.selectedDir = parentDir(path); // the create context follows the selection + resetSearch(); + clearDiscovery(); + state.collapsedCards.clear(); // per-note fold state belongs to the note we just left + state.contextMenu = null; +} + +/** Forget the side pane's discovery — the Similar list and the `explain` read. */ +function clearDiscovery(): void { + state.similar = []; + clearConnections(); +} + +/** Forget the `explain` half of discovery: connections, resource links, unresolved. */ +function clearConnections(): void { + state.connections = []; + state.resourceLinks = []; + state.unresolved = []; +} + +/** Adopt a note's `explain` read into the Connections section. */ +function adoptExplain(explain: ExplainView): void { + state.connections = explain.connections; + state.resourceLinks = explain.resources; + state.unresolved = explain.unresolved; +} + +/** Put a resource card in the pane, with its picture (null when there is none to show — + * `loadResourceImage`). The two travel together so the card never shows another + * file's picture. */ +function adoptResource(resource: ResourceExplainView, picture: string | null): void { + state.currentResource = resource; + state.resourceImage = picture; +} + // User navigation to a note (tree, wikilink, backlink, similar card, search result). // Mid-edit navigation flushes the buffer and leaves edit mode first; a conflict keeps // the editor — and the user's buffer — alive instead. A successful load records the @@ -762,19 +804,9 @@ async function loadResource(path: string, commit: (path: string) => void): Promi render(); try { const resource = await api.explainResource(path); - state.currentResource = resource; - state.resourceImage = await loadResourceImage(resource); + adoptResource(resource, await loadResourceImage(resource)); state.current = null; - commit(resource.path); - expandAncestors(resource.path); - state.selectedDir = parentDir(resource.path); // the create context follows the selection - resetSearch(); - state.similar = []; - state.connections = []; - state.resourceLinks = []; - state.unresolved = []; - state.collapsedCards.clear(); - state.contextMenu = null; + enterDocument(resource.path, commit); state.discoveringSimilar = false; state.discoveringConnections = false; return true; @@ -1842,8 +1874,7 @@ async function executeMove(node: TreeNodeRef, to: string): Promise { } if (openResourcePath !== null) { const moved = await api.explainResource(openResourcePath); - state.currentResource = moved; - state.resourceImage = await loadResourceImage(moved); + adoptResource(moved, await loadResourceImage(moved)); } await loadNotes(); if (openNotePath !== null) await refreshDiscovery(); // backlinks may show new paths @@ -1952,10 +1983,7 @@ async function executeDelete(node: TreeNodeRef): Promise { state.current = null; state.currentResource = null; state.resourceImage = null; - state.similar = []; - state.connections = []; - state.resourceLinks = []; - state.unresolved = []; + clearDiscovery(); state.discoveringSimilar = false; state.discoveringConnections = false; } @@ -2015,17 +2043,11 @@ async function refreshDiscovery(): Promise { const connections = api .explain(n.path) .then((explain) => { - if (!stale()) { - state.connections = explain.connections; - state.resourceLinks = explain.resources; - state.unresolved = explain.unresolved; - } + if (!stale()) adoptExplain(explain); }) .catch((e) => { if (!stale()) { - state.connections = []; - state.resourceLinks = []; - state.unresolved = []; + clearConnections(); flash(errText(e)); } }) @@ -2155,16 +2177,21 @@ function toggleChat(): void { closeChat(); return; } + openChat(); + focusChatInput(); + void refreshChatSetup(); +} + +/** Open the pane. Chat and search both own the whole column, one at a time (chat.ts's + * header), so search goes. The render is explicit, rather than leaning on + * `clearSearch`'s own repaint: a caller that focuses the composer next needs it to + * exist, and a paint that happens only as somebody else's side effect is one refactor + * away from not happening. The panes are memoized, so a second render over identical + * HTML costs nothing. */ +function openChat(): void { state.chatOpen = true; - // Chat and search both own the whole column, one at a time (chat.ts's header). clearSearch(); - // Explicitly, rather than leaning on `clearSearch`'s own repaint: `focusChatInput` - // needs the composer to exist, and a paint that happens only as somebody else's side - // effect is one refactor away from not happening. The panes are memoized, so a second - // render over identical HTML costs nothing. render(); - focusChatInput(); - void refreshChatSetup(); } /** Close the pane. A streaming answer is stopped first — a pane you can't see must not @@ -2188,8 +2215,7 @@ function focusChatInput(): void { * pane down with it: an unknown setup reads as the "loading" state, which is honest. */ async function refreshChatSetup(): Promise { try { - state.chatSetup = await api.chatSetup(); - state.chatCloud = state.chatSetup.cloud; + adoptChatSetup(await api.chatSetup()); } catch (e) { flash(errText(e)); return; @@ -2235,17 +2261,11 @@ async function askWhy(candidate: { path: string; title: string | null }): Promis if (state.chatStreaming !== null) flash("B2 is still answering. Press Esc to stop it."); return; } - if (!state.chatOpen) { - state.chatOpen = true; - clearSearch(); - render(); - } + if (!state.chatOpen) openChat(); // The probe first: with no model to answer, the pane's setup card is the useful thing // to show, and a turn sent anyway would only add a failure under it. await refreshChatSetup(); - const ready = - chatEmptyState({ hasVault: state.vaultRoot !== null, setup: state.chatSetup }) === "ready"; - if (!ready || state.current?.path !== anchor.path) return; + if (!chatReady(state) || state.current?.path !== anchor.path) return; // `false`: this turn was not typed, so a question half-written in the composer stays. await runChatTurn( whyQuestion(candidate, anchor), @@ -2423,12 +2443,6 @@ function setChatModelTyped(typed: boolean): void { document.getElementById("settings-chat-model")?.focus(); } -/** Ollama's OpenAI-compatible endpoint — the **Local** configuration's starting point, and - * the only place the frontend spells it. The host's `b2_llm::DEFAULT_BASE_URL` is the - * authority (it is what an unset endpoint resolves to); this is the field's seed when the - * user presses *Local* after typing a cloud URL. */ -const LOCAL_CHAT_ENDPOINT = "http://localhost:11434/v1"; - /** Settings → Chat: save the endpoint/model/key and re-probe, so "Save and test" is one * act. The key is sent only when the user typed one — an untouched field must not clear * a key that is already in force (the host applies the same rule). */ @@ -2456,18 +2470,11 @@ async function saveChatConfig(): Promise { capField?.focus(); return; } - try { - state.chatSetup = await api.setChatConfig(url, model, key, cap.send); - state.chatCloud = state.chatSetup.cloud; - flash( - state.chatSetup.state === "ready" - ? `Chat model saved — connected to ${state.chatSetup.model}.` - : (state.chatSetup.message ?? "Chat settings saved."), - ); - } catch (e) { - flash(errText(e)); - } - render(); + await applyChatConfig(url, model, key, cap.send, (setup) => + setup.state === "ready" + ? `Chat model saved — connected to ${setup.model}.` + : (setup.message ?? "Chat settings saved."), + ); } /** The setup card's installed-model list: pick one and it becomes the configured model. @@ -2478,7 +2485,13 @@ async function useChatModel(model: string): Promise { // an emptied field returns to the environment's value — so sending it here would // quietly reset a configured endpoint back to the default as a side effect of picking // a model off the card. The key is `null` in the other sense: untouched, so kept. - await applyChatConfig(state.chatSetup?.base_url ?? null, model, null, `Chat model set to ${model}.`); + await applyChatConfig( + state.chatSetup?.base_url ?? null, + model, + null, + null, + () => `Chat model set to ${model}.`, + ); } /** @@ -2493,49 +2506,51 @@ async function useChatModel(model: string): Promise { * the copy beside the button says so. */ async function clearChatKey(): Promise { - try { - state.chatSetup = await api.setChatConfig( - state.chatSetup?.base_url ?? null, - state.chatSetup?.model ?? null, - "", - ); - state.chatCloud = state.chatSetup.cloud; + await applyChatConfig( + state.chatSetup?.base_url ?? null, + state.chatSetup?.model ?? null, + "", + null, // Removal is all-or-nothing host-side, so the returned source *is* the // outcome — no separate success flag to keep in step. A key still reported // as stored/session means the Keychain refused to let go, and saying // "removed" there would be the one lie this button must never tell: the key // would be back at the next launch. (`environment` is neither outcome — B2 // never had standing over that key, and the panel's copy says so.) - const source = state.chatSetup.api_key_source; - flash( - source === "stored" || source === "session" + (setup) => + setup.api_key_source === "stored" || setup.api_key_source === "session" ? "Couldn’t remove the key — your Keychain refused. It is still saved." : "API key removed.", - ); - } catch (e) { - flash(errText(e)); - } - render(); + ); } /** Save a chat configuration, re-probe, and say what happened — the shared tail of every - * path that changes it (Save, the card's model picker, Remove key). */ + * path that changes it (Save, the card's model picker, Remove key). `said` words the + * outcome from the setup the host sends back; a refusal says the host's own sentence. */ async function applyChatConfig( baseUrl: string | null, model: string | null, apiKey: string | null, - ok: string, + maxToolCalls: string | null, + said: (setup: ChatSetup) => string, ): Promise { try { - state.chatSetup = await api.setChatConfig(baseUrl, model, apiKey); - state.chatCloud = state.chatSetup.cloud; - flash(ok); + const setup = await api.setChatConfig(baseUrl, model, apiKey, maxToolCalls); + adoptChatSetup(setup); + flash(said(setup)); } catch (e) { flash(errText(e)); } render(); } +/** Adopt what the host says the chat provider is now. The Local/Cloud switch follows the + * configuration — it is a view of it, until the user flips it to start another. */ +function adoptChatSetup(setup: ChatSetup): void { + state.chatSetup = setup; + state.chatCloud = setup.cloud; +} + function openLinkModal(path: string, title: string): void { // A committed link rewrites the open note's frontmatter — the exact bytes the // mini-editor is holding — so resolve that edit before offering to link. @@ -3112,16 +3127,11 @@ async function switchVault(): Promise { // flag and bail. Not awaited here: the UI reset below must not block on a wind-down. const departing = indexingRun; state.vaultRoot = info.root; // set now so the departing run's guards bail promptly - state.semantic = info.semantic; - state.notesEmbedded = info.notes_embedded; - state.notesTotal = info.notes_total; + adoptCoverage(info); state.current = null; state.currentResource = null; state.resourceImage = null; - state.similar = []; - state.connections = []; - state.resourceLinks = []; - state.unresolved = []; + clearDiscovery(); resetSearch(); // The conversation is grounded in the vault we just left — every citation in it // names a path that means nothing here (a note's identity is its path, L1). Dropping @@ -3166,14 +3176,21 @@ async function refreshEmbedStatus(forRoot: string | null): Promise { try { const info = await api.vaultInfo(); if (state.vaultRoot !== forRoot) return; - state.semantic = info.semantic; - state.notesEmbedded = info.notes_embedded; - state.notesTotal = info.notes_total; + adoptCoverage(info); } catch { // ignore — coverage is a hint, never worth surfacing an error over } } +/** Adopt a `VaultInfo`'s embedding coverage: whether semantic ranking is live, and how + * many notes are embedded of how many. The root is the caller's to set — only a boot or + * a switch changes it. */ +function adoptCoverage(info: VaultInfo): void { + state.semantic = info.semantic; + state.notesEmbedded = info.notes_embedded; + state.notesTotal = info.notes_total; +} + // The in-flight background index — a manual Reindex (`doReindex`), an auto-index on // open (`autoIndexOnOpen`), or a trailing embed after a save (`runTrailingEmbed`) — or // null when idle. A vault switch cancels the run host-side (choose_vault → @@ -3192,6 +3209,36 @@ function trackIndexing(run: Promise): void { indexingRun = done; } +/** Start an index run: the meter comes up empty, with no cancel pending. The caller + * picks the repaint (a full one, or just the meter). */ +function beginIndexRun(): void { + state.reindexing = true; + state.reindexProgress = null; + state.reindexCancelling = false; +} + +/** End an index run — every run's `finally`: the meter goes, and the app repaints. */ +function endIndexRun(): void { + state.reindexing = false; + state.reindexProgress = null; + state.reindexCancelling = false; + render(); +} + +/** + * After a run: the open note's vectors exist now, so refresh discovery for `similar` to + * rank with — and re-read the note first, unless an editor (body or frontmatter) holds + * it: adopting a fresh revision under an open buffer could regress the save chain into a + * false conflict, or let its save clobber what changed on disk. + */ +async function refreshOpenNoteAfterIndex(): Promise { + if (!state.current) return; + if (!state.editing && !state.fmEditing) { + state.current = await api.readNote(state.current.path); + } + await refreshDiscovery(); +} + // Reindex as project → embed, sequenced here (Shape A, docs/index-engine.md): // the fast, model-free `project` completes the keyword + graph index, the tree // paints immediately, and only then does the slow, cancellable `embed` stream behind @@ -3202,9 +3249,7 @@ function trackIndexing(run: Promise): void { async function doReindex(): Promise { if (state.reindexing) return; // single-in-flight (the host also guards embed) const startedRoot = state.vaultRoot; // guard against a vault switch mid-run - state.reindexing = true; - state.reindexProgress = null; - state.reindexCancelling = false; + beginIndexRun(); render(); try { // Phase 1 — projection (fast, no model): notes, keyword index, and graph are @@ -3260,24 +3305,11 @@ async function doReindex(): Promise { ? `Embedded ${r.embedded}/${p.indexed} note(s) — cancelled. Re-run to finish the rest.${skipped}` : `Indexed ${p.indexed} note(s) — ${r.embedded} embedded.${skipped}`, ); - if (state.current) { - // Projection may have stamped the open note on disk; re-read it, and refresh - // discovery now that vectors exist for `similar` to rank with. Not mid-edit: - // the editor's revision chain owns the note then (an indexed note is already - // stamped), and adopting a re-read racing an in-flight save could regress the - // chain into a false conflict. - if (!state.editing && !state.fmEditing) { - state.current = await api.readNote(state.current.path); - } - await refreshDiscovery(); - } + await refreshOpenNoteAfterIndex(); } catch (e) { if (state.vaultRoot === startedRoot) flash(errText(e)); } finally { - state.reindexing = false; - state.reindexProgress = null; - state.reindexCancelling = false; - render(); + endIndexRun(); } } @@ -3304,9 +3336,7 @@ async function autoIndexOnOpen(startedRoot: string | null): Promise { // one still gets its keyword + graph index below (project is model-free). if (!needsProject && !state.semantic) return; - state.reindexing = true; - state.reindexProgress = null; - state.reindexCancelling = false; + beginIndexRun(); render(); try { if (needsProject) { @@ -3325,23 +3355,15 @@ async function autoIndexOnOpen(startedRoot: string | null): Promise { // switch reloads the new vault, so leave the one we're departing untouched (spec §6). if (r.cancelled && !state.reindexCancelling) return; await refreshEmbedStatus(startedRoot); - // If the user opened a note while embedding ran, its vectors exist now — re-read it - // (projection may have stamped it) and refresh discovery so `similar` can rank. Not - // under a live editor (body or frontmatter): adopting a fresh revision beneath an - // open buffer would let its save silently clobber what changed on disk — the same - // carve-out as reconcile and doReindex. - if (state.current && !state.editing && !state.fmEditing) { - state.current = await api.readNote(state.current.path); - await refreshDiscovery(); - } + // If the user opened a note while embedding ran, its vectors exist now. Unlike + // doReindex this skips discovery too while an editor is open — the run was unasked + // for, so it leaves a pane someone is working in entirely alone. + if (!state.editing && !state.fmEditing) await refreshOpenNoteAfterIndex(); } catch { // Silent by design (§7.2): the user didn't ask for this run, so a missing model or a // lost race just leaves the vault keyword-first; the pending set heals on the next run. } finally { - state.reindexing = false; - state.reindexProgress = null; - state.reindexCancelling = false; - render(); + endIndexRun(); } } @@ -3807,9 +3829,7 @@ async function refreshConnections(): Promise { try { const explain = await api.explain(cur.path); if (state.current?.path !== cur.path) return; // navigated away meanwhile - state.connections = explain.connections; - state.resourceLinks = explain.resources; - state.unresolved = explain.unresolved; + adoptExplain(explain); render(); } catch { // deliberately silent @@ -3836,9 +3856,7 @@ async function runTrailingEmbed(): Promise { // missing-vector set is DB-derived, so any later embed/reindex heals it (split §7.2). if (state.reindexing || state.vaultRoot === null) return; const startedRoot = state.vaultRoot; - state.reindexing = true; - state.reindexProgress = null; - state.reindexCancelling = false; + beginIndexRun(); paintReindex(); try { await embedWithProgress(startedRoot); @@ -3848,10 +3866,7 @@ async function runTrailingEmbed(): Promise { // Refused (ReindexInFlight race) or failed (e.g. no model provisioned): skip // silently — the user didn't ask for this run, and the pending set heals. } finally { - state.reindexing = false; - state.reindexProgress = null; - state.reindexCancelling = false; - render(); + endIndexRun(); } } @@ -4407,10 +4422,7 @@ async function reconcileExternalChange(): Promise { // The bytes are re-read too: an external edit can rewrite the picture in place // without the path ever changing, and a stale `data:` URL would show the old one. const picture = await loadResourceImage(fresh); - if (state.currentResource?.path === cur.path) { - state.currentResource = fresh; - state.resourceImage = picture; - } + if (state.currentResource?.path === cur.path) adoptResource(fresh, picture); } catch { if (state.currentResource?.path === cur.path) { flash("This file is no longer on disk — it was moved or removed."); @@ -5926,9 +5938,7 @@ async function boot(): Promise { try { const info = await api.vaultInfo(); state.vaultRoot = info.root; - state.semantic = info.semantic; - state.notesEmbedded = info.notes_embedded; - state.notesTotal = info.notes_total; + adoptCoverage(info); // Populate the file tree so the vault is navigable before anything is opened. await loadNotes(); } catch (e) { diff --git a/ui/src/render.ts b/ui/src/render.ts index c8697e4..6ad4293 100644 --- a/ui/src/render.ts +++ b/ui/src/render.ts @@ -79,8 +79,10 @@ import { OLLAMA_CLOUD_URL, OLLAMA_QUICKSTART_URL, PULL_PLACEHOLDER, + LOCAL_CHAT_ENDPOINT, STREAMING_ROW_KEY, - chatEmptyState, + chatReady, + chatStateOf, citationRowKey, formatModelSize, pullCommand, @@ -1116,15 +1118,13 @@ function chatPaneHtml(state: AppState, roving: string | null): string { // No composer until there is something to answer with: a disabled field under a card // that says why is chrome with nothing behind it, and one more stop for a keyboard user // to walk past on the way to the fix the card is pointing at. - const ready = - chatEmptyState({ hasVault: state.vaultRoot !== null, setup: state.chatSetup }) === "ready"; - return head + chatStageHtml(state, roving) + (ready ? chatComposerHtml(state) : ""); + return head + chatStageHtml(state, roving) + (chatReady(state) ? chatComposerHtml(state) : ""); } /** The conversation, or the state that stands in for one (chat.ts's `chatEmptyState` * makes that choice once, so the paint doesn't re-derive it branch by branch). */ function chatStageHtml(state: AppState, roving: string | null): string { - switch (chatEmptyState({ hasVault: state.vaultRoot !== null, setup: state.chatSetup })) { + switch (chatStateOf(state)) { case "no-vault": return `

Open a vault to chat with your notes.

`; case "loading": @@ -1932,7 +1932,7 @@ function chatPanelHtml(state: AppState): string { ${chatModelFieldHtml(state, setup)} ${key} From e30455d02ea2317887e6c20946a3c6e1223c1cf5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:26:36 +0000 Subject: [PATCH 04/30] refactor(ui): one tree/path helper per rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The contextmenu handler, the drag start and the drop target rebuilt a tree row's node ref by hand; they use treeRowRef (and a TREE_ROW selector) now, with the folder a row stands for from move.ts's folderContext. - openDocPath (state.ts) for the pane's document path. - revealDir replaces expandAncestors (a re-implementation of dirChain) and the five hand-written dirChain loops. - isWithin (move.ts) for "this folder or under it", written four ways before; moveRefusal says why a destination is refused, canMoveInto is defined from it, and the Move… modal reads the reason instead of re-deriving it. - render.ts uses baseName instead of split("/").pop(). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/main.ts | 112 ++++++++++++++++++-------------------------- ui/src/move.test.ts | 37 +++++++++++++++ ui/src/move.ts | 39 +++++++++++---- ui/src/render.ts | 18 ++++--- ui/src/state.ts | 5 ++ 5 files changed, 127 insertions(+), 84 deletions(-) diff --git a/ui/src/main.ts b/ui/src/main.ts index 9a49c4b..5d4021d 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -30,13 +30,22 @@ import { tooltips, } from "@codemirror/view"; import { api, errText, isWriteConflict } from "./api"; -import { state, type AppState, type SideSection, type ThemePref, type TreeNodeRef } from "./state"; +import { + openDocPath, + state, + type AppState, + type SideSection, + type ThemePref, + type TreeNodeRef, +} from "./state"; import { dirChain, joinPath, normalizeName, parentDir } from "./newentry"; import { systemPath } from "./copypath"; import { bytesToBase64, importSummary, planImport } from "./importfiles"; import { baseName, canMoveInto, + folderContext, + isWithin, moveDestination, type NodeKind, refKind, @@ -529,15 +538,11 @@ function flash(msg: string): void { // --- actions -------------------------------------------------------------------- -// Expand every folder on the way to `path` so the file tree reveals it — used when a -// note is opened from search/wikilink/discovery, not just by clicking it in the tree. -function expandAncestors(path: string): void { - const parts = path.split("/"); - let dir = ""; - for (const seg of parts.slice(0, -1)) { - dir = dir ? `${dir}/${seg}` : seg; - state.expandedDirs.add(dir); - } +/** Unfold every folder down to and including `dir`, so the file tree shows what is in + * it — a document opened from search, a wikilink or discovery, a fresh create, a + * rename's input, a move's destination. */ +function revealDir(dir: string): void { + for (const d of dirChain(dir)) state.expandedDirs.add(d); } // Load the vault listing for the file tree — all three lists fetched before any @@ -612,7 +617,7 @@ async function loadNote(ref: string, commit: (path: string) => void): Promise void): void { commit(path); - expandAncestors(path); + revealDir(parentDir(path)); state.selectedDir = parentDir(path); // the create context follows the selection resetSearch(); clearDiscovery(); @@ -903,11 +908,15 @@ function treeRows(): TreeRow[] { return visibleRows(buildTree(state.notes, state.resources, state.dirs), state.expandedDirs); } +/** What a file-tree row is, as a selector: every row carries its vault path in + * `data-tree-row` (render.ts), whatever kind of node it is. */ +const TREE_ROW = ".tree-row[data-tree-row]"; + /** The DOM row for a vault path. Looked up by data attribute rather than by CSS * selector because a filename may contain anything a selector would choke on. */ function treeRowEl(path: string | null): HTMLElement | null { if (path === null) return null; - const rows = el("tree-pane").querySelectorAll(".tree-row[data-tree-row]"); + const rows = el("tree-pane").querySelectorAll(TREE_ROW); for (const row of rows) if (row.dataset.treeRow === path) return row; return null; } @@ -921,7 +930,7 @@ function rovingRowEl(): HTMLElement | null { function focusedTreeRow(): HTMLElement | null { const active = document.activeElement; if (!(active instanceof HTMLElement)) return null; - return active.closest("#tree-pane .tree-row[data-tree-row]"); + return active.closest(`#tree-pane ${TREE_ROW}`); } /** A tree row element as the node ref that rename / move / delete all speak. */ @@ -1214,7 +1223,7 @@ function captureReturnFocus(): (() => void) | null { // Unscoped on purpose: `active` is usually *detached* by now, so an ancestor-matching // selector (`#tree-pane .tree-row`) would find nothing. `data-tree-row` is emitted by // the tree and nowhere else, so it identifies a row on its own. - const row = active.closest(".tree-row[data-tree-row]"); + const row = active.closest(TREE_ROW); if (row) { const path = row.dataset.treeRow ?? null; return () => (treeRowEl(path) ?? rovingRowEl())?.focus(); @@ -1562,7 +1571,7 @@ function startTreeCreate(kind: "note" | "folder", dir: string): void { if (state.vaultRoot === null) return; state.contextMenu = null; state.treeCreate = { kind, dir }; - for (const d of dirChain(dir)) state.expandedDirs.add(d); // reveal the target folder + revealDir(dir); // reveal the target folder render(); // paintTree focuses the fresh input } @@ -1592,7 +1601,7 @@ async function commitTreeCreate(raw: string, open: boolean): Promise { try { const report = await api.createDir(path); const refreshed = await loadNotes(); // re-lists structure from disk — the folder is real - for (const d of dirChain(report.dir)) state.expandedDirs.add(d); + revealDir(report.dir); state.selectedDir = report.dir; // the natural next step is a note inside it if (refreshed) flash(`Created ${report.dir}/.`); else render(); // the refresh failure already toasted; still repaint the expansion @@ -1754,7 +1763,7 @@ async function pickAndImport(dir: string): Promise { */ async function finishImport(dir: string, imported: string[], refused: string[]): Promise { if (imported.length > 0) { - for (const d of dirChain(dir)) state.expandedDirs.add(d); + revealDir(dir); state.selectedDir = dir; await loadNotes(); void refreshEmbedStatus(state.vaultRoot); // the N/M denominator grew (#26) @@ -1782,7 +1791,7 @@ let moveInFlight = false; function startTreeRename(node: TreeNodeRef): void { state.contextMenu = null; state.treeRename = node; - for (const d of dirChain(parentDir(node.path))) state.expandedDirs.add(d); + revealDir(parentDir(node.path)); render(); // paintTree focuses the input and selects the prefilled name } @@ -1830,7 +1839,7 @@ async function executeMove(node: TreeNodeRef, to: string): Promise { } // If the open document is affected, flush and close the editor first so the save // chain never targets the old path (a conflict keeps the editor and aborts the move). - const curPath = state.current?.path ?? state.currentResource?.path ?? null; + const curPath = openDocPath(state); const affected = curPath !== null && (node.nodeKind === "folder" ? remapPath(curPath, node.path, to) !== null : curPath === node.path); @@ -1864,7 +1873,7 @@ async function executeMove(node: TreeNodeRef, to: string): Promise { [...state.expandedDirs].map((d) => remapPath(d, from, to) ?? d), ); state.selectedDir = remapPath(state.selectedDir, from, to) ?? state.selectedDir; - for (const d of dirChain(parentDir(to))) state.expandedDirs.add(d); + revealDir(parentDir(to)); const openNotePath = state.current ? remapPath(state.current.path, from, to) : null; const openResourcePath = state.currentResource ? remapPath(state.currentResource.path, from, to) @@ -1922,7 +1931,7 @@ function requestDelete(node: TreeNodeRef): void { /** Forget tree state pointing into a deleted folder subtree. */ function dropDirState(dir: string): void { - const gone = (d: string) => d === dir || d.startsWith(`${dir}/`); + const gone = (d: string) => isWithin(d, dir); state.expandedDirs = new Set([...state.expandedDirs].filter((d) => !gone(d))); if (gone(state.selectedDir)) state.selectedDir = parentDir(dir); } @@ -1941,12 +1950,10 @@ async function executeDelete(node: TreeNodeRef): Promise { // If the open document dies with the delete, close the editor first so no save // chain targets a file that's about to be removed (a conflict aborts the delete, // keeping the buffer alive — the executeMove posture). - const curPath = state.current?.path ?? state.currentResource?.path ?? null; + const curPath = openDocPath(state); const affected = curPath !== null && - (node.nodeKind === "folder" - ? curPath === node.path || curPath.startsWith(`${node.path}/`) - : curPath === node.path); + (node.nodeKind === "folder" ? isWithin(curPath, node.path) : curPath === node.path); if (affected && !fmEditGuard()) return; if (affected && state.editing && !(await closeEditor())) return; @@ -5055,25 +5062,11 @@ function wireEvents(): void { const target = e.target as HTMLElement; if (target.closest("#tree-pane") && state.vaultRoot !== null) { e.preventDefault(); - const dirRow = target.closest("[data-dir]"); - const fileRow = target.closest("[data-open], [data-open-resource]"); - const dir = dirRow - ? (dirRow.dataset.dir ?? "") - : fileRow - ? parentDir(fileRow.dataset.open ?? fileRow.dataset.openResource ?? "") - : ""; - // Over a concrete row, the menu also targets that node (Rename / Move…). - const node: TreeNodeRef | null = dirRow - ? { path: dirRow.dataset.dir ?? "", nodeKind: "folder", label: baseName(dirRow.dataset.dir ?? "") } - : fileRow?.dataset.open - ? { path: fileRow.dataset.open, nodeKind: "note", label: baseName(fileRow.dataset.open) } - : fileRow?.dataset.openResource - ? { - path: fileRow.dataset.openResource, - nodeKind: "resource", - label: baseName(fileRow.dataset.openResource), - } - : null; + // Over a concrete row, the menu also targets that node (Rename / Move…); over the + // pane's empty space, only the vault root. + const row = target.closest(TREE_ROW); + const node = row ? treeRowRef(row) : null; + const dir = node ? folderContext(node.path, node.nodeKind) : ""; state.selectedDir = dir; openTreeMenu(e.clientX, e.clientY, dir, node && node.path ? node : null); return; @@ -5096,7 +5089,7 @@ function wireEvents(): void { // The inline create/rename inputs live in this pane but are text entry — they own // their keys (Enter/Escape), handled with the other text surfaces below. if (e.target instanceof HTMLInputElement) return; - const row = (e.target as HTMLElement).closest(".tree-row[data-tree-row]"); + const row = (e.target as HTMLElement).closest(TREE_ROW); if (!row) return; const path = row.dataset.treeRow ?? ""; const rows = treeRows(); @@ -5433,7 +5426,7 @@ function wireEvents(): void { if (row) { e.preventDefault(); const node = treeRowRef(row); - const dir = node.nodeKind === "folder" ? node.path : parentDir(node.path); + const dir = folderContext(node.path, node.nodeKind); state.selectedDir = dir; const box = row.getBoundingClientRect(); openTreeMenu(box.left + 12, box.bottom, dir, node.path ? node : null); @@ -5725,12 +5718,10 @@ function wireEvents(): void { const dropTargetOf = (target: HTMLElement): { el: Element; dir: string } | null => { const pane = target.closest("#tree-pane"); if (!pane) return null; - const dirRow = target.closest("[data-dir]"); - if (dirRow) return { el: dirRow, dir: dirRow.dataset.dir ?? "" }; - const fileRow = target.closest("[data-open], [data-open-resource]"); - if (fileRow) - return { el: fileRow, dir: parentDir(fileRow.dataset.open ?? fileRow.dataset.openResource ?? "") }; - return { el: pane, dir: "" }; + const row = target.closest(TREE_ROW); + if (!row) return { el: pane, dir: "" }; + const node = treeRowRef(row); + return { el: row, dir: folderContext(node.path, node.nodeKind) }; }; document.addEventListener("dragstart", (e) => { @@ -5753,20 +5744,9 @@ function wireEvents(): void { return; } if (!target.closest("#tree-pane")) return; - const dirRow = target.closest("[data-dir]"); - const noteRow = target.closest("[data-open]"); - const resRow = target.closest("[data-open-resource]"); - treeDrag = dirRow?.dataset.dir - ? { path: dirRow.dataset.dir, nodeKind: "folder", label: baseName(dirRow.dataset.dir) } - : noteRow?.dataset.open - ? { path: noteRow.dataset.open, nodeKind: "note", label: baseName(noteRow.dataset.open) } - : resRow?.dataset.openResource - ? { - path: resRow.dataset.openResource, - nodeKind: "resource", - label: baseName(resRow.dataset.openResource), - } - : null; + const row = target.closest(TREE_ROW); + const node = row ? treeRowRef(row) : null; + treeDrag = node && node.path ? node : null; if (!treeDrag) return; if (e.dataTransfer) { e.dataTransfer.setData("text/plain", treeDrag.path); diff --git a/ui/src/move.test.ts b/ui/src/move.test.ts index ab02903..680793a 100644 --- a/ui/src/move.test.ts +++ b/ui/src/move.test.ts @@ -5,7 +5,10 @@ import { allDirs, baseName, canMoveInto, + folderContext, + isWithin, moveDestination, + moveRefusal, refKind, remapPath, renameDestination, @@ -141,4 +144,38 @@ check("a leading-dot dotfile has no stem, so it is a note ref", () => { equal(refKind(".gitignore"), "note", "dotfile: empty stem"); }); +// --- isWithin / folderContext / moveRefusal ------------------------------------------ + +check("a path is within a folder when it is the folder or under it — by segment", () => { + assert(isWithin("a/b", "a/b"), "the folder itself"); + assert(isWithin("a/b/c.md", "a/b"), "a file inside"); + assert(isWithin("a/b/c/d", "a/b"), "deep inside"); + assert(!isWithin("a/bc", "a/b"), "a prefix-sharing sibling is not inside"); + assert(!isWithin("a", "a/b"), "the parent is not inside"); +}); + +check("a folder is its own context; a file's is its parent's", () => { + equal(folderContext("a/b", "folder"), "a/b", "folder"); + equal(folderContext("a/b/c.md", "note"), "a/b", "note"); + equal(folderContext("img/p.png", "resource"), "img", "resource"); + equal(folderContext("root.md", "note"), "", "a root file's context is the root"); +}); + +check("a refused move says why, and canMoveInto is exactly its absence", () => { + equal(moveRefusal("concepts/memory.md", "note", "concepts"), "current-folder", "own parent"); + equal(moveRefusal("a/b", "folder", "a"), "current-folder", "a folder's own parent"); + equal(moveRefusal("a/b", "folder", "a/b"), "inside-itself", "into itself"); + equal(moveRefusal("a/b", "folder", "a/b/c"), "inside-itself", "into a descendant"); + equal(moveRefusal("a/b", "folder", "a/bc"), null, "a prefix-sharing sibling"); + equal(moveRefusal("a/b.md", "note", "a/b"), null, "a note never refuses as inside-itself"); + for (const [src, kind, dest] of [ + ["a/b", "folder", "a/b"], + ["a/b", "folder", "c"], + ["x.md", "note", ""], + ["x.md", "note", "d"], + ] as const) { + assert(canMoveInto(src, kind, dest) === (moveRefusal(src, kind, dest) === null), `${src} → ${dest}`); + } +}); + console.log(`move.test: ${passed} checks passed`); diff --git a/ui/src/move.ts b/ui/src/move.ts index f283205..b7cef11 100644 --- a/ui/src/move.ts +++ b/ui/src/move.ts @@ -63,22 +63,45 @@ export function renameDestination(path: string, kind: NodeKind, raw: string): st return dest === path ? null : dest; } +/** + * Is `path` the folder `dir` itself or anything inside it? Segment-aware, so a + * prefix-sharing sibling (`notes2/x` against `notes`) is not inside. `dir` names a + * folder, never the vault root (`""`): nothing is "within" the root in this sense. + */ +export function isWithin(path: string, dir: string): boolean { + return path === dir || path.startsWith(`${dir}/`); +} + +/** The folder a tree node stands for when something is created, dropped or imported + * "here": a folder is its own context, a file its parent's. */ +export function folderContext(path: string, kind: NodeKind): string { + return kind === "folder" ? path : parentDir(path); +} + /** The destination path for "move `srcPath` into `destDir`" — same name, new folder. */ export function moveDestination(srcPath: string, destDir: string): string { return joinPath(destDir, baseName(srcPath)); } +/** Why a destination can't take a node: it is already there, or it is the folder + * being moved (or inside it). */ +export type MoveRefusal = "current-folder" | "inside-itself"; + /** - * Whether dropping / moving `srcPath` (of `kind`) into `destDir` is a real move: - * false for its current folder (a no-op) and, for a folder, for itself or any - * of its own descendants (the host refuses those too — this keeps the gesture - * honest before the IPC round-trip). + * Why dropping / moving `srcPath` (of `kind`) into `destDir` is not a real move, or + * null when it is: its current folder is a no-op, and a folder can't go into itself or + * any of its own descendants (the host refuses those too — this keeps the gesture + * honest before the IPC round-trip, and lets the Move… modal say which). */ +export function moveRefusal(srcPath: string, kind: NodeKind, destDir: string): MoveRefusal | null { + if (destDir === parentDir(srcPath)) return "current-folder"; + if (kind === "folder" && isWithin(destDir, srcPath)) return "inside-itself"; + return null; +} + +/** Whether moving `srcPath` into `destDir` is a real move (`moveRefusal` has no reason). */ export function canMoveInto(srcPath: string, kind: NodeKind, destDir: string): boolean { - if (destDir === parentDir(srcPath)) return false; - if (kind === "folder" && (destDir === srcPath || destDir.startsWith(`${srcPath}/`))) - return false; - return true; + return moveRefusal(srcPath, kind, destDir) === null; } /** diff --git a/ui/src/render.ts b/ui/src/render.ts index 6ad4293..a55445b 100644 --- a/ui/src/render.ts +++ b/ui/src/render.ts @@ -24,8 +24,8 @@ import { WIKILINK_ANCHORED, type EmbedImages, } from "./embeds.ts"; -import { RELATION_VERBS, type AppState, type SideSection } from "./state.ts"; -import { allDirs, canMoveInto, renamePrefill } from "./move.ts"; +import { RELATION_VERBS, openDocPath, type AppState, type SideSection } from "./state.ts"; +import { allDirs, baseName, moveRefusal, renamePrefill } from "./move.ts"; import { shouldPromptEmbedInstall } from "./embedreminder.ts"; import { STRENGTH_MIN_CANDIDATES, strengthBand } from "./strength.ts"; import { @@ -155,7 +155,7 @@ const wikilink: TokenizerAndRendererExtension = { // The alt text is the filename: it is all B2 knows about the picture, and it is what // the embed would have read as had the bytes not arrived (the resource card's viewer // makes the same choice, for the same reason). - const name = target.split("/").pop() ?? target; + const name = baseName(target); const width = typeof token.width === "number" ? ` width="${token.width}"` : ""; return `${anchor}${escapeHtml( name, @@ -423,7 +423,7 @@ export function treePaneHtml(state: AppState): string { const roving = rovingPath( visibleRows(tree, state.expandedDirs), state.treeFocus, - state.current?.path ?? state.currentResource?.path ?? null, + openDocPath(state), ); const body = treeChildrenHtml(tree, state, 0, roving); if (!body) @@ -604,7 +604,7 @@ function resourceCardHtml(r: ResourceExplainView, image: string | null): string }) .join(` : `

No notes link to this file yet.

`; - const name = r.path.split("/").pop() ?? r.path; + const name = baseName(r.path); // The alt text is the filename: the `

` right above already says it, so a screen // reader that reads both is repeating itself rather than being told nothing — and B2 // has no description of the picture to offer that would be truer than its name. @@ -2628,11 +2628,9 @@ function moveModalHtml(state: AppState): string { const rows = dirs .map((dir) => { const label = dir === "" ? "vault root" : `${dir}/`; - if (!canMoveInto(t.path, t.nodeKind, dir)) { - const why = - t.nodeKind === "folder" && (dir === t.path || dir.startsWith(`${t.path}/`)) - ? "inside the folder being moved" - : "current folder"; + const refusal = moveRefusal(t.path, t.nodeKind, dir); + if (refusal !== null) { + const why = refusal === "inside-itself" ? "inside the folder being moved" : "current folder"; return `
${escapeHtml(label)} — ${why}
`; } return ``; diff --git a/ui/src/state.ts b/ui/src/state.ts index a5254ec..90288d5 100644 --- a/ui/src/state.ts +++ b/ui/src/state.ts @@ -401,6 +401,11 @@ export interface AppState { status: string | null; } +/** The path of the document the note pane shows — a note or a resource — or null. */ +export function openDocPath(s: AppState): string | null { + return s.current?.path ?? s.currentResource?.path ?? null; +} + export const state: AppState = { vaultRoot: null, semantic: true, From ac26f5da6327e3d167e6365eac74fed2dcba8129 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:29:11 +0000 Subject: [PATCH 05/30] refactor(ui): one coverage classifier behind the five honest surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The search caveat, the Similar empty state, the graph's ghost hint, Settings → Index and chat's retrieval note each re-derived the embedding tiers from the three raw numbers. coverage.ts reads them once — the model as its own fact, the embedded fraction as a tier — and each surface keeps its own copy and its own order of asking. Every surface's output was snapshotted across a state matrix before and after (224 cases, identical); coverage.test.ts pins the classifier and each surface's copy per tier. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/chat.ts | 19 +++--- ui/src/coverage.test.ts | 124 ++++++++++++++++++++++++++++++++++++++++ ui/src/coverage.ts | 37 ++++++++++++ ui/src/render.ts | 49 +++++++++------- 4 files changed, 200 insertions(+), 29 deletions(-) create mode 100644 ui/src/coverage.test.ts create mode 100644 ui/src/coverage.ts diff --git a/ui/src/chat.ts b/ui/src/chat.ts index e81f8b3..f70cd5e 100644 --- a/ui/src/chat.ts +++ b/ui/src/chat.ts @@ -30,6 +30,7 @@ // human-invoked export, and is not MVP (GH #151's open question 2). import type { SideRow } from "./sidenav.ts"; +import { coverage } from "./coverage.ts"; import type { AnswerView, ChatSetup, @@ -289,14 +290,18 @@ export function retrievalNote(s: { notesEmbedded: number; notesTotal: number; }): string { - if (s.notesTotal === 0) return ""; - if (!s.semantic) + const c = coverage(s); + if (c.embedded === "empty") return ""; + if (!c.model) return "Answers are grounded by keyword search only — the embedding model isn’t installed."; - if (s.notesEmbedded === 0) - return "Answers are grounded by keyword search for now — this vault isn’t embedded yet."; - if (s.notesEmbedded < s.notesTotal) - return `Keyword-first grounding — ${s.notesEmbedded}/${s.notesTotal} notes embedded. Reindex to fill the rest.`; - return ""; + switch (c.embedded) { + case "none": + return "Answers are grounded by keyword search for now — this vault isn’t embedded yet."; + case "partial": + return `Keyword-first grounding — ${c.n}/${c.m} notes embedded. Reindex to fill the rest.`; + case "all": + return ""; + } } /** The one command the setup card tells a human to run, spelled here so the label, the diff --git a/ui/src/coverage.test.ts b/ui/src/coverage.test.ts new file mode 100644 index 0000000..15b0fe4 --- /dev/null +++ b/ui/src/coverage.test.ts @@ -0,0 +1,124 @@ +// The coverage classifier (coverage.ts), and the five surfaces that speak from it, pinned. +// +// The surfaces are pinned by what they *say* at each tier rather than by how they ask, +// because they deliberately ask in different orders (coverage.ts's header): chat is +// silent about an empty vault even with no model, search names the missing model first. + +import { strict as assert } from "node:assert"; +import test from "node:test"; +import { JSDOM } from "jsdom"; +import { coverage } from "./coverage.ts"; +import { retrievalNote } from "./chat.ts"; +import { modalHtml, notePaneHtml, sidePaneHtml } from "./render.ts"; +import { state, type AppState } from "./state.ts"; +import type { NoteView } from "./types.ts"; + +// The note pane renders a body through the sanitizer, which needs a DOM (render.test.ts). +(globalThis as unknown as { window: unknown }).window = new JSDOM("").window; + +const cov = (semantic: boolean, notesEmbedded: number, notesTotal: number) => ({ + semantic, + notesEmbedded, + notesTotal, +}); + +test("the tiers fall where the numbers say", () => { + assert.equal(coverage(cov(true, 0, 0)).embedded, "empty"); + assert.equal(coverage(cov(true, 0, 5)).embedded, "none"); + assert.equal(coverage(cov(true, 3, 5)).embedded, "partial"); + assert.equal(coverage(cov(true, 5, 5)).embedded, "all"); + assert.equal(coverage(cov(true, 6, 5)).embedded, "all", "more embedded than listed reads as all"); + assert.equal(coverage(cov(true, 2, 0)).embedded, "empty", "no notes is empty, whatever the count says"); +}); + +test("the model is its own fact, never folded into a tier", () => { + const c = coverage(cov(false, 3, 5)); + assert.equal(c.model, false); + assert.equal(c.embedded, "partial"); + assert.deepEqual([c.n, c.m], [3, 5]); +}); + +// --- the surfaces -------------------------------------------------------------------- + +const note = { + path: "notes/a.md", + title: "a", + type: null, + created: null, + updated: null, + tags: [], + body: "hello", + frontmatter: null, + frontmatter_readable: true, + revision: "r0", +} as NoteView; + +const app = (semantic: boolean, n: number, m: number, over: Partial = {}): AppState => + ({ ...state, vaultRoot: "/v", semantic, notesEmbedded: n, notesTotal: m, ...over }) as AppState; + +/** The search pane's caveat line, as painted. */ +const caveat = (semantic: boolean, n: number, m: number): string => { + const html = sidePaneHtml(app(semantic, n, m, { searchQuery: "q", searchResults: [] })); + return /

for “q”(.*?)<\/p>/s.exec(html)?.[1] ?? "(no sub line)"; +}; + +test("search: the missing model first, then the fraction; silent when semantic is live", () => { + assert.equal(caveat(false, 0, 0), " · keyword only (run b2 init for semantic)"); + assert.equal(caveat(true, 0, 0), ""); + assert.equal(caveat(true, 0, 5), " · keyword-only for now (0/5 embedded — Reindex)"); + assert.equal(caveat(true, 3, 5), " · keyword-first (3/5 embedded)"); + assert.equal(caveat(true, 5, 5), ""); +}); + +test("chat: silent about an empty vault, even with no model", () => { + assert.equal(retrievalNote(cov(false, 0, 0)), ""); + assert.equal( + retrievalNote(cov(false, 0, 5)), + "Answers are grounded by keyword search only — the embedding model isn’t installed.", + ); + assert.equal( + retrievalNote(cov(true, 0, 5)), + "Answers are grounded by keyword search for now — this vault isn’t embedded yet.", + ); + assert.equal( + retrievalNote(cov(true, 3, 5)), + "Keyword-first grounding — 3/5 notes embedded. Reindex to fill the rest.", + ); + assert.equal(retrievalNote(cov(true, 5, 5)), ""); +}); + +test("the graph: a ghost hint for no model or an unfinished vault, else nothing", () => { + const hint = (semantic: boolean, n: number, m: number): string => { + const html = notePaneHtml(app(semantic, n, m, { current: note, graphOpen: true, similar: [] })); + return /

(.*?)<\/div>/s.exec(html)?.[1] ?? ""; + }; + assert.ok(hint(false, 5, 5).startsWith("ghost connections need the semantic model")); + assert.equal(hint(true, 0, 5), "ghosts appear once the vault is embedded — Reindex"); + assert.equal(hint(true, 3, 5), "ghosts appear once the vault is embedded — Reindex"); + assert.equal(hint(true, 5, 5), ""); + assert.equal(hint(true, 0, 0), ""); +}); + +test("the Similar section: its empty state names only the missing model", () => { + const empty = (semantic: boolean) => + sidePaneHtml(app(semantic, 0, 5, { current: note, similar: [], discoveringSimilar: false })); + assert.ok(empty(false).includes("Semantic similarity is off")); + assert.ok(!empty(true).includes("Semantic similarity is off")); + assert.ok(empty(true).includes("Nothing unlinked has stored vectors to compare")); +}); + +test("Settings → Index: indexed and embedded are two different states", () => { + const line = (semantic: boolean, n: number, m: number, vaultRoot: string | null = "/v") => { + const html = modalHtml(app(semantic, n, m, { vaultRoot, settingsOpen: true, settingsTab: "index" })); + return /

(.*?)<\/p>/s.exec(html)?.[1] ?? "(none)"; + }; + assert.equal(line(true, 0, 0, null), "No vault is open."); + assert.equal(line(false, 0, 0), "Nothing indexed yet — B2 indexes a vault when you open it."); + assert.equal( + line(false, 0, 1), + "1 note indexed for keyword search. The embedding model isn’t installed, so none are embedded.", + ); + assert.equal(line(true, 5, 5), "5 notes indexed, all embedded."); + assert.equal(line(true, 3, 5), "5 notes indexed · 3/5 embedded."); + assert.equal(line(true, 0, 5), "5 notes indexed · 0/5 embedded."); +}); diff --git a/ui/src/coverage.ts b/ui/src/coverage.ts new file mode 100644 index 0000000..37c9157 --- /dev/null +++ b/ui/src/coverage.ts @@ -0,0 +1,37 @@ +// How much of the vault semantic ranking can see — the one classifier behind every +// surface that has to be honest about it (#26). +// +// Five surfaces say something about embedding coverage: the search caveat, the Similar +// section's empty state, the graph's ghost hint, Settings → Index, and chat's retrieval +// note. Each used to re-derive the tiers from the three raw numbers, and they didn't all +// agree on where the lines fell. The *reading* is shared, so it lives here; the *copy* is +// each surface's own, and so is the order it asks in — chat says nothing about an empty +// vault even without a model, while search names the missing model first. That is why +// this returns both facts rather than one collapsed tier: a surface picks which one it +// asks about first, and the classifier never has to know. +// +// Pure, so node tests it straight off the source (`npm test`). + +/** How much of the vault has vectors, whatever the model's state. */ +export type Embedded = + | "empty" // nothing projected yet: no notes to embed + | "none" // notes, none embedded + | "partial" // some embedded — the vector half is still filling, or a run stopped short + | "all"; // every note embedded — semantic ranking sees the whole vault + +export interface Coverage { + /** Is the real embedding model installed (`VaultInfo.semantic`)? */ + readonly model: boolean; + readonly embedded: Embedded; + /** Notes embedded, and notes in all — for the surfaces that print the fraction. */ + readonly n: number; + readonly m: number; +} + +/** Read a vault's coverage off the three numbers `VaultInfo` carries. */ +export function coverage(s: { semantic: boolean; notesEmbedded: number; notesTotal: number }): Coverage { + const n = s.notesEmbedded; + const m = s.notesTotal; + const embedded: Embedded = m === 0 ? "empty" : n >= m ? "all" : n === 0 ? "none" : "partial"; + return { model: s.semantic, embedded, n, m }; +} diff --git a/ui/src/render.ts b/ui/src/render.ts index a55445b..b1fee99 100644 --- a/ui/src/render.ts +++ b/ui/src/render.ts @@ -27,6 +27,7 @@ import { import { RELATION_VERBS, openDocPath, type AppState, type SideSection } from "./state.ts"; import { allDirs, baseName, moveRefusal, renamePrefill } from "./move.ts"; import { shouldPromptEmbedInstall } from "./embedreminder.ts"; +import { coverage } from "./coverage.ts"; import { STRENGTH_MIN_CANDIDATES, strengthBand } from "./strength.ts"; import { EXPLAIN_PAIRS_SHOWN, @@ -706,14 +707,17 @@ function sideTab(key: string, roving: string | null): string { // • model, partly embedded → "keyword-first (N/M embedded)" (vector half still filling) // • model, fully embedded → "" (ranking is fully semantic; no caveat) function searchCaveat(state: AppState): string { - if (!state.semantic) - return " · keyword only (run b2 init for semantic)"; - const n = state.notesEmbedded; - const m = state.notesTotal; - if (m === 0 || n >= m) return ""; // empty vault, or every note embedded — semantic is live - return n === 0 - ? ` · keyword-only for now (0/${m} embedded — Reindex)` - : ` · keyword-first (${n}/${m} embedded)`; + const c = coverage(state); + if (!c.model) return " · keyword only (run b2 init for semantic)"; + switch (c.embedded) { + case "empty": + case "all": + return ""; // empty vault, or every note embedded — semantic is live + case "none": + return ` · keyword-only for now (0/${c.m} embedded — Reindex)`; + case "partial": + return ` · keyword-first (${c.n}/${c.m} embedded)`; + } } // The install banner — the prominent, persistent counterpart to the small search caveat @@ -919,7 +923,7 @@ function similarSectionHtml(state: AppState, roving: string | null): string { head + `

` ); - if (!state.semantic) + if (!coverage(state).model) return ( head + `

Semantic similarity is off — run b2 init then Reindex.

` @@ -1534,15 +1538,16 @@ function nodeGroupHtml(n: GraphNode, edges: GraphEdge[], order: number): string `; } -/** The honest ghost-halo caveat (mirrors `searchCaveat`'s tiers, #26): why there are +/** The honest ghost-halo caveat (coverage.ts's tiers, like `searchCaveat`, #26): why there are * no ghosts right now, or null when there are (or when silence is the honest state). */ function ghostHintHtml(state: AppState): string { if (state.similar.length > 0) return ""; if (state.discoveringSimilar) return `
scanning for latent connections…
`; - if (!state.semantic) + const c = coverage(state); + if (!c.model) return `
ghost connections need the semantic model — run b2 init, then Reindex
`; - if (state.notesTotal > 0 && state.notesEmbedded < state.notesTotal) + if (c.embedded === "none" || c.embedded === "partial") return `
ghosts appear once the vault is embedded — Reindex
`; return ""; } @@ -2186,16 +2191,16 @@ function indexPanelHtml(state: AppState): string { const disabled = reindexDisabled(state); // The same honesty as the search caveat (#26): "indexed" and "embedded" are two different // states, and a projected-but-unembedded vault must never read as finished. - const coverage = ((): string => { + const summary = ((): string => { if (state.vaultRoot === null) return "No vault is open."; - if (state.notesTotal === 0) - return "Nothing indexed yet — B2 indexes a vault when you open it."; - const n = state.notesTotal; - if (!state.semantic) - return `${n} note${n === 1 ? "" : "s"} indexed for keyword search. The embedding model isn’t installed, so none are embedded.`; - return state.notesEmbedded >= n - ? `${n} note${n === 1 ? "" : "s"} indexed, all embedded.` - : `${n} note${n === 1 ? "" : "s"} indexed · ${state.notesEmbedded}/${n} embedded.`; + const c = coverage(state); + if (c.embedded === "empty") return "Nothing indexed yet — B2 indexes a vault when you open it."; + const notes = `${c.m} note${c.m === 1 ? "" : "s"}`; + if (!c.model) + return `${notes} indexed for keyword search. The embedding model isn’t installed, so none are embedded.`; + return c.embedded === "all" + ? `${notes} indexed, all embedded.` + : `${notes} indexed · ${c.n}/${c.m} embedded.`; })(); // While a run is live the panel carries the meter itself. It used to point at the top // bar's ("Progress and Cancel are in the top bar"), which was true while Settings was a @@ -2213,7 +2218,7 @@ function indexPanelHtml(state: AppState): string { : `Rarely needed — B2 indexes on open and as you save.`; return `
Vault index

The index is a disposable projection of your Markdown — delete it and a reindex rebuilds it identically.

-

${escapeHtml(coverage)}

+

${escapeHtml(summary)}

From 21f9021f276a3e8ffdec2b6e5a5dfdc2dd5255b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:30:49 +0000 Subject: [PATCH 06/30] refactor(llm,embed): own the fake switches, the wiring rule and the setup sentences b2-llm and b2-embed now answer the questions both adapters were each answering for themselves: - `fake_requested()` + `ENV_LLM`/`ENV_EMBEDDER`, and b2-llm's `FAKE_NOTICE`, so B2_LLM=fake / B2_EMBEDDER=fake mean one thing. - `b2_llm::provider` / `probed_provider` and `b2_embed::embedder_for`: the one wiring rule (fake when forced, else the real thing) in one place. - `unreachable_message` / `model_missing_message` are public (the unused detail parameter dropped), and the Ollama sentence uses `pull_command`. The non-Ollama model-missing sentence now says "Load it there." Smaller clean-ups on the way through: - sse: EOF is dispatched as a final blank line instead of a copy of that branch; a tool call under assembly is a named `PartialCall`. - One `is_valid_tool_call_cap` predicate behind both cap checks. - setup: host_of/origin_of share `scheme_and_authority`; `installed_models` returns Option; `setup` is a private module and `is_local`, `tier_for_ram`, `system_ram_gb` are private; the unused `OpenAiCompatProvider::config()` is gone. - b2-embed: HF_ENDPOINT is passed into `from_raw` (testable, pinned); the Metal fallback reports via tracing (b2::embed) instead of eprintln; single and batch embedding share one forward/CLS/normalize step (the single path still encodes one unpadded sequence); one cache-path join; `find_model`, `model_dir`, `sanitize`, `default_cache_dir` internal. `LocalEmbedder::configured_model()` exposes the untagged repo id. - quickstart documents source/cache_dir/query_prefix and HF_ENDPOINT. - wire.rs: drop an unused `mut` clippy now flags. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- Cargo.lock | 1 + crates/b2-embed/Cargo.toml | 3 + crates/b2-embed/src/config.rs | 75 +++++++++++++----- crates/b2-embed/src/lib.rs | 28 ++++++- crates/b2-embed/src/model.rs | 64 ++++++++++------ crates/b2-llm/src/lib.rs | 65 +++++++++++++--- crates/b2-llm/src/provider.rs | 8 +- crates/b2-llm/src/setup.rs | 138 +++++++++++++++++----------------- crates/b2-llm/src/sse.rs | 80 ++++++++++---------- crates/b2-llm/tests/wire.rs | 2 +- docs/quickstart.md | 9 ++- 11 files changed, 302 insertions(+), 171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 179359d..771cd90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -402,6 +402,7 @@ dependencies = [ "thiserror 2.0.18", "tokenizers", "toml 0.8.23", + "tracing", ] [[package]] diff --git a/crates/b2-embed/Cargo.toml b/crates/b2-embed/Cargo.toml index 2e415d0..f3a9c3f 100644 --- a/crates/b2-embed/Cargo.toml +++ b/crates/b2-embed/Cargo.toml @@ -33,6 +33,9 @@ serde_json = "1.0.150" toml = "0.8" dirs = "6.0" thiserror = "2.0.18" +# The Metal-fallback warning, on the `b2::embed` target: a library reports through the +# adapter's subscriber rather than printing into its output. +tracing = "0.1.41" # On macOS, link Apple's Accelerate (vDSP/BLAS) for candle's CPU matmuls. Without # it candle falls back to a naive kernel that makes bge-base inference ~10-100× diff --git a/crates/b2-embed/src/config.rs b/crates/b2-embed/src/config.rs index 5b7f40e..e0f9e6f 100644 --- a/crates/b2-embed/src/config.rs +++ b/crates/b2-embed/src/config.rs @@ -2,7 +2,7 @@ //! //! Resolution order: the TOML at `$XDG_CONFIG_HOME/b2/config.toml` (if present) //! over compiled defaults; the `HF_ENDPOINT` env var (the standard Hugging Face -//! mirror knob) overrides the endpoint on top. A vault with no config file gets a +//! mirror knob) supplies the endpoint when the file's `source` names none. A vault with no config file gets a //! working default — zero-config is the happy path (the fail-fast config rule //! applies to the *model files*, via `b2 init`, not to this file). @@ -58,7 +58,7 @@ pub const AVAILABLE_MODELS: &[ModelInfo] = &[ /// The registry entry for `id`, or `None` if it isn't a model B2 supports — the guard /// [`EmbedConfig::set_model`] uses to refuse writing a config the loader can't provision. -pub fn find_model(id: &str) -> Option<&'static ModelInfo> { +fn find_model(id: &str) -> Option<&'static ModelInfo> { AVAILABLE_MODELS.iter().find(|m| m.id == id) } @@ -82,6 +82,10 @@ pub struct ModelChoice { /// index-engine.md §5). Documents are embedded verbatim. Empty ⇒ symmetric. pub const DEFAULT_QUERY_PREFIX: &str = "Represent this sentence for searching relevant passages: "; +/// The standard Hugging Face mirror knob: the endpoint `b2 init` downloads from when +/// the config's `source` names none. +const ENV_HF_ENDPOINT: &str = "HF_ENDPOINT"; + /// Where the model files are fetched from when `b2 init` provisions them. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Source { @@ -128,7 +132,8 @@ struct RawEmbedder { impl EmbedConfig { /// Load from the standard config path, falling back to defaults where the file - /// (or any field) is absent. `HF_ENDPOINT` overrides the mirror endpoint last. + /// (or any field) is absent. `HF_ENDPOINT` supplies the mirror endpoint when the + /// file's `source` names none. pub fn load() -> Result { let raw = match Self::config_path() { Some(p) if p.is_file() => { @@ -138,17 +143,25 @@ impl EmbedConfig { } _ => RawFile::default(), }; - Ok(Self::from_raw(raw.embedder)) + Ok(Self::from_raw( + raw.embedder, + std::env::var(ENV_HF_ENDPOINT).ok(), + )) } - /// The standard config file location: `$XDG_CONFIG_HOME/b2/config.toml`. + /// The standard config file location: `$XDG_CONFIG_HOME/b2/config.toml`. Public + /// so an adapter can name the file in a "check your config" message. pub fn config_path() -> Option { dirs::config_dir().map(|d| d.join("b2").join("config.toml")) } - fn from_raw(e: RawEmbedder) -> Self { + /// The file's fields over the defaults. `hf_endpoint` is the `HF_ENDPOINT` value, + /// passed in rather than read here so resolution is testable without mutating a + /// process-global that parallel tests share (`b2_llm::LlmConfig::resolve`'s reason). + /// Blank reads as unset. + fn from_raw(e: RawEmbedder, hf_endpoint: Option) -> Self { let model = e.model.unwrap_or_else(|| DEFAULT_MODEL.to_string()); - let endpoint_env = std::env::var("HF_ENDPOINT").ok().filter(|s| !s.is_empty()); + let endpoint_env = hf_endpoint.filter(|s| !s.trim().is_empty()); let source = Self::resolve_source(&model, e.source, endpoint_env); let cache_dir = e .cache_dir @@ -186,10 +199,17 @@ impl EmbedConfig { } } - /// The flat directory the model's files live in under the cache — predictable - /// (`/`) so "is it installed?" is a plain file check. - pub fn model_dir(&self) -> PathBuf { - self.cache_dir.join(sanitize(&self.model)) + /// The flat directory the configured model's files live in under the cache — + /// predictable (`/`) so "is it installed?" is a plain + /// file check. + pub(crate) fn model_dir(&self) -> PathBuf { + self.model_dir_for(&self.model) + } + + /// [`model_dir`](Self::model_dir) for any registry id — the one place the cache + /// path is joined, so the loader and the picker's installed flag agree. + fn model_dir_for(&self, model: &str) -> PathBuf { + self.cache_dir.join(sanitize(model)) } /// Whether the model with repo id `model` is already provisioned in *this* config's @@ -198,7 +218,7 @@ impl EmbedConfig { /// choices are installed vs. still need `b2 init`. Uses the config's own `cache_dir` /// so a custom cache is honored, not just the default. pub fn is_model_provisioned(&self, model: &str) -> bool { - crate::model::files_present(&self.cache_dir.join(sanitize(model))) + crate::model::files_present(&self.model_dir_for(model)) } /// The full registry annotated against this config — the data the settings picker @@ -269,7 +289,7 @@ impl EmbedConfig { /// Default XDG cache: `~/.local/share/b2/models` (falls back to `./.b2-models` only /// if no home/data dir is discoverable, which is not expected on a normal machine). -pub fn default_cache_dir() -> PathBuf { +fn default_cache_dir() -> PathBuf { dirs::data_dir() .map(|d| d.join("b2").join("models")) .unwrap_or_else(|| PathBuf::from(".b2-models")) @@ -280,7 +300,7 @@ fn is_url(s: &str) -> bool { } /// Make a filesystem-safe directory name from a repo id (`a/b` → `a_b`). -pub fn sanitize(model: &str) -> String { +fn sanitize(model: &str) -> String { model .chars() .map(|c| { @@ -299,7 +319,7 @@ mod tests { #[test] fn defaults_when_no_file() { - let c = EmbedConfig::from_raw(RawEmbedder::default()); + let c = EmbedConfig::from_raw(RawEmbedder::default(), None); assert_eq!(c.model, DEFAULT_MODEL); assert!(matches!(c.source, Source::Hf { endpoint: None, .. })); assert!(c.query_prefix.starts_with("Represent")); @@ -311,7 +331,8 @@ mod tests { source: Some("https://hf-mirror.com".into()), ..Default::default() }; - let c = EmbedConfig::from_raw(e); + // The file's mirror wins over the environment's. + let c = EmbedConfig::from_raw(e, Some("https://env-mirror.example".into())); assert_eq!( c.source, Source::Hf { @@ -321,6 +342,24 @@ mod tests { ); } + #[test] + fn hf_endpoint_is_the_mirror_when_the_file_names_none() { + let c = EmbedConfig::from_raw( + RawEmbedder::default(), + Some("https://env-mirror.example".into()), + ); + assert_eq!( + c.source, + Source::Hf { + repo: DEFAULT_MODEL.into(), + endpoint: Some("https://env-mirror.example".into()) + } + ); + // An exported-but-blank variable means unset to a shell user. + let blank = EmbedConfig::from_raw(RawEmbedder::default(), Some(" ".into())); + assert!(matches!(blank.source, Source::Hf { endpoint: None, .. })); + } + #[test] fn alternate_repo_source() { let e = RawEmbedder { @@ -328,7 +367,7 @@ mod tests { ..Default::default() }; // model id stays what `model` says; source repo is the override. - let c = EmbedConfig::from_raw(e); + let c = EmbedConfig::from_raw(e, None); assert!(matches!(c.source, Source::Hf { repo, .. } if repo == "BAAI/bge-small-en-v1.5")); } @@ -358,7 +397,7 @@ mod tests { fn model_choices_flag_current_and_installed() { let tmp = tempfile::TempDir::new().unwrap(); // Default model, cache pointed at an empty temp dir → current but not installed. - let mut c = EmbedConfig::from_raw(RawEmbedder::default()); + let mut c = EmbedConfig::from_raw(RawEmbedder::default(), None); c.cache_dir = tmp.path().to_path_buf(); let choices = c.model_choices(); assert_eq!(choices.len(), AVAILABLE_MODELS.len()); diff --git a/crates/b2-embed/src/lib.rs b/crates/b2-embed/src/lib.rs index 5f6e395..d42ca23 100644 --- a/crates/b2-embed/src/lib.rs +++ b/crates/b2-embed/src/lib.rs @@ -14,12 +14,34 @@ mod config; mod model; mod provision; -pub use config::{ - find_model, EmbedConfig, ModelChoice, ModelInfo, Source, AVAILABLE_MODELS, DEFAULT_MODEL, -}; +pub use config::{EmbedConfig, ModelChoice, ModelInfo, Source, AVAILABLE_MODELS, DEFAULT_MODEL}; pub use model::{active_device_label, LocalEmbedder}; pub use provision::{provision, ProvisionReport}; +/// The environment variable that forces the deterministic fake embedder everywhere: +/// `B2_EMBEDDER=fake`, the offline/dev mode both adapters honour and their suites run +/// under. +pub const ENV_EMBEDDER: &str = "B2_EMBEDDER"; + +/// Whether `B2_EMBEDDER=fake` is in force. Read in one place so the CLI and the desktop +/// cannot disagree about what the switch means. +pub fn fake_requested() -> bool { + std::env::var_os(ENV_EMBEDDER).is_some_and(|v| v == "fake") +} + +/// The embedder a command should open its vault with — the adapters' one wiring rule. +/// +/// A command that embeds (`needs_semantic`) gets the real, configured [`LocalEmbedder`], +/// failing fast with [`EmbedError::NotProvisioned`] ("run `b2 init`") when it is absent. +/// Everything else — and every command under [`fake_requested`] — gets `None`, which the +/// caller reads as "open with the core's fake", so no model is needed just to read. +pub fn embedder_for(needs_semantic: bool) -> Result> { + if !needs_semantic || fake_requested() { + return Ok(None); + } + LocalEmbedder::load(&EmbedConfig::load()?).map(Some) +} + /// Errors from provisioning/loading the local model. Embed-*time* failures map into /// [`b2_core::Error::Embed`] so the index path surfaces one error type; the /// setup-time errors here carry the actionable "run `b2 init`" guidance. diff --git a/crates/b2-embed/src/model.rs b/crates/b2-embed/src/model.rs index ee77748..39b5434 100644 --- a/crates/b2-embed/src/model.rs +++ b/crates/b2-embed/src/model.rs @@ -9,7 +9,7 @@ use candle_core::{Device, IndexOp, Tensor}; use candle_nn::VarBuilder; use candle_transformers::models::bert::{BertModel, Config, DTYPE}; use tokenizers::{ - PaddingParams, PaddingStrategy, Tokenizer, TruncationDirection, TruncationParams, + Encoding, PaddingParams, PaddingStrategy, Tokenizer, TruncationDirection, TruncationParams, TruncationStrategy, }; @@ -34,6 +34,10 @@ pub struct LocalEmbedder { model: BertModel, tokenizer: Tokenizer, device: Device, + /// The configured repo id (`config.model`), untagged by device — what the + /// adapters attribute embedding cost to. + configured_model: String, + /// The id recorded as `meta.embed_model_id`: the repo id, tagged by device. model_id: String, dim: usize, query_prefix: String, @@ -103,35 +107,35 @@ impl LocalEmbedder { model, tokenizer, device, + configured_model: config.model.clone(), model_id, dim, query_prefix: config.query_prefix.clone(), }) } - /// The pooled, L2-normalized embedding of `text`. CLS pooling (row 0) — what - /// bge is trained for; normalized so the index's L2 distance ranks by cosine. + /// The configured model's repo id, **without** the device tag + /// [`model_id`](Embedder::model_id) carries — the id the config and the settings + /// picker speak, so an adapter's per-model bookkeeping keys on the same name. + pub fn configured_model(&self) -> &str { + &self.configured_model + } + + /// The pooled, L2-normalized embedding of one `text`: a single, unpadded encode + /// through [`forward_cls`](Self::forward_cls). Deliberately not a batch of one — a + /// query takes the plain single-sequence path. fn embed_inner(&self, text: &str) -> candle_core::Result> { let enc = self .tokenizer .encode(text, true) .map_err(|e| candle_core::Error::Msg(format!("tokenize: {e}")))?; - let ids = enc.get_ids(); - let input_ids = Tensor::new(ids, &self.device)?.unsqueeze(0)?; - let token_type_ids = input_ids.zeros_like()?; - let attention_mask = Tensor::new(enc.get_attention_mask(), &self.device)?.unsqueeze(0)?; - // [1, seq, hidden] → CLS token → [hidden] - let hidden = self - .model - .forward(&input_ids, &token_type_ids, Some(&attention_mask))?; - let cls = hidden.i((0, 0))?; - let v: Vec = cls.to_vec1()?; - Ok(l2_normalize(&v)) + self.forward_cls(std::slice::from_ref(&enc))? + .pop() + .ok_or_else(|| candle_core::Error::Msg("the model returned no embedding".into())) } - /// Embed a batch in one forward pass: tokenize+pad to the batch's longest, run - /// `[B, L]` through BERT, take each row's CLS token (position 0, unaffected by - /// right-padding), and L2-normalize. One matmul over `B` texts is far cheaper on + /// Embed a batch in one forward pass: tokenize+pad to the batch's longest, then + /// [`forward_cls`](Self::forward_cls). One matmul over `B` texts is far cheaper on /// CPU than `B` single passes — the reindex win. Equivalent, per row, to /// [`embed_inner`](Self::embed_inner). fn embed_batch_inner(&self, texts: &[&str]) -> candle_core::Result>> { @@ -142,12 +146,21 @@ impl LocalEmbedder { .tokenizer .encode_batch(texts.to_vec(), true) .map_err(|e| candle_core::Error::Msg(format!("tokenize: {e}")))?; + self.forward_cls(&encs) + } + + /// The step both paths share: stack same-length encodings into `[B, L]`, run them + /// through BERT, take each row's CLS token (position 0 — what bge is trained for, and + /// unaffected by right-padding), and L2-normalize so the index's L2 distance ranks by + /// cosine. + fn forward_cls(&self, encs: &[Encoding]) -> candle_core::Result>> { let batch = encs.len(); - // Padding made every encoding the same length. + // A single encode is unpadded; a batch was padded to its longest. Either way + // every encoding here has the same length. let seq = encs.first().map_or(0, |e| e.get_ids().len()); let mut ids = Vec::with_capacity(batch * seq); let mut mask = Vec::with_capacity(batch * seq); - for e in &encs { + for e in encs { ids.extend_from_slice(e.get_ids()); mask.extend_from_slice(e.get_attention_mask()); } @@ -158,8 +171,7 @@ impl LocalEmbedder { let hidden = self .model .forward(&input_ids, &token_type_ids, Some(&attention_mask))?; - let cls = hidden.i((.., 0))?; - let rows: Vec> = cls.to_vec2()?; + let rows: Vec> = hidden.i((.., 0))?.to_vec2()?; Ok(rows.iter().map(|r| l2_normalize(r)).collect()) } } @@ -202,14 +214,18 @@ fn l2_normalize(v: &[f32]) -> Vec { /// Try to open the Metal GPU — only when compiled `--features metal` (candle's /// `metal_is_available()` is literally `cfg!(feature = "metal")`), and never a hard /// requirement: any failure returns `None` and the caller uses the CPU (GH #40). `announce` -/// gates the one-line fallback notice so the load path can warn while the cheap capability -/// probe ([`active_device_label`]) stays silent. No `unwrap`: a failed `new_metal` is a soft +/// gates the fallback warning (a `b2::embed` tracing event) so the load path can warn while +/// the cheap capability probe ([`active_device_label`]) stays silent. No `unwrap`: a failed `new_metal` is a soft /// degrade (no-panic rule), not a load error. fn open_metal(announce: bool) -> Option { if candle_core::utils::metal_is_available() { match Device::new_metal(0) { Ok(d) => return Some(d), - Err(e) if announce => eprintln!("note: Metal GPU unavailable ({e}); embedding on CPU"), + Err(e) if announce => tracing::warn!( + target: "b2::embed", + error = %e, + "Metal GPU unavailable; embedding on CPU" + ), Err(_) => {} } } diff --git a/crates/b2-llm/src/lib.rs b/crates/b2-llm/src/lib.rs index 8956c48..b2c2745 100644 --- a/crates/b2-llm/src/lib.rs +++ b/crates/b2-llm/src/lib.rs @@ -4,10 +4,10 @@ //! never sees it — flow ④ is built and tested against `FakeLlm`. //! //! Decisions (GH #151, cut as GH #154): -//! - **Build our own.** The MVP wire surface is *one* endpoint shape — +//! - **Build our own.** The wire surface is *one* endpoint shape — //! `POST {base}/chat/completions` with `stream: true`, SSE frames until `[DONE]` — so a -//! client library would buy a dependency tree, not leverage. The question re-opens -//! *inside this crate*, behind the unchanged trait, when the agent loop is cut. +//! client library would buy a dependency tree, not leverage. Tool calls (ADR-0022) rode +//! in on that same stream as `delta.tool_calls`, so they did not change the answer. //! - **Sync end-to-end, over `ureq`** (ADR-0011): no runtime, no async-to-sync bridge, and //! **cancellation is just returning early from a blocking read loop**. //! - **Any OpenAI-compatible URL.** Ollama is the *guided* default; LM Studio, llama.cpp, @@ -20,17 +20,57 @@ //! nothing is cached, and swapping models never touches the index. mod provider; -pub mod setup; +mod setup; mod sse; +use b2_core::llm::{FakeLlm, LlmProvider}; use serde::Serialize; pub use provider::OpenAiCompatProvider; pub use setup::{ - is_local, is_ollama, probe_setup, pull_command, refusal_message, ChatSetup, ChatState, - ModelTier, OllamaModel, OllamaSetup, ToolCallCap, MODEL_TIERS, OLLAMA_INSTALL_URL, + is_ollama, model_missing_message, probe_setup, pull_command, refusal_message, + unreachable_message, ChatSetup, ChatState, ModelTier, OllamaModel, OllamaSetup, ToolCallCap, + MODEL_TIERS, OLLAMA_INSTALL_URL, }; +/// The environment variable that swaps in the deterministic [`FakeLlm`]: `B2_LLM=fake`, +/// `B2_EMBEDDER=fake`'s sibling for the chat seam. +pub const ENV_LLM: &str = "B2_LLM"; + +/// What an adapter says when [`fake_requested`] is in force — never overstate what +/// answered. One sentence, so the CLI's stderr note and the desktop's setup card agree. +pub const FAKE_NOTICE: &str = "The fake chat provider is in use (B2_LLM=fake) — answers are \ + deterministic test scaffolding, not a model."; + +/// Whether `B2_LLM=fake` is in force. Read in one place so the two adapters cannot +/// disagree about what the switch means. +pub fn fake_requested() -> bool { + std::env::var_os(ENV_LLM).is_some_and(|v| v == "fake") +} + +/// The provider a chat command talks to: the fake under [`fake_requested`], else the real +/// client over `config`. **No probe** — for a caller that has already checked the endpoint +/// (the desktop probes once, when its chat surface opens) or wants none. +pub fn provider(config: LlmConfig) -> Box { + if fake_requested() { + Box::new(FakeLlm) + } else { + Box::new(OpenAiCompatProvider::new(config)) + } +} + +/// [`provider`], **probed** before it is returned — the `b2 init` posture applied to chat: +/// a stopped server is an [`LlmError`] before the question, never a surprise after it. The +/// fake has nothing to probe. +pub fn probed_provider(config: LlmConfig) -> Result, LlmError> { + if fake_requested() { + return Ok(Box::new(FakeLlm)); + } + let provider = OpenAiCompatProvider::new(config); + provider.probe()?; + Ok(Box::new(provider)) +} + /// Where the Ollama daemon serves its OpenAI-compatible surface. The *guided* /// default per GH #151: local-first, no key, and the runtime B2's onboarding /// speaks natively — while the chat path itself stays generic `/v1`. @@ -74,7 +114,14 @@ pub fn parse_max_tool_calls(raw: &str) -> Option { raw.trim() .parse::() .ok() - .filter(|n| (1..=MAX_TOOL_CALLS_CEILING).contains(n)) + .filter(|&n| is_valid_tool_call_cap(n)) +} + +/// The range every tool-call cap must fall in, from any source: 1 (zero would refuse every +/// call) to [`MAX_TOOL_CALLS_CEILING`]. Shared by the parser and +/// [`LlmConfig::with_max_tool_calls`], so the two can never accept different values. +fn is_valid_tool_call_cap(n: usize) -> bool { + (1..=MAX_TOOL_CALLS_CEILING).contains(&n) } /// Where the bearer token in force came from. A **fact about the key, never the key** — @@ -243,7 +290,7 @@ impl LlmConfig { /// hand-edited settings file cannot lift the ceiling the parser enforces. #[must_use] pub fn with_max_tool_calls(mut self, max_tool_calls: Option) -> Self { - if let Some(n) = max_tool_calls.filter(|n| (1..=MAX_TOOL_CALLS_CEILING).contains(n)) { + if let Some(n) = max_tool_calls.filter(|&n| is_valid_tool_call_cap(n)) { self.max_tool_calls = n; } self @@ -310,7 +357,7 @@ pub enum LlmError { /// 404 here is a base URL that isn't a chat API (the `…/v1X` typo), while a 404 /// mid-answer is the server's own "model not found". An adapter that had to guess would /// give the wrong fix half the time, so the distinction is carried in the type — and - /// turned into a sentence once, by [`crate::setup::refusal_message`]. + /// turned into a sentence once, by [`crate::refusal_message`]. #[error("the model server at {endpoint} refused a probe (HTTP {status}): {message}")] Refused { endpoint: String, diff --git a/crates/b2-llm/src/provider.rs b/crates/b2-llm/src/provider.rs index c7a59f7..318ec98 100644 --- a/crates/b2-llm/src/provider.rs +++ b/crates/b2-llm/src/provider.rs @@ -67,12 +67,6 @@ impl OpenAiCompatProvider { Self { config, agent } } - /// The configuration this provider speaks to — what an adapter names in its - /// "can't reach the model server at …" message. - pub fn config(&self) -> &LlmConfig { - &self.config - } - /// Fail fast, before a human waits: `GET {base}/models` — the `b2 init` posture applied /// to the second seam (ADR-0020's "never a surprise mid-command"). It answers two /// questions and deliberately no others: @@ -84,7 +78,7 @@ impl OpenAiCompatProvider { /// cannot be told apart from the far commoner mistake it silently blessed: a wrong /// path (`…:11434/v1X` came back as *Connected* and failed at the first question). /// The tolerance survives where the evidence supports it — a **2xx** whose body isn't - /// a model list — and [`crate::setup`] is where a refusal becomes advice. + /// a model list — and [`crate::refusal_message`] is where a refusal becomes advice. /// 2. **Does it serve the configured model?** Checked only when the response *parsed* /// as a non-empty model list, and leniently: the check may only ever catch a real /// mistake, never invent one, because a false refusal would block a working setup. diff --git a/crates/b2-llm/src/setup.rs b/crates/b2-llm/src/setup.rs index 5bf3a06..6776327 100644 --- a/crates/b2-llm/src/setup.rs +++ b/crates/b2-llm/src/setup.rs @@ -11,7 +11,7 @@ //! daemon isn't running" is the answer the card is asking for. Adapter-level throughout, so //! nothing here is recorded in the vault or the index. -use crate::{ApiKeySource, LlmConfig, LlmError, OpenAiCompatProvider}; +use crate::{ApiKeySource, LlmConfig, LlmError, OpenAiCompatProvider, FAKE_NOTICE}; use serde::{Deserialize, Serialize}; use std::time::Duration; @@ -21,8 +21,8 @@ const TAGS_PATH: &str = "/api/tags"; /// Where a human who has *no* Ollama is sent — the quickstart rather than the product /// page: someone reading this has already found out nothing is listening, so they need the -/// page with the install command on it. One constant because two adapters print it, and a -/// link that drifts between them is one they get wrong. +/// page with the install command on it. One constant, printed by [`unreachable_message`] +/// for both adapters, so the link cannot drift between them. pub const OLLAMA_INSTALL_URL: &str = "https://docs.ollama.com/quickstart"; /// The port Ollama serves on. Recognizing it is what decides whether Ollama's own @@ -51,7 +51,7 @@ pub fn is_ollama(base_url: &str) -> bool { /// Permissive about *how* loopback is spelled, because guessing "cloud" for a local URL /// only shows a warning the user doesn't need, while guessing "local" for a remote one /// would hide one they do. So it is a **membership** test: only known loopback spellings. -pub fn is_local(base_url: &str) -> bool { +fn is_local(base_url: &str) -> bool { let host = host_of(base_url); host == "localhost" || host == "::1" @@ -71,21 +71,23 @@ fn is_loopback_v4(host: &str) -> bool { matches!(host.parse::(), Ok(ip) if ip.is_loopback()) } +/// A URL's scheme (when it names one) and its authority — everything between `://` and +/// the first `/`, `?` or `#`. Hand-rolled rather than a URL crate: this crate's whole +/// posture is one wire shape and no dependency tree, and [`host_of`] and [`origin_of`] +/// need only these two pieces. +fn scheme_and_authority(url: &str) -> (Option<&str>, &str) { + let (scheme, rest) = match url.split_once("://") { + Some((s, r)) => (Some(s), r), + None => (None, url), + }; + (scheme, rest.split(['/', '?', '#']).next().unwrap_or("")) +} + /// The host portion of a URL, lowercased and without scheme, port, path or -/// credentials. Hand-rolled rather than a URL crate: this crate's whole posture -/// is one wire shape and no dependency tree, and the two callers here need only -/// the authority's host. +/// credentials. fn host_of(url: &str) -> String { - let rest = url - .split_once("://") - .map(|(_, r)| r) - .unwrap_or(url) - .to_ascii_lowercase(); - let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); - let authority = authority - .rsplit_once('@') - .map(|(_, h)| h) - .unwrap_or(authority); + let lowered = scheme_and_authority(url).1.to_ascii_lowercase(); + let authority = lowered.rsplit_once('@').map(|(_, h)| h).unwrap_or(&lowered); // IPv6 literals keep their brackets; everything else drops a `:port`. match authority.strip_prefix('[') { Some(v6) => format!("[{}]", v6.split(']').next().unwrap_or("")), @@ -114,12 +116,8 @@ fn ollama_root(base_url: &str) -> String { /// `scheme://authority` — the URL with every path segment dropped. A missing /// scheme reads as `http`, which is what an Ollama URL without one means. fn origin_of(url: &str) -> String { - let (scheme, rest) = match url.split_once("://") { - Some((s, r)) => (s, r), - None => ("http", url), - }; - let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); - format!("{scheme}://{authority}") + let (scheme, authority) = scheme_and_authority(url); + format!("{}://{authority}", scheme.unwrap_or("http")) } /// How ready chat is, right now — the setup card's top-level branch. @@ -199,7 +197,7 @@ pub const MODEL_TIERS: [ModelTier; 3] = [ /// The rung `ram_gb` sits on — the highest tier whose floor it meets. Total /// installed memory, not free: the question is what the machine can host, and a /// suggestion that changed with whatever else is open would be noise. -pub fn tier_for_ram(ram_gb: u64) -> &'static ModelTier { +fn tier_for_ram(ram_gb: u64) -> &'static ModelTier { MODEL_TIERS .iter() .rev() @@ -300,11 +298,7 @@ impl ChatSetup { cloud: false, api_key_source: ApiKeySource::None, state: ChatState::Fake, - message: Some( - "The fake chat provider is in use (B2_LLM=fake) — answers are deterministic test \ - scaffolding, not a model." - .to_string(), - ), + message: Some(FAKE_NOTICE.to_string()), available: Vec::new(), ollama: None, tool_calls: ToolCallCap::of(config), @@ -346,9 +340,9 @@ pub fn probe_setup(config: &LlmConfig) -> ChatSetup { )), Vec::new(), ), - Err(e) => ( + Err(_) => ( ChatState::Unreachable, - Some(unreachable_message(&config.base_url, &e)), + Some(unreachable_message(&config.base_url)), Vec::new(), ), }; @@ -365,10 +359,12 @@ pub fn probe_setup(config: &LlmConfig) -> ChatSetup { } } -/// E4's own sentence, and the reason [`is_ollama`] exists: `ollama serve` is the -/// fix for Ollama and no help whatever for LM Studio, llama.cpp, vLLM or a cloud -/// endpoint — all of which the same setting points at. -fn unreachable_message(base_url: &str, _detail: &LlmError) -> String { +/// E4's own sentence — nothing is listening at `base_url` — and the reason [`is_ollama`] +/// exists: `ollama serve` is the fix for Ollama and no help whatever for LM Studio, +/// llama.cpp, vLLM or a cloud endpoint, all of which the same setting points at. Phrased +/// here so both adapters say it the same way; an adapter appends only its own way of +/// changing the endpoint (a CLI flag, a Settings field). +pub fn unreachable_message(base_url: &str) -> String { if is_ollama(base_url) { format!( "Can't reach the model server at {base_url} — is Ollama running? \ @@ -427,11 +423,18 @@ pub fn refusal_message( } } -fn model_missing_message(model: &str, base_url: &str) -> String { +/// The server at `base_url` is up but doesn't serve `model` ([`LlmError::ModelMissing`]) — +/// the most common local-setup mistake. [`unreachable_message`]'s sibling: Ollama gets its +/// own fix, everything else the general one, and an adapter appends only how *it* picks +/// another model. +pub fn model_missing_message(model: &str, base_url: &str) -> String { if is_ollama(base_url) { - format!("The model server doesn't have '{model}'. Pull it with `ollama pull {model}`.") + format!( + "The model server doesn't have '{model}'. Pull it with `{}`.", + pull_command(model) + ) } else { - format!("The model server at {base_url} doesn't serve '{model}'.") + format!("The model server at {base_url} doesn't serve '{model}'. Load it there.") } } @@ -453,7 +456,7 @@ fn ollama_setup(base_url: &str) -> OllamaSetup { let ram_gb = system_ram_gb(); OllamaSetup { root, - running: installed.is_ok(), + running: installed.is_some(), installed: installed.unwrap_or_default(), ram_gb, tiers: MODEL_TIERS.to_vec(), @@ -485,29 +488,36 @@ struct TagsDetails { parameter_size: Option, } -/// Every model the daemon at `root` has installed. `Err(())` means the native API didn't +/// Every model the daemon at `root` has installed. `None` means the native API didn't /// answer — not that nothing is installed, which is a different card. Its own agent rather /// than the provider's: a different API on a different path, wanted at a shorter timeout, /// and it must never send the bearer token a cloud configuration might carry. -fn installed_models(root: &str) -> Result, ()> { +fn installed_models(root: &str) -> Option> { let url = format!("{root}{TAGS_PATH}"); let agent = ureq::AgentBuilder::new() .timeout_connect(TAGS_TIMEOUT) .timeout_read(TAGS_TIMEOUT) .build(); - let body = match agent.get(&url).timeout(TAGS_TIMEOUT).call() { - Ok(r) => r.into_string().map_err(|e| { - tracing::debug!(target: "b2::llm", error = %e, url, "unreadable model inventory"); - })?, - Err(e) => { + let response = agent + .get(&url) + .timeout(TAGS_TIMEOUT) + .call() + .map_err(|e| { tracing::debug!(target: "b2::llm", error = %e, url, "no native Ollama inventory"); - return Err(()); - } - }; - let parsed: TagsResponse = serde_json::from_str(&body).map_err(|e| { - tracing::debug!(target: "b2::llm", error = %e, url, "unparseable model inventory"); - })?; - Ok(models_from(parsed)) + }) + .ok()?; + let body = response + .into_string() + .map_err(|e| { + tracing::debug!(target: "b2::llm", error = %e, url, "unreadable model inventory"); + }) + .ok()?; + let parsed: TagsResponse = serde_json::from_str(&body) + .map_err(|e| { + tracing::debug!(target: "b2::llm", error = %e, url, "unparseable model inventory"); + }) + .ok()?; + Some(models_from(parsed)) } /// Ollama's inventory shape, narrowed to the three fields the card shows. Split @@ -535,7 +545,7 @@ fn models_from(parsed: TagsResponse) -> Vec { /// bring a tree for one number), and best-effort by construction: the only thing /// it feeds is a *non-binding suggestion*, so `None` costs the card its /// highlighted row and nothing else. -pub fn system_ram_gb() -> Option { +fn system_ram_gb() -> Option { // Rounded, not truncated. The tiers are floors, and a machine sold as "16 GB" // does not always report 16 GiB of usable memory — Linux's `MemTotal` excludes // what the firmware reserved, so it reads ~15.6, and truncation would drop such @@ -713,8 +723,6 @@ mod tests { assert_eq!(tier_for_ram(0).model, MODEL_TIERS[0].model); } - /// The setup view is what crosses to a webview, so the one thing it must - /// never carry is the bearer token — only whether there is one. /// The only real parsing in this module, and the reason `models_from` is split /// out of the HTTP call: a nameless entry is dropped rather than painted as a /// blank row, and an entry with no `details` keeps its name instead of going @@ -747,6 +755,8 @@ mod tests { assert!(models_from(serde_json::from_str("{}").unwrap()).is_empty()); } + /// The setup view is what crosses to a webview, so the one thing it must + /// never carry is the bearer token — only whether there is one. #[test] fn the_setup_view_reports_a_key_without_carrying_it() { let config = LlmConfig { @@ -815,24 +825,12 @@ mod tests { #[test] fn messages_name_ollamas_own_fix_only_for_ollama() { - let ollama = unreachable_message( - "http://localhost:11434/v1", - &LlmError::Unreachable { - endpoint: "http://localhost:11434/v1".into(), - detail: "connection refused".into(), - }, - ); + let ollama = unreachable_message("http://localhost:11434/v1"); assert!(ollama.contains("ollama serve"), "{ollama}"); // The other half of "nothing is listening": the daemon may not be installed at // all, and the fix for that is a page, not a command. assert!(ollama.contains(OLLAMA_INSTALL_URL), "{ollama}"); - let other = unreachable_message( - "http://localhost:1234/v1", - &LlmError::Unreachable { - endpoint: "http://localhost:1234/v1".into(), - detail: "connection refused".into(), - }, - ); + let other = unreachable_message("http://localhost:1234/v1"); assert!(!other.to_lowercase().contains("ollama"), "{other}"); assert!( model_missing_message("llama3.2", "http://localhost:11434/v1") diff --git a/crates/b2-llm/src/sse.rs b/crates/b2-llm/src/sse.rs index e9d4c38..cc28cd2 100644 --- a/crates/b2-llm/src/sse.rs +++ b/crates/b2-llm/src/sse.rs @@ -54,33 +54,11 @@ pub(crate) fn stream_completion( let read = reader .read_line(&mut line) .map_err(|e| LlmError::Stream(format!("could not read the stream: {e}")))?; - if read == 0 { - // EOF. A pending event has no blank line to close it, so dispatch it - // before deciding how the stream ended. - match dispatch(&data, &mut text, &mut calls, on_token)? { - Step::Done => return Ok(completed(text, calls)), - Step::Cancelled => return Ok(cancelled(text)), - Step::Finished => finished = true, - Step::Continue => {} - } - if !finished { - tracing::debug!( - target: "b2::llm", - chars = text.len(), - "the model stream ended without [DONE]; reporting a partial answer" - ); - } - // Tool calls from a stream that was cut off are not run: half a call's - // arguments is not a call. - return Ok(if finished { - completed(text, calls) - } else { - cancelled(text) - }); - } - + // EOF closes a pending event exactly as a blank line would: a server that + // stops mid-event has still sent that event. + let eof = read == 0; let field = line.trim_end_matches(['\n', '\r']); - if field.is_empty() { + if eof || field.is_empty() { // End of event: dispatch what was accumulated, then start the next. match dispatch(&data, &mut text, &mut calls, on_token)? { Step::Done => return Ok(completed(text, calls)), @@ -88,6 +66,22 @@ pub(crate) fn stream_completion( Step::Finished => finished = true, Step::Continue => {} } + if eof { + if !finished { + tracing::debug!( + target: "b2::llm", + chars = text.len(), + "the model stream ended without [DONE]; reporting a partial answer" + ); + } + // Tool calls from a stream that was cut off are not run: half a call's + // arguments is not a call. + return Ok(if finished { + completed(text, calls) + } else { + cancelled(text) + }); + } data.clear(); continue; } @@ -185,11 +179,20 @@ fn cancelled(text: String) -> Completion { } } +/// One tool call under assembly: what the fragments so far have said about it. +#[derive(Debug, Default)] +struct PartialCall { + id: Option, + name: String, + /// The arguments as JSON text, concatenated fragment by fragment. + arguments: String, +} + /// Tool calls under assembly, slotted by the wire's `index`. A delta with an index fills /// (or extends) that slot; a delta without one is a whole call and takes the next slot. #[derive(Debug)] struct ToolCallParts { - slots: Vec<(Option, String, String)>, // (id, name, arguments) + slots: Vec, /// The most slots this reply may fill ([`crate::LlmConfig::max_tool_calls`]). max: usize, } @@ -207,8 +210,8 @@ impl ToolCallParts { // A *different* id on a slot that already names a call is a new call, whatever // the index says: some servers send every whole call as `index: 0`, and merging // them would run one tool named after two. - if let (Some(id), Some((Some(held), name, _))) = (&part.id, self.slots.get(at)) { - if held != id && !name.is_empty() { + if let (Some(id), Some(slot)) = (&part.id, self.slots.get(at)) { + if slot.id.as_ref().is_some_and(|held| held != id) && !slot.name.is_empty() { at = self.slots.len(); } } @@ -233,17 +236,17 @@ impl ToolCallParts { return Ok(()); }; if part.id.is_some() { - slot.0 = part.id; + slot.id = part.id; } if let Some(function) = part.function { if let Some(name) = function.name { - slot.1.push_str(&name); + slot.name.push_str(&name); } match function.arguments { - Some(serde_json::Value::String(text)) => slot.2.push_str(&text), + Some(serde_json::Value::String(text)) => slot.arguments.push_str(&text), // Arguments sent as an object rather than as JSON text (Ollama's native // shape, and some `/v1` shims): re-serialize so the seam stays text. - Some(other) if !other.is_null() => slot.2.push_str(&other.to_string()), + Some(other) if !other.is_null() => slot.arguments.push_str(&other.to_string()), _ => {} } } @@ -257,13 +260,14 @@ impl ToolCallParts { self.slots .into_iter() .enumerate() - .filter(|(_, (_, name, _))| !name.is_empty()) - .map(|(i, (id, name, arguments))| ToolCall { - id: id + .filter(|(_, call)| !call.name.is_empty()) + .map(|(i, call)| ToolCall { + id: call + .id .filter(|id| !id.is_empty()) .unwrap_or_else(|| format!("call_{i}")), - name, - arguments, + name: call.name, + arguments: call.arguments, }) .collect() } diff --git a/crates/b2-llm/tests/wire.rs b/crates/b2-llm/tests/wire.rs index 43d8d28..9c047dd 100644 --- a/crates/b2-llm/tests/wire.rs +++ b/crates/b2-llm/tests/wire.rs @@ -320,7 +320,7 @@ fn a_reply_past_the_configured_tool_call_cap_fails_with_a_typed_error() { }) }; let req = build_request("q", &[], Vec::new()); - let mut run = |p: &OpenAiCompatProvider| p.complete(&req, &mut |_| ControlFlow::Continue(())); + let run = |p: &OpenAiCompatProvider| p.complete(&req, &mut |_| ControlFlow::Continue(())); let err = run(&capped(2)).unwrap_err(); assert!( diff --git a/docs/quickstart.md b/docs/quickstart.md index 1da1261..ad71966 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -467,6 +467,12 @@ which model built your vectors: [embedder] model = "BAAI/bge-base-en-v1.5" # default — 768-dim, ~440 MB, the better ranker # model = "BAAI/bge-small-en-v1.5" # 384-dim, ~130 MB, faster to download and embed +# source = "https://hf-mirror.com" # where `b2 init` fetches from: a Hugging Face mirror URL, +# # or a local folder holding the model files (offline install) +# cache_dir = "/path/to/models" # where models are kept (default: /b2/models) +# query_prefix = "..." # the instruction prepended to search queries only; +# # "" embeds queries like documents. Leave it unless you +# # know the model wants a different one ``` Those two are the supported models, the same pair the desktop app's Settings → Embedding @@ -476,8 +482,9 @@ they are, `search` refuses rather than mixing embedding spaces. | Environment variable | Effect | |---|---| -| `B2_VAULT_PATH` | Vault root, so commands find it without `-C`. An explicit `-C`/`--vault` overrides it. Read-only commands fall back to the current dir; commands that write (`reindex`/`add`/`mv`/`link`) require it explicitly | +| `B2_VAULT_PATH` | Vault root, so commands find it without `-C`. An explicit `-C`/`--vault` overrides it. Read-only commands fall back to the current dir; commands that write (`reindex`/`add`/`write`/`mv`/`rm`/`link`) require it explicitly | | `B2_EMBEDDER=fake` | Offline mode: deterministic non-semantic embedder. Search runs keyword-only | +| `HF_ENDPOINT` | The standard Hugging Face mirror setting: the endpoint `b2 init` downloads from when the config's `source` names none | | `B2_LLM_URL` / `B2_LLM_MODEL` | The OpenAI-compatible chat endpoint + model for `ask`/`chat` (defaults: `http://localhost:11434/v1`, Ollama's, and `llama3.2`). The `--llm-url`/`--llm-model` flags beat the env, which beats the default | | `B2_LLM_API_KEY` | Bearer token for a cloud chat endpoint. An env var, never a flag, because a key in a flag is a key in `ps`. The desktop stores its key in the macOS Keychain instead | | `B2_LLM=fake` | The deterministic chat provider: `B2_EMBEDDER=fake`'s sibling for `ask`/`chat` | From 64427356038409692d95fbaeb2a5f7250bf8b0d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:31:48 +0000 Subject: [PATCH 07/30] =?UTF-8?q?refactor(core):=20split=20the=20Vault=20f?= =?UTF-8?q?a=C3=A7ade=20into=20views,=20flow=20=E2=91=A3=20and=20snippets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vault.rs had grown to 2,300 lines holding three modules' worth of code. The view types move to vault/views.rs (re-exported, so b2_core::vault::X paths are unchanged), ask + the Why? tool turn to vault/chat.rs, and the one-line excerpt helpers to snippet.rs. An orphaned doc comment that rustdoc was attaching to ToolDesk goes back onto snippet(). While moving, the façade stops repeating itself: - write and write_frontmatter share one save_guarded (resolve, revision guard, splice, write, re-project, read the revision back). - One require_resource check and one refuse_resource_anchor guard replace five hand-written copies. - A search hit resolves in one chunks JOIN notes read (db::chunk_hit) instead of three or four, so the note and chunk come from one snapshot. - A neighbor's title and created date are one read (db::note_header). - neighbor_paths / shared_neighbors / note_summary replace the neighbor sets and NoteSummary builds written out by hand in why_similar and explain_similar; ToolDesk no longer copies the pair WhyFacts carries. - query_snippet windows on search::query_terms, the tokenizer the lexical half ranked with, instead of a second one. - AddReport and ReindexProgress are re-exported from vault like every other report. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/db.rs | 61 +- crates/b2-core/src/lib.rs | 1 + crates/b2-core/src/snippet.rs | 64 ++ crates/b2-core/src/vault.rs | 1324 +++-------------------------- crates/b2-core/src/vault/chat.rs | 552 ++++++++++++ crates/b2-core/src/vault/views.rs | 477 +++++++++++ 6 files changed, 1267 insertions(+), 1212 deletions(-) create mode 100644 crates/b2-core/src/snippet.rs create mode 100644 crates/b2-core/src/vault/chat.rs create mode 100644 crates/b2-core/src/vault/views.rs diff --git a/crates/b2-core/src/db.rs b/crates/b2-core/src/db.rs index 220d6b6..d1e4125 100644 --- a/crates/b2-core/src/db.rs +++ b/crates/b2-core/src/db.rs @@ -936,12 +936,7 @@ pub fn rebuild_fts(conn: &Connection, tokenizer: FtsTokenizer) -> Result<()> { /// Whether the embedding space (the `embeddings` table) currently exists. pub fn embedding_space_exists(conn: &Connection) -> Result { - let n: i64 = conn.query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'embeddings'", - [], - |r| r.get(0), - )?; - Ok(n > 0) + table_exists(conn, "embeddings") } /// Ensure the vector tables exist, recording `(embed_model_id, embed_dim)` in `meta`. If @@ -990,8 +985,10 @@ pub fn ensure_embedding_space(conn: &Connection, model_id: &str, dim: usize) -> /// once under the write lock. Identity first: a differing model settles it without the /// `sqlite_master` lookup. fn embedding_space_matches(conn: &Connection, model_id: &str, dim: usize) -> Result { - let unchanged = meta_value(conn, "embed_model_id")?.as_deref() == Some(model_id) - && meta_value(conn, "embed_dim")?.as_deref() == Some(dim.to_string().as_str()); + let unchanged = matches!( + recorded_embedder(conn)?, + Some((m, d)) if m == model_id && d == dim + ); Ok(unchanged && embedding_space_exists(conn)?) } @@ -1066,8 +1063,38 @@ pub fn chunk_note_map(conn: &Connection) -> Result> { Ok(rows.collect::>>()?) } -/// A chunk's text (None if the chunk id is unknown) — the search-hit → snippet -/// resolution the CLI shows. +/// A ranked chunk resolved for display in one read: its note, that note's title, and the +/// chunk's heading breadcrumb and text. One statement, so the note and the chunk come +/// from the same snapshot: a hit is either whole or `None` (GH #137). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChunkHit { + pub note_path: String, + pub title: Option, + pub heading_path: Option, + pub text: String, +} + +/// [`ChunkHit`] for `chunk_id`, `None` if the chunk or its note is gone. +pub fn chunk_hit(conn: &Connection, chunk_id: i64) -> Result> { + Ok(conn + .query_row( + "SELECT c.note_path, n.title, c.heading_path, c.text + FROM chunks c JOIN notes n ON n.path = c.note_path + WHERE c.id = ?1", + [chunk_id], + |r| { + Ok(ChunkHit { + note_path: r.get(0)?, + title: r.get(1)?, + heading_path: r.get(2)?, + text: r.get(3)?, + }) + }, + ) + .optional()?) +} + +/// A chunk's text (None if the chunk id is unknown) — a similar card's evidence passage. pub fn chunk_text(conn: &Connection, chunk_id: i64) -> Result> { Ok(conn .query_row("SELECT text FROM chunks WHERE id = ?1", [chunk_id], |r| { @@ -1250,18 +1277,18 @@ pub fn note_title(conn: &Connection, note_path: &str) -> Result> .flatten()) } -/// A note's `created` date (`None` if absent or unset), resolved from the -/// projection (GH #22): a neighbor is dated for display without an adapter ever -/// re-reading the file just for a date. -pub fn note_created(conn: &Connection, note_path: &str) -> Result> { +/// A note's `(title, created)` in one read (both `None` if the note is absent), resolved +/// from the projection (GH #22): a neighbor is titled and dated for display without an +/// adapter ever re-reading the file. +pub fn note_header(conn: &Connection, note_path: &str) -> Result<(Option, Option)> { Ok(conn .query_row( - "SELECT created FROM notes WHERE path = ?1", + "SELECT title, created FROM notes WHERE path = ?1", [note_path], - |r| r.get::<_, Option>(0), + |r| Ok((r.get(0)?, r.get(1)?)), ) .optional()? - .flatten()) + .unwrap_or_default()) } /// Every indexed note's `(path, title)`, ordered by `path` — the flat listing diff --git a/crates/b2-core/src/lib.rs b/crates/b2-core/src/lib.rs index 8667b43..f33207a 100644 --- a/crates/b2-core/src/lib.rs +++ b/crates/b2-core/src/lib.rs @@ -23,6 +23,7 @@ pub mod relation; pub mod resource; pub mod rm; pub mod search; +mod snippet; pub mod vault; pub use db::{open, SCHEMA_VERSION}; diff --git a/crates/b2-core/src/snippet.rs b/crates/b2-core/src/snippet.rs new file mode 100644 index 0000000..1f599f3 --- /dev/null +++ b/crates/b2-core/src/snippet.rs @@ -0,0 +1,64 @@ +//! One-line excerpts of chunk text for display: a search hit's snippet, a similar card's +//! evidence, a citation's excerpt. + +/// Longest snippet (in chars) shown for a search hit, so a result stays one line. +const SNIPPET_CHARS: usize = 160; + +/// Flatten a chunk's text to a single whitespace-collapsed line. +fn flatten(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +/// The head of a flattened chunk, bounded to one line. +fn head_snippet(flat: &str) -> String { + if flat.chars().count() <= SNIPPET_CHARS { + flat.to_string() + } else { + let cut: String = flat.chars().take(SNIPPET_CHARS).collect(); + format!("{}…", cut.trim_end()) + } +} + +/// Collapse a chunk's text to a single-line, length-bounded snippet (its head). Used +/// where there is no query to center on (e.g. `similar`'s evidence passage). +pub(crate) fn snippet(text: &str) -> String { + head_snippet(&flatten(text)) +} + +/// Like [`snippet`] but windows the excerpt around the first query-term match, so a +/// section-sized chunk still surfaces the matched text instead of only its head. +/// Falls back to the head when no term matches or the match is already in view — a +/// pure vector hit keeps the head. +pub(crate) fn query_snippet(text: &str, query: &str) -> String { + let flat = flatten(text); + if flat.chars().count() <= SNIPPET_CHARS { + return flat; + } + let lower = flat.to_lowercase(); + // The same tokenizer the lexical half matched with, so the window lands on a term + // that actually ranked the hit. + let match_pos = crate::search::query_terms(query) + .iter() + .filter(|t| t.chars().count() >= 2) + .filter_map(|t| { + let byte = lower.find(&t.to_lowercase())?; + Some(lower[..byte].chars().count()) + }) + .min(); + // A little lead-in so the match is not flush against the ellipsis. + const LEAD: usize = 24; + let Some(pos) = match_pos.filter(|p| *p > LEAD) else { + return head_snippet(&flat); + }; + let chars: Vec = flat.chars().collect(); + // `pos` indexes the lowercased text, whose length can differ from `flat` for + // exotic Unicode; clamp so the slice below can never go out of range. + let start = (pos - LEAD).min(chars.len()); + let end = (start + SNIPPET_CHARS).min(chars.len()); + let mut out = String::from("…"); + out.extend(&chars[start..end]); + if end < chars.len() { + out.push('…'); + } + out +} diff --git a/crates/b2-core/src/vault.rs b/crates/b2-core/src/vault.rs index 8041af3..cfbff7f 100644 --- a/crates/b2-core/src/vault.rs +++ b/crates/b2-core/src/vault.rs @@ -9,8 +9,7 @@ //! (ADR-0005). Under the fake, `search`'s BM25 half is still real but the vector //! half is not semantic — callers must not overstate it. -use crate::add::{self, AddReport}; -use crate::chat; +use crate::add; use crate::chunk::ChunkConfig; use crate::db; use crate::dirs; @@ -19,22 +18,27 @@ use crate::embed::{Embedder, FakeEmbedder}; use crate::error::{Error, Result}; use crate::graph::{self, Direction}; use crate::import; -use crate::llm::{ChatRequest, ChatTurn, ContextPassage, LlmProvider, ToolCall, ToolExchange}; use crate::mv; use crate::rm; +use crate::snippet::{query_snippet, snippet}; use crate::{ingest, note, relation, search}; use rusqlite::Connection; -use serde::Serialize; use std::collections::BTreeSet; use std::fs; use std::ops::ControlFlow; use std::path::{Path, PathBuf}; +mod chat; +mod views; + +pub use views::*; + // Report types the façade returns — re-exported so adapters name them through the // one typed contract. +pub use crate::add::AddReport; pub use crate::dirs::DirCreateReport; pub use crate::import::ImportReport; -pub use crate::ingest::SkippedNote; +pub use crate::ingest::{ReindexProgress, SkippedNote}; pub use crate::mv::DirMoveReport; pub use crate::mv::{MoveReport, ResourceMoveReport}; pub use crate::rm::{DeleteReport, DirDeleteReport, ResourceDeleteReport}; @@ -43,9 +47,6 @@ pub use crate::rm::{DeleteReport, DirDeleteReport, ResourceDeleteReport}; /// brings its own (768); a model or dim swap re-embeds on `reindex` (ADR-0007). const EMBED_DIM: usize = 64; -/// Longest snippet (in chars) shown for a search hit, so a result stays one line. -const SNIPPET_CHARS: usize = 160; - /// Headroom [`Vault::search_chunks`] keeps over `limit`. A small constant, because /// façade headroom is multiplied: each unit buys [`search::pool_size`] more /// candidates from each signal, and candidate width changes answers (GH #142). @@ -111,472 +112,6 @@ pub struct Vault { chunk_config: ChunkConfig, } -/// What `reindex` did: notes projected, and how many were actually (re)embedded -/// (the rest reused their vectors). It reports no vault writes because there are -/// none — a reindex reads (ADR-0004). -/// -/// `cancelled` marks a cooperative cancel of the embed phase; the counts then -/// describe the partial work truthfully and the index is still consistent (keyword -/// and graph complete, a prefix embedded). Always `false` for -/// [`reindex`](Vault::reindex). -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ReindexReport { - pub indexed: usize, - pub embedded: usize, - pub cancelled: bool, - /// Files skipped as unreadable this run — one bad file never fails the pass. - pub skipped: Vec, - /// Ghost rows pruned: notes whose files were deleted outside b2 (#31), so - /// incremental equals a from-scratch rebuild. - pub notes_pruned: usize, - pub resources_indexed: usize, - pub resources_pruned: usize, -} - -/// What [`project`](Vault::project) did — the model-free half of a reindex. No -/// embed counts: projection never touches vectors. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ProjectReport { - pub indexed: usize, - /// Files skipped as unreadable this pass, so an adapter can say which and why. - pub skipped: Vec, - /// Ghost rows pruned: notes whose files were deleted outside b2 (#31). - pub notes_pruned: usize, - pub resources_indexed: usize, - pub resources_pruned: usize, -} - -/// What [`embed`](Vault::embed) did — the model-bound half of a reindex: notes -/// whose missing vectors were filled, and whether a cooperative cancel cut the pass -/// short (the counts stay truthful, and a re-run embeds exactly the remainder). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -pub struct EmbedReport { - pub embedded: usize, - pub cancelled: bool, -} - -/// The vault's embedding coverage — the honest "N/M embedded" signal (#26). -/// Model-free: a pure count over the projection, so an adapter can say -/// "keyword-only for now" precisely without loading a model. `embedded < total` -/// means [`search`](Vault::search) is running keyword-first over the remainder; -/// `embedded == 0` is a fully keyword-only vault. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -pub struct EmbedStatus { - /// Notes with no chunk awaiting a vector. A note with **no chunks at all** — an empty - /// body — counts here: there is no vector it could be waiting for, so it must be able - /// to reach the numerator (see [`crate::db::embed_progress`]). - pub embedded: usize, - /// Every projected note (the denominator). - pub total: usize, -} - -/// What a reindex **would** do — the `reindex --dry-run` preview, computed -/// read-only. The `would_*` keys are the honesty signal: this is a forecast. -/// -/// It forecasts work and nothing else. The dry-run's old columns (which notes would -/// be stamped, which files collide) existed because a real run wrote to the vault; -/// it no longer does (ADR-0004), so only the embedding is left to size. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ReindexPlan { - /// Notes a real reindex would project (every `.md` file the walk collects). - pub would_index: usize, - /// …of which this many would be (re)embedded. - pub would_embed: usize, -} - -/// One neighbor of a note, resolved for display: the note at the other end of an -/// edge, with its path + title, so the adapter stays a dumb printer. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct NeighborView { - /// The other note's vault-relative path — its identity (ADR-0003). - pub path: String, - pub title: Option, - /// The stored relation verb (outbound direction of the edge). - pub relation: String, - /// `"outbound"` (this note → other) or `"inbound"`. - pub direction: String, - /// Display label: the verb outbound, its inverse inbound (ADR-0010). - pub label: String, - pub explanation: Option, - /// Edge origin: `"inline"` (a body link) or `"frontmatter"` (ADR-0010). - pub origin: String, - /// The other note's `created` date, resolved from the projection (GH #22). - pub created: Option, -} - -/// One outbound link a note authors at a **resource** (an image, a PDF — any -/// non-`.md` vault file), resolved for display. Surfaced on [`ExplainView`] so a -/// note's file links are visible from the note's side, not only as the resource's -/// backlinks (GH #22). Distinct from [`NeighborView`]: a resource has no title and -/// authors no edges, so these are always outbound. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ResourceLinkView { - pub path: String, - /// Its inventory class (`image`/`pdf`/`html`/`text`/`media`/`binary`). - pub class: String, - pub relation: String, - pub origin: String, - /// The authored caption (alt text / `|caption`), if any. - pub caption: Option, - /// Whether the link is an embed (`![…]` / `![[…]]`). - pub embed: bool, - pub explanation: Option, -} - -/// One outbound link a note authored that resolves to **nothing** — no note and no -/// resource at its target (a `[[Hermes]]` naming a *folder*, or a typo). A note is -/// one `.md` file, so a folder is never a valid target; rather than drop such a link -/// B2 surfaces it as unresolved, so it reads as broken rather than missing (GH #12). -/// It has no `path` — that is the whole point. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct UnresolvedLink { - /// The target exactly as written in the Markdown (`[[target]]`). - pub target: String, - pub relation: String, - pub origin: String, - pub explanation: Option, -} - -/// A note's full connection picture for `b2 explain`: the note itself, every active -/// connection with its "why", its outbound resource links, and any unresolved -/// outbound links. A thin header over [`NeighborView`] — it reuses the per-edge -/// shape `neighbors` returns rather than a parallel one. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ExplainView { - pub path: String, - pub title: Option, - /// Outbound edges first, then inbound (as [`graph::neighbors`] orders them). - pub connections: Vec, - /// Outbound links at resources — the third target kind (GH #22). - pub resources: Vec, - /// Outbound links that resolved to nothing (GH #12). - pub unresolved: Vec, -} - -/// A note's content + display metadata for a reader. Carries the note's identity, -/// the frontmatter fields worth showing a human, and the **raw Markdown body read -/// from disk** (the source of truth, not the projection) so an adapter renders -/// Markdown itself. A pure read — no embedding. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct NoteView { - pub path: String, - pub title: Option, - pub r#type: Option, - pub created: Option, - pub updated: Option, - pub tags: Vec, - /// The note's Markdown body (frontmatter stripped), verbatim from disk. - pub body: String, - /// The raw frontmatter YAML **verbatim** (the text between the `---` fences, - /// fences excluded), not a re-serialization of the projected fields above — so - /// `b2_relations:` and any keys B2 doesn't model show as written. - pub frontmatter: Option, - /// Whether that block *reads* as YAML metadata - /// ([`note::ParsedNote::frontmatter_readable`]): `false` means the raw bytes - /// above are shown verbatim but the projected fields came back empty. Every - /// read passes through here, so an external hand-edit surfaces the same warning - /// as an in-app save (GH #79). - pub frontmatter_readable: bool, - /// blake3 of the **raw file bytes** at read time — the save-guard token - /// [`write`](Vault::write) validates, so a save can never silently clobber an - /// external edit. Whole-file, so *any* out-of-band change conflicts honestly. - pub revision: String, -} - -/// One note's identity for a listing — `path` + `title`, with **no body**: enough -/// to show and open a note, cheap enough to fetch the whole vault at once. The body -/// is a separate [`read`](Vault::read) when a note is opened. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct NoteSummary { - pub path: String, - pub title: Option, -} - -/// One resource's identity for the file tree — the per-kind sibling of -/// [`NoteSummary`], never a union type. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ResourceSummary { - pub path: String, - pub class: String, - pub size: i64, - pub mtime: Option, -} - -/// The resource fallback card's data: inventory metadata plus inbound backlinks. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ResourceExplainView { - pub path: String, - pub class: String, - pub size: i64, - pub mtime: Option, - pub content_hash: String, - pub backlinks: Vec, -} - -/// One note that links at a resource, with the edge's authored context. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ResourceBacklink { - pub path: String, - pub title: Option, - pub r#type: String, - pub caption: Option, - pub embed: bool, -} - -/// One search hit, resolved to the note it belongs to with a text snippet. -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct SearchResult { - pub path: String, - pub title: Option, - /// Fused relevance score; higher is better. - pub score: f64, - /// A one-line excerpt of the matched chunk. - pub snippet: String, -} - -/// A search's evidence reading — [`Vault::search_evidence`]'s return, and what a -/// surface needs to decide what it vouches for (ADR-0015). -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct SearchEvidenceView { - /// The served results, whole and in fused order — evidence never reorders or - /// removes a row. - pub results: Vec, - /// Does the vault hold positive evidence for this query at the active model's - /// bar? `None` = no calibrated bar for this embedder, so no verdict is offered - /// rather than one guessed. - pub vouched: Option, - /// Chunks in the index — the scale every term's weight is read against. - pub chunk_total: usize, - /// Every query term with its document frequency, in query order. - pub terms: Vec, - /// Best cosine between the query and any chunk vector — the dense half's - /// absolute claim. `None` on a projected-but-unembedded vault. - pub best_cos: Option, -} - -/// One served result with the provenance RRF discarded — which lists ranked its -/// chunk, and how near its vector actually was. -/// -/// The query-level verdict on [`SearchEvidenceView`] is what ADR-0015's "no matches" -/// rests on; **this** is what the per-hit tail bake-off is argued from (GH #206). -/// That bake-off has run — the labels carry the per-hit depth (`tail_relevant`) — -/// and **no tail fold shipped**: the fused order is not an evidence order, so every -/// admissible prefix cut proved vacuous, and the tail complaint is ordering work -/// (the reranker seam), not disclosure work. This stays an instrument reading, -/// re-priced on every `make eval` run. -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct EvidencedResult { - #[serde(flatten)] - pub result: SearchResult, - /// 0-based rank in the BM25 list; `None` = the lexical half never ranked it. - pub bm25_rank: Option, - /// 0-based rank in the dense list; `None` = the vector half never ranked it, - /// or never ran. - pub vector_rank: Option, - /// This chunk's cosine to the query; `None` whenever `vector_rank` is. - pub cos: Option, -} - -/// One query term's lexical reading (see [`SearchEvidenceView`]). -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct QueryTermView { - pub term: String, - /// Chunks matching this term alone; `0` = the vault has never seen the word. - pub df: usize, - /// This term's weight in the coverage reading — `ln((chunks+1)/(df+1))`; near - /// zero for a ubiquitous word, largest for one the vault has never seen. - pub idf: f64, -} - -/// One **chunk-level** search hit — the sub-note view of [`search`](Vault::search). -/// Same retrieval, but ranked chunks are returned as-is instead of deduped up to -/// notes, so a caller can see *which passage* matched and at what rank. The client -/// is the out-of-CI retrieval eval (ADR-0013): note-rank scoring is blind to -/// sub-note quality, which is exactly what chunking levers move. Carries the chunk's -/// **full text**, which the eval's containment scoring anchors on. -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct ChunkSearchResult { - pub path: String, - /// The chunk's heading breadcrumb (`"Fermentation > Vegetables"`), when the - /// chunker recorded one. - pub heading_path: Option, - /// Fused relevance score; higher is better. - pub score: f64, - /// The chunk's stored text, verbatim. - pub text: String, -} - -/// The answer to one grounded-chat ask — flow ④'s display view: the model's -/// streamed text with its `[n]` citation markers resolved back to the vault. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct AnswerView { - /// The answer, verbatim as streamed — including any marker that did *not* - /// resolve. Model output is untrusted content, but it is never rewritten here; - /// an unmatched marker is simply absent from `citations`. - pub answer: String, - /// The resolved citations, ascending by marker; one entry per **distinct** - /// marker that names a real passage. - pub citations: Vec, - /// `true` when the caller's callback broke the stream mid-answer: `answer` then - /// holds the partial text honestly, and citations resolve over what arrived. - pub cancelled: bool, - /// The B2 tools that ran to produce this answer, in order — empty for a plain - /// [`ask`](Vault::ask), which retrieves once and offers the model none. Serialized - /// only when present, so existing JSON consumers see no change. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub tools: Vec, -} - -/// One tool run during a tool-using chat turn — shown beside the answer so a human can -/// see what the explanation was built from. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ToolUseView { - /// The tool's name (`b2_passage_pairs`, `b2_read`, …). - pub name: String, - /// The call's arguments, as JSON text — the model's own, verbatim, for a call it made. - pub arguments: String, - /// `true` for the lookup B2 made itself before asking the model anything; `false` - /// for a call the model chose. - pub seeded: bool, -} - -/// One resolved `[n]` citation: the passage's note, plus a one-line excerpt of the -/// cited passage as display evidence. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct Citation { - /// The marker as it appears in the answer text (1-based passage number). - pub marker: usize, - /// Vault-relative path of the cited note — its identity (ADR-0003), and what an - /// adapter opens on click. A note renamed between answer and click goes stale, - /// exactly as any path handle does. - pub path: String, - /// A one-line excerpt of the cited passage (its head, length-bounded). - pub excerpt: String, -} - -/// One semantically-similar candidate for `b2 similar`: a note near the anchor in -/// embedding space that is **not** already connected to it, resolved for display -/// with the passage that made it similar. The machine finds the candidate, the human -/// decides whether to `link` it (ADR-0009). -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct SimilarView { - pub path: String, - pub title: Option, - /// Best chunk-pair similarity to the anchor; higher is nearer. - pub score: f64, - /// A one-line excerpt of the candidate chunk that achieved `score` — the - /// evidence for *why* it surfaced. - pub evidence: String, - /// How far this candidate stands above the anchor's own candidate population — - /// its best-passage z (GH #192), and the one honest input for a displayed - /// *strength* band (GH #150). Non-increasing down the row order, so the band - /// never contradicts the ranking. It **gates nothing** (ADR-0014): the band is a - /// within-list grading, never a verdict on existence. The unit is load-bearing — - /// a band calibrated in the retired centroid unit grades every card down - /// (GH #182) — so a surface reads its landmarks off `make eval`'s calibration - /// block. `None` when no statistic was computed (a fake-embedded space, a pool - /// under the statistics minimum, or zero variance), which is the adapters' cue - /// to say the list is *ungraded* rather than let bare cards read as uniformly - /// weak; serialized only when present, so older JSON consumers see no change. - #[serde(skip_serializing_if = "Option::is_none")] - pub z: Option, -} - -/// **Explain** for a *Similar & unlinked* card (GH #236): where one note stands in an -/// anchor's discovery field, and the passage pairs behind it. Model-free and read from -/// the same computation as [`Vault::similar`], so a served row's rank, z and best pair are -/// exactly the card's. Raw distances are never the point: every grade is a z on the same -/// yardstick as the strength band. -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct SimilarExplainView { - pub anchor: NoteSummary, - pub candidate: NoteSummary, - /// The list length the explanation is read at: the surface's own, so `served` - /// answers "was this a card?". - pub limit: usize, - pub standing: SimilarStanding, - /// The candidate's z when it was scored in a graded field (the card's band input). - pub z: Option, - /// The candidate's rank judged by whole-note average (stage 1), when it entered - /// stage 1. Far worse than a ranked standing's `rank` means one section carries the - /// match: a buried gem. - pub centroid_rank: Option, - /// Every scored note's z, nearest first: the field the band is relative to. Empty - /// when ungraded (a fake space, a small pool, no spread). - pub population: Vec, - /// One pair per candidate passage, nearest first, each matched to the anchor passage - /// nearest to it. Empty when either side has no vectors. - pub pairs: Vec, - /// Notes both sides already link with (either direction). - pub shared_neighbors: Vec, -} - -/// Where a note stands in an anchor's discovery field: why it is a card, or why not. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SimilarStanding { - /// The candidate is the anchor itself. - SameNote, - /// The anchor has no stored vectors yet: nothing to compare from. - AnchorUnembedded, - /// Directly linked (either direction), so discovery leaves it out as already known. - Linked, - /// The candidate has no stored vectors yet. - Unembedded, - /// Its whole-note rank fell past the first-pass shortlist of `shortlist` notes, so - /// it was never scored passage by passage. - NotShortlisted { shortlist: usize }, - /// Scored: `rank` (1-based) of `of` scored notes; `served` when `rank <= limit`. - Ranked { - rank: usize, - of: usize, - served: bool, - }, -} - -/// One passage pair: a candidate passage and the anchor passage nearest to it. -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct PassagePairView { - pub anchor: PassageView, - pub candidate: PassageView, - /// Negated L2, higher is nearer: [`SimilarView::score`]'s unit. - pub score: f64, - /// The pair's z on the card's yardstick, `None` when ungraded. - pub z: Option, - /// Both passages hold the same text (a template, a copy): a perfect match that says - /// nothing about what the notes are about. - pub identical: bool, -} - -/// One passage, as stored in the index. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct PassageView { - /// The chunk's heading breadcrumb (`"Fermentation > Vegetables"`), when it has one. - pub heading_path: Option, - /// The chunk's stored text, verbatim. - pub text: String, -} - -/// What [`write`](Vault::write) did: the saved note's path and the **new revision** -/// (blake3 of the final on-disk bytes) — the token the editor chains its next save -/// on, so sequential saves never self-conflict. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct WriteReport { - pub path: String, - pub revision: String, -} - -/// What `b2 link` did: the committed typed edge, resolved for display. `created` is -/// `false` when the directed `(src, dst, type)` edge already existed, so nothing was -/// written (the command is idempotent). -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct LinkReport { - pub src_path: String, - pub dst_path: String, - pub relation: String, - pub created: bool, -} - impl Vault { /// Open the vault rooted at `vault_root` with the deterministic [`FakeEmbedder`] /// — the default for tests/dev. Creating `/.b2/` if absent; idempotent. @@ -750,8 +285,7 @@ impl Vault { fn neighbors_of(&self, note_path: &str) -> Result> { let mut out = Vec::new(); for n in graph::neighbors(&self.conn, note_path)? { - let title = db::note_title(&self.conn, &n.other)?; - let created = db::note_created(&self.conn, &n.other)?; + let (title, created) = db::note_header(&self.conn, &n.other)?; out.push(NeighborView { path: n.other, title, @@ -876,29 +410,10 @@ impl Vault { /// an external write trips the guard. pub fn write(&self, note_ref: &str, body: &str, base_revision: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "write", note = note_ref).entered(); - let path = self.resolve_ref(note_ref)?; - let abs = self.root.join(&path); - let raw = fs::read_to_string(&abs)?; - - // The guard: the bytes the edit was based on must still be the bytes on disk. - if revision_of(&raw) != base_revision { - return Err(Error::WriteConflict(path)); - } - - // Markdown first: the byte-honest splice (frontmatter bytes untouched). - let mut parsed = note::parse(&raw); - parsed.replace_body(body); - fs::write(&abs, parsed.as_str())?; - - // Re-project model-free, through the ordinary path. - ingest::project_file(self.ctx(), &path)?; - - // Read the final bytes back rather than assume them: the revision the editor - // chains on must describe the file, not our intent. - let final_raw = fs::read_to_string(&abs)?; - Ok(WriteReport { - path, - revision: revision_of(&final_raw), + // The byte-honest splice: frontmatter bytes untouched. + self.save_guarded(note_ref, base_revision, |parsed| { + parsed.replace_body(body); + Ok(()) }) } @@ -923,28 +438,43 @@ impl Vault { ) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "write_frontmatter", note = note_ref) .entered(); + self.save_guarded(note_ref, base_revision, |parsed| { + if frontmatter + .lines() + .any(|l| l.trim_end_matches('\r') == "---") + { + return Err(Error::Frontmatter( + "a `---` line inside frontmatter would end the block early".into(), + )); + } + parsed.replace_frontmatter(frontmatter); + Ok(()) + }) + } + + /// The shared shape of the two editor saves: resolve, check that the file on disk + /// still hashes to `base_revision` (else [`Error::WriteConflict`], nothing written), + /// apply `splice` (which may refuse, also before any byte reaches disk), write, and + /// re-project **model-free** through the ordinary path. + /// + /// The returned revision is read back from disk rather than assumed: the token the + /// editor chains its next save on must describe the file, not our intent. + fn save_guarded( + &self, + note_ref: &str, + base_revision: &str, + splice: impl FnOnce(&mut note::ParsedNote) -> Result<()>, + ) -> Result { let path = self.resolve_ref(note_ref)?; let abs = self.root.join(&path); let raw = fs::read_to_string(&abs)?; - - // The guard: the bytes the edit was based on must still be the bytes on disk. if revision_of(&raw) != base_revision { return Err(Error::WriteConflict(path)); } - if frontmatter - .lines() - .any(|l| l.trim_end_matches('\r') == "---") - { - return Err(Error::Frontmatter( - "a `---` line inside frontmatter would end the block early".into(), - )); - } - let mut parsed = note::parse(&raw); - parsed.replace_frontmatter(frontmatter); + splice(&mut parsed)?; fs::write(&abs, parsed.as_str())?; ingest::project_file(self.ctx(), &path)?; - let final_raw = fs::read_to_string(&abs)?; Ok(WriteReport { path, @@ -1006,8 +536,7 @@ impl Vault { /// engine imposes. pub fn read_resource_bytes(&self, path: &str) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "read_resource_bytes", path).entered(); - db::resource_detail(&self.conn, path)? - .ok_or_else(|| Error::ResourceNotFound(path.to_string()))?; + self.require_resource(path)?; Ok(fs::read(self.root.join(path))?) } @@ -1017,8 +546,7 @@ impl Vault { /// [`Error::ResourceNotFound`] when it is not inventoried. pub fn explain_resource(&self, path: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "explain_resource", path).entered(); - let detail = db::resource_detail(&self.conn, path)? - .ok_or_else(|| Error::ResourceNotFound(path.to_string()))?; + let detail = self.require_resource(path)?; let backlinks = db::inbound_resource_edges(&self.conn, path)? .into_iter() .map(|b| ResourceBacklink { @@ -1047,9 +575,7 @@ impl Vault { pub fn move_resource(&self, path: &str, to: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "mv_resource", from = path, to).entered(); - if db::resource_detail(&self.conn, path)?.is_none() { - return Err(Error::ResourceNotFound(path.to_string())); - } + self.require_resource(path)?; mv::move_resource(self.embed_ctx(), path, to) } @@ -1060,8 +586,8 @@ impl Vault { /// **Keyword-first fallback:** when the vector space does not exist yet — a /// projected-but-unembedded vault — this runs BM25-only rather than returning /// nothing, so a vault is searchable the moment [`project`](Self::project) - /// finishes. A never-indexed vault yields no hits and no error; callers should - /// consult the `semantic` flag to present keyword-only results honestly. + /// finishes. A never-indexed vault yields no hits and no error; callers read + /// [`embed_status`](Self::embed_status) to present keyword-only results honestly. /// /// A `limit` of 0 short-circuits ahead of [`retrieve`](Self::retrieve), so it /// costs no query embedding and no [`Error::ModelMismatch`] either: that guard @@ -1072,11 +598,7 @@ impl Vault { return Ok(Vec::new()); } let hits = self.retrieve(query, note_hit_pool(limit))?.hits; - Ok(self - .resolve_note_hits(hits, query, limit)? - .into_iter() - .map(|(result, _)| result) - .collect()) + self.note_results(hits, query, limit) } /// [`search`](Self::search)'s dense half alone — vector KNN resolved to notes, @@ -1102,11 +624,7 @@ impl Vault { query, note_hit_pool(limit), )?; - Ok(self - .resolve_note_hits(hits, query, limit)? - .into_iter() - .map(|(result, _)| result) - .collect()) + self.note_results(hits, query, limit) } /// [`search`](Self::search) with the **evidence behind it** (ADR-0015): the same @@ -1208,8 +726,23 @@ impl Vault { }) } - /// The note-resolution tail shared by [`search`](Self::search) and - /// [`search_vector_only`](Self::search_vector_only): dedup chunk hits to their + /// [`resolve_note_hits`](Self::resolve_note_hits) without the provenance — the + /// shared tail of [`search`](Self::search) and + /// [`search_vector_only`](Self::search_vector_only). + fn note_results( + &self, + hits: Vec, + query: &str, + limit: usize, + ) -> Result> { + Ok(self + .resolve_note_hits(hits, query, limit)? + .into_iter() + .map(|(result, _)| result) + .collect()) + } + + /// The note-resolution tail of every note-level search: dedup chunk hits to their /// best-scoring note, resolve path + title + query-windowed snippet, stop at /// `limit`. fn resolve_note_hits( @@ -1226,22 +759,18 @@ impl Vault { if out.iter().any(|(r, _)| r.path == hit.note_path) { continue; // note already represented by a higher-scoring chunk } - // The note row can only be missing on a torn read; drop the hit rather - // than name a note that is gone — the pool has headroom to backfill it - // (GH #137). - if !db::note_exists(&self.conn, &hit.note_path)? { + // The chunk or its note can only be missing on a torn read; drop the hit + // rather than name a note that is gone — the pool has headroom to backfill + // it (GH #137). + let Some(found) = db::chunk_hit(&self.conn, hit.chunk_id)? else { continue; - } - let title = db::note_title(&self.conn, &hit.note_path)?; - let snippet = db::chunk_text(&self.conn, hit.chunk_id)? - .map(|t| query_snippet(&t, query)) - .unwrap_or_default(); + }; out.push(( SearchResult { - path: hit.note_path, - title, + snippet: query_snippet(&found.text, query), + path: found.note_path, + title: found.title, score: hit.score, - snippet, }, hit.provenance, )); @@ -1266,528 +795,25 @@ impl Vault { if out.len() == limit { break; } - // Either lookup can miss only on an inconsistent index; drop such a hit - // rather than emit a half-resolved one — a rank slot with an empty path - // would read as a real result. The pool backfills it (GH #137). - if !db::note_exists(&self.conn, &hit.note_path)? { - continue; - } - let Some((heading_path, text)) = db::chunk_detail(&self.conn, hit.chunk_id)? else { + // The lookup misses only on a torn read; drop such a hit rather than emit a + // half-resolved one — a rank slot with an empty path would read as a real + // result. The pool backfills it (GH #137). + let Some(found) = db::chunk_hit(&self.conn, hit.chunk_id)? else { continue; }; out.push(ChunkSearchResult { - path: hit.note_path, - heading_path, + path: found.note_path, + heading_path: found.heading_path, score: hit.score, - text, + text: found.text, }); } Ok(out) } - /// Flow ④ — grounded chat over the vault: condense → retrieve → assemble → - /// stream → cite, orchestrated here over the core logic in [`crate::chat`]. The - /// provider is injected **per call**: chat is its sole consumer and, unlike the - /// embedder, it carries no index identity (contrast ADR-0007), so nothing about - /// it belongs on the open vault. - /// - /// - **Condense** (multi-turn only): one provider call rewrites the follow-up - /// into a standalone retrieval query; on failure it degrades to the raw - /// question, so that step can never break chat. - /// - **Retrieve**: [`search_chunks`](Self::search_chunks) at - /// [`chat::ASK_PASSAGES`], holding `search`'s posture — chat is a reader. - /// - **Stream**: tokens flow up through `on_token` as they arrive; returning - /// `ControlFlow::Break(())` cancels at token granularity and the result reports - /// it ([`AnswerView::cancelled`]). - /// - **Cite**: distinct `[n]` markers resolve to `(path, excerpt)`; a - /// hallucinated marker resolves to nothing, and the text is never rewritten. - /// - /// Nothing model-derived is stored anywhere, and history is the caller's, - /// session-only. Errors: retrieval as `search` raises it; a failed *answer* call - /// as [`Error::Llm`](crate::Error::Llm). - pub fn ask( - &self, - llm: &dyn LlmProvider, - question: &str, - history: &[ChatTurn], - on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, - ) -> Result { - let _op = tracing::debug_span!( - target: "b2::vault", - "ask", - question, - multi_turn = !history.is_empty() - ) - .entered(); - let query = if history.is_empty() { - question.to_string() - } else { - chat::condense_query(llm, question, history) - }; - let passages: Vec = self - .search_chunks(&query, chat::ASK_PASSAGES)? - .into_iter() - .map(|c| ContextPassage { - path: c.path, - heading_path: c.heading_path, - text: c.text, - }) - .collect(); - tracing::debug!( - target: "b2::chat", - passages = passages.len(), - "retrieved grounding passages" - ); - let req = chat::build_request(question, history, passages); - self.stream_answer(llm, &req, on_token) - } - - /// **Why was this suggested?** — the chat answer behind one *Similar & unlinked* - /// row: explain, grounded and cited, why `candidate_ref` surfaced for `anchor_ref`. - /// - /// A **tool-using** turn (ADR-0022). The model is offered B2's read-only tools - /// ([`chat::why_tools`]: `b2_passage_pairs`, `b2_similar`, `b2_neighbors`, `b2_read`) - /// and makes the lookups itself; each call is one read on this façade, run here and - /// replayed to the model with its result. Every argument defaults to this turn's - /// pair, so `{}` is always a valid call. Three things bound that: - /// - /// - **Round 1 is the lookup round.** Its text is never streamed: a model that - /// answers there answered from nothing. If it calls no tool — or the call fails, - /// most often a model with no tool support — the turn degrades to - /// [`chat::build_why_request`]: B2 makes the pair lookup and hands the evidence over - /// in one plain grounded request. If it calls tools but skips the pair lookup, B2 - /// appends that call itself (`seeded`), because the matched pairs are what the row - /// was ranked on and no explanation is written without them. - /// - **The loop is bounded** at [`chat::MAX_TOOL_ROUNDS`] model calls and - /// [`chat::MAX_CALLS_PER_ROUND`] tool calls each, and the last round offers no - /// tools, so it can only answer. - /// - **The row's position rides in the prompt** — rank and strength from - /// [`similar`](Self::similar) at the `limit` the surface showed, so the explanation - /// describes the card that was clicked. - /// - /// One consequence of the hidden first round: a cancel lands when the first visible - /// token arrives, not during the lookups. - /// - /// A tool call is model output, so it is untrusted: an unknown tool, malformed - /// arguments or an unknown note is answered with an `error:` *result* the model can - /// read, never a failed turn. Every passage a tool hands over is numbered once on one - /// ledger, and the answer's `[n]` markers resolve against it. Streaming, cancellation - /// and the [`Error::Llm`] normalization are [`ask`](Self::ask)'s — chat is a reader - /// here too, and nothing is stored. [`Error::NoteNotFound`] for an unknown ref on - /// either side. - pub fn why_similar( - &self, - llm: &dyn LlmProvider, - anchor_ref: &str, - candidate_ref: &str, - limit: usize, - on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, - ) -> Result { - let _op = tracing::debug_span!( - target: "b2::vault", - "why_similar", - anchor = anchor_ref, - candidate = candidate_ref, - limit - ) - .entered(); - let anchor = self.resolve_ref(anchor_ref)?; - let candidate = self.resolve_ref(candidate_ref)?; - - // The row as the surface showed it: same call, same limit, so the same rank + z. - let served = self.similar(&anchor, limit)?; - let row = served.iter().position(|s| s.path == candidate); - let anchor_neighbors: BTreeSet = graph::neighbors(&self.conn, &anchor)? - .into_iter() - .map(|n| n.other) - .collect(); - let candidate_neighbors: BTreeSet = graph::neighbors(&self.conn, &candidate)? - .into_iter() - .map(|n| n.other) - .collect(); - - let mut desk = ToolDesk { - anchor: anchor.clone(), - candidate: candidate.clone(), - limit, - chunk_ids: Vec::new(), - passages: Vec::new(), - }; - let mut facts = chat::WhyFacts { - anchor_title: db::note_title(&self.conn, &anchor)?, - candidate_title: db::note_title(&self.conn, &candidate)?, - rank: row.map(|i| (i + 1, served.len())), - z: row.and_then(|i| served.get(i)).and_then(|s| s.z), - linked: anchor_neighbors.contains(&candidate), - shared_neighbors: anchor_neighbors - .intersection(&candidate_neighbors) - .cloned() - .collect(), - // The handoff's half, filled only if the turn degrades to it. - embedded: true, - pairs: Vec::new(), - anchor_path: anchor, - candidate_path: candidate, - }; - - let mut req = - chat::build_why_agent_request(&facts, chat::why_tools(), Vec::new(), Vec::new()); - let mut tools_used: Vec = Vec::new(); - let mut answer = String::new(); - let mut cancelled = false; - for round in 1..=chat::MAX_TOOL_ROUNDS { - if round == chat::MAX_TOOL_ROUNDS { - req.tools.clear(); // nothing left to do but answer - } - // Round 1 is the lookup round, and its text is never shown: a model that - // answers there has answered from nothing, and one that calls tools says at - // most "let me check". From round 2 on, tokens stream as they arrive. - let completion = if round == 1 { - llm.complete(&req, &mut |_| ControlFlow::Continue(())) - } else { - llm.complete(&req, on_token) - }; - let completion = match completion { - Ok(c) if round == 1 && c.tool_calls.is_empty() => { - tracing::debug!( - target: "b2::chat", - "the model called no tool; handing the evidence over instead" - ); - return self.why_handoff(llm, &mut facts, desk, tools_used, on_token); - } - Ok(c) => c, - // Never degraded: a reply past the tool-call cap is a broken or hostile - // server, and a quiet handoff would hide it. - Err(e @ Error::ToolCallLimit { .. }) => return Err(e), - // The lookup round failed: most often a model with no tool support. - // If the server is simply down, the handoff fails the same way, honestly. - Err(e) if round == 1 => { - tracing::debug!( - target: "b2::chat", - error = %e, - "the tool round failed; handing the evidence over without tools" - ); - return self.why_handoff(llm, &mut facts, desk, tools_used, on_token); - } - Err(e) => return Err(llm_error(e)), - }; - if round > 1 { - answer.push_str(&completion.text); - } - if completion.cancelled { - cancelled = true; - break; - } - if completion.tool_calls.is_empty() || req.tools.is_empty() { - break; - } - for call in completion - .tool_calls - .into_iter() - .take(chat::MAX_CALLS_PER_ROUND) - { - let result = self.run_tool(&mut desk, &call); - tracing::debug!( - target: "b2::chat", - tool = call.name, - failed = result.starts_with("error:"), - "ran a tool for the model" - ); - tools_used.push(ToolUseView { - name: call.name.clone(), - arguments: call.arguments.clone(), - seeded: false, - }); - req.exchanges.push(ToolExchange { call, result }); - } - // Whatever the model chose to look up, the matched pairs for *this* pair are - // what the row was ranked on. If the lookup round skipped them, B2 makes that - // call itself, so no explanation is written without them on the desk. - if round == 1 && desk.passages.is_empty() { - let seeded = self.seed_pairs(&mut desk, &facts)?; - tools_used.push(ToolUseView { - name: seeded.call.name.clone(), - arguments: seeded.call.arguments.clone(), - seeded: true, - }); - req.exchanges.push(ToolExchange { - call: seeded.call, - result: seeded.result, - }); - } - req.passages.clone_from(&desk.passages); - } - Ok(AnswerView { - citations: cite(&answer, &desk.passages), - answer, - cancelled, - tools: tools_used, - }) - } - - /// B2's own `b2_passage_pairs` call for the turn's pair, as the exchange it is. - fn seed_pairs(&self, desk: &mut ToolDesk, facts: &chat::WhyFacts) -> Result { - let call = ToolCall { - id: "b2_seed_1".to_string(), - name: chat::TOOL_PASSAGE_PAIRS.to_string(), - arguments: serde_json::json!({ - "note": facts.anchor_path, - "candidate": facts.candidate_path, - }) - .to_string(), - }; - let (result, pairs) = - self.tool_passage_pairs(desk, &facts.anchor_path, &facts.candidate_path)?; - Ok(SeededLookup { - call, - result, - pairs, - }) - } - - /// The degrade of a tool-using why-turn: B2 makes the pair lookup itself and hands - /// the evidence over in one plain grounded request — for a model with no tool - /// support, and for one that was offered tools and used none. - fn why_handoff( - &self, - llm: &dyn LlmProvider, - facts: &mut chat::WhyFacts, - mut desk: ToolDesk, - mut tools_used: Vec, - on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, - ) -> Result { - let seeded = self.seed_pairs(&mut desk, facts)?; - tools_used.push(ToolUseView { - name: seeded.call.name, - arguments: seeded.call.arguments, - seeded: true, - }); - facts.embedded = !seeded.pairs.is_empty(); - facts.pairs = seeded.pairs; - let req = chat::build_why_request(facts, desk.passages); - let mut view = self.stream_answer(llm, &req, on_token)?; - view.tools = tools_used; - Ok(view) - } - - /// Run one tool call for the model and render its result as text. Infallible by - /// design: the call is model output, so every failure — an unknown tool, arguments - /// that aren't a JSON object, a note that doesn't exist, an index error — becomes an - /// `error:` result the model reads, and the turn carries on. - fn run_tool(&self, desk: &mut ToolDesk, call: &ToolCall) -> String { - let args: serde_json::Value = if call.arguments.trim().is_empty() { - serde_json::json!({}) - } else { - match serde_json::from_str(&call.arguments) { - Ok(v @ serde_json::Value::Object(_)) => v, - _ => return "error: the arguments were not a JSON object".to_string(), - } - }; - let text_arg = |key: &str| args.get(key).and_then(|v| v.as_str()).map(str::to_string); - let note = text_arg("note"); - let ran = match call.name.as_str() { - chat::TOOL_PASSAGE_PAIRS => { - let note = note.unwrap_or_else(|| desk.anchor.clone()); - let candidate = text_arg("candidate").unwrap_or_else(|| desk.candidate.clone()); - self.resolve_ref(¬e).and_then(|n| { - let c = self.resolve_ref(&candidate)?; - Ok(self.tool_passage_pairs(desk, &n, &c)?.0) - }) - } - chat::TOOL_SIMILAR => { - let note = note.unwrap_or_else(|| desk.anchor.clone()); - self.tool_similar(¬e, desk.limit) - } - chat::TOOL_NEIGHBORS => { - let note = note.unwrap_or_else(|| desk.anchor.clone()); - self.tool_neighbors(¬e) - } - chat::TOOL_READ => { - // The open note is already on the human's screen; the one worth reading - // by default is the suggestion. - let note = note.unwrap_or_else(|| desk.candidate.clone()); - let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0); - self.tool_read(desk, ¬e, offset as usize) - } - other => return format!("error: unknown tool `{other}`"), - }; - ran.unwrap_or_else(|e| match e { - Error::NoteNotFound(n) => format!("error: note not found: {n}"), - // Internals stay out of the model's context as they stay out of the UI. - _ => "error: B2 could not run that lookup".to_string(), - }) - } - - /// `b2_passage_pairs`: the matched pairs between two notes, then each passage they - /// name that the ledger had not handed over yet. Also returns the pairs over the - /// ledger's numbering, which the seeded call's facts need. - fn tool_passage_pairs( - &self, - desk: &mut ToolDesk, - note: &str, - candidate: &str, - ) -> Result<(String, Vec)> { - let mut pairs = Vec::new(); - let mut fresh = Vec::new(); - for pair in discover::passage_pairs(&self.conn, note, candidate, chat::WHY_PAIRS)? { - let a = self.ledger_marker(desk, pair.anchor_chunk_id, note, &mut fresh)?; - let c = self.ledger_marker(desk, pair.candidate_chunk_id, candidate, &mut fresh)?; - // A miss means a torn read against a concurrent reindex; skip the pair rather - // than name a passage with no text. - if let (Some(a), Some(c)) = (a, c) { - pairs.push((a, c, pair.score)); - } - } - if pairs.is_empty() { - return Ok(( - format!( - "No matched passages: {note} or {candidate} has no stored vectors to \ - compare yet. A reindex fills them." - ), - pairs, - )); - } - let mut out: Vec = pairs - .iter() - .enumerate() - .map(|(i, (a, c, score))| chat::pair_line(i + 1, *a, *c, *score)) - .collect(); - out.push(String::new()); - out.extend(self.fresh_blocks(desk, &fresh)); - Ok((out.join("\n"), pairs)) - } - - /// `b2_similar`: the ranked list for a note, as `similar` serves it. - fn tool_similar(&self, note: &str, limit: usize) -> Result { - let rows = self.similar(note, limit)?; - if rows.is_empty() { - return Ok(format!( - "B2 has no similar unlinked notes to suggest for {note}." - )); - } - Ok(rows - .iter() - .enumerate() - .map(|(i, r)| { - let strength = - r.z.map(|z| format!(", strength z = {z:.2}")) - .unwrap_or_default(); - format!("#{} {}{strength} — {}", i + 1, r.path, r.evidence) - }) - .collect::>() - .join("\n")) - } - - /// `b2_neighbors`: a note's direct links, as `neighbors` serves them. - fn tool_neighbors(&self, note: &str) -> Result { - let rows = self.neighbors(note)?; - if rows.is_empty() { - return Ok(format!("{note} is not linked to any note.")); - } - Ok(rows - .iter() - .map(|n| { - let why = n - .explanation - .as_deref() - .map(|e| format!(" — {e}")) - .unwrap_or_default(); - format!("{} {} ({}){why}", n.label, n.path, n.direction) - }) - .collect::>() - .join("\n")) - } - - /// `b2_read`: up to [`chat::READ_PASSAGES`] of a note's passages from `offset`, each - /// numbered on the ledger so the model can cite what it read. - fn tool_read(&self, desk: &mut ToolDesk, note: &str, offset: usize) -> Result { - let path = self.resolve_ref(note)?; - let ids = db::note_chunk_ids(&self.conn, &path)?; - let mut fresh = Vec::new(); - let mut markers = Vec::new(); - for id in ids.iter().skip(offset).take(chat::READ_PASSAGES) { - if let Some(m) = self.ledger_marker(desk, *id, &path, &mut fresh)? { - markers.push(m); - } - } - if markers.is_empty() { - return Ok(format!("{path} has no passages at offset {offset}.")); - } - let mut out = vec![format!( - "{path}: passages {}–{} of {}, numbered {}.", - offset + 1, - offset + markers.len(), - ids.len(), - markers - .iter() - .map(|m| format!("[{m}]")) - .collect::>() - .join(" ") - )]; - out.push(String::new()); - out.extend(self.fresh_blocks(desk, &fresh)); - Ok(out.join("\n")) - } - - /// The ledger number of a chunk, handing it over (and recording it in `fresh`) the - /// first time it is seen. `None` when the chunk's row is gone. - fn ledger_marker( - &self, - desk: &mut ToolDesk, - chunk_id: i64, - path: &str, - fresh: &mut Vec, - ) -> Result> { - if let Some(i) = desk.chunk_ids.iter().position(|id| *id == chunk_id) { - return Ok(Some(i + 1)); - } - let Some((heading_path, text)) = db::chunk_detail(&self.conn, chunk_id)? else { - return Ok(None); - }; - desk.chunk_ids.push(chunk_id); - desk.passages.push(ContextPassage { - path: path.to_string(), - heading_path, - text, - }); - fresh.push(desk.passages.len()); - Ok(Some(desk.passages.len())) - } - - /// The text of the passages a tool call handed over for the first time. A passage - /// already on the ledger is named by its marker only — the model has its text. - fn fresh_blocks(&self, desk: &ToolDesk, fresh: &[usize]) -> Vec { - fresh - .iter() - .filter_map(|m| { - Some(chat::passage_block( - *m, - desk.passages.get(m.checked_sub(1)?)?, - )) - }) - .collect() - } - - /// The shared tail of [`ask`](Self::ask) and the handoff half of - /// [`why_similar`](Self::why_similar): stream the completion, then resolve the - /// answer's `[n]` markers against the request's own passages. - fn stream_answer( - &self, - llm: &dyn LlmProvider, - req: &ChatRequest, - on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, - ) -> Result { - let completion = llm.complete(req, on_token).map_err(llm_error)?; - Ok(AnswerView { - citations: cite(&completion.text, &req.passages), - answer: completion.text, - cancelled: completion.cancelled, - tools: Vec::new(), - }) - } - - /// The shared retrieval core of [`search`](Self::search) and - /// [`search_chunks`](Self::search_chunks): hybrid when the embedding space + /// The shared retrieval core of [`search`](Self::search), + /// [`search_chunks`](Self::search_chunks) and + /// [`search_evidence_excluding`](Self::search_evidence_excluding): hybrid when the embedding space /// exists (failing fast on a model mismatch), BM25-only on a /// projected-but-unembedded vault. fn retrieve(&self, query: &str, pool: usize) -> Result { @@ -1841,15 +867,7 @@ impl Vault { // Never claim a statistic over a fake-embedded space (`grades`). Grading changes // what the rows carry, never which rows exist (ADR-0014). let grade = self.grades()?; - // A resource anchor is honest, not silent: resources become discoverable once - // they have chunks + centroids. Until then an inventoried resource errs "not - // yet" — never an empty result — and an unknown path falls through to - // not-found. - if crate::resource::doc_kind(note_ref) == crate::resource::DocKind::Resource - && db::resource_detail(&self.conn, note_ref)?.is_some() - { - return Err(Error::ResourceUnsupported(note_ref.to_string())); - } + self.refuse_resource_anchor(note_ref)?; let anchor = self.resolve_ref(note_ref)?; let mut out = Vec::new(); for c in discover::candidates(&self.conn, &anchor, limit, grade)? { @@ -1890,11 +908,7 @@ impl Vault { limit ) .entered(); - if crate::resource::doc_kind(anchor_ref) == crate::resource::DocKind::Resource - && db::resource_detail(&self.conn, anchor_ref)?.is_some() - { - return Err(Error::ResourceUnsupported(anchor_ref.to_string())); - } + self.refuse_resource_anchor(anchor_ref)?; let anchor = self.resolve_ref(anchor_ref)?; let candidate = self.resolve_ref(candidate_ref)?; let ex = discover::explain(&self.conn, &anchor, &candidate, limit, self.grades()?)?; @@ -1933,20 +947,11 @@ impl Vault { let shared_neighbors = self .shared_neighbors(&anchor, &candidate)? .into_iter() - .map(|path| { - let title = db::note_title(&self.conn, &path)?; - Ok(NoteSummary { path, title }) - }) + .map(|path| self.note_summary(path)) .collect::>>()?; Ok(SimilarExplainView { - anchor: NoteSummary { - title: db::note_title(&self.conn, &anchor)?, - path: anchor, - }, - candidate: NoteSummary { - title: db::note_title(&self.conn, &candidate)?, - path: candidate, - }, + anchor: self.note_summary(anchor)?, + candidate: self.note_summary(candidate)?, limit, standing, z: ex.z, @@ -1973,15 +978,27 @@ impl Vault { .map(|(heading_path, text)| PassageView { heading_path, text })) } + /// A resolved note path with its title, for a view that names a note. + fn note_summary(&self, path: String) -> Result { + let title = db::note_title(&self.conn, &path)?; + Ok(NoteSummary { path, title }) + } + + /// Every note `note` is directly linked with, in either direction, by path. + fn neighbor_paths(&self, note: &str) -> Result> { + Ok(graph::neighbors(&self.conn, note)? + .into_iter() + .map(|n| n.other) + .collect()) + } + /// The notes both `a` and `b` are linked with, in either direction, by path. fn shared_neighbors(&self, a: &str, b: &str) -> Result> { - let of = |note: &str| -> Result> { - Ok(graph::neighbors(&self.conn, note)? - .into_iter() - .map(|n| n.other) - .collect()) - }; - Ok(of(a)?.intersection(&of(b)?).cloned().collect()) + Ok(self + .neighbor_paths(a)? + .intersection(&self.neighbor_paths(b)?) + .cloned() + .collect()) } /// Commit a typed connection `src --type--> dst` (`b2 link`, flow ③): append a @@ -2196,6 +1213,27 @@ impl Vault { .query_row("SELECT strftime('%Y-%m-%d','now')", [], |r| r.get(0))?) } + /// The inventory row for `path`, or [`Error::ResourceNotFound`]: the one existence + /// check every resource op makes before touching the filesystem, so a path a note + /// authored can never talk B2 into reading or moving a file outside the inventory. + fn require_resource(&self, path: &str) -> Result { + db::resource_detail(&self.conn, path)? + .ok_or_else(|| Error::ResourceNotFound(path.to_string())) + } + + /// Discovery's anchor guard. A resource anchor is honest, not silent: resources + /// become discoverable once they have chunks + centroids. Until then an inventoried + /// resource errs "not yet" ([`Error::ResourceUnsupported`]) — never an empty result — + /// and an unknown path falls through to the caller's not-found. + fn refuse_resource_anchor(&self, anchor_ref: &str) -> Result<()> { + if crate::resource::doc_kind(anchor_ref) == crate::resource::DocKind::Resource + && db::resource_detail(&self.conn, anchor_ref)?.is_some() + { + return Err(Error::ResourceUnsupported(anchor_ref.to_string())); + } + Ok(()) + } + /// Resolve a note reference to the indexed note's vault-relative path — which *is* /// its identity (ADR-0003), so this is one canonicalization rather than the /// two-step "is it an id? is it a path?" it replaced. The ref may be written with @@ -2212,107 +1250,3 @@ impl Vault { fn revision_of(raw: &str) -> String { blake3::hash(raw.as_bytes()).to_hex().to_string() } - -/// Flatten a chunk's text to a single whitespace-collapsed line. -fn flatten(text: &str) -> String { - text.split_whitespace().collect::>().join(" ") -} - -/// The head of a flattened chunk, bounded to one line. -fn head_snippet(flat: &str) -> String { - if flat.chars().count() <= SNIPPET_CHARS { - flat.to_string() - } else { - let cut: String = flat.chars().take(SNIPPET_CHARS).collect(); - format!("{}…", cut.trim_end()) - } -} - -/// Collapse a chunk's text to a single-line, length-bounded snippet (its head). Used -/// where there is no query to center on (e.g. `similar`'s evidence passage). -/// The state one tool-using turn carries between calls: whose turn it is (what `{}` -/// arguments default to) and the **passage ledger** — every passage a tool has handed -/// the model, numbered once in first-seen order, which is what `[n]` resolves against. -struct ToolDesk { - anchor: String, - candidate: String, - limit: usize, - /// Parallel to `passages`: the chunk behind each, so a passage is never numbered twice. - chunk_ids: Vec, - passages: Vec, -} - -/// B2's own pair lookup: the call as the model would have made it, its result text, and -/// the pairs over the ledger's numbering. -struct SeededLookup { - call: ToolCall, - result: String, - pairs: Vec, -} - -/// Normalize a provider failure: the trait returns the crate-wide `Result`, but every -/// failure of a model call is a failed model call, and adapters match [`Error::Llm`] for -/// the "can't reach the model server" message. Enforced here, not hoped for. -fn llm_error(e: Error) -> Error { - match e { - Error::Llm(_) | Error::ToolCallLimit { .. } => e, - other => Error::Llm(other.to_string()), - } -} - -/// Resolve an answer's distinct `[n]` markers against the passages it was grounded in. -fn cite(answer: &str, passages: &[ContextPassage]) -> Vec { - chat::cited_markers(answer, passages.len()) - .into_iter() - .filter_map(|marker| { - // 1-based marker to 0-based passage; skip rather than index, so even a - // broken invariant degrades to a missing citation. - let p = passages.get(marker.checked_sub(1)?)?; - Some(Citation { - marker, - path: p.path.clone(), - excerpt: snippet(&p.text), - }) - }) - .collect() -} - -fn snippet(text: &str) -> String { - head_snippet(&flatten(text)) -} - -/// Like [`snippet`] but windows the excerpt around the first query-term match, so a -/// section-sized chunk still surfaces the matched text instead of only its head. -/// Falls back to the head when no term matches or the match is already in view — a -/// pure vector hit keeps the head. -fn query_snippet(text: &str, query: &str) -> String { - let flat = flatten(text); - if flat.chars().count() <= SNIPPET_CHARS { - return flat; - } - let lower = flat.to_lowercase(); - let match_pos = query - .split(|c: char| !c.is_alphanumeric()) - .filter(|t| t.chars().count() >= 2) - .filter_map(|t| { - let byte = lower.find(&t.to_lowercase())?; - Some(lower[..byte].chars().count()) - }) - .min(); - // A little lead-in so the match is not flush against the ellipsis. - const LEAD: usize = 24; - let Some(pos) = match_pos.filter(|p| *p > LEAD) else { - return head_snippet(&flat); - }; - let chars: Vec = flat.chars().collect(); - // `pos` indexes the lowercased text, whose length can differ from `flat` for - // exotic Unicode; clamp so the slice below can never go out of range. - let start = (pos - LEAD).min(chars.len()); - let end = (start + SNIPPET_CHARS).min(chars.len()); - let mut out = String::from("…"); - out.extend(&chars[start..end]); - if end < chars.len() { - out.push('…'); - } - out -} diff --git a/crates/b2-core/src/vault/chat.rs b/crates/b2-core/src/vault/chat.rs new file mode 100644 index 0000000..689ad88 --- /dev/null +++ b/crates/b2-core/src/vault/chat.rs @@ -0,0 +1,552 @@ +//! Flow ④ on the façade: grounded chat ([`Vault::ask`]) and the one tool-using turn, +//! **Why?** on a *Similar & unlinked* card ([`Vault::why_similar`], ADR-0022). The prompt +//! text and tool schemas live in [`crate::chat`]; this is the orchestration over the +//! façade's own reads. Chat is a reader: nothing model-derived is stored. + +use super::{AnswerView, Citation, ToolUseView, Vault}; +use crate::chat; +use crate::db; +use crate::discover; +use crate::error::{Error, Result}; +use crate::llm::{ChatRequest, ChatTurn, ContextPassage, LlmProvider, ToolCall, ToolExchange}; +use crate::snippet::snippet; +use std::ops::ControlFlow; + +impl Vault { + /// Flow ④ — grounded chat over the vault: condense → retrieve → assemble → + /// stream → cite, orchestrated here over the core logic in [`crate::chat`]. The + /// provider is injected **per call**: chat is its sole consumer and, unlike the + /// embedder, it carries no index identity (contrast ADR-0007), so nothing about + /// it belongs on the open vault. + /// + /// - **Condense** (multi-turn only): one provider call rewrites the follow-up + /// into a standalone retrieval query; on failure it degrades to the raw + /// question, so that step can never break chat. + /// - **Retrieve**: [`search_chunks`](Self::search_chunks) at + /// [`chat::ASK_PASSAGES`], holding `search`'s posture — chat is a reader. + /// - **Stream**: tokens flow up through `on_token` as they arrive; returning + /// `ControlFlow::Break(())` cancels at token granularity and the result reports + /// it ([`AnswerView::cancelled`]). + /// - **Cite**: distinct `[n]` markers resolve to `(path, excerpt)`; a + /// hallucinated marker resolves to nothing, and the text is never rewritten. + /// + /// Nothing model-derived is stored anywhere, and history is the caller's, + /// session-only. Errors: retrieval as `search` raises it; a failed *answer* call + /// as [`Error::Llm`](crate::Error::Llm). + pub fn ask( + &self, + llm: &dyn LlmProvider, + question: &str, + history: &[ChatTurn], + on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, + ) -> Result { + let _op = tracing::debug_span!( + target: "b2::vault", + "ask", + question, + multi_turn = !history.is_empty() + ) + .entered(); + let query = if history.is_empty() { + question.to_string() + } else { + chat::condense_query(llm, question, history) + }; + let passages: Vec = self + .search_chunks(&query, chat::ASK_PASSAGES)? + .into_iter() + .map(|c| ContextPassage { + path: c.path, + heading_path: c.heading_path, + text: c.text, + }) + .collect(); + tracing::debug!( + target: "b2::chat", + passages = passages.len(), + "retrieved grounding passages" + ); + let req = chat::build_request(question, history, passages); + self.stream_answer(llm, &req, on_token) + } + + /// **Why was this suggested?** — the chat answer behind one *Similar & unlinked* + /// row: explain, grounded and cited, why `candidate_ref` surfaced for `anchor_ref`. + /// + /// A **tool-using** turn (ADR-0022). The model is offered B2's read-only tools + /// ([`chat::why_tools`]: `b2_passage_pairs`, `b2_similar`, `b2_neighbors`, `b2_read`) + /// and makes the lookups itself; each call is one read on this façade, run here and + /// replayed to the model with its result. Every argument defaults to this turn's + /// pair, so `{}` is always a valid call. Three things bound that: + /// + /// - **Round 1 is the lookup round.** Its text is never streamed: a model that + /// answers there answered from nothing. If it calls no tool — or the call fails, + /// most often a model with no tool support — the turn degrades to + /// [`chat::build_why_request`]: B2 makes the pair lookup and hands the evidence over + /// in one plain grounded request. If it calls tools but skips the pair lookup, B2 + /// appends that call itself (`seeded`), because the matched pairs are what the row + /// was ranked on and no explanation is written without them. + /// - **The loop is bounded** at [`chat::MAX_TOOL_ROUNDS`] model calls and + /// [`chat::MAX_CALLS_PER_ROUND`] tool calls each, and the last round offers no + /// tools, so it can only answer. + /// - **The row's position rides in the prompt** — rank and strength from + /// [`similar`](Self::similar) at the `limit` the surface showed, so the explanation + /// describes the card that was clicked. + /// + /// One consequence of the hidden first round: a cancel lands when the first visible + /// token arrives, not during the lookups. + /// + /// A tool call is model output, so it is untrusted: an unknown tool, malformed + /// arguments or an unknown note is answered with an `error:` *result* the model can + /// read, never a failed turn. Every passage a tool hands over is numbered once on one + /// ledger, and the answer's `[n]` markers resolve against it. Streaming, cancellation + /// and the [`Error::Llm`] normalization are [`ask`](Self::ask)'s — chat is a reader + /// here too, and nothing is stored. [`Error::NoteNotFound`] for an unknown ref on + /// either side. + pub fn why_similar( + &self, + llm: &dyn LlmProvider, + anchor_ref: &str, + candidate_ref: &str, + limit: usize, + on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, + ) -> Result { + let _op = tracing::debug_span!( + target: "b2::vault", + "why_similar", + anchor = anchor_ref, + candidate = candidate_ref, + limit + ) + .entered(); + let anchor = self.resolve_ref(anchor_ref)?; + let candidate = self.resolve_ref(candidate_ref)?; + + // The row as the surface showed it: same call, same limit, so the same rank + z. + let served = self.similar(&anchor, limit)?; + let row = served.iter().position(|s| s.path == candidate); + let mut desk = ToolDesk { + limit, + chunk_ids: Vec::new(), + passages: Vec::new(), + }; + let mut facts = chat::WhyFacts { + anchor_title: db::note_title(&self.conn, &anchor)?, + candidate_title: db::note_title(&self.conn, &candidate)?, + rank: row.map(|i| (i + 1, served.len())), + z: row.and_then(|i| served.get(i)).and_then(|s| s.z), + linked: self.neighbor_paths(&anchor)?.contains(&candidate), + shared_neighbors: self + .shared_neighbors(&anchor, &candidate)? + .into_iter() + .collect(), + // The handoff's half, filled only if the turn degrades to it. + embedded: true, + pairs: Vec::new(), + anchor_path: anchor, + candidate_path: candidate, + }; + + let mut req = + chat::build_why_agent_request(&facts, chat::why_tools(), Vec::new(), Vec::new()); + let mut tools_used: Vec = Vec::new(); + let mut answer = String::new(); + let mut cancelled = false; + for round in 1..=chat::MAX_TOOL_ROUNDS { + if round == chat::MAX_TOOL_ROUNDS { + req.tools.clear(); // nothing left to do but answer + } + // Round 1 is the lookup round, and its text is never shown: a model that + // answers there has answered from nothing, and one that calls tools says at + // most "let me check". From round 2 on, tokens stream as they arrive. + let completion = if round == 1 { + llm.complete(&req, &mut |_| ControlFlow::Continue(())) + } else { + llm.complete(&req, on_token) + }; + let completion = match completion { + Ok(c) if round == 1 && c.tool_calls.is_empty() => { + tracing::debug!( + target: "b2::chat", + "the model called no tool; handing the evidence over instead" + ); + return self.why_handoff(llm, &mut facts, desk, tools_used, on_token); + } + Ok(c) => c, + // Never degraded: a reply past the tool-call cap is a broken or hostile + // server, and a quiet handoff would hide it. + Err(e @ Error::ToolCallLimit { .. }) => return Err(e), + // The lookup round failed: most often a model with no tool support. + // If the server is simply down, the handoff fails the same way, honestly. + Err(e) if round == 1 => { + tracing::debug!( + target: "b2::chat", + error = %e, + "the tool round failed; handing the evidence over without tools" + ); + return self.why_handoff(llm, &mut facts, desk, tools_used, on_token); + } + Err(e) => return Err(llm_error(e)), + }; + if round > 1 { + answer.push_str(&completion.text); + } + if completion.cancelled { + cancelled = true; + break; + } + if completion.tool_calls.is_empty() || req.tools.is_empty() { + break; + } + for call in completion + .tool_calls + .into_iter() + .take(chat::MAX_CALLS_PER_ROUND) + { + let result = self.run_tool(&mut desk, &facts, &call); + tracing::debug!( + target: "b2::chat", + tool = call.name, + failed = result.starts_with("error:"), + "ran a tool for the model" + ); + tools_used.push(ToolUseView { + name: call.name.clone(), + arguments: call.arguments.clone(), + seeded: false, + }); + req.exchanges.push(ToolExchange { call, result }); + } + // Whatever the model chose to look up, the matched pairs for *this* pair are + // what the row was ranked on. If the lookup round skipped them, B2 makes that + // call itself, so no explanation is written without them on the desk. + if round == 1 && desk.passages.is_empty() { + let seeded = self.seed_pairs(&mut desk, &facts)?; + tools_used.push(ToolUseView { + name: seeded.call.name.clone(), + arguments: seeded.call.arguments.clone(), + seeded: true, + }); + req.exchanges.push(ToolExchange { + call: seeded.call, + result: seeded.result, + }); + } + req.passages.clone_from(&desk.passages); + } + Ok(AnswerView { + citations: cite(&answer, &desk.passages), + answer, + cancelled, + tools: tools_used, + }) + } + + /// B2's own `b2_passage_pairs` call for the turn's pair, as the exchange it is. + fn seed_pairs(&self, desk: &mut ToolDesk, facts: &chat::WhyFacts) -> Result { + let call = ToolCall { + id: "b2_seed_1".to_string(), + name: chat::TOOL_PASSAGE_PAIRS.to_string(), + arguments: serde_json::json!({ + "note": facts.anchor_path, + "candidate": facts.candidate_path, + }) + .to_string(), + }; + let (result, pairs) = + self.tool_passage_pairs(desk, &facts.anchor_path, &facts.candidate_path)?; + Ok(SeededLookup { + call, + result, + pairs, + }) + } + + /// The degrade of a tool-using why-turn: B2 makes the pair lookup itself and hands + /// the evidence over in one plain grounded request — for a model with no tool + /// support, and for one that was offered tools and used none. + fn why_handoff( + &self, + llm: &dyn LlmProvider, + facts: &mut chat::WhyFacts, + mut desk: ToolDesk, + mut tools_used: Vec, + on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, + ) -> Result { + let seeded = self.seed_pairs(&mut desk, facts)?; + tools_used.push(ToolUseView { + name: seeded.call.name, + arguments: seeded.call.arguments, + seeded: true, + }); + facts.embedded = !seeded.pairs.is_empty(); + facts.pairs = seeded.pairs; + let req = chat::build_why_request(facts, desk.passages); + let mut view = self.stream_answer(llm, &req, on_token)?; + view.tools = tools_used; + Ok(view) + } + + /// Run one tool call for the model and render its result as text. Infallible by + /// design: the call is model output, so every failure — an unknown tool, arguments + /// that aren't a JSON object, a note that doesn't exist, an index error — becomes an + /// `error:` result the model reads, and the turn carries on. + fn run_tool(&self, desk: &mut ToolDesk, facts: &chat::WhyFacts, call: &ToolCall) -> String { + let args: serde_json::Value = if call.arguments.trim().is_empty() { + serde_json::json!({}) + } else { + match serde_json::from_str(&call.arguments) { + Ok(v @ serde_json::Value::Object(_)) => v, + _ => return "error: the arguments were not a JSON object".to_string(), + } + }; + let text_arg = |key: &str| args.get(key).and_then(|v| v.as_str()).map(str::to_string); + let note = text_arg("note"); + let ran = match call.name.as_str() { + chat::TOOL_PASSAGE_PAIRS => { + let note = note.unwrap_or_else(|| facts.anchor_path.clone()); + let candidate = + text_arg("candidate").unwrap_or_else(|| facts.candidate_path.clone()); + self.resolve_ref(¬e).and_then(|n| { + let c = self.resolve_ref(&candidate)?; + Ok(self.tool_passage_pairs(desk, &n, &c)?.0) + }) + } + chat::TOOL_SIMILAR => { + let note = note.unwrap_or_else(|| facts.anchor_path.clone()); + self.tool_similar(¬e, desk.limit) + } + chat::TOOL_NEIGHBORS => { + let note = note.unwrap_or_else(|| facts.anchor_path.clone()); + self.tool_neighbors(¬e) + } + chat::TOOL_READ => { + // The open note is already on the human's screen; the one worth reading + // by default is the suggestion. + let note = note.unwrap_or_else(|| facts.candidate_path.clone()); + let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0); + self.tool_read(desk, ¬e, offset as usize) + } + other => return format!("error: unknown tool `{other}`"), + }; + ran.unwrap_or_else(|e| match e { + Error::NoteNotFound(n) => format!("error: note not found: {n}"), + // Internals stay out of the model's context as they stay out of the UI. + _ => "error: B2 could not run that lookup".to_string(), + }) + } + + /// `b2_passage_pairs`: the matched pairs between two notes, then each passage they + /// name that the ledger had not handed over yet. Also returns the pairs over the + /// ledger's numbering, which the seeded call's facts need. + fn tool_passage_pairs( + &self, + desk: &mut ToolDesk, + note: &str, + candidate: &str, + ) -> Result<(String, Vec)> { + let mut pairs = Vec::new(); + let mut fresh = Vec::new(); + for pair in discover::passage_pairs(&self.conn, note, candidate, chat::WHY_PAIRS)? { + let a = self.ledger_marker(desk, pair.anchor_chunk_id, note, &mut fresh)?; + let c = self.ledger_marker(desk, pair.candidate_chunk_id, candidate, &mut fresh)?; + // A miss means a torn read against a concurrent reindex; skip the pair rather + // than name a passage with no text. + if let (Some(a), Some(c)) = (a, c) { + pairs.push((a, c, pair.score)); + } + } + if pairs.is_empty() { + return Ok(( + format!( + "No matched passages: {note} or {candidate} has no stored vectors to \ + compare yet. A reindex fills them." + ), + pairs, + )); + } + let mut out: Vec = pairs + .iter() + .enumerate() + .map(|(i, (a, c, score))| chat::pair_line(i + 1, *a, *c, *score)) + .collect(); + out.push(String::new()); + out.extend(self.fresh_blocks(desk, &fresh)); + Ok((out.join("\n"), pairs)) + } + + /// `b2_similar`: the ranked list for a note, as `similar` serves it. + fn tool_similar(&self, note: &str, limit: usize) -> Result { + let rows = self.similar(note, limit)?; + if rows.is_empty() { + return Ok(format!( + "B2 has no similar unlinked notes to suggest for {note}." + )); + } + Ok(rows + .iter() + .enumerate() + .map(|(i, r)| { + let strength = + r.z.map(|z| format!(", strength z = {z:.2}")) + .unwrap_or_default(); + format!("#{} {}{strength} — {}", i + 1, r.path, r.evidence) + }) + .collect::>() + .join("\n")) + } + + /// `b2_neighbors`: a note's direct links, as `neighbors` serves them. + fn tool_neighbors(&self, note: &str) -> Result { + let rows = self.neighbors(note)?; + if rows.is_empty() { + return Ok(format!("{note} is not linked to any note.")); + } + Ok(rows + .iter() + .map(|n| { + let why = n + .explanation + .as_deref() + .map(|e| format!(" — {e}")) + .unwrap_or_default(); + format!("{} {} ({}){why}", n.label, n.path, n.direction) + }) + .collect::>() + .join("\n")) + } + + /// `b2_read`: up to [`chat::READ_PASSAGES`] of a note's passages from `offset`, each + /// numbered on the ledger so the model can cite what it read. + fn tool_read(&self, desk: &mut ToolDesk, note: &str, offset: usize) -> Result { + let path = self.resolve_ref(note)?; + let ids = db::note_chunk_ids(&self.conn, &path)?; + let mut fresh = Vec::new(); + let mut markers = Vec::new(); + for id in ids.iter().skip(offset).take(chat::READ_PASSAGES) { + if let Some(m) = self.ledger_marker(desk, *id, &path, &mut fresh)? { + markers.push(m); + } + } + if markers.is_empty() { + return Ok(format!("{path} has no passages at offset {offset}.")); + } + let mut out = vec![format!( + "{path}: passages {}–{} of {}, numbered {}.", + offset + 1, + offset + markers.len(), + ids.len(), + markers + .iter() + .map(|m| format!("[{m}]")) + .collect::>() + .join(" ") + )]; + out.push(String::new()); + out.extend(self.fresh_blocks(desk, &fresh)); + Ok(out.join("\n")) + } + + /// The ledger number of a chunk, handing it over (and recording it in `fresh`) the + /// first time it is seen. `None` when the chunk's row is gone. + fn ledger_marker( + &self, + desk: &mut ToolDesk, + chunk_id: i64, + path: &str, + fresh: &mut Vec, + ) -> Result> { + if let Some(i) = desk.chunk_ids.iter().position(|id| *id == chunk_id) { + return Ok(Some(i + 1)); + } + let Some((heading_path, text)) = db::chunk_detail(&self.conn, chunk_id)? else { + return Ok(None); + }; + desk.chunk_ids.push(chunk_id); + desk.passages.push(ContextPassage { + path: path.to_string(), + heading_path, + text, + }); + fresh.push(desk.passages.len()); + Ok(Some(desk.passages.len())) + } + + /// The text of the passages a tool call handed over for the first time. A passage + /// already on the ledger is named by its marker only — the model has its text. + fn fresh_blocks(&self, desk: &ToolDesk, fresh: &[usize]) -> Vec { + fresh + .iter() + .filter_map(|m| { + Some(chat::passage_block( + *m, + desk.passages.get(m.checked_sub(1)?)?, + )) + }) + .collect() + } + + /// The shared tail of [`ask`](Self::ask) and the handoff half of + /// [`why_similar`](Self::why_similar): stream the completion, then resolve the + /// answer's `[n]` markers against the request's own passages. + fn stream_answer( + &self, + llm: &dyn LlmProvider, + req: &ChatRequest, + on_token: &mut dyn FnMut(&str) -> ControlFlow<()>, + ) -> Result { + let completion = llm.complete(req, on_token).map_err(llm_error)?; + Ok(AnswerView { + citations: cite(&completion.text, &req.passages), + answer: completion.text, + cancelled: completion.cancelled, + tools: Vec::new(), + }) + } +} + +/// The state one tool-using turn carries between calls: the list length the surface +/// showed, and the **passage ledger** — every passage a tool has handed the model, +/// numbered once in first-seen order, which is what `[n]` resolves against. Whose turn it +/// is (what `{}` arguments default to) rides on the turn's [`chat::WhyFacts`]. +struct ToolDesk { + limit: usize, + /// Parallel to `passages`: the chunk behind each, so a passage is never numbered twice. + chunk_ids: Vec, + passages: Vec, +} + +/// B2's own pair lookup: the call as the model would have made it, its result text, and +/// the pairs over the ledger's numbering. +struct SeededLookup { + call: ToolCall, + result: String, + pairs: Vec, +} + +/// Normalize a provider failure: the trait returns the crate-wide `Result`, but every +/// failure of a model call is a failed model call, and adapters match [`Error::Llm`] for +/// the "can't reach the model server" message. Enforced here, not hoped for. +fn llm_error(e: Error) -> Error { + match e { + Error::Llm(_) | Error::ToolCallLimit { .. } => e, + other => Error::Llm(other.to_string()), + } +} + +/// Resolve an answer's distinct `[n]` markers against the passages it was grounded in. +fn cite(answer: &str, passages: &[ContextPassage]) -> Vec { + chat::cited_markers(answer, passages.len()) + .into_iter() + .filter_map(|marker| { + // 1-based marker to 0-based passage; skip rather than index, so even a + // broken invariant degrades to a missing citation. + let p = passages.get(marker.checked_sub(1)?)?; + Some(Citation { + marker, + path: p.path.clone(), + excerpt: snippet(&p.text), + }) + }) + .collect() +} diff --git a/crates/b2-core/src/vault/views.rs b/crates/b2-core/src/vault/views.rs new file mode 100644 index 0000000..adfab12 --- /dev/null +++ b/crates/b2-core/src/vault/views.rs @@ -0,0 +1,477 @@ +//! The display-ready views the façade returns: what the CLI prints (human and `--json`) +//! and what the desktop reuses as its IPC contract (`ui/src/types.ts` mirrors them). + +use crate::ingest::SkippedNote; +use serde::Serialize; + +// Doc links only: the views describe what these produce. +#[cfg(doc)] +use super::Vault; +#[cfg(doc)] +use crate::{graph, note}; + +/// What `reindex` did: notes projected, and how many were actually (re)embedded +/// (the rest reused their vectors). It reports no vault writes because there are +/// none — a reindex reads (ADR-0004). +/// +/// `cancelled` marks a cooperative cancel of the embed phase; the counts then +/// describe the partial work truthfully and the index is still consistent (keyword +/// and graph complete, a prefix embedded). Always `false` for +/// [`reindex`](Vault::reindex). +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ReindexReport { + pub indexed: usize, + pub embedded: usize, + pub cancelled: bool, + /// Files skipped as unreadable this run — one bad file never fails the pass. + pub skipped: Vec, + /// Ghost rows pruned: notes whose files were deleted outside b2 (#31), so + /// incremental equals a from-scratch rebuild. + pub notes_pruned: usize, + pub resources_indexed: usize, + pub resources_pruned: usize, +} + +/// What [`project`](Vault::project) did — the model-free half of a reindex. No +/// embed counts: projection never touches vectors. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ProjectReport { + pub indexed: usize, + /// Files skipped as unreadable this pass, so an adapter can say which and why. + pub skipped: Vec, + /// Ghost rows pruned: notes whose files were deleted outside b2 (#31). + pub notes_pruned: usize, + pub resources_indexed: usize, + pub resources_pruned: usize, +} + +/// What [`embed`](Vault::embed) did — the model-bound half of a reindex: notes +/// whose missing vectors were filled, and whether a cooperative cancel cut the pass +/// short (the counts stay truthful, and a re-run embeds exactly the remainder). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct EmbedReport { + pub embedded: usize, + pub cancelled: bool, +} + +/// The vault's embedding coverage — the honest "N/M embedded" signal (#26). +/// Model-free: a pure count over the projection, so an adapter can say +/// "keyword-only for now" precisely without loading a model. `embedded < total` +/// means [`search`](Vault::search) is running keyword-first over the remainder; +/// `embedded == 0` is a fully keyword-only vault. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct EmbedStatus { + /// Notes with no chunk awaiting a vector. A note with **no chunks at all** — an empty + /// body — counts here: there is no vector it could be waiting for, so it must be able + /// to reach the numerator (see [`crate::db::embed_progress`]). + pub embedded: usize, + /// Every projected note (the denominator). + pub total: usize, +} + +/// What a reindex **would** do — the `reindex --dry-run` preview, computed +/// read-only. The `would_*` keys are the honesty signal: this is a forecast. +/// +/// It forecasts work and nothing else. The dry-run's old columns (which notes would +/// be stamped, which files collide) existed because a real run wrote to the vault; +/// it no longer does (ADR-0004), so only the embedding is left to size. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ReindexPlan { + /// Notes a real reindex would project (every `.md` file the walk collects). + pub would_index: usize, + /// …of which this many would be (re)embedded. + pub would_embed: usize, +} + +/// One neighbor of a note, resolved for display: the note at the other end of an +/// edge, with its path + title, so the adapter stays a dumb printer. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct NeighborView { + /// The other note's vault-relative path — its identity (ADR-0003). + pub path: String, + pub title: Option, + /// The stored relation verb (outbound direction of the edge). + pub relation: String, + /// `"outbound"` (this note → other) or `"inbound"`. + pub direction: String, + /// Display label: the verb outbound, its inverse inbound (ADR-0010). + pub label: String, + pub explanation: Option, + /// Edge origin: `"inline"` (a body link) or `"frontmatter"` (ADR-0010). + pub origin: String, + /// The other note's `created` date, resolved from the projection (GH #22). + pub created: Option, +} + +/// One outbound link a note authors at a **resource** (an image, a PDF — any +/// non-`.md` vault file), resolved for display. Surfaced on [`ExplainView`] so a +/// note's file links are visible from the note's side, not only as the resource's +/// backlinks (GH #22). Distinct from [`NeighborView`]: a resource has no title and +/// authors no edges, so these are always outbound. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResourceLinkView { + pub path: String, + /// Its inventory class (`image`/`pdf`/`html`/`text`/`media`/`binary`). + pub class: String, + pub relation: String, + pub origin: String, + /// The authored caption (alt text / `|caption`), if any. + pub caption: Option, + /// Whether the link is an embed (`![…]` / `![[…]]`). + pub embed: bool, + pub explanation: Option, +} + +/// One outbound link a note authored that resolves to **nothing** — no note and no +/// resource at its target (a `[[Hermes]]` naming a *folder*, or a typo). A note is +/// one `.md` file, so a folder is never a valid target; rather than drop such a link +/// B2 surfaces it as unresolved, so it reads as broken rather than missing (GH #12). +/// It has no `path` — that is the whole point. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct UnresolvedLink { + /// The target exactly as written in the Markdown (`[[target]]`). + pub target: String, + pub relation: String, + pub origin: String, + pub explanation: Option, +} + +/// A note's full connection picture for `b2 explain`: the note itself, every active +/// connection with its "why", its outbound resource links, and any unresolved +/// outbound links. A thin header over [`NeighborView`] — it reuses the per-edge +/// shape `neighbors` returns rather than a parallel one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExplainView { + pub path: String, + pub title: Option, + /// Outbound edges first, then inbound (as [`graph::neighbors`] orders them). + pub connections: Vec, + /// Outbound links at resources — the third target kind (GH #22). + pub resources: Vec, + /// Outbound links that resolved to nothing (GH #12). + pub unresolved: Vec, +} + +/// A note's content + display metadata for a reader. Carries the note's identity, +/// the frontmatter fields worth showing a human, and the **raw Markdown body read +/// from disk** (the source of truth, not the projection) so an adapter renders +/// Markdown itself. A pure read — no embedding. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct NoteView { + pub path: String, + pub title: Option, + pub r#type: Option, + pub created: Option, + pub updated: Option, + pub tags: Vec, + /// The note's Markdown body (frontmatter stripped), verbatim from disk. + pub body: String, + /// The raw frontmatter YAML **verbatim** (the text between the `---` fences, + /// fences excluded), not a re-serialization of the projected fields above — so + /// `b2_relations:` and any keys B2 doesn't model show as written. + pub frontmatter: Option, + /// Whether that block *reads* as YAML metadata + /// ([`note::ParsedNote::frontmatter_readable`]): `false` means the raw bytes + /// above are shown verbatim but the projected fields came back empty. Every + /// read passes through here, so an external hand-edit surfaces the same warning + /// as an in-app save (GH #79). + pub frontmatter_readable: bool, + /// blake3 of the **raw file bytes** at read time — the save-guard token + /// [`write`](Vault::write) validates, so a save can never silently clobber an + /// external edit. Whole-file, so *any* out-of-band change conflicts honestly. + pub revision: String, +} + +/// One note's identity for a listing — `path` + `title`, with **no body**: enough +/// to show and open a note, cheap enough to fetch the whole vault at once. The body +/// is a separate [`read`](Vault::read) when a note is opened. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct NoteSummary { + pub path: String, + pub title: Option, +} + +/// One resource's identity for the file tree — the per-kind sibling of +/// [`NoteSummary`], never a union type. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResourceSummary { + pub path: String, + pub class: String, + pub size: i64, + pub mtime: Option, +} + +/// The resource fallback card's data: inventory metadata plus inbound backlinks. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResourceExplainView { + pub path: String, + pub class: String, + pub size: i64, + pub mtime: Option, + pub content_hash: String, + pub backlinks: Vec, +} + +/// One note that links at a resource, with the edge's authored context. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResourceBacklink { + pub path: String, + pub title: Option, + pub r#type: String, + pub caption: Option, + pub embed: bool, +} + +/// One search hit, resolved to the note it belongs to with a text snippet. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SearchResult { + pub path: String, + pub title: Option, + /// Fused relevance score; higher is better. + pub score: f64, + /// A one-line excerpt of the matched chunk. + pub snippet: String, +} + +/// A search's evidence reading — [`Vault::search_evidence`]'s return, and what a +/// surface needs to decide what it vouches for (ADR-0015). +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SearchEvidenceView { + /// The served results, whole and in fused order — evidence never reorders or + /// removes a row. + pub results: Vec, + /// Does the vault hold positive evidence for this query at the active model's + /// bar? `None` = no calibrated bar for this embedder, so no verdict is offered + /// rather than one guessed. + pub vouched: Option, + /// Chunks in the index — the scale every term's weight is read against. + pub chunk_total: usize, + /// Every query term with its document frequency, in query order. + pub terms: Vec, + /// Best cosine between the query and any chunk vector — the dense half's + /// absolute claim. `None` on a projected-but-unembedded vault. + pub best_cos: Option, +} + +/// One served result with the provenance RRF discarded — which lists ranked its +/// chunk, and how near its vector actually was. +/// +/// The query-level verdict on [`SearchEvidenceView`] is what ADR-0015's "no matches" +/// rests on; **this** is what the per-hit tail bake-off is argued from (GH #206). +/// That bake-off has run — the labels carry the per-hit depth (`tail_relevant`) — +/// and **no tail fold shipped**: the fused order is not an evidence order, so every +/// admissible prefix cut proved vacuous, and the tail complaint is ordering work +/// (the reranker seam), not disclosure work. This stays an instrument reading, +/// re-priced on every `make eval` run. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct EvidencedResult { + #[serde(flatten)] + pub result: SearchResult, + /// 0-based rank in the BM25 list; `None` = the lexical half never ranked it. + pub bm25_rank: Option, + /// 0-based rank in the dense list; `None` = the vector half never ranked it, + /// or never ran. + pub vector_rank: Option, + /// This chunk's cosine to the query; `None` whenever `vector_rank` is. + pub cos: Option, +} + +/// One query term's lexical reading (see [`SearchEvidenceView`]). +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct QueryTermView { + pub term: String, + /// Chunks matching this term alone; `0` = the vault has never seen the word. + pub df: usize, + /// This term's weight in the coverage reading — `ln((chunks+1)/(df+1))`; near + /// zero for a ubiquitous word, largest for one the vault has never seen. + pub idf: f64, +} + +/// One **chunk-level** search hit — the sub-note view of [`search`](Vault::search). +/// Same retrieval, but ranked chunks are returned as-is instead of deduped up to +/// notes, so a caller can see *which passage* matched and at what rank. The client +/// is the out-of-CI retrieval eval (ADR-0013): note-rank scoring is blind to +/// sub-note quality, which is exactly what chunking levers move. Carries the chunk's +/// **full text**, which the eval's containment scoring anchors on. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ChunkSearchResult { + pub path: String, + /// The chunk's heading breadcrumb (`"Fermentation > Vegetables"`), when the + /// chunker recorded one. + pub heading_path: Option, + /// Fused relevance score; higher is better. + pub score: f64, + /// The chunk's stored text, verbatim. + pub text: String, +} + +/// The answer to one grounded-chat ask — flow ④'s display view: the model's +/// streamed text with its `[n]` citation markers resolved back to the vault. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct AnswerView { + /// The answer, verbatim as streamed — including any marker that did *not* + /// resolve. Model output is untrusted content, but it is never rewritten here; + /// an unmatched marker is simply absent from `citations`. + pub answer: String, + /// The resolved citations, ascending by marker; one entry per **distinct** + /// marker that names a real passage. + pub citations: Vec, + /// `true` when the caller's callback broke the stream mid-answer: `answer` then + /// holds the partial text honestly, and citations resolve over what arrived. + pub cancelled: bool, + /// The B2 tools that ran to produce this answer, in order — empty for a plain + /// [`ask`](Vault::ask), which retrieves once and offers the model none. Serialized + /// only when present, so existing JSON consumers see no change. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tools: Vec, +} + +/// One tool run during a tool-using chat turn — shown beside the answer so a human can +/// see what the explanation was built from. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ToolUseView { + /// The tool's name (`b2_passage_pairs`, `b2_read`, …). + pub name: String, + /// The call's arguments, as JSON text — the model's own, verbatim, for a call it made. + pub arguments: String, + /// `true` for the lookup B2 made itself before asking the model anything; `false` + /// for a call the model chose. + pub seeded: bool, +} + +/// One resolved `[n]` citation: the passage's note, plus a one-line excerpt of the +/// cited passage as display evidence. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Citation { + /// The marker as it appears in the answer text (1-based passage number). + pub marker: usize, + /// Vault-relative path of the cited note — its identity (ADR-0003), and what an + /// adapter opens on click. A note renamed between answer and click goes stale, + /// exactly as any path handle does. + pub path: String, + /// A one-line excerpt of the cited passage (its head, length-bounded). + pub excerpt: String, +} + +/// One semantically-similar candidate for `b2 similar`: a note near the anchor in +/// embedding space that is **not** already connected to it, resolved for display +/// with the passage that made it similar. The machine finds the candidate, the human +/// decides whether to `link` it (ADR-0009). +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SimilarView { + pub path: String, + pub title: Option, + /// Best chunk-pair similarity to the anchor; higher is nearer. + pub score: f64, + /// A one-line excerpt of the candidate chunk that achieved `score` — the + /// evidence for *why* it surfaced. + pub evidence: String, + /// How far this candidate stands above the anchor's own candidate population — + /// its best-passage z (GH #192), and the one honest input for a displayed + /// *strength* band (GH #150). Non-increasing down the row order, so the band + /// never contradicts the ranking. It **gates nothing** (ADR-0014): the band is a + /// within-list grading, never a verdict on existence. The unit is load-bearing — + /// a band calibrated in the retired centroid unit grades every card down + /// (GH #182) — so a surface reads its landmarks off `make eval`'s calibration + /// block. `None` when no statistic was computed (a fake-embedded space, a pool + /// under the statistics minimum, or zero variance), which is the adapters' cue + /// to say the list is *ungraded* rather than let bare cards read as uniformly + /// weak; serialized only when present, so older JSON consumers see no change. + #[serde(skip_serializing_if = "Option::is_none")] + pub z: Option, +} + +/// **Explain** for a *Similar & unlinked* card (GH #236): where one note stands in an +/// anchor's discovery field, and the passage pairs behind it. Model-free and read from +/// the same computation as [`Vault::similar`], so a served row's rank, z and best pair are +/// exactly the card's. Raw distances are never the point: every grade is a z on the same +/// yardstick as the strength band. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SimilarExplainView { + pub anchor: NoteSummary, + pub candidate: NoteSummary, + /// The list length the explanation is read at: the surface's own, so `served` + /// answers "was this a card?". + pub limit: usize, + pub standing: SimilarStanding, + /// The candidate's z when it was scored in a graded field (the card's band input). + pub z: Option, + /// The candidate's rank judged by whole-note average (stage 1), when it entered + /// stage 1. Far worse than a ranked standing's `rank` means one section carries the + /// match: a buried gem. + pub centroid_rank: Option, + /// Every scored note's z, nearest first: the field the band is relative to. Empty + /// when ungraded (a fake space, a small pool, no spread). + pub population: Vec, + /// One pair per candidate passage, nearest first, each matched to the anchor passage + /// nearest to it. Empty when either side has no vectors. + pub pairs: Vec, + /// Notes both sides already link with (either direction). + pub shared_neighbors: Vec, +} + +/// Where a note stands in an anchor's discovery field: why it is a card, or why not. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SimilarStanding { + /// The candidate is the anchor itself. + SameNote, + /// The anchor has no stored vectors yet: nothing to compare from. + AnchorUnembedded, + /// Directly linked (either direction), so discovery leaves it out as already known. + Linked, + /// The candidate has no stored vectors yet. + Unembedded, + /// Its whole-note rank fell past the first-pass shortlist of `shortlist` notes, so + /// it was never scored passage by passage. + NotShortlisted { shortlist: usize }, + /// Scored: `rank` (1-based) of `of` scored notes; `served` when `rank <= limit`. + Ranked { + rank: usize, + of: usize, + served: bool, + }, +} + +/// One passage pair: a candidate passage and the anchor passage nearest to it. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PassagePairView { + pub anchor: PassageView, + pub candidate: PassageView, + /// Negated L2, higher is nearer: [`SimilarView::score`]'s unit. + pub score: f64, + /// The pair's z on the card's yardstick, `None` when ungraded. + pub z: Option, + /// Both passages hold the same text (a template, a copy): a perfect match that says + /// nothing about what the notes are about. + pub identical: bool, +} + +/// One passage, as stored in the index. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct PassageView { + /// The chunk's heading breadcrumb (`"Fermentation > Vegetables"`), when it has one. + pub heading_path: Option, + /// The chunk's stored text, verbatim. + pub text: String, +} + +/// What [`write`](Vault::write) did: the saved note's path and the **new revision** +/// (blake3 of the final on-disk bytes) — the token the editor chains its next save +/// on, so sequential saves never self-conflict. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WriteReport { + pub path: String, + pub revision: String, +} + +/// What `b2 link` did: the committed typed edge, resolved for display. `created` is +/// `false` when the directed `(src, dst, type)` edge already existed, so nothing was +/// written (the command is idempotent). +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct LinkReport { + pub src_path: String, + pub dst_path: String, + pub relation: String, + pub created: bool, +} From 7f0b07866d24e6fba04105861dfc68791559ce3b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:35:24 +0000 Subject: [PATCH 08/30] refactor(ui): split wireEvents, tabulate the context menu, pull find + editor commands out of main.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - wireEvents was one 1,350-line function; it is now a list of named wiring functions (focus memory, clicks, context menu, tree keys, side keys, find bar, forms, chords, mouse history, window blur, drags) called in the old registration order. Settings' click branch is settingsClick. - The context-menu click branch's thirteen near-identical blocks are three tables of MenuItem (attr, closeFirst, act) and one contextMenuClick. - findfield.ts: the editor find StateField, its effect and decorations. - editorcmds.ts: format, list shift, table, rich paste and [[ completion, each split into a pure plan over an EditorState and a run… that dispatches it. Both modules have EditorState tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/droplink.test.ts | 2 +- ui/src/droplink.ts | 2 +- ui/src/editorcmds.test.ts | 86 ++++++ ui/src/editorcmds.ts | 197 +++++++++++++ ui/src/findfield.test.ts | 63 ++++ ui/src/findfield.ts | 61 ++++ ui/src/main.ts | 606 +++++++++++++++----------------------- 7 files changed, 646 insertions(+), 371 deletions(-) create mode 100644 ui/src/editorcmds.test.ts create mode 100644 ui/src/editorcmds.ts create mode 100644 ui/src/findfield.test.ts create mode 100644 ui/src/findfield.ts diff --git a/ui/src/droplink.test.ts b/ui/src/droplink.test.ts index 8f75389..43aa7e3 100644 --- a/ui/src/droplink.test.ts +++ b/ui/src/droplink.test.ts @@ -138,7 +138,7 @@ check("a line that merely ends in an inline span still accepts", () => { }); check("the code check reads a position, not the selection", () => { - // main.ts's cursor-shaped `inCodeContext` delegates to this: a drop names a place the + // editorcmds.ts's cursor-shaped `inCodeContext` delegates to this: a drop names a place the // caret isn't, so the position has to be the argument rather than an implicit read. // Unlike the drop's own question, this one counts an inline span — pasted HTML inside // one must stay literal (paste.ts), which is a different rule for a different reason. diff --git a/ui/src/droplink.ts b/ui/src/droplink.ts index 11de629..243e627 100644 --- a/ui/src/droplink.ts +++ b/ui/src/droplink.ts @@ -65,7 +65,7 @@ export function lineDrop(lineText: string, target: string): { offset: number; in /** * Is `pos` inside code — a fence, an indented block, or an inline span? The read the rich - * paste and the list keys make (main.ts's `inCodeContext` delegates here), taking a + * paste and the list keys make (editorcmds.ts's `inCodeContext` delegates here), taking a * position rather than reading the selection, because a *drop* names a place the cursor * isn't. */ diff --git a/ui/src/editorcmds.test.ts b/ui/src/editorcmds.test.ts new file mode 100644 index 0000000..54a4784 --- /dev/null +++ b/ui/src/editorcmds.test.ts @@ -0,0 +1,86 @@ +// The editor's commands (editorcmds.ts), as plans over real EditorStates parsed by the +// editor's own language config — the droplink.test.ts rig, for its reason: a mocked tree +// would let a code fence or a nested list go unrecognized here and undetected there. +// The engines underneath (format.ts, list.ts, paste.ts) have their own suites; what is +// pinned here is the adapter's half — which transaction, and when it declines. + +import { strict as assert } from "node:assert"; +import test from "node:test"; +import { markdown, markdownLanguage } from "@codemirror/lang-markdown"; +import { EditorState, type TransactionSpec } from "@codemirror/state"; +import { + formatPlan, + inCodeContext, + inListItem, + insertTablePlan, + listShiftPlan, + pastePlan, +} from "./editorcmds.ts"; +import { BOLD } from "./format.ts"; +import { indentList } from "./list.ts"; +import { wikilink } from "./livepreview.ts"; + +const LANG = markdown({ base: markdownLanguage, extensions: [wikilink] }); + +/** A state with the caret (or selection) where `|` marks are: one bar is a caret, two a + * selection between them. */ +function at(marked: string): EditorState { + const first = marked.indexOf("|"); + const second = marked.indexOf("|", first + 1); + const doc = marked.replace(/\|/g, ""); + const selection = + second < 0 ? { anchor: first } : { anchor: first, head: second - 1 }; + return EditorState.create({ doc, selection, extensions: [LANG] }); +} + +const applied = (s: EditorState, spec: TransactionSpec): string => s.update(spec).state.doc.toString(); + +test("⌘B wraps the selection", () => { + const s = at("a |word| b"); + assert.equal(applied(s, formatPlan(s, BOLD)), "a **word** b"); +}); + +test("Tab nests a list item under the one above it", () => { + const s = at("- one\n- |two"); + const plan = listShiftPlan(s, indentList); + assert.ok(typeof plan !== "boolean", "a transaction"); + assert.equal(applied(s, plan as TransactionSpec), "- one\n - two"); +}); + +test("Tab outside a list is declined, so it walks the focus ring", () => { + assert.equal(listShiftPlan(at("just |prose"), indentList), false); +}); + +test("Tab in a fence is declined before the engine is asked", () => { + const s = at("```\n- |item\n```"); + assert.equal(inCodeContext(s), true); + assert.equal(listShiftPlan(s, indentList), false); +}); + +test("Tab on the first item is claimed and does nothing", () => { + assert.equal(listShiftPlan(at("- |one\n- two"), indentList), true); +}); + +test("a list item behind a quote is still a list item to the tree", () => { + assert.equal(inListItem(at("> - |a")), true); + assert.equal(inListItem(at("plain |text")), false); +}); + +test("⌘T drops a table at the caret", () => { + const s = at("|"); + const doc = applied(s, insertTablePlan(s)); + assert.ok(doc.includes("|"), doc); + assert.ok(doc.split("\n").length >= 4, "header, rule, two rows"); +}); + +test("rich paste converts formatted HTML, and declines what has nothing to add", () => { + const s = at("|"); + const plan = pastePlan(s, "

bold

", "bold"); + assert.ok(plan !== null); + assert.equal(applied(s, plan), "**bold**"); + assert.equal(pastePlan(s, "", "plain"), null, "no HTML flavor: CodeMirror's paste"); +}); + +test("rich paste keeps its hands off code", () => { + assert.equal(pastePlan(at("```\n|\n```"), "

bold

", "bold"), null); +}); diff --git a/ui/src/editorcmds.ts b/ui/src/editorcmds.ts new file mode 100644 index 0000000..55b9309 --- /dev/null +++ b/ui/src/editorcmds.ts @@ -0,0 +1,197 @@ +// The editor's commands — the CodeMirror adapters over the pure engines (format.ts, +// list.ts, paste.ts, wikicomplete.ts). Each is split in two: a *plan* that reads an +// EditorState and says what should happen (a transaction, or a decline), and a `run…` +// that dispatches it on a view. The plans are pure, so node tests them on real +// EditorStates parsed by the app's own grammar (editorcmds.test.ts, the droplink.test.ts +// rig); main.ts only builds the keymap and the extensions out of the `run…` halves. + +import type { CompletionContext, CompletionResult } from "@codemirror/autocomplete"; +import { syntaxTree } from "@codemirror/language"; +import { EditorSelection, type EditorState, type TransactionSpec } from "@codemirror/state"; +import { EditorView } from "@codemirror/view"; +import { inCodeAt } from "./droplink.ts"; +import { insertTable, toggleInline, type InlineFormat } from "./format.ts"; +import type { ListEdit } from "./list.ts"; +import { markdownForPaste } from "./paste.ts"; +import type { NoteSummary, ResourceSummary } from "./types.ts"; +import { wikiCandidates, wikiInsertion, wikiQueryAt } from "./wikicomplete.ts"; + +// --- formatting (⌘B / ⌘I, …) ---------------------------------------------------------- + +/** + * Toggle an inline format over every selection range. `changeByRange` runs the toggle per + * range and maps the coordinates, so multi-cursor edits come free. The keymap derives + * from the `FORMATS` table paired with each row's chord from the keyboard registry + * (`format.`, bindings.ts), so adding a format is one new row in each. + */ +export function formatPlan(state: EditorState, fmt: InlineFormat): TransactionSpec { + const doc = state.doc.toString(); + return state.changeByRange((range) => { + const r = toggleInline(doc, range.from, range.to, fmt); + return { changes: r.changes, range: EditorSelection.range(r.selFrom, r.selTo) }; + }); +} + +export function runFormat(view: EditorView, fmt: InlineFormat): boolean { + view.dispatch(formatPlan(view.state, fmt)); + return true; +} + +// --- list nesting (Tab / ⇧Tab) -------------------------------------------------------- + +/** list.ts's nest or lift, as the key hands it over. */ +export type ListShift = (doc: string, from: number, to: number) => ListEdit | null; + +/** + * Tab / ⇧Tab — nest or lift out the list item(s) the selection covers: a transaction to + * dispatch, `true` to claim the key and do nothing, or `false` to decline it. The engine + * is list.ts; this is the CodeMirror half, `formatPlan`'s pattern one construct up. + * + * Declining matters twice over. A `null` from the engine usually means the caret is not + * in a list, and declining there is what leaves Tab walking the focus ring through the + * rest of the app — the reason this isn't `indentWithTab`, which claims the key outright + * and takes the keyboard's way out of the buffer with it. The one exception is a list the + * engine can't see — `inListItem` gives the syntax tree the last word, so the no-ejection + * contract holds there too. And a caret inside code declines before the engine is asked + * at all: a `- item` line in a fence is text, not structure, and `inCodeContext` is the + * same read the rich paste makes to keep its hands off code. + * + * One range rather than `changeByRange`: an indent moves every offset after it, so a + * second cursor's edit would be computed against a document the first has already + * shifted. Multi-cursor nesting is a gesture nobody makes; ⌘B's is one they do. + */ +export function listShiftPlan(state: EditorState, shift: ListShift): TransactionSpec | boolean { + if (inCodeContext(state)) return false; + const { from, to } = state.selection.main; + const r = shift(state.doc.toString(), from, to); + if (!r) return inListItem(state); + // Claimed but inert — the first item of a list has nothing to nest under. Swallowing + // the key is the point (list.ts's header): a gesture that sometimes ejects you from + // the buffer is worse than one that sometimes does nothing. + if (r.changes.length === 0) return true; + return { + changes: r.changes, + selection: EditorSelection.range(r.selFrom, r.selTo), + scrollIntoView: true, + }; +} + +export function runListShift(view: EditorView, shift: ListShift): boolean { + const plan = listShiftPlan(view.state, shift); + if (typeof plan === "boolean") return plan; + view.dispatch(plan); + return true; +} + +/** Is the cursor inside a list item the *scanner* can't see? list.ts reads only lists at + * the top level of the note — `> - a` is a bullet behind a container prefix it doesn't + * parse — so on its null the tree gets the last word before the key is handed back to + * the focus ring. Claimed-but-inert there: the caret is visibly on a list item, and + * ejecting from it would break the gesture's contract even where the edit itself isn't + * built yet (list.ts's header). */ +export function inListItem(state: EditorState): boolean { + const at = syntaxTree(state).resolveInner(state.selection.main.from, -1); + for (let n: typeof at | null = at; n; n = n.parent) { + if (n.name === "ListItem") return true; + } + return false; +} + +// --- ⌘T: a table ---------------------------------------------------------------------- + +/** A fresh 3-column table (header + two rows) at the cursor, caret in the first cell. + * A block insert, not an inline toggle, so it's its own binding over the pure + * `insertTable` (format.ts). */ +export function insertTablePlan(state: EditorState): TransactionSpec { + const { from, to } = state.selection.main; + const r = insertTable(state.doc.toString(), from, to); + return { + changes: r.changes, + selection: EditorSelection.range(r.selFrom, r.selTo), + scrollIntoView: true, + }; +} + +export function runInsertTable(view: EditorView): boolean { + view.dispatch(insertTablePlan(view.state)); + return true; +} + +// --- rich paste ------------------------------------------------------------------------- +// +// A copy from a web page carries a `text/html` flavor next to `text/plain`; CodeMirror's +// own paste takes the plain one, which is why every heading, bold and list used to vanish +// on the way in. This converts the HTML to Markdown instead, and *declines* in two cases, +// leaving CodeMirror's paste to run untouched: +// - the cursor sits in code, where pasted text must stay literal; +// - the HTML carried no formatting the plain flavor didn't already have (paste.ts's +// `markdownForPaste` returns null) — pasting escaped Markdown there would be a loss. +// The third way out is the ⌘⇧V chord (main.ts `pastePlain`), which does its own plain paste. + +/** Is the cursor inside code — a fenced block, an indented block, or an inline span? + * The read itself is droplink.ts's (by position, since a *drop* names a place the cursor + * isn't); this is the cursor's spelling of the same question. */ +export function inCodeContext(state: EditorState): boolean { + return inCodeAt(state, state.selection.main.from); +} + +/** The paste as Markdown, or null to decline and let CodeMirror paste the plain text. */ +export function pastePlan(state: EditorState, html: string, plain: string): TransactionSpec | null { + if (inCodeContext(state)) return null; + const md = markdownForPaste(html, plain); + if (md === null) return null; + return { ...state.replaceSelection(md), scrollIntoView: true, userEvent: "input.paste" }; +} + +function handlePaste(event: ClipboardEvent, view: EditorView): boolean { + const data = event.clipboardData; + if (!data) return false; + const plan = pastePlan(view.state, data.getData("text/html"), data.getData("text/plain")); + if (plan === null) return false; + event.preventDefault(); + view.dispatch(plan); + return true; +} + +/** Web-page formatting survives the clipboard; an unformatted paste still takes + * CodeMirror's own path. */ +export const richPaste = EditorView.domEventHandlers({ paste: handlePaste }); + +// --- `[[` completion -------------------------------------------------------------------- + +/** The vault's lists the picker ranks over — already loaded in the app's state, so a + * query is an in-memory scan. */ +export interface WikiInventory { + readonly notes: NoteSummary[]; + readonly resources: ResourceSummary[]; +} + +/** + * Wikilink completion (the Obsidian gesture): typing `[[` opens a picker over the vault's + * notes + files. The logic — trigger detection, ranking, bracket-closing — is the pure + * wikicomplete.ts; this is only the CodeMirror adapter. `filter: false` because the + * ranking is ours (title-prefix > title > path), and no `validFor` so each keystroke + * re-queries it. `inventory` is read per query, so the lists are always the live ones. + */ +export function wikiCompletionSource( + inventory: () => WikiInventory, +): (ctx: CompletionContext) => CompletionResult | null { + return (ctx) => { + const line = ctx.state.doc.lineAt(ctx.pos); + const found = wikiQueryAt(line.text.slice(0, ctx.pos - line.from)); + if (!found) return null; + const { notes, resources } = inventory(); + const options = wikiCandidates(notes, resources, found.query).map((c) => ({ + label: c.label, + detail: c.detail, + apply: (view: EditorView, _completion: unknown, from: number, to: number) => { + const { insert, cursor } = wikiInsertion(c.target, view.state.sliceDoc(to, to + 2)); + view.dispatch({ + changes: { from, to, insert }, + selection: { anchor: from + cursor }, + }); + }, + })); + return { from: line.from + found.from, options, filter: false }; + }; +} diff --git a/ui/src/findfield.test.ts b/ui/src/findfield.test.ts new file mode 100644 index 0000000..2058028 --- /dev/null +++ b/ui/src/findfield.test.ts @@ -0,0 +1,63 @@ +// Find-in-note's editor engine (findfield.ts), on real EditorStates — the droplink.test.ts +// rig. What's pinned is the field's contract with the bar: an effect sets the query and +// the wanted active match, a doc edit re-derives the matches and keeps the active one +// where the user was, and null closes it. + +import { strict as assert } from "node:assert"; +import test from "node:test"; +import { EditorState } from "@codemirror/state"; +import { findDecorations, findField, setFindEffect } from "./findfield.ts"; + +const doc = (text: string) => EditorState.create({ doc: text, extensions: [findField] }); +const find = (s: EditorState) => s.field(findField); + +test("closed until a query arrives", () => { + assert.equal(find(doc("cat cat")), null); +}); + +test("an effect sets the query, finds every match, and clamps the active one", () => { + const s = doc("cat dog cat").update({ effects: setFindEffect.of({ query: "cat", active: 5 }) }).state; + const f = find(s); + assert.equal(f?.query, "cat"); + assert.deepEqual(f?.matches.map((m) => m.from), [0, 8]); + assert.equal(f?.active, 1, "clamped to the last match"); +}); + +test("no match is active -1", () => { + const s = doc("cat").update({ effects: setFindEffect.of({ query: "dog", active: 0 }) }).state; + assert.equal(find(s)?.active, -1); +}); + +test("a doc edit re-derives the matches and re-anchors the active one", () => { + let s = doc("cat dog cat").update({ effects: setFindEffect.of({ query: "cat", active: 1 }) }).state; + // Typing at the start shifts both matches; the active one follows the one it was on. + s = s.update({ changes: { from: 0, insert: "a " } }).state; + const f = find(s); + assert.deepEqual(f?.matches.map((m) => m.from), [2, 10]); + assert.equal(f?.active, 1); + // A new match typed in counts at once. + s = s.update({ changes: { from: s.doc.length, insert: " cat" } }).state; + assert.equal(find(s)?.matches.length, 3); +}); + +test("a transaction that neither edits nor sets leaves the value as it was", () => { + const s = doc("cat").update({ effects: setFindEffect.of({ query: "cat", active: 0 }) }).state; + const after = s.update({ selection: { anchor: 1 } }).state; + assert.equal(find(after), find(s)); +}); + +test("null closes it", () => { + let s = doc("cat").update({ effects: setFindEffect.of({ query: "cat", active: 0 }) }).state; + s = s.update({ effects: setFindEffect.of(null) }).state; + assert.equal(find(s), null); +}); + +test("every match paints, and the active one is marked apart", () => { + const s = doc("cat dog cat").update({ effects: setFindEffect.of({ query: "cat", active: 0 }) }).state; + const classes: string[] = []; + findDecorations(find(s)).between(0, s.doc.length, (_from, _to, d) => { + classes.push(String(d.spec.class)); + }); + assert.deepEqual(classes, ["find-match is-active", "find-match"]); + assert.equal(findDecorations(null).size, 0); +}); diff --git a/ui/src/findfield.ts b/ui/src/findfield.ts new file mode 100644 index 0000000..45790bb --- /dev/null +++ b/ui/src/findfield.ts @@ -0,0 +1,61 @@ +// Find-in-note's editor engine (⌘F while editing): the match state as a CodeMirror +// StateField, so the highlights re-derive on every doc change — typing with the bar open +// keeps them honest, with no listener → dispatch round-trip. +// +// findbar.ts owns the matching and stepping; this owns only how that state lives in an +// EditorState and paints as decorations. main.ts dispatches `setFindEffect` and reads the +// field back to mirror it into the bar. No DOM, so node tests it on real EditorStates +// (`npm test`), the droplink.test.ts rig. + +import { StateEffect, StateField } from "@codemirror/state"; +import { Decoration, type DecorationSet, EditorView } from "@codemirror/view"; +import { activeAfter, findMatches, type Match } from "./findbar.ts"; + +/** The bar's query, its matches in the current doc, and which one is active (-1: none). */ +export interface EditorFind { + readonly query: string; + readonly matches: Match[]; + readonly active: number; +} + +/** Set the query and the wanted active match — or null to close the bar's engine. */ +export const setFindEffect = StateEffect.define<{ query: string; active: number } | null>(); + +const findMark = Decoration.mark({ class: "find-match" }); +const findMarkActive = Decoration.mark({ class: "find-match is-active" }); + +/** Every match as a mark, the active one distinguished. */ +export function findDecorations(v: EditorFind | null): DecorationSet { + if (!v) return Decoration.none; + return Decoration.set( + v.matches.map((m, i) => (i === v.active ? findMarkActive : findMark).range(m.from, m.to)), + ); +} + +export const findField = StateField.define({ + create: () => null, + update(value, tr) { + let next = value; + for (const ef of tr.effects) { + if (ef.is(setFindEffect)) + next = ef.value && { query: ef.value.query, matches: [], active: ef.value.active }; + } + if (!next) return null; + if (next === value && !tr.docChanged) return value; + const matches = findMatches(tr.newDoc.toString(), next.query); + const active = + next !== value || value === null + ? matches.length === 0 + ? -1 + : Math.max(0, Math.min(next.active, matches.length - 1)) + : // A doc edit: re-anchor on where the old active match ended up. + activeAfter( + matches, + value.active >= 0 && value.matches[value.active] + ? tr.changes.mapPos(value.matches[value.active].from) + : 0, + ); + return { query: next.query, matches, active }; + }, + provide: (f) => EditorView.decorations.from(f, findDecorations), +}); diff --git a/ui/src/main.ts b/ui/src/main.ts index 5d4021d..7aca2f7 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -5,25 +5,16 @@ // flow, never engine logic. import "../style.css"; -import { - autocompletion, - type CompletionContext, - type CompletionResult, -} from "@codemirror/autocomplete"; +import { autocompletion } from "@codemirror/autocomplete"; import { history } from "@codemirror/commands"; import { markdown, markdownLanguage } from "@codemirror/lang-markdown"; -import { syntaxHighlighting, syntaxTree } from "@codemirror/language"; +import { syntaxHighlighting } from "@codemirror/language"; import { Compartment, - EditorSelection, - type EditorState, StateEffect, - StateField, type Extension, } from "@codemirror/state"; import { - Decoration, - type DecorationSet, EditorView, type KeyBinding, keymap, @@ -34,6 +25,7 @@ import { openDocPath, state, type AppState, + type ContextMenuState, type SideSection, type ThemePref, type TreeNodeRef, @@ -83,19 +75,18 @@ import { inlineImagePlan, } from "./embeds"; import { b2Highlighter, highlightCodeBlocks, resolveLang } from "./highlight"; -import { noteTarget, wikiCandidates, wikiInsertion, wikiQueryAt } from "./wikicomplete"; +import { noteTarget } from "./wikicomplete"; import { CARD_DRAG_MIME, cardDrop, type DraggedCard, - inCodeAt, insertDrop, planDrop, setDropTarget, withoutCard, } from "./droplink"; -import { FORMATS, insertTable, toggleInline, type InlineFormat } from "./format"; -import { indentList, outdentList, type ListEdit } from "./list"; +import { FORMATS } from "./format"; +import { indentList, outdentList } from "./list"; import { activeBindings, canonicalKey, @@ -120,10 +111,17 @@ import { capture, PROBE_AFTER_MS, silenceHint } from "./recorder"; import { STOCK_EDITOR_KEYMAP } from "./editorkeys"; import { menuDrift } from "./menukeys"; import { HOLD_MS, type HoldEvent, type HoldPhase, holdStep } from "./cmdhold"; -import { markdownForPaste } from "./paste"; +import { + richPaste, + runFormat, + runInsertTable, + runListShift, + wikiCompletionSource, +} from "./editorcmds"; import { icon } from "./icons"; import { editDoneTitle, shellHints } from "./hints"; import { activeAfter, countLabel, FIND_CAP, findMatches, locate, stepActive, type Match } from "./findbar"; +import { findField, setFindEffect } from "./findfield"; import { BOUNDS, initPanes, visiblePanes } from "./panes"; import { DEFAULT_ZOOM, @@ -900,8 +898,8 @@ function inTextEntry(): boolean { // live here: the file tree navigates by arrow key (the ARIA `tree` pattern, walking the // row order treenav.ts also paints), every overlay takes focus on open and gives it back // on close, and every mouse-only gesture — the right-click menu above all — has a key -// that reaches it. The chords themselves are wired in the global keydown handler at the -// bottom of `wireEvents`; what's here is the focus bookkeeping they share. +// that reaches it. The chords themselves are wired in the global keydown handler, +// `wireChords`; what's here is the focus bookkeeping they share. /** Every row the tree currently paints, in paint order — the list the arrows walk. */ function treeRows(): TreeRow[] { @@ -1164,7 +1162,7 @@ function wireCmdHold(): void { // The releases the keyboard never delivers. macOS stops sending key events to a window // that isn't key, so ⌘⇥ into another app, Spotlight, or Hide takes the ⌘ keyup with it — // and a sheet whose only exit is an event that will never arrive is a sheet stuck on - // screen. (A second `blur` listener, deliberately: the one in `wireEvents` is the + // screen. (A second `blur` listener, deliberately: the one in `wireWindowBlur` is the // editor's flush point and the recorder's silence probe, and stapling an unrelated // third job onto it would hide this one from anyone reading either.) window.addEventListener("blur", () => cmdHoldEvent({ kind: "release" })); @@ -1205,7 +1203,7 @@ function overlayFocusables(): HTMLElement[] { * gone: `render()` swaps `#modal-root`/`#menu-root` wholesale, so `document.activeElement` * has fallen back to `` and the trigger is unrecoverable. `focusin` fires long * before that, and *destroying* a focused node fires no `focusin` (only blur), so this - * still names the trigger at the moment we come to remember it. Set in `wireEvents`. + * still names the trigger at the moment we come to remember it. Set in `wireFocusMemory`. */ let lastFocused: HTMLElement | SVGElement | null = null; @@ -1550,6 +1548,78 @@ function closeContextMenu(): void { render(); } +/** + * One right-click menu item: the attribute its button carries, and what choosing it does. + * `closeFirst` is for an act that would leave the menu up over its result — a copy whose + * confirmation is the status line, an OS picker, a navigation — so the menu goes before + * it runs. An act that opens a surface of its own (rename, Move…, a create, a delete) + * clears the menu itself. The act's argument is read off the menu before it closes. + */ +interface MenuItem { + readonly attr: string; + readonly closeFirst: boolean; + readonly act: (arg: A) => void; +} + +type CardMenu = Extract; + +/** The tree menu's items that act on the row it was opened over. */ +const TREE_ROW_ITEMS: readonly MenuItem[] = [ + { attr: "data-ctx-rename", closeFirst: false, act: startTreeRename }, + { attr: "data-ctx-move", closeFirst: false, act: openMoveModal }, + { attr: "data-ctx-copy-vault-path", closeFirst: true, act: (node) => void copyPath(node.path) }, + { + attr: "data-ctx-copy-system-path", + closeFirst: true, + // Both entry points refuse to open this menu without a vault, so the root is set + // here — but the absolute path is the root's to give, and inventing one for a + // vault-less window is not this item's call. + act: (node) => { + if (state.vaultRoot !== null) void copyPath(systemPath(state.vaultRoot, node.path)); + }, + }, + { attr: "data-ctx-delete", closeFirst: false, act: requestDelete }, +]; + +/** The tree menu's items that act on the folder it was opened in. */ +const TREE_DIR_ITEMS: readonly MenuItem[] = [ + { attr: "data-ctx-new-note", closeFirst: false, act: (dir) => startTreeCreate("note", dir) }, + { attr: "data-ctx-new-folder", closeFirst: false, act: (dir) => startTreeCreate("folder", dir) }, + // The picker is modal to the OS — the menu goes first. + { attr: "data-ctx-import", closeFirst: true, act: (dir) => void pickAndImport(dir) }, +]; + +/** A discovery card's (or graph ghost's) items. */ +const CARD_ITEMS: readonly MenuItem[] = [ + { attr: "data-ctx-open", closeFirst: true, act: (m) => void openNote(m.path) }, + // The drag's keyboard half — aimed at the caret's line. + { attr: "data-ctx-insert", closeFirst: true, act: (m) => insertCardLink(m.path) }, + { attr: "data-ctx-link", closeFirst: true, act: (m) => openLinkModal(m.path, m.title ?? "") }, + { attr: "data-ctx-explain", closeFirst: true, act: (m) => void openExplain(m.path) }, + { attr: "data-ctx-why", closeFirst: true, act: (m) => void askWhy({ path: m.path, title: m.title }) }, +]; + +/** A click while a right-click menu is up: its own items act; any other click only + * dismisses it. */ +function contextMenuClick(target: HTMLElement): void { + const menu = state.contextMenu; + if (!menu) return; + const chose = (items: readonly MenuItem[], arg: A): boolean => { + const item = items.find((i) => target.closest(`[${i.attr}]`) !== null); + if (!item) return false; + if (item.closeFirst) closeContextMenu(); + item.act(arg); + return true; + }; + if (menu.kind === "tree") { + if (menu.node && chose(TREE_ROW_ITEMS, menu.node)) return; + if (chose(TREE_DIR_ITEMS, menu.dir)) return; + } else if (chose(CARD_ITEMS, menu)) { + return; + } + closeContextMenu(); +} + // --- tree creation: new note / new folder (left nav) ------------------------------ // // The create affordances: the tree-head icons, ⌘N / ⇧⌘N, and the tree's right-click @@ -1635,7 +1705,7 @@ async function commitTreeCreate(raw: string, open: boolean): Promise { // --- tree import: files from outside the vault ------------------------------------ // // Two gestures, one outcome: drag files from Finder onto a folder row (the pointer -// path, wired in wireEvents) or pick them in an OS dialog from the tree's right-click +// path, wired in wireDrags) or pick them in an OS dialog from the tree's right-click // menu (the keyboard path — K1: a drag is pointer-only, so it can't be the only way // in). Both place the files through `Vault::import_file`/`import_path`, which copies // the bytes verbatim and projects them — adding nothing to either: a `.md` lands as a @@ -3439,97 +3509,9 @@ function enterEdit(): void { mountEditor(n.body); } -// Wikilink completion (the Obsidian gesture): typing `[[` opens a picker over the -// vault's notes + files. The logic — trigger detection, ranking, bracket-closing — -// is the pure wikicomplete.ts; this is only the CodeMirror adapter. `filter: false` -// because the ranking is ours (title-prefix > title > path), and no `validFor` so -// each keystroke re-queries it — the lists live in `state` and are already loaded, -// so a query is just an in-memory scan. -function wikiSource(ctx: CompletionContext): CompletionResult | null { - const line = ctx.state.doc.lineAt(ctx.pos); - const found = wikiQueryAt(line.text.slice(0, ctx.pos - line.from)); - if (!found) return null; - const options = wikiCandidates(state.notes, state.resources, found.query).map((c) => ({ - label: c.label, - detail: c.detail, - apply: (view: EditorView, _completion: unknown, from: number, to: number) => { - const { insert, cursor } = wikiInsertion(c.target, view.state.sliceDoc(to, to + 2)); - view.dispatch({ - changes: { from, to, insert }, - selection: { anchor: from + cursor }, - }); - }, - })); - return { from: line.from + found.from, options, filter: false }; -} - -// Formatting chords (⌘B/⌘I, …) — the CodeMirror adapter over the pure format.ts -// engine. The keymap derives from the `FORMATS` table paired with each row's chord from -// the keyboard registry (`format.`, bindings.ts), so adding a format is one new row -// in each and no wiring here. `changeByRange` runs the toggle per selection range and -// maps the coordinates, so multi-cursor edits come free. -function runFormat(view: EditorView, fmt: InlineFormat): boolean { - const doc = view.state.doc.toString(); - view.dispatch( - view.state.changeByRange((range) => { - const r = toggleInline(doc, range.from, range.to, fmt); - return { changes: r.changes, range: EditorSelection.range(r.selFrom, r.selTo) }; - }), - ); - return true; -} - -/** - * Tab / ⇧Tab — nest or lift out the list item(s) the selection covers. The engine is - * list.ts; this is the CodeMirror half, the `runFormat` pattern one construct up. - * - * Declining matters twice over. A `null` from the engine usually means the caret is not - * in a list, and returning false there is what leaves Tab walking the focus ring through - * the rest of the app — the reason this isn't `indentWithTab`, which claims the key - * outright and takes the keyboard's way out of the buffer with it. The one exception is - * a list the engine can't see — `inListItem` below gives the syntax tree the last word, - * so the no-ejection contract holds there too. And a caret inside code declines before - * the engine is asked at all: a `- item` line in a fence is text, not structure, and - * `inCodeContext` is the same read the rich paste makes to keep its hands off code. - * - * One range rather than `changeByRange`: an indent moves every offset after it, so a - * second cursor's edit would be computed against a document the first has already - * shifted. Multi-cursor nesting is a gesture nobody makes; ⌘B's is one they do. - */ -function runListShift( - view: EditorView, - shift: (doc: string, from: number, to: number) => ListEdit | null, -): boolean { - if (inCodeContext(view.state)) return false; - const { from, to } = view.state.selection.main; - const r = shift(view.state.doc.toString(), from, to); - if (!r) return inListItem(view.state); - // Claimed but inert — the first item of a list has nothing to nest under. Swallowing - // the key is the point (list.ts's header): a gesture that sometimes ejects you from - // the buffer is worse than one that sometimes does nothing. - if (r.changes.length > 0) { - view.dispatch({ - changes: r.changes, - selection: EditorSelection.range(r.selFrom, r.selTo), - scrollIntoView: true, - }); - } - return true; -} +// `[[` completion over the vault's lists as they stand (editorcmds.ts). +const wikiSource = wikiCompletionSource(() => state); -/** Is the cursor inside a list item the *scanner* can't see? list.ts reads only lists at - * the top level of the note — `> - a` is a bullet behind a container prefix it doesn't - * parse — so on its null the tree gets the last word before the key is handed back to - * the focus ring. Claimed-but-inert there: the caret is visibly on a list item, and - * ejecting from it would break the gesture's contract even where the edit itself isn't - * built yet (list.ts's header). */ -function inListItem(state: EditorState): boolean { - const at = syntaxTree(state).resolveInner(state.selection.main.from, -1); - for (let n: typeof at | null = at; n; n = n.parent) { - if (n.name === "ListItem") return true; - } - return false; -} /** * B2's own chords inside the editor, read from the **live** registry each time. @@ -3580,52 +3562,7 @@ function b2EditorKeymap(): KeyBinding[] { const keysCompartment = new Compartment(); const editorKeymap = (): Extension => keymap.of([...b2EditorKeymap(), ...STOCK_EDITOR_KEYMAP]); -// ⌘T — drop a fresh 3-column table (header + two rows) at the cursor, caret in the -// first cell. Block insert, not an inline toggle, so it's its own binding over the pure -// `insertTable` (format.ts). -function runInsertTable(view: EditorView): boolean { - const { from, to } = view.state.selection.main; - const r = insertTable(view.state.doc.toString(), from, to); - view.dispatch({ - changes: r.changes, - selection: EditorSelection.range(r.selFrom, r.selTo), - scrollIntoView: true, - }); - return true; -} - -// Rich paste — the CodeMirror adapter over the pure paste.ts. A copy from a web page -// carries a `text/html` flavor next to `text/plain`; CodeMirror's own paste takes the -// plain one, which is why every heading, bold and list used to vanish on the way in. -// This converts the HTML to Markdown instead, and *declines* in two cases, leaving -// CodeMirror's paste to run untouched: -// - the cursor sits in code, where pasted text must stay literal; -// - the HTML carried no formatting the plain flavor didn't already have (paste.ts's -// `markdownForPaste` returns null) — pasting escaped Markdown there would be a loss. -// The third way out is the ⌘⇧V chord below, which does its own plain paste. - -/** Is the cursor inside code — a fenced block, an indented block, or an inline span? - * The read itself is droplink.ts's (by position, since a *drop* names a place the cursor - * isn't); this is the cursor's spelling of the same question. */ -function inCodeContext(state: EditorState): boolean { - return inCodeAt(state, state.selection.main.from); -} - -function handlePaste(event: ClipboardEvent, view: EditorView): boolean { - const data = event.clipboardData; - if (!data || inCodeContext(view.state)) return false; - const md = markdownForPaste(data.getData("text/html"), data.getData("text/plain")); - if (md === null) return false; - event.preventDefault(); - view.dispatch({ - ...view.state.replaceSelection(md), - scrollIntoView: true, - userEvent: "input.paste", - }); - return true; -} -const richPaste = EditorView.domEventHandlers({ paste: handlePaste }); /** * ⌘⇧V — paste as plain text, the escape hatch from the conversion above. @@ -3699,7 +3636,7 @@ function mountEditor(body: string): void { // order decides who wins. keysCompartment.of(editorKeymap()), EditorView.lineWrapping, - // Web-page formatting survives the clipboard (see `richPaste` above); an + // Web-page formatting survives the clipboard (editorcmds.ts `richPaste`); an // unformatted paste still takes CodeMirror's own path. richPaste, // A discovery card dragged in from the right column — the drop preview and the @@ -4081,46 +4018,7 @@ let findRanges: globalThis.Range[] = []; /** The doc the bar is bound to — navigating anywhere else closes it (syncFind). */ let findDocKey: string | null = null; -// The editor engine: match state lives in a StateField so the decorations re-derive on -// every doc change — typing with the bar open keeps the highlights honest, with no -// listener→dispatch round-trip. -type EditorFind = { query: string; matches: Match[]; active: number }; -const setFindEffect = StateEffect.define<{ query: string; active: number } | null>(); -const findMark = Decoration.mark({ class: "find-match" }); -const findMarkActive = Decoration.mark({ class: "find-match is-active" }); -const findField = StateField.define({ - create: () => null, - update(value, tr) { - let next = value; - for (const ef of tr.effects) { - if (ef.is(setFindEffect)) - next = ef.value && { query: ef.value.query, matches: [], active: ef.value.active }; - } - if (!next) return null; - if (next === value && !tr.docChanged) return value; - const matches = findMatches(tr.newDoc.toString(), next.query); - const active = - next !== value || value === null - ? matches.length === 0 - ? -1 - : Math.max(0, Math.min(next.active, matches.length - 1)) - : // A doc edit: re-anchor on where the old active match ended up. - activeAfter( - matches, - value.active >= 0 && value.matches[value.active] - ? tr.changes.mapPos(value.matches[value.active].from) - : 0, - ); - return { query: next.query, matches, active }; - }, - provide: (f) => - EditorView.decorations.from(f, (v): DecorationSet => { - if (!v) return Decoration.none; - return Decoration.set( - v.matches.map((m, i) => (i === v.active ? findMarkActive : findMark).range(m.from, m.to)), - ); - }), -}); +// The editor engine is findfield.ts's StateField. /** What the note pane is showing, as an identity — null means "nothing findable" * (empty pane, or the graph, which has no text to find in). */ @@ -4544,9 +4442,124 @@ function paintShellHints(): void { } } +/** + * A click inside Settings. Every branch here is Settings' own, and the caller returns + * after it whatever happened: a click inside the surface does nothing else. + */ +function settingsClick(target: HTMLElement): void { + const tab = target.closest("[data-settings-tab]"); + if (tab) { + const id = tab.dataset.settingsTab ?? null; + if (isSettingsTab(id)) selectSettingsTab(id, false); + return; + } + if (target.closest("#settings-provision")) { + void provisionModel(); + return; + } + // Settings → Chat. The Local/Cloud segments are a *view* of the endpoint (render.ts + // says why), so pressing one rewrites the URL field to that configuration's starting + // point and shows or hides the key + its privacy copy — the consent moment is the + // configuration moment (M5). + const chatMode = target.closest("[data-chat-mode]"); + if (chatMode) { + setChatMode(chatMode.dataset.chatMode === "cloud"); + return; + } + if (target.closest("#settings-chat-save")) { + void saveChatConfig(); + return; + } + // The Model field's two shapes (render.ts's `chatModelFieldHtml`). Neither saves: + // this only decides whether the field is a list of what the daemon has or a box for + // a name it doesn't have yet. + if (target.closest("[data-chat-model-custom]")) { + setChatModelTyped(true); + return; + } + if (target.closest("[data-chat-model-pick]")) { + setChatModelTyped(false); + return; + } + const useModel = target.closest("[data-chat-use-model]"); + if (useModel) { + void useChatModel(useModel.dataset.chatUseModel ?? ""); + return; + } + if (target.closest("[data-chat-clear-key]")) { + void clearChatKey(); + return; + } + // Settings → Index: the manual Reindex, which used to be a top-bar button. Handled + // in here because this branch returns unconditionally — a click inside the dialog + // never reaches the shell's handlers below. The dialog deliberately stays open: the + // run's meter and its Cancel are in the top bar, one Esc away, and closing a dialog + // out from under the button you just pressed hides the result of pressing it. + if (target.closest("#reindex")) { + trackIndexing(doReindex()); + return; + } + // …and the Cancel beside it while a run is live. It is the top bar's Cancel in a + // second place, not a second behaviour — the bar itself is behind this surface now. + if (target.closest("[data-cancel-reindex]")) { + void cancelReindex(); + return; + } + const themeBtn = target.closest("[data-theme-choice]"); + if (themeBtn) { + const choice = themeBtn.dataset.themeChoice ?? null; + if (isThemePref(choice)) setTheme(choice); + return; + } + // Settings → Keyboard: a chord chip opens the recorder on that command; the strip's + // own buttons commit, back out, or restore a default. Checked before Done/backdrop + // so a click inside the strip is never read as "close the dialog". + const chip = target.closest("[data-rebind]"); + if (chip) { + const id = chip.dataset.rebind ?? ""; + if (findBinding(activeBindings(), id)) startRecording(id as BindingId); + return; + } + if (target.closest("#keys-save")) { + commitRecording(); + return; + } + if (target.closest("#keys-cancel")) { + stopRecording(); + return; + } + if (target.closest("#keys-reset-one")) { + if (state.recorder) resetChord(state.recorder.id); + return; + } + if (target.closest("#keys-reset-all")) { + resetAllChords(); + return; + } + if (target.closest("[data-settings-close]")) closeSettings(); +} + +/** Every listener the app registers, in registration order — which matters only where two + * share an event and a target (`wireCmdHold`'s keydown before `wireChords`'), but is kept + * exactly as it was everywhere, so a split never reorders what a user can feel. */ function wireEvents(): void { wireCmdHold(); // hold ⌘ and the app says what ⌘ does — a spectator, so it goes on first - + wireFocusMemory(); + wireFmErrorClear(); + wireClicks(); + wireContextMenu(); + wireTreeKeys(); + wireSideKeys(); + wireMenuDismissal(); + wireFindBar(); + wireForms(); + wireChords(); + wireMouseHistory(); + wireWindowBlur(); + wireDrags(); +} + +function wireFocusMemory(): void { // Remember where the keyboard is, continuously (K1). `syncOverlayFocus` needs the // element that *triggered* an overlay, and by the time it runs that element has been // swapped out of the DOM — see `lastFocused`. Capture-phase isn't needed (`focusin` @@ -4557,7 +4570,9 @@ function wireEvents(): void { lastFocused = t; } }); +} +function wireFmErrorClear(): void { // Typing in the frontmatter mini-editor clears its inline error — the message // belonged to the save attempt that failed. Delegated (like the clicks below) // because the textarea renders dynamically. @@ -4566,7 +4581,9 @@ function wireEvents(): void { hideFmError(); } }); +} +function wireClicks(): void { // Delegated clicks for everything that renders dynamically. document.addEventListener("click", (e) => { const target = e.target as HTMLElement; @@ -4574,87 +4591,7 @@ function wireEvents(): void { // An open right-click menu owns the next click: its own items act, any other // click merely dismisses it (a menu-dismissing click isn't also a card click). if (state.contextMenu) { - const menu = state.contextMenu; - if (menu.kind === "tree") { - if (menu.node && target.closest("[data-ctx-rename]")) { - startTreeRename(menu.node); // clears the menu itself - return; - } - if (menu.node && target.closest("[data-ctx-move]")) { - openMoveModal(menu.node); - return; - } - // The two copy actions. Both close the menu first — the copy is instant and - // its confirmation is the status line, so leaving the menu up over it would - // be the only thing hiding the answer. - if (menu.node && target.closest("[data-ctx-copy-vault-path]")) { - const p = menu.node.path; - closeContextMenu(); - void copyPath(p); - return; - } - if (menu.node && target.closest("[data-ctx-copy-system-path]")) { - const p = menu.node.path; - const root = state.vaultRoot; - closeContextMenu(); - // Both entry points refuse to open this menu without a vault, so `root` is - // set here — but the absolute path is the root's to give, and inventing one - // for a vault-less window is not this handler's call. - if (root !== null) void copyPath(systemPath(root, p)); - return; - } - if (menu.node && target.closest("[data-ctx-delete]")) { - requestDelete(menu.node); // clears the menu itself - return; - } - if (target.closest("[data-ctx-new-note]")) { - startTreeCreate("note", menu.dir); // clears the menu itself - return; - } - if (target.closest("[data-ctx-new-folder]")) { - startTreeCreate("folder", menu.dir); - return; - } - if (target.closest("[data-ctx-import]")) { - const dir = menu.dir; - closeContextMenu(); // the picker is modal to the OS — let the menu go first - void pickAndImport(dir); - return; - } - closeContextMenu(); - return; - } - if (target.closest("[data-ctx-open]")) { - const p = menu.path; - closeContextMenu(); - void openNote(p); - return; - } - if (target.closest("[data-ctx-insert]")) { - const p = menu.path; - closeContextMenu(); - insertCardLink(p); // the drag's keyboard half — aimed at the caret's line - return; - } - if (target.closest("[data-ctx-link]")) { - const { path, title } = menu; - closeContextMenu(); - openLinkModal(path, title ?? ""); - return; - } - if (target.closest("[data-ctx-explain]")) { - const p = menu.path; - closeContextMenu(); - void openExplain(p); - return; - } - if (target.closest("[data-ctx-why]")) { - const { path, title } = menu; - closeContextMenu(); - void askWhy({ path, title }); - return; - } - closeContextMenu(); + contextMenuClick(target); return; } @@ -4751,96 +4688,7 @@ function wireEvents(): void { // up. There is no click-outside to close on any more — the surface is the whole window // (render.ts) — so the ways out are Done and Escape. if (state.settingsOpen) { - const tab = target.closest("[data-settings-tab]"); - if (tab) { - const id = tab.dataset.settingsTab ?? null; - if (isSettingsTab(id)) selectSettingsTab(id, false); - return; - } - if (target.closest("#settings-provision")) { - void provisionModel(); - return; - } - // Settings → Chat. The Local/Cloud segments are a *view* of the endpoint (render.ts - // says why), so pressing one rewrites the URL field to that configuration's starting - // point and shows or hides the key + its privacy copy — the consent moment is the - // configuration moment (M5). - const chatMode = target.closest("[data-chat-mode]"); - if (chatMode) { - setChatMode(chatMode.dataset.chatMode === "cloud"); - return; - } - if (target.closest("#settings-chat-save")) { - void saveChatConfig(); - return; - } - // The Model field's two shapes (render.ts's `chatModelFieldHtml`). Neither saves: - // this only decides whether the field is a list of what the daemon has or a box for - // a name it doesn't have yet. - if (target.closest("[data-chat-model-custom]")) { - setChatModelTyped(true); - return; - } - if (target.closest("[data-chat-model-pick]")) { - setChatModelTyped(false); - return; - } - const useModel = target.closest("[data-chat-use-model]"); - if (useModel) { - void useChatModel(useModel.dataset.chatUseModel ?? ""); - return; - } - if (target.closest("[data-chat-clear-key]")) { - void clearChatKey(); - return; - } - // Settings → Index: the manual Reindex, which used to be a top-bar button. Handled - // in here because this branch returns unconditionally — a click inside the dialog - // never reaches the shell's handlers below. The dialog deliberately stays open: the - // run's meter and its Cancel are in the top bar, one Esc away, and closing a dialog - // out from under the button you just pressed hides the result of pressing it. - if (target.closest("#reindex")) { - trackIndexing(doReindex()); - return; - } - // …and the Cancel beside it while a run is live. It is the top bar's Cancel in a - // second place, not a second behaviour — the bar itself is behind this surface now. - if (target.closest("[data-cancel-reindex]")) { - void cancelReindex(); - return; - } - const themeBtn = target.closest("[data-theme-choice]"); - if (themeBtn) { - const choice = themeBtn.dataset.themeChoice ?? null; - if (isThemePref(choice)) setTheme(choice); - return; - } - // Settings → Keyboard: a chord chip opens the recorder on that command; the strip's - // own buttons commit, back out, or restore a default. Checked before Done/backdrop - // so a click inside the strip is never read as "close the dialog". - const chip = target.closest("[data-rebind]"); - if (chip) { - const id = chip.dataset.rebind ?? ""; - if (findBinding(activeBindings(), id)) startRecording(id as BindingId); - return; - } - if (target.closest("#keys-save")) { - commitRecording(); - return; - } - if (target.closest("#keys-cancel")) { - stopRecording(); - return; - } - if (target.closest("#keys-reset-one")) { - if (state.recorder) resetChord(state.recorder.id); - return; - } - if (target.closest("#keys-reset-all")) { - resetAllChords(); - return; - } - if (target.closest("[data-settings-close]")) closeSettings(); + settingsClick(target); return; // clicks inside Settings do nothing else } @@ -5051,7 +4899,9 @@ function wireEvents(): void { return; } }); +} +function wireContextMenu(): void { // Right-click surfaces. The file tree's default menu is taken over wholesale: // New note / New folder, contextual on the row under the cursor — a folder row // targets itself, a file row its parent folder, the pane's empty space the vault @@ -5076,7 +4926,9 @@ function wireEvents(): void { e.preventDefault(); openCardMenu(e.clientX, e.clientY, card.dataset.cardPath ?? "", card.dataset.cardTitle ?? ""); }); +} +function wireTreeKeys(): void { // The file tree's own keyboard, the ARIA `tree` pattern (K1, GH #78). Bound to the // pane rather than the document so it answers *before* the global chords below, and // only while the keyboard is actually on a row. The moves themselves are pure and @@ -5122,7 +4974,9 @@ function wireEvents(): void { } } }); +} +function wireSideKeys(): void { // Discovery's own keyboard — the *same* ARIA `tree` pattern as the file tree (K1, GH #78), // bound to the pane so it answers before the global chords. The moves are pure and tested // (sidenav.ts `sideArrowMove`); this half is the DOM and the folding, plus one wrinkle the @@ -5175,13 +5029,17 @@ function wireEvents(): void { open.click(); } }); +} +function wireMenuDismissal(): void { // The floating menu is positioned at fixed viewport coords, so any scroll or resize // strands it — dismiss rather than let it hover over the wrong card. Capture-phase so // a scroll inside the side pane (which doesn't bubble) is still caught. document.addEventListener("scroll", closeContextMenu, true); window.addEventListener("resize", closeContextMenu); +} +function wireFindBar(): void { // The find bar is static shell chrome — direct listeners, not delegation. mousedown // preventDefault keeps focus in the find input across button clicks, so Enter keeps // stepping without a re-click. @@ -5196,7 +5054,9 @@ function wireEvents(): void { btn.addEventListener("mousedown", (e) => e.preventDefault()); btn.addEventListener("click", act); } +} +function wireForms(): void { // Search on submit (Enter). document.addEventListener("submit", (e) => { if ((e.target as HTMLElement).id === "search-form") { @@ -5247,7 +5107,9 @@ function wireEvents(): void { void commitTreeRename((t as HTMLInputElement).value); } }); +} +function wireChords(): void { // The app's chords. Every `isBound(e, …)` below asks the keyboard registry // (bindings.ts) whether this keystroke is that command — the registry owns *what* the // chord is, and the sheet in Settings → Keyboard is projected from the same table, so @@ -5652,7 +5514,9 @@ function wireEvents(): void { void navGo(back ? -1 : 1); } }); +} +function wireMouseHistory(): void { // Mouse back/forward buttons (W3C numbering: 3 back, 4 forward) walk the history // too. `auxclick` fires only for non-primary buttons, so this never doubles the // click delegation above. @@ -5661,7 +5525,9 @@ function wireEvents(): void { e.preventDefault(); void navGo(e.button === 3 ? -1 : 1); }); +} +function wireWindowBlur(): void { // Losing window focus is a flush point: the buffer lands on disk before the user // looks at (or edits in) anything else. // @@ -5678,7 +5544,9 @@ function wireEvents(): void { render(); } }); +} +function wireDrags(): void { // --- tree drag-and-drop --------------------------------------------------------- // // Two drags land here, and they are told apart by `treeDrag` being set: a **tree From 1e24a7620d8dc64daf92dbab2713274b9e9f6363 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:38:11 +0000 Subject: [PATCH 09/30] refactor(cli): split main.rs into domain modules A pure move: no line of behaviour changes. main.rs keeps the module doc, `main`, `dispatch` and `print_json`; the rest goes where its domain is: - args.rs Cli, Command, LlmArgs, AGENT_HELP, the vault-root rules - logging.rs init_logging - wiring.rs open_vault / open_llm and the fake switches - error.rs CliError, user_message and its unit tests - reindex.rs init, reindex, status, the lock and --cancel - chat.rs ask, why, chat and the streamed rendering - read.rs neighbors, explain, search, similar (+ --explain), the presentation helpers and the search_report tests - write.rs add, write, mv, rm, link and their tallies - cancel.rs the CANCEL/ANSWERING flags and cancel_flow, which both reindex.rs and chat.rs read Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-cli/src/args.rs | 256 +++++ crates/b2-cli/src/cancel.rs | 33 + crates/b2-cli/src/chat.rs | 232 +++++ crates/b2-cli/src/error.rs | 225 +++++ crates/b2-cli/src/logging.rs | 63 ++ crates/b2-cli/src/main.rs | 1809 +--------------------------------- crates/b2-cli/src/read.rs | 456 +++++++++ crates/b2-cli/src/reindex.rs | 345 +++++++ crates/b2-cli/src/wiring.rs | 52 + crates/b2-cli/src/write.rs | 197 ++++ 10 files changed, 1878 insertions(+), 1790 deletions(-) create mode 100644 crates/b2-cli/src/args.rs create mode 100644 crates/b2-cli/src/cancel.rs create mode 100644 crates/b2-cli/src/chat.rs create mode 100644 crates/b2-cli/src/error.rs create mode 100644 crates/b2-cli/src/logging.rs create mode 100644 crates/b2-cli/src/read.rs create mode 100644 crates/b2-cli/src/reindex.rs create mode 100644 crates/b2-cli/src/wiring.rs create mode 100644 crates/b2-cli/src/write.rs diff --git a/crates/b2-cli/src/args.rs b/crates/b2-cli/src/args.rs new file mode 100644 index 0000000..5f8c8cf --- /dev/null +++ b/crates/b2-cli/src/args.rs @@ -0,0 +1,256 @@ +//! The command line itself: the clap definitions, the agent help text, and the rule +//! for which vault root a command runs against. + +use crate::error::CliError; +use clap::{Args, Parser, Subcommand}; +use std::path::{Path, PathBuf}; + +/// The retrieval loop, stated where an agent will actually read it (`b2 --help`) — +/// the toolset carrying its own usage instructions, the way an MCP server ships its +/// tool descriptions. The loop itself lives in the *caller*: B2's part is dumb, +/// composable reads (ADR-0012), and the vault being plain Markdown on disk is what +/// makes "open the file yourself" a step no B2 command needs to exist for. +pub const AGENT_HELP: &str = "\ +For agents (--json): + Every command takes --json, and the vault is plain Markdown on disk — a result's + `path` is a real file, readable and greppable directly. The retrieval loop: + + 1. b2 search \"\" --json Hybrid keyword+semantic search: the ranked rows + plus an evidence verdict. `\"vouched\": false` + means the vault holds no evidence for this query + — report \"no matches\" instead of using the rows. + 2. Read the files it names Paths are vault-relative; open them for the full + note, not just the matched snippet. + 3. b2 neighbors/explain Follow the typed graph around a promising note; + b2 similar ranks unlinked semantic + neighbors. + 4. Refine and search again Pass --exclude (repeatable) for notes + already inspected, so follow-ups surface fresh + material. Fewer results than asked = the head of + the ranking is spent; refine the query."; + +#[derive(Parser)] +#[command( + name = "b2", + version, + about = "B2 — explore a Markdown vault's typed graph and search from the terminal", + after_long_help = AGENT_HELP +)] +pub struct Cli { + /// Vault root (the folder of Markdown). The index lives in `/.b2/`. + /// Set it with `-C ` or `$B2_VAULT_PATH` (the flag wins). Read-only commands + /// fall back to the current dir; commands that write (`reindex`/`add`/`mv`/`rm`/ + /// `link`) require it explicitly, so they can never silently touch the wrong directory. + #[arg(short = 'C', long = "vault", global = true, env = "B2_VAULT_PATH")] + pub vault: Option, + + /// Emit machine-readable JSON instead of human-readable text. + #[arg(long, global = true)] + pub json: bool, + + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand)] +pub enum Command { + /// Download + verify the embedding model into the shared cache (one-time setup). + Init, + /// Re-project every note under the vault into the index. Reads your vault and + /// writes only to `.b2/`. Incremental by default: notes whose content is + /// unchanged keep their vectors. + Reindex { + /// Vault root (overrides --vault / $B2_VAULT_PATH). Required — no cwd default. + vault: Option, + /// Re-embed every note, even unchanged ones (a full rebuild in place). + #[arg(long)] + force: bool, + /// Preview what a reindex would do without touching the index — how many + /// notes it would project and embed. + #[arg(long)] + dry_run: bool, + /// Stop a reindex already running on this vault — the way to reach one + /// backgrounded with `b2 reindex &`, which has no controlling terminal for + /// Ctrl-C. It stops after the current batch, leaving the same consistent, + /// re-runnable partial index a foreground Ctrl-C does. + #[arg(long, conflicts_with_all = ["force", "dry_run"])] + cancel: bool, + }, + /// Report embedding coverage — how many notes are embedded (so semantic ranking + /// is live vs. keyword-only) and whether a reindex is currently running, with the + /// process id to stop it by. A pure, model-free read; handy after kicking off a + /// slow reindex in the background with `b2 reindex &`. + Status, + /// Create a new note and project it into the index (it's immediately in the + /// graph + searchable). PATH is vault-relative (the `.md` extension is optional). + Add { + /// Where the new note goes: a vault-relative path (`.md` optional). + path: String, + /// The note's human title (frontmatter `title:`). + #[arg(long)] + title: Option, + /// Initial body content (Markdown). Omit for an empty note to fill in later. + #[arg(long)] + content: Option, + }, + /// Replace a note's **body** with Markdown piped on stdin — the CLI/agent editing + /// surface (the counterpart to the desktop editor's save). NOTE is a vault-relative + /// path; the note must already exist (`b2 add` creates one). Pipe the body + /// **alone** — the frontmatter (including B2's managed `b2_relations:`) is + /// left untouched, so no `---` block. The note is re-projected (chunks + FTS + graph) + /// so the index stays live; a later `b2 reindex` fills the changed body's vectors. + Write { + /// The note whose body to overwrite: a vault-relative path. + note: String, + }, + /// Show a note's typed neighbors. NOTE is a vault-relative path. + Neighbors { note: String }, + /// Explain a note's connections — every typed edge and its "why". NOTE is a + /// vault-relative path. + Explain { note: String }, + /// Move/rename a note, file, or folder and rewrite every inbound link to + /// point at the new path. An existing directory moves as a whole folder + /// (everything under it, one rename). + Mv { + /// What to move: a vault-relative path (note, file, or folder). + from: String, + /// The new vault-relative path (for a note, the `.md` extension is optional). + to: String, + }, + /// Delete a note, file, or folder from the vault *and* the disk. Inbound links + /// are never rewritten — they dangle and surface as unresolved (`b2 explain`). + /// An existing directory deletes as a whole folder and requires --recursive. + Rm { + /// What to delete: a vault-relative path (note, file, or folder). + target: String, + /// Required to delete a folder (and everything inside it, unindexed files too). + #[arg(short = 'r', long)] + recursive: bool, + }, + /// Hybrid keyword+semantic search across the vault. A query the vault + /// holds no evidence for answers "no matches" rather than serving its nearest + /// rows (invariants.md D2); `--json` is an **object** — the rows plus that + /// verdict, which has nowhere to live in a bare list. + Search { + query: String, + /// Maximum number of notes to return. + #[arg(long, default_value_t = 10)] + limit: usize, + /// Leave a note out of the results: a vault-relative path exactly as a + /// previous search served it (repeatable). The follow-up-search flag for + /// agent loops — pass the notes already inspected so a re-query surfaces + /// fresh material instead of the same head. It subtracts rows only: the + /// evidence verdict still reads the whole vault, and a heavily-excluded + /// query may serve fewer than LIMIT notes — the cue to refine the query + /// rather than page deeper. + #[arg(long, value_name = "PATH")] + exclude: Vec, + }, + /// Surface the notes most semantically similar to NOTE that you haven't linked + /// yet — connection discovery, ranked nearest first. NOTE is a vault-relative + /// path. A local read over stored vectors (run `b2 reindex` with the real + /// model first). Similarity is relative to your vault: the list is always + /// the ranked nearest, and you are the judge of which are worth a link. + Similar { + /// The note to find similar notes for: a vault-relative path. + note: String, + /// Maximum number of similar notes to return. + #[arg(long, default_value_t = 10)] + limit: usize, + /// Explain one note against NOTE's list instead of printing the list: where it + /// stands (and why, if it is not a card), and the passage pairs behind it. No + /// model call. Read at `--limit`, so the rank is the one the list shows. + #[arg(long, value_name = "OTHER")] + explain: Option, + }, + /// Commit a typed connection SRC → DST into SRC's frontmatter `b2_relations:`. + /// SRC and DST are each a vault-relative path. + Link { + /// The source note (the edge points *from* it): a vault-relative path. + src: String, + /// The target note (the edge points *to* it): a vault-relative path. + dst: String, + /// The relation verb (a core verb: references/supports/contradicts). + #[arg(long = "type", default_value = "references")] + edge_type: String, + /// Optional explanation — trailing text shown after the link. + #[arg(long)] + explanation: Option, + }, + /// Ask one question about your vault and stream a grounded answer. The model + /// answers **only** from passages retrieved out of your notes and cites them by + /// `[n]`; `--json` emits the answer as a JSON Lines event stream (one token + /// event per token, then the final answer with its citations resolved). + /// Needs a model server — Ollama by default (see `--llm-url`). + Ask { + /// The question, in plain language. + question: String, + #[command(flatten)] + llm: LlmArgs, + }, + /// Explain why CANDIDATE shows up in `b2 similar NOTE` — a grounded, cited answer + /// streamed from the chat model, which is given B2's read-only tools (the matched + /// passages, the similar list, a note's links, reading a note) and looks up what it + /// needs. The answer lists the tools that ran. A model with no tool support is handed + /// the matched passages instead. `--json` streams events as `ask` does. + /// Needs a model server — Ollama by default (see `--llm-url`). + Why { + /// The note the list was for: a vault-relative path. + note: String, + /// The suggested note to explain: a vault-relative path. + candidate: String, + /// The length of the `similar` list the rank is quoted against. + #[arg(long, default_value_t = 10)] + limit: usize, + #[command(flatten)] + llm: LlmArgs, + }, + /// Interactive grounded chat about your vault — the same answers as `ask`, with + /// follow-up questions that remember the conversation. Ctrl-C stops an answer + /// mid-stream (the partial text stands); `/exit` or Ctrl-D leaves. History is + /// session-only: nothing about a chat is ever written to your notes or the index. + Chat { + #[command(flatten)] + llm: LlmArgs, + }, +} + +/// Which model to chat with, and where it lives — shared by `ask` and `chat`. Precedence +/// is the `B2_VAULT_PATH` convention exactly: an explicit flag beats the environment beats +/// the default. Resolution lives once in `b2_llm::LlmConfig`, so the desktop's settings +/// layer over the same base — which is why these are plain options, not clap `env` args. +#[derive(Args, Debug)] +pub struct LlmArgs { + /// The OpenAI-compatible base URL of your model server + /// [env: B2_LLM_URL, default: http://localhost:11434/v1 — Ollama's]. + /// Any compatible endpoint works: LM Studio, llama.cpp, vLLM, or a cloud + /// provider's (which sends your question and the retrieved passages to them — + /// pair it with B2_LLM_API_KEY). + #[arg(long = "llm-url", value_name = "URL")] + pub llm_url: Option, + /// The chat model id, as the server names it + /// [env: B2_LLM_MODEL, default: llama3.2]. Pull it first, e.g. `ollama pull + /// llama3.2`. + #[arg(long = "llm-model", value_name = "MODEL")] + pub llm_model: Option, +} + +impl Cli { + /// The vault root for **read-only** commands (`search`, `neighbors`, `explain`, + /// `similar`): the `-C`/`$B2_VAULT_PATH` value if given, else the current directory. + /// A pure read can't pollute anything, so the cwd convenience is safe here. + pub fn vault_or_cwd(&self) -> &Path { + self.vault.as_deref().unwrap_or_else(|| Path::new(".")) + } + + /// The vault root for commands that **write** (`reindex`, `add`, `mv`, `rm`, `link`): + /// `positional` wins, then `-C`/`$B2_VAULT_PATH`; with none, error rather than + /// silently mutating the current directory, where a stale binary or mistyped var would + /// otherwise leave a stray `.b2/`. The write-side counterpart to + /// [`vault_or_cwd`](Self::vault_or_cwd). + pub fn require_vault<'a>(&'a self, positional: Option<&'a Path>) -> Result<&'a Path, CliError> { + positional + .or(self.vault.as_deref()) + .ok_or(CliError::VaultRequired) + } +} diff --git a/crates/b2-cli/src/cancel.rs b/crates/b2-cli/src/cancel.rs new file mode 100644 index 0000000..ca82f88 --- /dev/null +++ b/crates/b2-cli/src/cancel.rs @@ -0,0 +1,33 @@ +//! The cooperative-cancel flags a SIGINT handler flips, read by the two long loops a +//! command can stop part-way: the reindex embed loop and the chat token stream. + +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Set by the SIGINT handler installed for a long-running command — by Ctrl-C in the +/// foreground, or (for a `reindex`) by another process's `b2 reindex --cancel` (GH #55), +/// which raises the *same* signal so both reach here. +/// +/// Two loops read it through the same [`ControlFlow`] seam: the reindex embed loop at +/// every batch boundary, stopping *after* the current batch so a cancel leaves a +/// consistent, re-runnable index; and the chat surfaces' token callback at every token, +/// where stopping renders the partial answer honestly. `chat` clears it before each turn. +pub static CANCEL: AtomicBool = AtomicBool::new(false); + +/// Whether `b2 chat` is streaming an answer right now — which is what decides +/// what Ctrl-C *means* in the REPL: cancel this answer, or leave. Only `chat` +/// maintains it (see its handler); every other command's SIGINT meaning is +/// unconditional. +pub static ANSWERING: AtomicBool = AtomicBool::new(false); + +/// Map the Ctrl-C flag onto the cooperative-cancel signal the engine's loops read — +/// [`ControlFlow::Break`] once a cancel has been requested, else +/// [`ControlFlow::Continue`]. Shared by `reindex`'s two progress closures and by the +/// chat surfaces' token callback. +pub fn cancel_flow() -> ControlFlow<()> { + if CANCEL.load(Ordering::SeqCst) { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } +} diff --git a/crates/b2-cli/src/chat.rs b/crates/b2-cli/src/chat.rs new file mode 100644 index 0000000..fd9a35d --- /dev/null +++ b/crates/b2-cli/src/chat.rs @@ -0,0 +1,232 @@ +//! The chat surfaces: `ask`, `why` and the `chat` REPL, and the streamed rendering +//! they share. + +use crate::args::{Cli, LlmArgs}; +use crate::cancel::{cancel_flow, ANSWERING, CANCEL}; +use crate::error::{user_message, CliError}; +use crate::wiring::{open_llm, open_vault, use_fake_llm}; +use b2_core::llm::{ChatTurn, LlmProvider}; +use b2_core::vault::{AnswerView, Vault}; +use std::io::{IsTerminal, Write}; +use std::ops::ControlFlow; +use std::sync::atomic::Ordering; + +pub fn cmd_ask(cli: &Cli, question: &str, llm_args: &LlmArgs) -> Result<(), CliError> { + // The provider first, deliberately: it is the cheap check, and loading the real + // embedder below takes seconds. A stopped model server should cost a round trip, + // not a model load followed by a failure. + let llm = open_llm(llm_args)?; + // Retrieval embeds the question for the vector half → the real model, like `search`. + let vault = open_vault(cli.vault_or_cwd(), true)?; + // Ctrl-C cancels the answer at the next token rather than killing the process, so + // what already streamed stays on screen and is reported as partial. + let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); + ask_streamed(&vault, llm.as_ref(), question, &[], cli.json)?; + if !cli.json { + note_fake_llm(); + } + Ok(()) +} + +pub fn cmd_why( + cli: &Cli, + note: &str, + candidate: &str, + limit: usize, + llm_args: &LlmArgs, +) -> Result<(), CliError> { + let llm = open_llm(llm_args)?; + // A pure read over stored vectors, like `similar`: nothing embeds a query, so the + // real model is never loaded. + let vault = open_vault(cli.vault_or_cwd(), false)?; + let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); + let json = cli.json; + let answer = vault.why_similar(llm.as_ref(), note, candidate, limit, &mut |token| { + stream_token(token, json) + })?; + finish_answer(&answer, json); + if !json { + note_fake_llm(); + } + Ok(()) +} + +pub fn cmd_chat(cli: &Cli, llm_args: &LlmArgs) -> Result<(), CliError> { + let llm = open_llm(llm_args)?; + let vault = open_vault(cli.vault_or_cwd(), true)?; + // Ctrl-C means two different things in a REPL, and a handler that only ever meant one + // would trap the user: mid-answer it cancels the stream (the partial text stands), but + // at an idle prompt it must still be the way out, since swallowing it would leave + // `/exit` and Ctrl-D as the only exits. `ctrlc` runs this on its own thread. + let _ = ctrlc::set_handler(|| { + if ANSWERING.load(Ordering::SeqCst) { + CANCEL.store(true, Ordering::SeqCst); + } else { + // 128 + SIGINT, the shell's own convention for "interrupted". + std::process::exit(130); + } + }); + // Prompts and the banner are chrome for a human at a terminal: on stderr so + // stdout stays answers, and only when there's a terminal there to read them + // (the `reindex` progress-line rule). + let interactive = !cli.json && std::io::stderr().is_terminal(); + if interactive { + eprintln!( + "Grounded chat over your notes — answers come only from what B2 retrieves \ + from them, cited by [n]." + ); + eprintln!( + "Model: {}. Ctrl-C stops an answer; /exit or Ctrl-D leaves. \ + Nothing here is saved.", + llm.model_id() + ); + } + if !cli.json { + note_fake_llm(); + } + // Session-only history (S4): the turns live in this Vec and die with the process — + // a persisted transcript would be B2-derived state outside the Markdown. + let mut history: Vec = Vec::new(); + let stdin = std::io::stdin(); + loop { + if interactive { + eprint!("\nyou> "); + let _ = std::io::stderr().flush(); + } + let mut line = String::new(); + if stdin.read_line(&mut line)? == 0 { + // Ctrl-D / end of a piped script. + break; + } + let question = line.trim(); + if question.is_empty() { + continue; + } + if matches!(question, "/exit" | "/quit") { + break; + } + // A fresh cancel budget per turn: the Ctrl-C that stopped the *last* answer + // must not cancel this one before its first token. + CANCEL.store(false, Ordering::SeqCst); + ANSWERING.store(true, Ordering::SeqCst); + let turn = ask_streamed(&vault, llm.as_ref(), question, &history, cli.json); + ANSWERING.store(false, Ordering::SeqCst); + match turn { + Ok(answer) => { + // A cancelled answer goes into the history too: the human saw that + // text, so a follow-up referring to it ("go on") must be read + // against what was actually said, not against a turn we pretend + // never happened. + history.push(ChatTurn::user(question)); + history.push(ChatTurn::assistant(&answer.answer)); + } + // A failed turn ends the turn, not the session: a model server that + // hiccuped is worth retyping a question at, not worth losing the + // conversation over. The message is the same one the process would + // have exited with. + Err(e) => eprintln!("{}", user_message(&e)), + } + } + Ok(()) +} + +/// One grounded ask, rendered as it arrives — the shared body of `ask` and each `chat` +/// turn (flow ④'s streaming contract: every surface streams). Under `--json` this is a +/// **JSON Lines event stream** — one `token` event per token, then one `answer` event +/// carrying the [`AnswerView`] — so an agent sees the answer forming and still gets the +/// resolved citations as data. +fn ask_streamed( + vault: &Vault, + llm: &dyn LlmProvider, + question: &str, + history: &[ChatTurn], + json: bool, +) -> Result { + let answer = vault.ask(llm, question, history, &mut |token| { + stream_token(token, json) + })?; + finish_answer(&answer, json); + Ok(answer) +} + +/// Render one streamed token — the framing `ask`, `chat` and `why` share — and report +/// whether Ctrl-C has asked the stream to stop. +fn stream_token(token: &str, json: bool) -> ControlFlow<()> { + if json { + print_event(&AskEvent::Token { text: token }); + } else { + print!("{token}"); + // Streaming is the point: an unflushed line renders in one lump. + let _ = std::io::stdout().flush(); + } + cancel_flow() +} + +/// Close a streamed answer: the final `answer` event under `--json`, else the sources. +fn finish_answer(answer: &AnswerView, json: bool) { + if json { + print_event(&AskEvent::Answer(answer)); + } else { + print_answer_tail(answer); + } +} + +/// One line of the `--json` ask stream. The framing is the CLI's; the payload of +/// the final event is `b2-core`'s [`AnswerView`], which is the standing +/// convention (the view types are the adapters' shared contract — the desktop +/// carries the same two facts as Tauri events plus a command return). +#[derive(serde::Serialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum AskEvent<'a> { + /// A token, exactly as it arrived from the model. + Token { text: &'a str }, + /// The finished answer: text, resolved citations, and whether it was cut short. + Answer(&'a AnswerView), +} + +/// Print one event as a single line of JSON — **not** `print_json`, which is +/// pretty-printed: this is a stream, so one object per line is the contract. +fn print_event(event: &AskEvent) { + if let Ok(line) = serde_json::to_string(event) { + println!("{line}"); + } +} + +/// The human-readable tail of an answer: end the streamed line, then the sources +/// the model cited, then — honestly — whether the answer is the whole of one. +fn print_answer_tail(answer: &AnswerView) { + println!(); + if !answer.citations.is_empty() { + println!("\nSources:"); + for c in &answer.citations { + println!(" [{}] {}", c.marker, c.path); + if !c.excerpt.is_empty() { + println!(" {}", c.excerpt); + } + } + } + if !answer.tools.is_empty() { + println!("\nB2 tools used:"); + for t in &answer.tools { + // A lookup B2 made itself is marked, so the list never overstates what the + // model chose to do. + let by = if t.seeded { " (made by B2)" } else { "" }; + println!(" {} {}{by}", t.name, t.arguments); + } + } + if answer.cancelled { + eprintln!("(stopped early — the answer above is partial.)"); + } +} + +/// Never overstate what answered (the `search` fake-embedder caveat, applied to +/// the chat seam): under `B2_LLM=fake` the "answer" is deterministic scaffolding, +/// not a model. On stderr, so stdout stays the answer. +fn note_fake_llm() { + if use_fake_llm() { + eprintln!( + "note: the fake chat provider is in use (B2_LLM=fake) — answers are \ + deterministic test scaffolding, not a model." + ); + } +} diff --git a/crates/b2-cli/src/error.rs b/crates/b2-cli/src/error.rs new file mode 100644 index 0000000..96cf6f9 --- /dev/null +++ b/crates/b2-cli/src/error.rs @@ -0,0 +1,225 @@ +//! The CLI's error type and its one translation into a user-facing sentence. + +use b2_embed::EmbedError; +use b2_llm::{is_ollama, refusal_message, LlmError, OLLAMA_INSTALL_URL}; + +/// The CLI's error, composing the two crates it drives. Kept internal; `user_message` +/// turns it into a generic, actionable, no-internals-leaked line (logging policy). +/// `#[from]` supplies the `?` conversions; `transparent` defers `Display` to the +/// inner error (only ever surfaced under `B2_DEBUG`). +#[derive(Debug, thiserror::Error)] +pub enum CliError { + #[error(transparent)] + Core(#[from] b2_core::Error), + #[error(transparent)] + Embed(#[from] EmbedError), + /// A setup-time chat failure the adapter can act on — an unreachable model + /// server, an un-pulled model. Call-time failures arrive as + /// [`b2_core::Error::Llm`] instead (the seam collapses them to a message). + #[error(transparent)] + Llm(#[from] LlmError), + #[error(transparent)] + Serde(#[from] serde_json::Error), + /// A filesystem error creating `.b2/` or opening the reindex lock file. + #[error(transparent)] + Io(#[from] std::io::Error), + /// `reindex` was run with no vault at all (no positional, no `-C`, no + /// `$B2_VAULT_PATH`) — refuse rather than silently index the current directory. + #[error("no vault specified")] + VaultRequired, + /// Another `reindex` already holds the single-in-flight lock on this vault. + #[error("a reindex is already running")] + ReindexRunning, + /// `reindex --cancel` found no run in flight on this vault — nothing to signal. + #[error("no reindex is running")] + NoReindexRunning, + /// `reindex --cancel` found a run in flight whose lock names no readable pid, so + /// there is no address to signal (see [`ReindexHolder::pid`]). + #[error("the running reindex recorded no pid")] + ReindexPidUnknown, + /// `rm` was pointed at a folder without `--recursive` — refuse rather than + /// silently remove a whole subtree (the CLI's stand-in for the desktop's + /// confirm dialog; there is no interactive prompt to give an agent). + #[error("folder delete requires --recursive: {0}")] + RecursiveRequired(String), + /// `write` was invoked with stdin attached to a terminal (nothing piped) — refuse + /// rather than hang waiting for hand-typed input; the new body must be piped in. + #[error("no body piped on stdin")] + StdinRequired, +} + +// `is_ollama` — "does this endpoint look like the Ollama daemon" — decides whether +// Ollama's own commands belong in an error message (`--llm-url` also points at LM Studio, +// llama.cpp, vLLM and cloud endpoints, where "run `ollama serve`" is advice about the +// wrong program). The rule lives in `b2-llm` (GH #155), where guided setup needs the same +// answer for a bigger decision, so one rule with two callers cannot drift. + +/// The head of a model server's own model list, for "…or pick one it already +/// serves". Bounded: a local runtime holds a handful, but a cloud endpoint lists +/// hundreds, and a hundred-model line is not a hint. +fn model_hint(available: &[String]) -> Option { + const SHOWN: usize = 6; + if available.is_empty() { + return None; + } + let head = available.iter().take(SHOWN).cloned().collect::>(); + Some(if available.len() > SHOWN { + format!("{}, …", head.join(", ")) + } else { + head.join(", ") + }) +} + +/// Translate an internal error into a generic, actionable, user-facing message — +/// never leaking sqlite/io/serde internals. Set `B2_DEBUG` to also print the detail. +pub fn user_message(err: &CliError) -> String { + let msg = match err { + CliError::Core(b2_core::Error::NoteNotFound(r)) => format!( + "Note not found: '{r}'. Check the path, and run `b2 reindex` first." + ), + CliError::Core(b2_core::Error::IndexTooNew { .. }) => { + "This vault's index was built by a newer version of B2 than this `b2`. Run the newer B2 (or upgrade this one) — the index was left untouched, so nothing needs rebuilding.".to_string() + } + CliError::Core(b2_core::Error::ModelMismatch { .. }) => { + "This vault's index was built with a different embedding model. Run `b2 reindex` to rebuild it.".to_string() + } + CliError::Embed(EmbedError::NotProvisioned { model, .. }) => format!( + "Embedding model '{model}' is not installed. Run `b2 init` to download it (or set B2_EMBEDDER=fake for an offline, non-semantic mode)." + ), + CliError::Embed(EmbedError::Download(_)) => { + "Could not download the embedding model. Check your network and try `b2 init` again.".to_string() + } + CliError::Core(b2_core::Error::MoveTargetExists(p)) => format!( + "Can't move: a file already exists at '{p}'. Choose a different destination." + ), + CliError::Core(b2_core::Error::MoveDestination(_)) => { + "That move destination isn't valid. Give a vault-relative path like `notes/new-name.md`.".to_string() + } + CliError::Core(b2_core::Error::MoveIncomplete { paths, .. }) => format!( + "The move failed, and B2 could not undo its link changes in: {}. Check the links in those files; `b2 reindex` then shows any that no longer resolve.", + paths.join(", ") + ), + CliError::Core(b2_core::Error::DirNotFound(p)) => format!( + "Folder not found: '{p}'. Check the path (folders are vault-relative, like `notes/archive`)." + ), + CliError::Core(b2_core::Error::AddTargetExists(p)) => format!( + "A note already exists at '{p}'. Choose a different path, or edit that note." + ), + CliError::Core(b2_core::Error::AddDestination(_)) => { + "That note path isn't valid. Give a vault-relative path like `notes/new-name.md`.".to_string() + } + CliError::Core(b2_core::Error::InvalidRelation(v)) => format!( + "'{v}' isn't a known relation type. Use one of: references, supports, contradicts." + ), + CliError::Core(b2_core::Error::WriteConflict(_)) => { + "This note changed on disk since it was opened. Reload the note, then reapply your edit.".to_string() + } + CliError::Core(b2_core::Error::Frontmatter(d)) => { + format!("Can't edit that frontmatter: {d}.") + } + CliError::Core(b2_core::Error::ResourceNotFound(r)) => format!( + "File not found in the vault: '{r}'. Check the path, and run `b2 reindex` first." + ), + CliError::Core(b2_core::Error::ResourceUnsupported(_)) => { + "Similarity for non-Markdown files isn't available yet — it arrives in a later release. `b2 explain ` shows its backlinks today.".to_string() + } + CliError::VaultRequired => { + "No vault specified. Point B2 at your vault with `-C `, or set B2_VAULT_PATH.".to_string() + } + CliError::ReindexRunning => { + "A reindex is already running on this vault. Wait for it to finish (check `b2 status`), or stop it with `b2 reindex --cancel`.".to_string() + } + CliError::NoReindexRunning => { + "No reindex is running on this vault — nothing to cancel. Check `b2 status`.".to_string() + } + CliError::ReindexPidUnknown => { + "A reindex is running on this vault, but it recorded no process id to signal. Try `b2 reindex --cancel` again in a moment, or stop the run from the shell it was started in.".to_string() + } + CliError::RecursiveRequired(p) => format!( + "'{p}' is a folder. Re-run with -r/--recursive to delete it and everything inside it." + ), + CliError::StdinRequired => { + "No body piped on stdin. Pipe the new note body in, e.g. `cat new-body.md | b2 write notes/foo`.".to_string() + } + // The E4 case chat is most likely to hit: nothing is serving the endpoint. + // Named by the endpoint the user actually configured — and *advised* by it + // too: `ollama serve` is the fix for Ollama and no help at all for LM + // Studio, llama.cpp, vLLM, or a cloud provider, all of which `--llm-url` + // supports. + CliError::Llm(LlmError::Unreachable { endpoint, .. }) if is_ollama(endpoint) => format!( + "Can't reach the model server at {endpoint} — is Ollama running? (`ollama serve`, or install: {OLLAMA_INSTALL_URL})" + ), + CliError::Llm(LlmError::Unreachable { endpoint, .. }) => format!( + "Can't reach the model server at {endpoint}. Start it, or point --llm-url (or B2_LLM_URL) at one that's running." + ), + // Something answered the probe and refused it — a *different* mistake from + // nothing listening, and one this used to report as success. The sentence is + // b2-llm's (`refusal_message`), so the CLI and the app say the same thing; the + // Ollama-root hint is the app's alone, since only its card has asked the daemon. + CliError::Llm(LlmError::Refused { endpoint, status, message }) => { + refusal_message(endpoint, *status, message, None) + } + CliError::Llm(LlmError::ModelMissing { model, endpoint, available }) => format!( + "Model '{model}' isn't available at {endpoint}. {}{}", + if is_ollama(endpoint) { + format!("Pull it with `ollama pull {model}`") + } else { + "Load it there".to_string() + }, + match model_hint(available) { + Some(list) => format!(", or point --llm-model at one it already serves ({list})."), + None => ".".to_string(), + } + ), + // Not the generic chat failure below: the server is up and the model is there, + // so that advice would mislead. The cap is the one fix on this side of the wire. + CliError::Core(b2_core::Error::ToolCallLimit { limit }) => format!( + "The chat model asked for more than {limit} tool calls in one reply, so b2 stopped it. Try again or use another model (--llm-model). If this model really needs more, raise {}.", + b2_llm::ENV_MAX_TOOL_CALLS + ), + // Every remaining chat failure — an HTTP refusal at probe time, a malformed + // stream — is one sentence with one fix, and the detail is a `B2_DEBUG` away. + CliError::Llm(_) | CliError::Core(b2_core::Error::Llm(_)) => { + "The model server couldn't answer. Check that it's running and that the model is installed (`ollama list`), then try again.".to_string() + } + _ => "Something went wrong. Please check the vault path and try again.".to_string(), + }; + if std::env::var_os("B2_DEBUG").is_some() { + // Every wrapper variant is `#[error(transparent)]` and every local variant + // carries its own `#[error("…")]` line, so the enum's own `Display` *is* the + // per-variant detail — no match needed. + let detail = err.to_string(); + format!("{msg}\n(debug: {detail})") + } else { + msg + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_blown_tool_call_cap_names_the_limit_and_the_variable_that_raises_it() { + let msg = user_message(&CliError::Core(b2_core::Error::ToolCallLimit { limit: 64 })); + assert!(msg.contains("64"), "{msg}"); + assert!(msg.contains("B2_LLM_MAX_TOOL_CALLS"), "{msg}"); + assert!( + !msg.contains("Check that it's running"), + "the server is fine — the generic chat advice would mislead: {msg}" + ); + } + + #[test] + fn a_move_that_could_not_be_undone_names_the_files_to_check() { + let msg = user_message(&CliError::Core(b2_core::Error::MoveIncomplete { + paths: vec!["b.md".into(), "notes/c.md".into()], + source: Box::new(b2_core::Error::Io(std::io::Error::other("disk full"))), + })); + assert!(msg.contains("in: b.md, notes/c.md."), "{msg}"); + assert!( + !msg.contains("disk full"), + "the cause stays internal: {msg}" + ); + } +} diff --git a/crates/b2-cli/src/logging.rs b/crates/b2-cli/src/logging.rs new file mode 100644 index 0000000..a4af68d --- /dev/null +++ b/crates/b2-cli/src/logging.rs @@ -0,0 +1,63 @@ +//! Opt-in structured debug logging for the CLI process. + +/// Opt-in structured debug logging: the kernel's `tracing` events — per-statement SQLite +/// timings, façade-op spans, flow milestones — rendered as **JSON Lines**, one flat object +/// per line, so a run's log pipes straight into jq/DuckDB while `--json` stdout stays pure +/// data. +/// +/// The sink is stderr by default; `B2_LOG_FILE=` writes there instead, in **append** +/// mode so successive runs accumulate into one dataset. A file is also the +/// guaranteed-pure capture: stderr can interleave human notices with the JSONL. +/// +/// `B2_LOG` holds a tracing filter directive; `B2_DEBUG` or `B2_LOG_FILE` without it +/// implies **`b2=debug`** — the kernel's own targets only. That scoping is what keeps the +/// dataset reportable now that a chat command links an HTTP client: `ureq` logs through +/// the `log` bridge in a foreign shape, and a bare `debug` would fold it into the same +/// file. Opt into the firehose with an explicit `B2_LOG=debug`. With none of the three +/// set, no subscriber is installed and the instrumentation stays inert. +pub fn init_logging() { + let log_file = std::env::var_os("B2_LOG_FILE"); + let directive = match std::env::var("B2_LOG") { + Ok(v) if !v.trim().is_empty() => v, + _ if std::env::var_os("B2_DEBUG").is_some() || log_file.is_some() => "b2=debug".to_string(), + _ => return, + }; + let filter = match tracing_subscriber::EnvFilter::try_new(&directive) { + Ok(f) => f, + Err(e) => { + eprintln!("warning: invalid B2_LOG filter '{directive}' ({e}); using 'debug'"); + tracing_subscriber::EnvFilter::new("debug") + } + }; + let builder = tracing_subscriber::fmt() + .json() + // Event fields at the top level of each object (not nested under "fields") + // — what makes `jq '.duration_us'`-style reporting one-liners work. + .flatten_event(true) + // Close events give each façade-op span its measured duration; the clock + // lives here in the adapter, keeping b2-core itself wall-clock-free. + .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE) + .with_current_span(true) + .with_span_list(false) + .with_ansi(false) + .with_env_filter(filter); + // A CLI run is short-lived and single-threaded at the log site, so a plain + // `Mutex` writer suffices — no async appender needed. + match log_file { + Some(p) => match std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(std::path::Path::new(&p)) + { + Ok(file) => builder.with_writer(std::sync::Mutex::new(file)).init(), + Err(e) => { + eprintln!( + "warning: cannot open B2_LOG_FILE '{}' ({e}); logging to stderr", + p.to_string_lossy() + ); + builder.with_writer(std::io::stderr).init(); + } + }, + None => builder.with_writer(std::io::stderr).init(), + } +} diff --git a/crates/b2-cli/src/main.rs b/crates/b2-cli/src/main.rs index 13776d8..a20ebdb 100644 --- a/crates/b2-cli/src/main.rs +++ b/crates/b2-cli/src/main.rs @@ -9,302 +9,27 @@ //! (ADR-0020). `B2_EMBEDDER=fake` forces the deterministic fake — an offline/dev mode, //! and what the CLI suite runs under. -use b2_core::embed::Embedder; -use b2_core::llm::{ChatTurn, FakeLlm, LlmProvider}; -use b2_core::resource::{doc_kind, DocKind}; -use b2_core::vault::{AnswerView, SearchEvidenceView, SimilarStanding, Vault}; -use b2_embed::{provision, EmbedConfig, EmbedError, LocalEmbedder}; -use b2_llm::{ - is_ollama, refusal_message, LlmConfig, LlmError, OpenAiCompatProvider, OLLAMA_INSTALL_URL, -}; -use clap::{Args, Parser, Subcommand}; -use std::fs::{File, OpenOptions}; -use std::io::{IsTerminal, Read, Seek, SeekFrom, Write}; -use std::ops::ControlFlow; -use std::path::{Path, PathBuf}; +mod args; +mod cancel; +mod chat; +mod error; +mod logging; +mod read; +mod reindex; +mod wiring; +mod write; + +use args::{Cli, Command}; +use chat::{cmd_ask, cmd_chat, cmd_why}; +use clap::Parser; +use error::{user_message, CliError}; +use read::{cmd_explain, cmd_explain_similar, cmd_neighbors, cmd_search, cmd_similar}; +use reindex::{cmd_init, cmd_reindex, cmd_status}; use std::process::ExitCode; -use std::sync::atomic::{AtomicBool, Ordering}; - -/// Set by the SIGINT handler installed for a long-running command — by Ctrl-C in the -/// foreground, or (for a `reindex`) by another process's `b2 reindex --cancel` (GH #55), -/// which raises the *same* signal so both reach here. -/// -/// Two loops read it through the same [`ControlFlow`] seam: the reindex embed loop at -/// every batch boundary, stopping *after* the current batch so a cancel leaves a -/// consistent, re-runnable index; and the chat surfaces' token callback at every token, -/// where stopping renders the partial answer honestly. `chat` clears it before each turn. -static CANCEL: AtomicBool = AtomicBool::new(false); - -/// Whether `b2 chat` is streaming an answer right now — which is what decides -/// what Ctrl-C *means* in the REPL: cancel this answer, or leave. Only `chat` -/// maintains it (see its handler); every other command's SIGINT meaning is -/// unconditional. -static ANSWERING: AtomicBool = AtomicBool::new(false); - -/// Map the Ctrl-C flag onto the cooperative-cancel signal the engine's loops read — -/// [`ControlFlow::Break`] once a cancel has been requested, else -/// [`ControlFlow::Continue`]. Shared by `reindex`'s two progress closures and by the -/// chat surfaces' token callback. -fn cancel_flow() -> ControlFlow<()> { - if CANCEL.load(Ordering::SeqCst) { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } -} - -/// The retrieval loop, stated where an agent will actually read it (`b2 --help`) — -/// the toolset carrying its own usage instructions, the way an MCP server ships its -/// tool descriptions. The loop itself lives in the *caller*: B2's part is dumb, -/// composable reads (ADR-0012), and the vault being plain Markdown on disk is what -/// makes "open the file yourself" a step no B2 command needs to exist for. -const AGENT_HELP: &str = "\ -For agents (--json): - Every command takes --json, and the vault is plain Markdown on disk — a result's - `path` is a real file, readable and greppable directly. The retrieval loop: - - 1. b2 search \"\" --json Hybrid keyword+semantic search: the ranked rows - plus an evidence verdict. `\"vouched\": false` - means the vault holds no evidence for this query - — report \"no matches\" instead of using the rows. - 2. Read the files it names Paths are vault-relative; open them for the full - note, not just the matched snippet. - 3. b2 neighbors/explain Follow the typed graph around a promising note; - b2 similar ranks unlinked semantic - neighbors. - 4. Refine and search again Pass --exclude (repeatable) for notes - already inspected, so follow-ups surface fresh - material. Fewer results than asked = the head of - the ranking is spent; refine the query."; - -#[derive(Parser)] -#[command( - name = "b2", - version, - about = "B2 — explore a Markdown vault's typed graph and search from the terminal", - after_long_help = AGENT_HELP -)] -struct Cli { - /// Vault root (the folder of Markdown). The index lives in `/.b2/`. - /// Set it with `-C ` or `$B2_VAULT_PATH` (the flag wins). Read-only commands - /// fall back to the current dir; commands that write (`reindex`/`add`/`mv`/`rm`/ - /// `link`) require it explicitly, so they can never silently touch the wrong directory. - #[arg(short = 'C', long = "vault", global = true, env = "B2_VAULT_PATH")] - vault: Option, - - /// Emit machine-readable JSON instead of human-readable text. - #[arg(long, global = true)] - json: bool, - - #[command(subcommand)] - command: Command, -} - -#[derive(Subcommand)] -enum Command { - /// Download + verify the embedding model into the shared cache (one-time setup). - Init, - /// Re-project every note under the vault into the index. Reads your vault and - /// writes only to `.b2/`. Incremental by default: notes whose content is - /// unchanged keep their vectors. - Reindex { - /// Vault root (overrides --vault / $B2_VAULT_PATH). Required — no cwd default. - vault: Option, - /// Re-embed every note, even unchanged ones (a full rebuild in place). - #[arg(long)] - force: bool, - /// Preview what a reindex would do without touching the index — how many - /// notes it would project and embed. - #[arg(long)] - dry_run: bool, - /// Stop a reindex already running on this vault — the way to reach one - /// backgrounded with `b2 reindex &`, which has no controlling terminal for - /// Ctrl-C. It stops after the current batch, leaving the same consistent, - /// re-runnable partial index a foreground Ctrl-C does. - #[arg(long, conflicts_with_all = ["force", "dry_run"])] - cancel: bool, - }, - /// Report embedding coverage — how many notes are embedded (so semantic ranking - /// is live vs. keyword-only) and whether a reindex is currently running, with the - /// process id to stop it by. A pure, model-free read; handy after kicking off a - /// slow reindex in the background with `b2 reindex &`. - Status, - /// Create a new note and project it into the index (it's immediately in the - /// graph + searchable). PATH is vault-relative (the `.md` extension is optional). - Add { - /// Where the new note goes: a vault-relative path (`.md` optional). - path: String, - /// The note's human title (frontmatter `title:`). - #[arg(long)] - title: Option, - /// Initial body content (Markdown). Omit for an empty note to fill in later. - #[arg(long)] - content: Option, - }, - /// Replace a note's **body** with Markdown piped on stdin — the CLI/agent editing - /// surface (the counterpart to the desktop editor's save). NOTE is a vault-relative - /// path; the note must already exist (`b2 add` creates one). Pipe the body - /// **alone** — the frontmatter (including B2's managed `b2_relations:`) is - /// left untouched, so no `---` block. The note is re-projected (chunks + FTS + graph) - /// so the index stays live; a later `b2 reindex` fills the changed body's vectors. - Write { - /// The note whose body to overwrite: a vault-relative path. - note: String, - }, - /// Show a note's typed neighbors. NOTE is a vault-relative path. - Neighbors { note: String }, - /// Explain a note's connections — every typed edge and its "why". NOTE is a - /// vault-relative path. - Explain { note: String }, - /// Move/rename a note, file, or folder and rewrite every inbound link to - /// point at the new path. An existing directory moves as a whole folder - /// (everything under it, one rename). - Mv { - /// What to move: a vault-relative path (note, file, or folder). - from: String, - /// The new vault-relative path (for a note, the `.md` extension is optional). - to: String, - }, - /// Delete a note, file, or folder from the vault *and* the disk. Inbound links - /// are never rewritten — they dangle and surface as unresolved (`b2 explain`). - /// An existing directory deletes as a whole folder and requires --recursive. - Rm { - /// What to delete: a vault-relative path (note, file, or folder). - target: String, - /// Required to delete a folder (and everything inside it, unindexed files too). - #[arg(short = 'r', long)] - recursive: bool, - }, - /// Hybrid keyword+semantic search across the vault. A query the vault - /// holds no evidence for answers "no matches" rather than serving its nearest - /// rows (invariants.md D2); `--json` is an **object** — the rows plus that - /// verdict, which has nowhere to live in a bare list. - Search { - query: String, - /// Maximum number of notes to return. - #[arg(long, default_value_t = 10)] - limit: usize, - /// Leave a note out of the results: a vault-relative path exactly as a - /// previous search served it (repeatable). The follow-up-search flag for - /// agent loops — pass the notes already inspected so a re-query surfaces - /// fresh material instead of the same head. It subtracts rows only: the - /// evidence verdict still reads the whole vault, and a heavily-excluded - /// query may serve fewer than LIMIT notes — the cue to refine the query - /// rather than page deeper. - #[arg(long, value_name = "PATH")] - exclude: Vec, - }, - /// Surface the notes most semantically similar to NOTE that you haven't linked - /// yet — connection discovery, ranked nearest first. NOTE is a vault-relative - /// path. A local read over stored vectors (run `b2 reindex` with the real - /// model first). Similarity is relative to your vault: the list is always - /// the ranked nearest, and you are the judge of which are worth a link. - Similar { - /// The note to find similar notes for: a vault-relative path. - note: String, - /// Maximum number of similar notes to return. - #[arg(long, default_value_t = 10)] - limit: usize, - /// Explain one note against NOTE's list instead of printing the list: where it - /// stands (and why, if it is not a card), and the passage pairs behind it. No - /// model call. Read at `--limit`, so the rank is the one the list shows. - #[arg(long, value_name = "OTHER")] - explain: Option, - }, - /// Commit a typed connection SRC → DST into SRC's frontmatter `b2_relations:`. - /// SRC and DST are each a vault-relative path. - Link { - /// The source note (the edge points *from* it): a vault-relative path. - src: String, - /// The target note (the edge points *to* it): a vault-relative path. - dst: String, - /// The relation verb (a core verb: references/supports/contradicts). - #[arg(long = "type", default_value = "references")] - edge_type: String, - /// Optional explanation — trailing text shown after the link. - #[arg(long)] - explanation: Option, - }, - /// Ask one question about your vault and stream a grounded answer. The model - /// answers **only** from passages retrieved out of your notes and cites them by - /// `[n]`; `--json` emits the answer as a JSON Lines event stream (one token - /// event per token, then the final answer with its citations resolved). - /// Needs a model server — Ollama by default (see `--llm-url`). - Ask { - /// The question, in plain language. - question: String, - #[command(flatten)] - llm: LlmArgs, - }, - /// Explain why CANDIDATE shows up in `b2 similar NOTE` — a grounded, cited answer - /// streamed from the chat model, which is given B2's read-only tools (the matched - /// passages, the similar list, a note's links, reading a note) and looks up what it - /// needs. The answer lists the tools that ran. A model with no tool support is handed - /// the matched passages instead. `--json` streams events as `ask` does. - /// Needs a model server — Ollama by default (see `--llm-url`). - Why { - /// The note the list was for: a vault-relative path. - note: String, - /// The suggested note to explain: a vault-relative path. - candidate: String, - /// The length of the `similar` list the rank is quoted against. - #[arg(long, default_value_t = 10)] - limit: usize, - #[command(flatten)] - llm: LlmArgs, - }, - /// Interactive grounded chat about your vault — the same answers as `ask`, with - /// follow-up questions that remember the conversation. Ctrl-C stops an answer - /// mid-stream (the partial text stands); `/exit` or Ctrl-D leaves. History is - /// session-only: nothing about a chat is ever written to your notes or the index. - Chat { - #[command(flatten)] - llm: LlmArgs, - }, -} - -/// Which model to chat with, and where it lives — shared by `ask` and `chat`. Precedence -/// is the `B2_VAULT_PATH` convention exactly: an explicit flag beats the environment beats -/// the default. Resolution lives once in `b2_llm::LlmConfig`, so the desktop's settings -/// layer over the same base — which is why these are plain options, not clap `env` args. -#[derive(Args, Debug)] -struct LlmArgs { - /// The OpenAI-compatible base URL of your model server - /// [env: B2_LLM_URL, default: http://localhost:11434/v1 — Ollama's]. - /// Any compatible endpoint works: LM Studio, llama.cpp, vLLM, or a cloud - /// provider's (which sends your question and the retrieved passages to them — - /// pair it with B2_LLM_API_KEY). - #[arg(long = "llm-url", value_name = "URL")] - llm_url: Option, - /// The chat model id, as the server names it - /// [env: B2_LLM_MODEL, default: llama3.2]. Pull it first, e.g. `ollama pull - /// llama3.2`. - #[arg(long = "llm-model", value_name = "MODEL")] - llm_model: Option, -} - -impl Cli { - /// The vault root for **read-only** commands (`search`, `neighbors`, `explain`, - /// `similar`): the `-C`/`$B2_VAULT_PATH` value if given, else the current directory. - /// A pure read can't pollute anything, so the cwd convenience is safe here. - fn vault_or_cwd(&self) -> &Path { - self.vault.as_deref().unwrap_or_else(|| Path::new(".")) - } - - /// The vault root for commands that **write** (`reindex`, `add`, `mv`, `rm`, `link`): - /// `positional` wins, then `-C`/`$B2_VAULT_PATH`; with none, error rather than - /// silently mutating the current directory, where a stale binary or mistyped var would - /// otherwise leave a stray `.b2/`. The write-side counterpart to - /// [`vault_or_cwd`](Self::vault_or_cwd). - fn require_vault<'a>(&'a self, positional: Option<&'a Path>) -> Result<&'a Path, CliError> { - positional - .or(self.vault.as_deref()) - .ok_or(CliError::VaultRequired) - } -} +use write::{cmd_add, cmd_link, cmd_mv, cmd_rm, cmd_write}; fn main() -> ExitCode { - init_logging(); + logging::init_logging(); let cli = Cli::parse(); match dispatch(&cli) { Ok(()) => ExitCode::SUCCESS, @@ -315,68 +40,6 @@ fn main() -> ExitCode { } } -/// Opt-in structured debug logging: the kernel's `tracing` events — per-statement SQLite -/// timings, façade-op spans, flow milestones — rendered as **JSON Lines**, one flat object -/// per line, so a run's log pipes straight into jq/DuckDB while `--json` stdout stays pure -/// data. -/// -/// The sink is stderr by default; `B2_LOG_FILE=` writes there instead, in **append** -/// mode so successive runs accumulate into one dataset. A file is also the -/// guaranteed-pure capture: stderr can interleave human notices with the JSONL. -/// -/// `B2_LOG` holds a tracing filter directive; `B2_DEBUG` or `B2_LOG_FILE` without it -/// implies **`b2=debug`** — the kernel's own targets only. That scoping is what keeps the -/// dataset reportable now that a chat command links an HTTP client: `ureq` logs through -/// the `log` bridge in a foreign shape, and a bare `debug` would fold it into the same -/// file. Opt into the firehose with an explicit `B2_LOG=debug`. With none of the three -/// set, no subscriber is installed and the instrumentation stays inert. -fn init_logging() { - let log_file = std::env::var_os("B2_LOG_FILE"); - let directive = match std::env::var("B2_LOG") { - Ok(v) if !v.trim().is_empty() => v, - _ if std::env::var_os("B2_DEBUG").is_some() || log_file.is_some() => "b2=debug".to_string(), - _ => return, - }; - let filter = match tracing_subscriber::EnvFilter::try_new(&directive) { - Ok(f) => f, - Err(e) => { - eprintln!("warning: invalid B2_LOG filter '{directive}' ({e}); using 'debug'"); - tracing_subscriber::EnvFilter::new("debug") - } - }; - let builder = tracing_subscriber::fmt() - .json() - // Event fields at the top level of each object (not nested under "fields") - // — what makes `jq '.duration_us'`-style reporting one-liners work. - .flatten_event(true) - // Close events give each façade-op span its measured duration; the clock - // lives here in the adapter, keeping b2-core itself wall-clock-free. - .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE) - .with_current_span(true) - .with_span_list(false) - .with_ansi(false) - .with_env_filter(filter); - // A CLI run is short-lived and single-threaded at the log site, so a plain - // `Mutex` writer suffices — no async appender needed. - match log_file { - Some(p) => match std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(std::path::Path::new(&p)) - { - Ok(file) => builder.with_writer(std::sync::Mutex::new(file)).init(), - Err(e) => { - eprintln!( - "warning: cannot open B2_LOG_FILE '{}' ({e}); logging to stderr", - p.to_string_lossy() - ); - builder.with_writer(std::io::stderr).init(); - } - }, - None => builder.with_writer(std::io::stderr).init(), - } -} - /// The thin router: each subcommand's whole behavior lives in its `cmd_*` fn below; /// this match only destructures the parsed args and forwards them. fn dispatch(cli: &Cli) -> Result<(), CliError> { @@ -433,1441 +96,7 @@ fn dispatch(cli: &Cli) -> Result<(), CliError> { /// Print `value` as pretty JSON on stdout — the one `--json` output path, shared by /// every subcommand. -fn print_json(value: &T) -> Result<(), CliError> { +pub fn print_json(value: &T) -> Result<(), CliError> { println!("{}", serde_json::to_string_pretty(value)?); Ok(()) } - -fn cmd_init(json: bool) -> Result<(), CliError> { - // Global, per-machine setup — no vault involved. - let config = EmbedConfig::load()?; - let report = provision(&config, |line| eprintln!("{line}"))?; - if json { - print_json(&report)?; - } else if report.already_present { - println!("Model '{}' is already installed.", report.model); - } else { - println!( - "Installed '{}' ({} dims). Run `b2 reindex` to embed your vault.", - report.model, report.dim - ); - } - Ok(()) -} - -fn cmd_reindex( - cli: &Cli, - vault: Option<&Path>, - force: bool, - dry_run: bool, - cancel: bool, -) -> Result<(), CliError> { - // Reindex writes an index → require an explicit vault (positional wins), - // never a silent cwd fallback. See `Cli::require_vault`. - let root = cli.require_vault(vault)?; - if cancel { - // Signals another process and returns; it never opens the vault (no - // model load, no index read) — the run being cancelled owns all of that. - return cancel_reindex(root, cli.json); - } - if dry_run { - // A dry-run neither embeds nor stamps → no model needed (open with - // the fake, like `neighbors`); it's a pure read, so there's no slow - // embed phase to show progress for. - let vault = open_vault(root, false)?; - let plan = vault.plan_reindex(force)?; - if cli.json { - print_json(&plan)?; - } else { - print_reindex_plan(&plan); - } - return Ok(()); - } - // Single-in-flight: take an advisory lock *before* the slow model load, so a second - // `b2 reindex` refuses cleanly instead of two processes writing the same index. - // Advisory, not a PID file: the OS frees it the instant the holder exits (crash, kill - // or Ctrl-C included), so nothing stale is left behind. Held until this fn ends. - let lock = open_reindex_lock(root)?; - match lock.try_lock() { - Ok(()) => {} - Err(std::fs::TryLockError::WouldBlock) => return Err(CliError::ReindexRunning), - Err(std::fs::TryLockError::Error(e)) => return Err(CliError::Io(e)), - } - // Now that the lock is ours, stamp who holds it: the address `b2 reindex - // --cancel` signals, and what `b2 status` prints so a manual `kill` stays - // available (GH #55). Best-effort — a failed write costs the cancel - // affordance, not the reindex. - let _ = record_reindex_pid(&lock); - // Reindex embeds every changed chunk → it needs the real model. - let vault = open_vault(root, true)?; - // Wire Ctrl-C to the cooperative-cancel flag now that the model is loaded and - // real embedding is next. (During the model load the default SIGINT still - // applies — nothing is written yet, so a hard stop there is safe.) Best-effort: - // if the handler can't be installed, Ctrl-C keeps its default (terminate), which - // still leaves a consistent index since edges + FTS land before any vectors. - let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); - // Embedding a large vault on CPU is slow; show a live progress line so it - // never looks frozen. Only on an interactive stderr (never in --json, and - // never when piped/captured) so machine output and tests stay clean. - let report = if cli.json || !std::io::stderr().is_terminal() { - vault.reindex_with_progress(force, &mut |_| cancel_flow())? - } else { - // Name the vault being indexed up front, then a live line that counts - // the notes actually (re)embedded — not every note, most of which an - // incremental run reuses untouched — with the current file + its chunks. - let shown = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); - eprintln!("Indexing {}", shown.display()); - let mut progressed = false; - let mut on_progress = |p: b2_core::ingest::ReindexProgress| { - progressed = true; - // \x1b[K clears any tail of a previous, longer line (paths vary in - // length); safe here because this branch only runs on a real terminal. - eprint!( - "\r embedding {}/{} · {} ({} chunk{})\x1b[K", - p.notes_embedded, - p.notes_to_embed, - p.note_path, - p.note_chunks, - if p.note_chunks == 1 { "" } else { "s" }, - ); - let _ = std::io::stderr().flush(); - // Stop after this batch if Ctrl-C was pressed, - // else carry on. The batch is already written above, so a cancel here - // never tears a write. - cancel_flow() - }; - let report = vault.reindex_with_progress(force, &mut on_progress)?; - if progressed { - eprintln!(); // close the progress line - } - report - }; - if cli.json { - print_json(&report)?; - } else { - print_reindex_report(&report); - } - Ok(()) -} - -/// The human-readable `reindex --dry-run` preview (the `--json` sibling prints the -/// plan itself). -fn print_reindex_plan(plan: &b2_core::vault::ReindexPlan) { - println!( - "Dry run: would index {} note(s) — {} to embed. Nothing in your vault is written either way.", - plan.would_index, plan.would_embed - ); -} - -/// The human-readable reindex summary: the one stdout line, then the stderr -/// notices — skipped files and the cancelled line. -fn print_reindex_report(report: &b2_core::vault::ReindexReport) { - println!( - "Indexed {} notes ({} embedded{}) and {} resources{}", - report.indexed, - report.embedded, - if report.notes_pruned > 0 { - format!(", {} pruned", report.notes_pruned) - } else { - String::new() - }, - report.resources_indexed, - if report.resources_pruned > 0 { - format!(" ({} pruned)", report.resources_pruned) - } else { - String::new() - } - ); - // One unreadable file no longer aborts the reindex — it is skipped and - // named here (to stderr, so it never pollutes the machine-readable stdout - // line above) with a short, file-level reason. - if !report.skipped.is_empty() { - eprintln!("Skipped {} unreadable file(s):", report.skipped.len()); - for s in &report.skipped { - eprintln!(" - {} ({})", s.path, s.reason); - } - } - // The counts above already report the partial work truthfully; add the - // one line that tells the user it was interrupted and is safe to resume. - if report.cancelled { - eprintln!( - "Cancelled — the index is consistent but only partly embedded. Re-run `b2 reindex` to finish the rest." - ); - } -} - -fn cmd_status(cli: &Cli) -> Result<(), CliError> { - // Read-only coverage report: how much of the vault is embedded (semantic - // ranking live vs. keyword-only) and whether a background reindex is in - // flight — the companion to backgrounding a slow reindex with `b2 reindex &`. - // A pure model-free DB read (#26): open with the fake. - let root = cli.vault_or_cwd(); - let vault = open_vault(root, false)?; - let status = vault.embed_status()?; - let holder = reindex_holder(root); - if cli.json { - print_json(&serde_json::json!({ - "embedded": status.embedded, - "total": status.total, - "reindex_running": holder.is_some(), - // The running process's id — `null` when nothing is running (and - // on the sliver of a moment before a fresh holder stamps it). - "reindex_pid": holder.as_ref().and_then(|h| h.pid), - }))?; - } else { - if status.total == 0 { - println!("No notes indexed yet. Run `b2 reindex` to build the index."); - } else if status.embedded == 0 { - println!( - "Embedded 0/{} notes — keyword-only. Run `b2 reindex` for semantic ranking.", - status.total - ); - } else if status.embedded < status.total { - println!( - "Embedded {}/{} notes — semantic ranking partial ({} still keyword-only).", - status.embedded, - status.total, - status.total - status.embedded - ); - } else { - println!( - "Embedded {}/{} notes — semantic ranking fully live.", - status.embedded, status.total - ); - } - // Name the process, not just the fact: `--cancel` is the supported stop, - // and the pid keeps a plain `kill -INT` as the documented fallback. - match holder.as_ref().map(|h| h.pid) { - Some(Some(pid)) => println!( - "A reindex is currently running (pid {pid}). Stop it with `b2 reindex --cancel` (or `kill -INT {pid}`)." - ), - Some(None) => { - println!("A reindex is currently running. Stop it with `b2 reindex --cancel`.") - } - None => {} - } - } - Ok(()) -} - -fn cmd_add( - cli: &Cli, - path: &str, - title: Option<&str>, - content: Option<&str>, -) -> Result<(), CliError> { - // Add writes a new note (and embeds its body) → require an explicit vault - // (no silent cwd), and it needs the real model like `reindex`/`mv`/`link`. - let vault = open_vault(cli.require_vault(None)?, true)?; - let report = vault.add_note(path, title, content)?; - if cli.json { - print_json(&report)?; - } else { - println!("Created {}.", report.path); - } - Ok(()) -} - -fn cmd_write(cli: &Cli, note: &str) -> Result<(), CliError> { - // A body splice + re-projection: **model-free** (like the desktop's save and - // `rm`), still an explicit vault like every write. Refuse an interactive - // terminal up front so the command never silently hangs waiting for - // hand-typed input — the new body is always *piped* (an agent, `cat file |`, …). - let stdin = std::io::stdin(); - if stdin.is_terminal() { - return Err(CliError::StdinRequired); - } - let vault = open_vault(cli.require_vault(None)?, false)?; - let mut body = String::new(); - stdin.lock().read_to_string(&mut body)?; - // Stateless one-shot: read the current on-disk revision and chain the write on - // it. A CLI holds no long-lived buffer, so there's no external-edit window to - // guard here — the content-hash guard exists for the desktop's in-memory - // buffer; for a one-shot, the contract is simply "the body becomes exactly - // this" (`Vault::write` still keeps the frontmatter bytes untouched). - let current = vault.read(note)?; - let report = vault.write(note, &body, ¤t.revision)?; - if cli.json { - print_json(&report)?; - } else { - println!("Wrote {} ({} bytes).", report.path, body.len()); - } - Ok(()) -} - -fn cmd_neighbors(cli: &Cli, note: &str) -> Result<(), CliError> { - // Neighbors is a pure graph query — it never embeds, so don't require - // the model (no needless `b2 init` just to explore the graph). - let vault = open_vault(cli.vault_or_cwd(), false)?; - let neighbors = vault.neighbors(note)?; - // Dangling outbound links (a `[[folder]]` or a typo) that resolve to no - // note or resource — surfaced, not dropped (GH #12). `--json` keeps its - // resolved-neighbors array contract; the full structured picture, - // including these, is `b2 explain --json`. - let unresolved = vault.unresolved_links(note)?; - if cli.json { - print_json(&neighbors)?; - } else if neighbors.is_empty() && unresolved.is_empty() { - println!("No neighbors."); - } else { - for n in &neighbors { - let arrow = arrow(&n.direction); - let name = display_name(n.title.as_deref(), &n.path); - let explanation = n - .explanation - .as_deref() - .map(|e| format!(" — {e}")) - .unwrap_or_default(); - println!("{arrow} {} {name} ({}){explanation}", n.label, n.path); - } - for u in &unresolved { - println!( - "⚠ {} [[{}]] — unresolved (no matching note or file)", - u.relation, u.target - ); - } - } - Ok(()) -} - -fn cmd_explain(cli: &Cli, note: &str) -> Result<(), CliError> { - // Explain is a pure graph read (edges + their explanations), no embed — - // like `neighbors`, it opens with the fake and needs no `b2 init`. - let vault = open_vault(cli.vault_or_cwd(), false)?; - // Kind dispatch by the argument's own shape (core's one rule, §9b #8): - // a resource arg gets the fallback card's view — metadata + backlinks. - if doc_kind(note) == DocKind::Resource { - let view = vault.explain_resource(note)?; - if cli.json { - print_json(&view)?; - } else { - println!("{} ({}, {} bytes)", view.path, view.class, view.size); - if view.backlinks.is_empty() { - println!("No backlinks yet."); - } else { - println!("Backlinks:"); - for b in &view.backlinks { - let name = display_name(b.title.as_deref(), &b.path); - let mut line = format!(" ← {name} ({}) {}", b.path, b.r#type); - decorate(&mut line, b.embed, b.caption.as_deref()); - println!("{line}"); - } - } - } - return Ok(()); - } - let view = vault.explain(note)?; - if cli.json { - print_json(&view)?; - } else { - let name = display_name(view.title.as_deref(), &view.path); - println!("{name} ({})", view.path); - if view.connections.is_empty() && view.resources.is_empty() && view.unresolved.is_empty() { - // Zero connections at all — nothing links to it and it links to - // nothing (an orphan; the kernel only surfaces, never archives). - println!("No connections yet."); - } else if !view.connections.is_empty() { - println!("Connections:"); - for c in &view.connections { - let arrow = arrow(&c.direction); - let target = display_name(c.title.as_deref(), &c.path); - println!( - " {arrow} {} {target} ({}) [{}]", - c.label, c.path, c.origin - ); - if let Some(why) = &c.explanation { - println!(" why: {why}"); - } - } - // If nothing points *at* the note, it's an orphan — surfaced, not - // acted on (invariants.md; files are only touched when asked). - if !view.connections.iter().any(|c| c.direction == "inbound") { - println!("No inbound links — this note is an orphan."); - } - } - // Outbound links at resources (images, PDFs, …) — the third target - // kind an edge can have, shown from the note's side (GH #22). - if !view.resources.is_empty() { - println!("Resource links:"); - for r in &view.resources { - let mut line = format!( - " → {} {} ({}) [{}]", - r.relation, r.path, r.class, r.origin - ); - decorate(&mut line, r.embed, r.caption.as_deref()); - println!("{line}"); - if let Some(why) = &r.explanation { - println!(" why: {why}"); - } - } - } - // Dangling outbound links (a `[[folder]]` or a typo): a note is one - // `.md` file, so these resolve to nothing — shown as broken rather - // than silently dropped (GH #12). - if !view.unresolved.is_empty() { - println!("Unresolved links:"); - for u in &view.unresolved { - println!( - " ⚠ {} [[{}]] (no matching note or file) [{}]", - u.relation, u.target, u.origin - ); - if let Some(why) = &u.explanation { - println!(" why: {why}"); - } - } - } - } - Ok(()) -} - -fn cmd_mv(cli: &Cli, from: &str, to: &str) -> Result<(), CliError> { - // A move rewrites files (and re-embeds them on re-projection) → require an - // explicit vault (no silent cwd), and it needs the real model the index was - // built with, like `reindex`/`add`/`link`. - let root = cli.require_vault(None)?; - let vault = open_vault(root, true)?; - // Kind dispatch (§9b #8): an existing directory moves as a folder - // (every file under it, one rename); otherwise the two file arms - // differ only in the report type they print. - // The human "Moved" line differs per arm, the rewrite tally is shared. - if is_dir_arg(root, from) { - let report = vault.move_dir(from, to)?; - if cli.json { - print_json(&report)?; - } else { - println!( - "Moved {}/ → {}/ ({} note(s), {} file(s))", - report.from, report.to, report.moved_notes, report.moved_resources - ); - print_rewrite_tally(report.links_rewritten, report.rewrote.len()); - } - } else if doc_kind(from) == DocKind::Resource { - let report = vault.move_resource(from, to)?; - if cli.json { - print_json(&report)?; - } else { - println!("Moved {} → {}", report.from, report.to); - print_rewrite_tally(report.links_rewritten, report.rewrote.len()); - } - } else { - let report = vault.move_note(from, to)?; - if cli.json { - print_json(&report)?; - } else { - println!("Moved {} → {}", report.from, report.to); - print_rewrite_tally(report.links_rewritten, report.rewrote.len()); - } - } - Ok(()) -} - -fn cmd_rm(cli: &Cli, target: &str, recursive: bool) -> Result<(), CliError> { - // A delete removes files and index rows but never rewrites a body - // (inbound links dangle, they aren't repaired) → **model-free**, like - // the desktop's delete; still an explicit vault, like every write. - let root = cli.require_vault(None)?; - let vault = open_vault(root, false)?; - // Kind dispatch (§9b #8), mirroring `mv`: an existing directory deletes - // as a folder — gated on --recursive, the CLI's stand-in for the - // desktop's confirm dialog — else the extension picks the file arm. - if is_dir_arg(root, target) { - if !recursive { - return Err(CliError::RecursiveRequired(target.to_string())); - } - let report = vault.delete_dir(target)?; - if cli.json { - print_json(&report)?; - } else { - println!( - "Deleted {}/ ({} note(s), {} file(s))", - report.dir, report.deleted_notes, report.deleted_resources - ); - print_dangled(&report.dangled); - } - } else if doc_kind(target) == DocKind::Resource { - let report = vault.delete_resource(target)?; - if cli.json { - print_json(&report)?; - } else { - println!("Deleted {}", report.path); - print_dangled(&report.dangled); - } - } else { - let report = vault.delete_note(target)?; - if cli.json { - print_json(&report)?; - } else { - println!("Deleted {}", report.path); - print_dangled(&report.dangled); - } - } - Ok(()) -} - -fn cmd_search(cli: &Cli, query: &str, limit: usize, exclude: &[String]) -> Result<(), CliError> { - // Search embeds the query for the vector half → it needs the real model. - let vault = open_vault(cli.vault_or_cwd(), true)?; - // The evidence read, not the bare list (invariants.md D2, GH #201/#202): the - // rows are identical and in the same order, and the verdict beside them is - // what lets this command say **"no matches"** honestly instead of serving - // `limit` confident-looking results for a query the vault holds nothing for. - // `--exclude` paths are the caller's subtraction, never the verdict's. - let view = vault.search_evidence_excluding(query, limit, exclude)?; - if cli.json { - // The whole view, verdict included. This is an OBJECT where `--json` used to be - // an array — a deliberate break (GH #202), because a query-level verdict has - // nowhere to live in a list of rows; the rows themselves stay additive. - // - // The JSON serves the rows even at `vouched: false`, where the human surface below - // shows none: an agent handed the rows *plus* an explicit verdict can be honest - // about them, where a human handed rows alone cannot. - print_json(&view)?; - } else { - println!("{}", search_report(&view, query)); - // Honesty (never overstate): with the fake embedder the vector half - // isn't semantic. Under the real model it is, so no caveat. Kept on - // stderr so stdout stays pure results. - if use_fake_embedder() { - eprintln!( - "note: keyword (BM25) ranking is live; semantic ranking is off (fake embedder)." - ); - } - } - Ok(()) -} - -/// The human-mode rendering of a search — ADR-0015's three verdict states as one pure -/// function (GH #202). -/// -/// Pure, and separate from [`cmd_search`], because the state that matters most is the one -/// the integration suite structurally cannot reach: `Some(false)` needs a *calibrated* -/// bar, and the fake embedder the suite runs under has none by design. A branch reachable -/// only under the real model is still the fast suite's to own once the rendering is -/// separated from the retrieval. -fn search_report(view: &SearchEvidenceView, query: &str) -> String { - match view.vouched { - // D2's "no matches", strict: the vault holds neither a lexical anchor - // nor semantic proximity clearing the model's bar, so the - // nearest-by-meaning rows are not shown at all — no reveal, no `--all`. - // Offering them behind a flag would still be this command putting them - // forward as candidates (GH #202, decision 1). - Some(false) => format!("No matches. Nothing in the vault matches “{query}”."), - // `Some(true)` — the vault vouches for these — and `None`, which is *no - // verdict at all*: the active embedder has no calibrated bar (the fake - // one, or any model until the harness measures it — M2), and reading - // that as "no matches" would blank a dev vault. Three states, three - // behaviors; `None` is never folded into `false`. - _ if view.results.is_empty() => "No results.".to_string(), - _ => view - .results - .iter() - .map(|r| { - let name = display_name(r.result.title.as_deref(), &r.result.path); - let head = format!("{:.4} {name} ({})", r.result.score, r.result.path); - if r.result.snippet.is_empty() { - head - } else { - format!("{head}\n {}", r.result.snippet) - } - }) - .collect::>() - .join("\n"), - } -} - -fn cmd_similar(cli: &Cli, note: &str, limit: usize) -> Result<(), CliError> { - // Candidate generation reads the *stored* vectors (no query embedding), so - // like `neighbors` it needs no live model — a prior `reindex` supplies them. - // Open with the fake; it's a pure, instant local read. (The z grading keys - // on the vault's RECORDED model id, so opening with the fake for reading - // never turns it off.) - let vault = open_vault(cli.vault_or_cwd(), false)?; - let results = vault.similar(note, limit)?; - if cli.json { - print_json(&results)?; - } else if limit == 0 { - // An ask for nothing yields nothing, silently: the empty-state copy - // below reads the candidate set, and a zero limit proves nothing about - // it — printing "nothing to compare" here would be the one way this - // command could still make a claim it can't check (GH #197). - } else if results.is_empty() { - // Two honest empty states, and neither claims "nothing relates" - // (GH #197): at a nonzero ask, an empty list means the candidate set is - // genuinely empty — nothing unlinked with stored vectors to compare — - // or the vault's similarity isn't semantic yet. - let status = vault.embed_status()?; - if status.embedded > 0 { - println!("Nothing unlinked has stored vectors to compare."); - } else { - println!( - "No similar notes. (If you haven't yet, run `b2 init` then `b2 reindex` so similarity is semantic.)" - ); - } - } else { - for r in &results { - let name = display_name(r.title.as_deref(), &r.path); - println!("{:.4} {name} ({})", r.score, r.path); - if !r.evidence.is_empty() { - println!(" {}", r.evidence); - } - } - // Nudge toward the commit step, on stderr so stdout stays pure results. - eprintln!("Commit one with: b2 link {note} --type "); - } - Ok(()) -} - -/// Longest passage excerpt `similar --explain` prints per side, so a pair stays readable -/// in a terminal. `--json` carries the full text. -const EXPLAIN_EXCERPT_CHARS: usize = 200; - -fn cmd_explain_similar(cli: &Cli, note: &str, other: &str, limit: usize) -> Result<(), CliError> { - // A pure read over stored vectors, like `similar`: the fake is enough to open with. - let vault = open_vault(cli.vault_or_cwd(), false)?; - let ex = vault.explain_similar(note, other, limit)?; - if cli.json { - return print_json(&ex); - } - let anchor = display_name(ex.anchor.title.as_deref(), &ex.anchor.path); - let candidate = display_name(ex.candidate.title.as_deref(), &ex.candidate.path); - println!( - "{candidate} ({}) against {anchor} ({})", - ex.candidate.path, ex.anchor.path - ); - let z = ex.z.map(|z| format!(", z {z:.2}")).unwrap_or_default(); - let standing = match ex.standing { - SimilarStanding::SameNote => "That is the same note.".to_string(), - SimilarStanding::AnchorUnembedded => format!( - "{anchor} has no stored vectors yet, so there is nothing to compare from. Run `b2 reindex`." - ), - SimilarStanding::Linked => { - "Already linked, so Similar leaves it out: it shows what you haven't connected." - .to_string() - } - SimilarStanding::Unembedded => { - "Not embedded yet, so it can't be compared. Run `b2 reindex`.".to_string() - } - SimilarStanding::NotShortlisted { shortlist } => format!( - "Never scored: judged as a whole note it ranks #{}, past the first-pass shortlist of {shortlist}.", - ex.centroid_rank.unwrap_or(0) - ), - SimilarStanding::Ranked { - rank, - of, - served: true, - } => format!("Shown at #{rank} of {limit} ({of} notes scored{z})."), - SimilarStanding::Ranked { - rank, - of, - served: false, - } => format!("Ranked #{rank} of {of} scored{z}: past the {limit} shown."), - }; - println!("{standing}"); - if let (Some(c), SimilarStanding::Ranked { rank, .. }) = (ex.centroid_rank, ex.standing) { - println!("Judged as a whole note it ranks #{c}; by its best passage, #{rank}."); - } - if !ex.shared_neighbors.is_empty() { - let names: Vec = ex - .shared_neighbors - .iter() - .map(|n| format!("{} ({})", display_name(n.title.as_deref(), &n.path), n.path)) - .collect(); - println!("Both link with: {}", names.join(", ")); - } - if !ex.pairs.is_empty() { - println!("\nPassage pairs, nearest first:"); - // Pairs carry a grade only when the list does; say why they don't, once, rather - // than print bare pairs that read as unmeasured by accident. - if ex.pairs.iter().all(|p| p.z.is_none()) { - println!( - " Ungraded: too few notes to compare against, or a vault indexed without the real model." - ); - } - } - for (i, p) in ex.pairs.iter().enumerate() { - let grade = p.z.map(|z| format!(" z {z:.2}")).unwrap_or_default(); - let same = if p.identical { " identical text" } else { "" }; - println!("{:>3}.{grade}{same}", i + 1); - for (side, passage) in [("this", &p.anchor), ("that", &p.candidate)] { - let heading = passage - .heading_path - .as_deref() - .map(|h| format!("[{h}] ")) - .unwrap_or_default(); - println!( - " {side}: {heading}{}", - excerpt(&passage.text, EXPLAIN_EXCERPT_CHARS) - ); - } - } - Ok(()) -} - -/// A passage flattened to one line and cut at `max` characters. -fn excerpt(text: &str, max: usize) -> String { - let flat = text.split_whitespace().collect::>().join(" "); - if flat.chars().count() <= max { - flat - } else { - format!("{}…", flat.chars().take(max).collect::().trim_end()) - } -} - -fn cmd_link( - cli: &Cli, - src: &str, - dst: &str, - edge_type: &str, - explanation: Option<&str>, -) -> Result<(), CliError> { - // Link writes the source note's frontmatter and re-projects it → require an - // explicit vault (no silent cwd), opening with the same real model the index - // was built with (like `add`/`mv`); a frontmatter-only edit won't re-embed. - let vault = open_vault(cli.require_vault(None)?, true)?; - let report = vault.link(src, dst, edge_type, explanation)?; - if cli.json { - print_json(&report)?; - } else if report.created { - println!( - "Linked {} —{}→ {}. Wrote the relation into the source note's frontmatter.", - report.src_path, report.relation, report.dst_path - ); - } else { - println!( - "Already linked {} —{}→ {}. Nothing changed.", - report.src_path, report.relation, report.dst_path - ); - } - Ok(()) -} - -fn cmd_ask(cli: &Cli, question: &str, llm_args: &LlmArgs) -> Result<(), CliError> { - // The provider first, deliberately: it is the cheap check, and loading the real - // embedder below takes seconds. A stopped model server should cost a round trip, - // not a model load followed by a failure. - let llm = open_llm(llm_args)?; - // Retrieval embeds the question for the vector half → the real model, like `search`. - let vault = open_vault(cli.vault_or_cwd(), true)?; - // Ctrl-C cancels the answer at the next token rather than killing the process, so - // what already streamed stays on screen and is reported as partial. - let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); - ask_streamed(&vault, llm.as_ref(), question, &[], cli.json)?; - if !cli.json { - note_fake_llm(); - } - Ok(()) -} - -fn cmd_why( - cli: &Cli, - note: &str, - candidate: &str, - limit: usize, - llm_args: &LlmArgs, -) -> Result<(), CliError> { - let llm = open_llm(llm_args)?; - // A pure read over stored vectors, like `similar`: nothing embeds a query, so the - // real model is never loaded. - let vault = open_vault(cli.vault_or_cwd(), false)?; - let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); - let json = cli.json; - let answer = vault.why_similar(llm.as_ref(), note, candidate, limit, &mut |token| { - stream_token(token, json) - })?; - finish_answer(&answer, json); - if !json { - note_fake_llm(); - } - Ok(()) -} - -fn cmd_chat(cli: &Cli, llm_args: &LlmArgs) -> Result<(), CliError> { - let llm = open_llm(llm_args)?; - let vault = open_vault(cli.vault_or_cwd(), true)?; - // Ctrl-C means two different things in a REPL, and a handler that only ever meant one - // would trap the user: mid-answer it cancels the stream (the partial text stands), but - // at an idle prompt it must still be the way out, since swallowing it would leave - // `/exit` and Ctrl-D as the only exits. `ctrlc` runs this on its own thread. - let _ = ctrlc::set_handler(|| { - if ANSWERING.load(Ordering::SeqCst) { - CANCEL.store(true, Ordering::SeqCst); - } else { - // 128 + SIGINT, the shell's own convention for "interrupted". - std::process::exit(130); - } - }); - // Prompts and the banner are chrome for a human at a terminal: on stderr so - // stdout stays answers, and only when there's a terminal there to read them - // (the `reindex` progress-line rule). - let interactive = !cli.json && std::io::stderr().is_terminal(); - if interactive { - eprintln!( - "Grounded chat over your notes — answers come only from what B2 retrieves \ - from them, cited by [n]." - ); - eprintln!( - "Model: {}. Ctrl-C stops an answer; /exit or Ctrl-D leaves. \ - Nothing here is saved.", - llm.model_id() - ); - } - if !cli.json { - note_fake_llm(); - } - // Session-only history (S4): the turns live in this Vec and die with the process — - // a persisted transcript would be B2-derived state outside the Markdown. - let mut history: Vec = Vec::new(); - let stdin = std::io::stdin(); - loop { - if interactive { - eprint!("\nyou> "); - let _ = std::io::stderr().flush(); - } - let mut line = String::new(); - if stdin.read_line(&mut line)? == 0 { - // Ctrl-D / end of a piped script. - break; - } - let question = line.trim(); - if question.is_empty() { - continue; - } - if matches!(question, "/exit" | "/quit") { - break; - } - // A fresh cancel budget per turn: the Ctrl-C that stopped the *last* answer - // must not cancel this one before its first token. - CANCEL.store(false, Ordering::SeqCst); - ANSWERING.store(true, Ordering::SeqCst); - let turn = ask_streamed(&vault, llm.as_ref(), question, &history, cli.json); - ANSWERING.store(false, Ordering::SeqCst); - match turn { - Ok(answer) => { - // A cancelled answer goes into the history too: the human saw that - // text, so a follow-up referring to it ("go on") must be read - // against what was actually said, not against a turn we pretend - // never happened. - history.push(ChatTurn::user(question)); - history.push(ChatTurn::assistant(&answer.answer)); - } - // A failed turn ends the turn, not the session: a model server that - // hiccuped is worth retyping a question at, not worth losing the - // conversation over. The message is the same one the process would - // have exited with. - Err(e) => eprintln!("{}", user_message(&e)), - } - } - Ok(()) -} - -/// One grounded ask, rendered as it arrives — the shared body of `ask` and each `chat` -/// turn (flow ④'s streaming contract: every surface streams). Under `--json` this is a -/// **JSON Lines event stream** — one `token` event per token, then one `answer` event -/// carrying the [`AnswerView`] — so an agent sees the answer forming and still gets the -/// resolved citations as data. -fn ask_streamed( - vault: &Vault, - llm: &dyn LlmProvider, - question: &str, - history: &[ChatTurn], - json: bool, -) -> Result { - let answer = vault.ask(llm, question, history, &mut |token| { - stream_token(token, json) - })?; - finish_answer(&answer, json); - Ok(answer) -} - -/// Render one streamed token — the framing `ask`, `chat` and `why` share — and report -/// whether Ctrl-C has asked the stream to stop. -fn stream_token(token: &str, json: bool) -> ControlFlow<()> { - if json { - print_event(&AskEvent::Token { text: token }); - } else { - print!("{token}"); - // Streaming is the point: an unflushed line renders in one lump. - let _ = std::io::stdout().flush(); - } - cancel_flow() -} - -/// Close a streamed answer: the final `answer` event under `--json`, else the sources. -fn finish_answer(answer: &AnswerView, json: bool) { - if json { - print_event(&AskEvent::Answer(answer)); - } else { - print_answer_tail(answer); - } -} - -/// One line of the `--json` ask stream. The framing is the CLI's; the payload of -/// the final event is `b2-core`'s [`AnswerView`], which is the standing -/// convention (the view types are the adapters' shared contract — the desktop -/// carries the same two facts as Tauri events plus a command return). -#[derive(serde::Serialize)] -#[serde(tag = "event", rename_all = "snake_case")] -enum AskEvent<'a> { - /// A token, exactly as it arrived from the model. - Token { text: &'a str }, - /// The finished answer: text, resolved citations, and whether it was cut short. - Answer(&'a AnswerView), -} - -/// Print one event as a single line of JSON — **not** `print_json`, which is -/// pretty-printed: this is a stream, so one object per line is the contract. -fn print_event(event: &AskEvent) { - if let Ok(line) = serde_json::to_string(event) { - println!("{line}"); - } -} - -/// The human-readable tail of an answer: end the streamed line, then the sources -/// the model cited, then — honestly — whether the answer is the whole of one. -fn print_answer_tail(answer: &AnswerView) { - println!(); - if !answer.citations.is_empty() { - println!("\nSources:"); - for c in &answer.citations { - println!(" [{}] {}", c.marker, c.path); - if !c.excerpt.is_empty() { - println!(" {}", c.excerpt); - } - } - } - if !answer.tools.is_empty() { - println!("\nB2 tools used:"); - for t in &answer.tools { - // A lookup B2 made itself is marked, so the list never overstates what the - // model chose to do. - let by = if t.seeded { " (made by B2)" } else { "" }; - println!(" {} {}{by}", t.name, t.arguments); - } - } - if answer.cancelled { - eprintln!("(stopped early — the answer above is partial.)"); - } -} - -/// Never overstate what answered (the `search` fake-embedder caveat, applied to -/// the chat seam): under `B2_LLM=fake` the "answer" is deterministic scaffolding, -/// not a model. On stderr, so stdout stays the answer. -fn note_fake_llm() { - if use_fake_llm() { - eprintln!( - "note: the fake chat provider is in use (B2_LLM=fake) — answers are \ - deterministic test scaffolding, not a model." - ); - } -} - -/// Pick + wire the chat provider — [`open_vault`]'s sibling for the second seam. -/// -/// `B2_LLM=fake` forces the deterministic [`FakeLlm`], the `B2_EMBEDDER=fake` sibling. -/// Otherwise the real client is built from the environment with this command's flags laid -/// over it, and **probed before anything else happens** — the `b2 init` posture applied to -/// chat: a stopped daemon is a sentence before the question, never a surprise after it. -fn open_llm(args: &LlmArgs) -> Result, CliError> { - if use_fake_llm() { - return Ok(Box::new(FakeLlm)); - } - let config = - LlmConfig::from_env().with_overrides(args.llm_url.as_deref(), args.llm_model.as_deref()); - let provider = OpenAiCompatProvider::new(config); - provider.probe()?; - Ok(Box::new(provider)) -} - -fn use_fake_llm() -> bool { - std::env::var_os("B2_LLM").is_some_and(|v| v == "fake") -} - -/// Open a vault with the appropriate embedder. `needs_semantic` commands (`reindex`, -/// `search`) load the real [`LocalEmbedder`] and **fail fast** with "run `b2 init`" if it -/// is absent; pure-graph commands pass `false` and use the fake, so no model is required -/// just to explore the graph. `B2_EMBEDDER=fake` forces the fake everywhere. -fn open_vault(root: &Path, needs_semantic: bool) -> Result { - if needs_semantic && !use_fake_embedder() { - let config = EmbedConfig::load()?; - let embedder = LocalEmbedder::load(&config)?; - Ok(Vault::open_with_embedder( - root, - Box::new(embedder) as Box, - )?) - } else { - Ok(Vault::open(root)?) - } -} - -fn use_fake_embedder() -> bool { - std::env::var_os("B2_EMBEDDER").is_some_and(|v| v == "fake") -} - -/// The presentation rule for naming a note: its title when it has one, else its -/// vault-relative path. -fn display_name<'a>(title: Option<&'a str>, path: &'a str) -> &'a str { - title.unwrap_or(path) -} - -/// The direction glyph: `→` for an outbound edge (this note → other), `←` inbound. -fn arrow(direction: &str) -> &'static str { - if direction == "outbound" { - "→" - } else { - "←" - } -} - -/// Append a resource line's decorations: the `(embed)` marker and the quoted caption. -fn decorate(line: &mut String, embed: bool, caption: Option<&str>) { - use std::fmt::Write as _; - if embed { - line.push_str(" (embed)"); - } - if let Some(c) = caption { - let _ = write!(line, " — \"{c}\""); - } -} - -/// Whether a `mv`/`rm` argument names an existing directory under `root` — the -/// kind-dispatch test that routes to the folder arm (a trailing `/` is tolerated). -fn is_dir_arg(root: &Path, arg: &str) -> bool { - root.join(arg.trim_end_matches('/')).is_dir() -} - -/// The `mv` tally, shared by its three arms: how many inbound link targets were -/// rewritten across how many files — or that nothing linked to the moved item. -fn print_rewrite_tally(links_rewritten: usize, rewrote_files: usize) { - if links_rewritten > 0 { - println!("Rewrote {links_rewritten} inbound link(s) across {rewrote_files} file(s)."); - } else { - println!("No inbound links to rewrite."); - } -} - -/// The `rm` tally, shared by its three arms: which surviving files' links now -/// dangle — or that none were affected. -fn print_dangled(dangled: &[String]) { - if dangled.is_empty() { - println!("No inbound links affected."); - } else { - println!( - "Links in {} file(s) now unresolved: {}", - dangled.len(), - dangled.join(", ") - ); - } -} - -/// Path to the single-in-flight advisory lock for `reindex`, under the disposable -/// index dir (`/.b2/reindex.lock`, gitignored along with the rest of `.b2/`). -fn reindex_lock_path(root: &Path) -> PathBuf { - root.join(".b2").join("reindex.lock") -} - -/// Open (creating if absent) the reindex lock file, ensuring `.b2/` exists first so -/// the lock can be taken *before* the index does on a first-ever reindex. The caller -/// takes the lock with [`File::try_lock`] and holds the returned handle for the run. -fn open_reindex_lock(root: &Path) -> Result { - std::fs::create_dir_all(root.join(".b2"))?; - Ok(OpenOptions::new() - .create(true) - // A lock file carries no contents we care about — never truncate it (that's a - // needless write, and could race a holder). Presence + the advisory lock is all. - .truncate(false) - .read(true) - .write(true) - .open(reindex_lock_path(root))?) -} - -/// Stamp this process's id into the reindex lock — **only ever called with the lock -/// held**, which is what makes truncate-then-write safe (`open_reindex_lock` deliberately -/// does not truncate, since that would race a holder). -/// -/// The pid outlives the run, harmlessly: [`reindex_holder`] reads it *only* when the lock -/// is contended, so the **lock**, never the file's contents, decides whether a run is in -/// flight. -fn record_reindex_pid(lock: &File) -> std::io::Result<()> { - let mut handle = lock; - handle.set_len(0)?; - handle.seek(SeekFrom::Start(0))?; - // A trailing newline so the file reads sanely under `cat`; parsing trims it. - writeln!(handle, "{}", std::process::id())?; - handle.flush() -} - -/// A reindex holding the lock, seen from another process. -#[derive(Debug)] -struct ReindexHolder { - /// The pid it stamped into the lock. `None` when the lock is held but no readable - /// pid is there — a holder that took the lock microseconds ago and hasn't written - /// yet, an older `b2` that stamped none, or an unreadable file. "Running" is the - /// lock's answer; the pid is the extra affordance that may be missing. - pid: Option, -} - -/// Peek at the reindex lock — the best-effort read behind `b2 status` and `b2 reindex -/// --cancel`: open the *existing* lock file and try to take it; a contended lock means -/// a run is in flight, and its pid is read from the file we could not lock. Never -/// creates the file (both callers are read-only on the vault), so a missing lock file -/// simply means no reindex has run; any I/O hiccup degrades to "not running". -fn reindex_holder(root: &Path) -> Option { - let mut file = OpenOptions::new() - .read(true) - .write(true) - .open(reindex_lock_path(root)) - .ok()?; - // If we *can* take it, nobody holds it; `file` drops here and releases at once. - if !matches!(file.try_lock(), Err(std::fs::TryLockError::WouldBlock)) { - return None; - } - let mut buf = String::new(); - let pid = file - .read_to_string(&mut buf) - .ok() - .and_then(|_| buf.trim().parse().ok()); - Some(ReindexHolder { pid }) -} - -/// `b2 reindex --cancel` (GH #55): stop a reindex on this vault by signalling the pid its -/// lock names. It adds **no cancellation machinery** — SIGINT is exactly what Ctrl-C -/// delivers, so the holder takes the shipped path to a consistent, re-runnable partial -/// index. That is the point: a run backgrounded with `b2 reindex &` has no controlling -/// terminal for Ctrl-C to reach. -/// -/// One window is inherent and accepted: a holder still loading the model hasn't installed -/// the handler yet, so SIGINT terminates it outright — the same as Ctrl-C there today, and -/// safe for the same reason (nothing is written until embedding starts). -fn cancel_reindex(root: &Path, json: bool) -> Result<(), CliError> { - let Some(holder) = reindex_holder(root) else { - return Err(CliError::NoReindexRunning); - }; - let Some(pid) = holder.pid else { - return Err(CliError::ReindexPidUnknown); - }; - signal_reindex(pid)?; - if json { - // `signalled`, not `cancelled`: the request landed; the run stops at its next - // batch boundary and reports the partial work itself (honest tense, like the - // dry-run's `would_*` keys). - print_json(&serde_json::json!({ - "signalled": true, - "pid": pid, - }))?; - } else { - println!( - "Cancelling the reindex on this vault (pid {pid}). It stops after the current batch, leaving a consistent index — re-run `b2 reindex` to finish." - ); - } - Ok(()) -} - -/// Send the cancel signal to a reindex holder. SIGINT, deliberately: the identical -/// signal a foreground Ctrl-C raises, so there is one cancel path, not two. -fn signal_reindex(pid: u32) -> Result<(), CliError> { - use nix::sys::signal::{kill, Signal}; - use nix::unistd::Pid; - // A pid that doesn't fit `pid_t` was never one we wrote — treat a garbled lock as - // nothing to cancel rather than signalling whatever the truncation would name. - let raw = i32::try_from(pid).map_err(|_| CliError::NoReindexRunning)?; - match kill(Pid::from_raw(raw), Signal::SIGINT) { - Ok(()) => Ok(()), - // The holder exited between the lock peek and the signal: there is nothing left - // to cancel — the outcome the user wanted, reported as such rather than as I/O. - Err(nix::errno::Errno::ESRCH) => Err(CliError::NoReindexRunning), - Err(e) => Err(CliError::Io(std::io::Error::from_raw_os_error(e as i32))), - } -} - -/// The CLI's error, composing the two crates it drives. Kept internal; `user_message` -/// turns it into a generic, actionable, no-internals-leaked line (logging policy). -/// `#[from]` supplies the `?` conversions; `transparent` defers `Display` to the -/// inner error (only ever surfaced under `B2_DEBUG`). -#[derive(Debug, thiserror::Error)] -enum CliError { - #[error(transparent)] - Core(#[from] b2_core::Error), - #[error(transparent)] - Embed(#[from] EmbedError), - /// A setup-time chat failure the adapter can act on — an unreachable model - /// server, an un-pulled model. Call-time failures arrive as - /// [`b2_core::Error::Llm`] instead (the seam collapses them to a message). - #[error(transparent)] - Llm(#[from] LlmError), - #[error(transparent)] - Serde(#[from] serde_json::Error), - /// A filesystem error creating `.b2/` or opening the reindex lock file. - #[error(transparent)] - Io(#[from] std::io::Error), - /// `reindex` was run with no vault at all (no positional, no `-C`, no - /// `$B2_VAULT_PATH`) — refuse rather than silently index the current directory. - #[error("no vault specified")] - VaultRequired, - /// Another `reindex` already holds the single-in-flight lock on this vault. - #[error("a reindex is already running")] - ReindexRunning, - /// `reindex --cancel` found no run in flight on this vault — nothing to signal. - #[error("no reindex is running")] - NoReindexRunning, - /// `reindex --cancel` found a run in flight whose lock names no readable pid, so - /// there is no address to signal (see [`ReindexHolder::pid`]). - #[error("the running reindex recorded no pid")] - ReindexPidUnknown, - /// `rm` was pointed at a folder without `--recursive` — refuse rather than - /// silently remove a whole subtree (the CLI's stand-in for the desktop's - /// confirm dialog; there is no interactive prompt to give an agent). - #[error("folder delete requires --recursive: {0}")] - RecursiveRequired(String), - /// `write` was invoked with stdin attached to a terminal (nothing piped) — refuse - /// rather than hang waiting for hand-typed input; the new body must be piped in. - #[error("no body piped on stdin")] - StdinRequired, -} - -// `is_ollama` — "does this endpoint look like the Ollama daemon" — decides whether -// Ollama's own commands belong in an error message (`--llm-url` also points at LM Studio, -// llama.cpp, vLLM and cloud endpoints, where "run `ollama serve`" is advice about the -// wrong program). The rule lives in `b2-llm` (GH #155), where guided setup needs the same -// answer for a bigger decision, so one rule with two callers cannot drift. - -/// The head of a model server's own model list, for "…or pick one it already -/// serves". Bounded: a local runtime holds a handful, but a cloud endpoint lists -/// hundreds, and a hundred-model line is not a hint. -fn model_hint(available: &[String]) -> Option { - const SHOWN: usize = 6; - if available.is_empty() { - return None; - } - let head = available.iter().take(SHOWN).cloned().collect::>(); - Some(if available.len() > SHOWN { - format!("{}, …", head.join(", ")) - } else { - head.join(", ") - }) -} - -/// Translate an internal error into a generic, actionable, user-facing message — -/// never leaking sqlite/io/serde internals. Set `B2_DEBUG` to also print the detail. -fn user_message(err: &CliError) -> String { - let msg = match err { - CliError::Core(b2_core::Error::NoteNotFound(r)) => format!( - "Note not found: '{r}'. Check the path, and run `b2 reindex` first." - ), - CliError::Core(b2_core::Error::IndexTooNew { .. }) => { - "This vault's index was built by a newer version of B2 than this `b2`. Run the newer B2 (or upgrade this one) — the index was left untouched, so nothing needs rebuilding.".to_string() - } - CliError::Core(b2_core::Error::ModelMismatch { .. }) => { - "This vault's index was built with a different embedding model. Run `b2 reindex` to rebuild it.".to_string() - } - CliError::Embed(EmbedError::NotProvisioned { model, .. }) => format!( - "Embedding model '{model}' is not installed. Run `b2 init` to download it (or set B2_EMBEDDER=fake for an offline, non-semantic mode)." - ), - CliError::Embed(EmbedError::Download(_)) => { - "Could not download the embedding model. Check your network and try `b2 init` again.".to_string() - } - CliError::Core(b2_core::Error::MoveTargetExists(p)) => format!( - "Can't move: a file already exists at '{p}'. Choose a different destination." - ), - CliError::Core(b2_core::Error::MoveDestination(_)) => { - "That move destination isn't valid. Give a vault-relative path like `notes/new-name.md`.".to_string() - } - CliError::Core(b2_core::Error::MoveIncomplete { paths, .. }) => format!( - "The move failed, and B2 could not undo its link changes in: {}. Check the links in those files; `b2 reindex` then shows any that no longer resolve.", - paths.join(", ") - ), - CliError::Core(b2_core::Error::DirNotFound(p)) => format!( - "Folder not found: '{p}'. Check the path (folders are vault-relative, like `notes/archive`)." - ), - CliError::Core(b2_core::Error::AddTargetExists(p)) => format!( - "A note already exists at '{p}'. Choose a different path, or edit that note." - ), - CliError::Core(b2_core::Error::AddDestination(_)) => { - "That note path isn't valid. Give a vault-relative path like `notes/new-name.md`.".to_string() - } - CliError::Core(b2_core::Error::InvalidRelation(v)) => format!( - "'{v}' isn't a known relation type. Use one of: references, supports, contradicts." - ), - CliError::Core(b2_core::Error::WriteConflict(_)) => { - "This note changed on disk since it was opened. Reload the note, then reapply your edit.".to_string() - } - CliError::Core(b2_core::Error::Frontmatter(d)) => { - format!("Can't edit that frontmatter: {d}.") - } - CliError::Core(b2_core::Error::ResourceNotFound(r)) => format!( - "File not found in the vault: '{r}'. Check the path, and run `b2 reindex` first." - ), - CliError::Core(b2_core::Error::ResourceUnsupported(_)) => { - "Similarity for non-Markdown files isn't available yet — it arrives in a later release. `b2 explain ` shows its backlinks today.".to_string() - } - CliError::VaultRequired => { - "No vault specified. Point B2 at your vault with `-C `, or set B2_VAULT_PATH.".to_string() - } - CliError::ReindexRunning => { - "A reindex is already running on this vault. Wait for it to finish (check `b2 status`), or stop it with `b2 reindex --cancel`.".to_string() - } - CliError::NoReindexRunning => { - "No reindex is running on this vault — nothing to cancel. Check `b2 status`.".to_string() - } - CliError::ReindexPidUnknown => { - "A reindex is running on this vault, but it recorded no process id to signal. Try `b2 reindex --cancel` again in a moment, or stop the run from the shell it was started in.".to_string() - } - CliError::RecursiveRequired(p) => format!( - "'{p}' is a folder. Re-run with -r/--recursive to delete it and everything inside it." - ), - CliError::StdinRequired => { - "No body piped on stdin. Pipe the new note body in, e.g. `cat new-body.md | b2 write notes/foo`.".to_string() - } - // The E4 case chat is most likely to hit: nothing is serving the endpoint. - // Named by the endpoint the user actually configured — and *advised* by it - // too: `ollama serve` is the fix for Ollama and no help at all for LM - // Studio, llama.cpp, vLLM, or a cloud provider, all of which `--llm-url` - // supports. - CliError::Llm(LlmError::Unreachable { endpoint, .. }) if is_ollama(endpoint) => format!( - "Can't reach the model server at {endpoint} — is Ollama running? (`ollama serve`, or install: {OLLAMA_INSTALL_URL})" - ), - CliError::Llm(LlmError::Unreachable { endpoint, .. }) => format!( - "Can't reach the model server at {endpoint}. Start it, or point --llm-url (or B2_LLM_URL) at one that's running." - ), - // Something answered the probe and refused it — a *different* mistake from - // nothing listening, and one this used to report as success. The sentence is - // b2-llm's (`refusal_message`), so the CLI and the app say the same thing; the - // Ollama-root hint is the app's alone, since only its card has asked the daemon. - CliError::Llm(LlmError::Refused { endpoint, status, message }) => { - refusal_message(endpoint, *status, message, None) - } - CliError::Llm(LlmError::ModelMissing { model, endpoint, available }) => format!( - "Model '{model}' isn't available at {endpoint}. {}{}", - if is_ollama(endpoint) { - format!("Pull it with `ollama pull {model}`") - } else { - "Load it there".to_string() - }, - match model_hint(available) { - Some(list) => format!(", or point --llm-model at one it already serves ({list})."), - None => ".".to_string(), - } - ), - // Not the generic chat failure below: the server is up and the model is there, - // so that advice would mislead. The cap is the one fix on this side of the wire. - CliError::Core(b2_core::Error::ToolCallLimit { limit }) => format!( - "The chat model asked for more than {limit} tool calls in one reply, so b2 stopped it. Try again or use another model (--llm-model). If this model really needs more, raise {}.", - b2_llm::ENV_MAX_TOOL_CALLS - ), - // Every remaining chat failure — an HTTP refusal at probe time, a malformed - // stream — is one sentence with one fix, and the detail is a `B2_DEBUG` away. - CliError::Llm(_) | CliError::Core(b2_core::Error::Llm(_)) => { - "The model server couldn't answer. Check that it's running and that the model is installed (`ollama list`), then try again.".to_string() - } - _ => "Something went wrong. Please check the vault path and try again.".to_string(), - }; - if std::env::var_os("B2_DEBUG").is_some() { - // Every wrapper variant is `#[error(transparent)]` and every local variant - // carries its own `#[error("…")]` line, so the enum's own `Display` *is* the - // per-variant detail — no match needed. - let detail = err.to_string(); - format!("{msg}\n(debug: {detail})") - } else { - msg - } -} - -/// The one unit-tested corner of this adapter: ADR-0015's three verdict states as -/// rendered by [`search_report`] (GH #202). -/// -/// The integration suite spawns the binary under `B2_EMBEDDER=fake`, whose embedder has no -/// calibrated bar — so `vouched` there is always `None` and the two states that matter are -/// unreachable from it. That is a fact about the model seam, not a hole to leave: the -/// rendering is pure, so it is tested here directly rather than hidden behind an -/// `#[ignore]` the suite would keep reporting as present. -#[cfg(test)] -mod tests { - use super::*; - use b2_core::vault::{EvidencedResult, SearchResult}; - - #[test] - fn a_blown_tool_call_cap_names_the_limit_and_the_variable_that_raises_it() { - let msg = user_message(&CliError::Core(b2_core::Error::ToolCallLimit { limit: 64 })); - assert!(msg.contains("64"), "{msg}"); - assert!(msg.contains("B2_LLM_MAX_TOOL_CALLS"), "{msg}"); - assert!( - !msg.contains("Check that it's running"), - "the server is fine — the generic chat advice would mislead: {msg}" - ); - } - - #[test] - fn a_move_that_could_not_be_undone_names_the_files_to_check() { - let msg = user_message(&CliError::Core(b2_core::Error::MoveIncomplete { - paths: vec!["b.md".into(), "notes/c.md".into()], - source: Box::new(b2_core::Error::Io(std::io::Error::other("disk full"))), - })); - assert!(msg.contains("in: b.md, notes/c.md."), "{msg}"); - assert!( - !msg.contains("disk full"), - "the cause stays internal: {msg}" - ); - } - - fn view(vouched: Option, n: usize) -> SearchEvidenceView { - SearchEvidenceView { - results: (0..n) - .map(|i| EvidencedResult { - result: SearchResult { - path: format!("notes/n{i}.md"), - title: Some(format!("Note {i}")), - score: 0.5, - snippet: "a matched line".to_string(), - }, - bm25_rank: Some(i), - vector_rank: Some(i), - cos: Some(0.7), - }) - .collect(), - vouched, - chunk_total: 100, - terms: Vec::new(), - best_cos: Some(0.7), - } - } - - #[test] - fn an_unvouched_query_shows_the_empty_state_and_none_of_its_rows() { - // Strict (GH #202, decision 1): the engine served rows, and the surface - // shows none of them — no reveal, no count of what is being withheld, - // since either would put them forward as candidates after all. - let out = search_report(&view(Some(false), 10), "Fasdfadsf"); - assert!(out.starts_with("No matches."), "{out}"); - assert!(!out.contains("notes/n0.md"), "no row leaks: {out}"); - assert!(out.contains("Fasdfadsf"), "names the query back: {out}"); - } - - #[test] - fn a_vouched_query_serves_its_rows() { - let out = search_report(&view(Some(true), 2), "memory"); - assert!( - out.contains("notes/n0.md") && out.contains("notes/n1.md"), - "{out}" - ); - assert!(out.contains("a matched line"), "snippets ride along: {out}"); - } - - #[test] - fn no_calibrated_bar_serves_its_rows_exactly_as_before() { - // `None` is *no verdict*, never "no matches" (M2): the fake embedder and - // every uncalibrated model land here, and folding it into `false` would - // blank the app on a dev vault. - let out = search_report(&view(None, 2), "memory"); - assert!(out.contains("notes/n0.md"), "{out}"); - assert!(!out.contains("No matches"), "{out}"); - } - - #[test] - fn an_empty_list_reads_as_no_results_whatever_the_verdict() { - // The genuinely-empty list keeps its own older copy, which claims - // nothing about evidence — an unbuilt index is not a judgment. - for vouched in [Some(true), None] { - let out = search_report(&view(vouched, 0), "memory"); - assert_eq!(out, "No results.", "{vouched:?}"); - } - } -} diff --git a/crates/b2-cli/src/read.rs b/crates/b2-cli/src/read.rs new file mode 100644 index 0000000..750f4bc --- /dev/null +++ b/crates/b2-cli/src/read.rs @@ -0,0 +1,456 @@ +//! The read commands: the graph (`neighbors`, `explain`), `search`, and discovery +//! (`similar`, `similar --explain`), with the presentation helpers they share. + +use crate::args::Cli; +use crate::error::CliError; +use crate::print_json; +use crate::wiring::{open_vault, use_fake_embedder}; +use b2_core::resource::{doc_kind, DocKind}; +use b2_core::vault::{SearchEvidenceView, SimilarStanding}; + +pub fn cmd_neighbors(cli: &Cli, note: &str) -> Result<(), CliError> { + // Neighbors is a pure graph query — it never embeds, so don't require + // the model (no needless `b2 init` just to explore the graph). + let vault = open_vault(cli.vault_or_cwd(), false)?; + let neighbors = vault.neighbors(note)?; + // Dangling outbound links (a `[[folder]]` or a typo) that resolve to no + // note or resource — surfaced, not dropped (GH #12). `--json` keeps its + // resolved-neighbors array contract; the full structured picture, + // including these, is `b2 explain --json`. + let unresolved = vault.unresolved_links(note)?; + if cli.json { + print_json(&neighbors)?; + } else if neighbors.is_empty() && unresolved.is_empty() { + println!("No neighbors."); + } else { + for n in &neighbors { + let arrow = arrow(&n.direction); + let name = display_name(n.title.as_deref(), &n.path); + let explanation = n + .explanation + .as_deref() + .map(|e| format!(" — {e}")) + .unwrap_or_default(); + println!("{arrow} {} {name} ({}){explanation}", n.label, n.path); + } + for u in &unresolved { + println!( + "⚠ {} [[{}]] — unresolved (no matching note or file)", + u.relation, u.target + ); + } + } + Ok(()) +} + +pub fn cmd_explain(cli: &Cli, note: &str) -> Result<(), CliError> { + // Explain is a pure graph read (edges + their explanations), no embed — + // like `neighbors`, it opens with the fake and needs no `b2 init`. + let vault = open_vault(cli.vault_or_cwd(), false)?; + // Kind dispatch by the argument's own shape (core's one rule, §9b #8): + // a resource arg gets the fallback card's view — metadata + backlinks. + if doc_kind(note) == DocKind::Resource { + let view = vault.explain_resource(note)?; + if cli.json { + print_json(&view)?; + } else { + println!("{} ({}, {} bytes)", view.path, view.class, view.size); + if view.backlinks.is_empty() { + println!("No backlinks yet."); + } else { + println!("Backlinks:"); + for b in &view.backlinks { + let name = display_name(b.title.as_deref(), &b.path); + let mut line = format!(" ← {name} ({}) {}", b.path, b.r#type); + decorate(&mut line, b.embed, b.caption.as_deref()); + println!("{line}"); + } + } + } + return Ok(()); + } + let view = vault.explain(note)?; + if cli.json { + print_json(&view)?; + } else { + let name = display_name(view.title.as_deref(), &view.path); + println!("{name} ({})", view.path); + if view.connections.is_empty() && view.resources.is_empty() && view.unresolved.is_empty() { + // Zero connections at all — nothing links to it and it links to + // nothing (an orphan; the kernel only surfaces, never archives). + println!("No connections yet."); + } else if !view.connections.is_empty() { + println!("Connections:"); + for c in &view.connections { + let arrow = arrow(&c.direction); + let target = display_name(c.title.as_deref(), &c.path); + println!( + " {arrow} {} {target} ({}) [{}]", + c.label, c.path, c.origin + ); + if let Some(why) = &c.explanation { + println!(" why: {why}"); + } + } + // If nothing points *at* the note, it's an orphan — surfaced, not + // acted on (invariants.md; files are only touched when asked). + if !view.connections.iter().any(|c| c.direction == "inbound") { + println!("No inbound links — this note is an orphan."); + } + } + // Outbound links at resources (images, PDFs, …) — the third target + // kind an edge can have, shown from the note's side (GH #22). + if !view.resources.is_empty() { + println!("Resource links:"); + for r in &view.resources { + let mut line = format!( + " → {} {} ({}) [{}]", + r.relation, r.path, r.class, r.origin + ); + decorate(&mut line, r.embed, r.caption.as_deref()); + println!("{line}"); + if let Some(why) = &r.explanation { + println!(" why: {why}"); + } + } + } + // Dangling outbound links (a `[[folder]]` or a typo): a note is one + // `.md` file, so these resolve to nothing — shown as broken rather + // than silently dropped (GH #12). + if !view.unresolved.is_empty() { + println!("Unresolved links:"); + for u in &view.unresolved { + println!( + " ⚠ {} [[{}]] (no matching note or file) [{}]", + u.relation, u.target, u.origin + ); + if let Some(why) = &u.explanation { + println!(" why: {why}"); + } + } + } + } + Ok(()) +} + +pub fn cmd_search( + cli: &Cli, + query: &str, + limit: usize, + exclude: &[String], +) -> Result<(), CliError> { + // Search embeds the query for the vector half → it needs the real model. + let vault = open_vault(cli.vault_or_cwd(), true)?; + // The evidence read, not the bare list (invariants.md D2, GH #201/#202): the + // rows are identical and in the same order, and the verdict beside them is + // what lets this command say **"no matches"** honestly instead of serving + // `limit` confident-looking results for a query the vault holds nothing for. + // `--exclude` paths are the caller's subtraction, never the verdict's. + let view = vault.search_evidence_excluding(query, limit, exclude)?; + if cli.json { + // The whole view, verdict included. This is an OBJECT where `--json` used to be + // an array — a deliberate break (GH #202), because a query-level verdict has + // nowhere to live in a list of rows; the rows themselves stay additive. + // + // The JSON serves the rows even at `vouched: false`, where the human surface below + // shows none: an agent handed the rows *plus* an explicit verdict can be honest + // about them, where a human handed rows alone cannot. + print_json(&view)?; + } else { + println!("{}", search_report(&view, query)); + // Honesty (never overstate): with the fake embedder the vector half + // isn't semantic. Under the real model it is, so no caveat. Kept on + // stderr so stdout stays pure results. + if use_fake_embedder() { + eprintln!( + "note: keyword (BM25) ranking is live; semantic ranking is off (fake embedder)." + ); + } + } + Ok(()) +} + +/// The human-mode rendering of a search — ADR-0015's three verdict states as one pure +/// function (GH #202). +/// +/// Pure, and separate from [`cmd_search`], because the state that matters most is the one +/// the integration suite structurally cannot reach: `Some(false)` needs a *calibrated* +/// bar, and the fake embedder the suite runs under has none by design. A branch reachable +/// only under the real model is still the fast suite's to own once the rendering is +/// separated from the retrieval. +fn search_report(view: &SearchEvidenceView, query: &str) -> String { + match view.vouched { + // D2's "no matches", strict: the vault holds neither a lexical anchor + // nor semantic proximity clearing the model's bar, so the + // nearest-by-meaning rows are not shown at all — no reveal, no `--all`. + // Offering them behind a flag would still be this command putting them + // forward as candidates (GH #202, decision 1). + Some(false) => format!("No matches. Nothing in the vault matches “{query}”."), + // `Some(true)` — the vault vouches for these — and `None`, which is *no + // verdict at all*: the active embedder has no calibrated bar (the fake + // one, or any model until the harness measures it — M2), and reading + // that as "no matches" would blank a dev vault. Three states, three + // behaviors; `None` is never folded into `false`. + _ if view.results.is_empty() => "No results.".to_string(), + _ => view + .results + .iter() + .map(|r| { + let name = display_name(r.result.title.as_deref(), &r.result.path); + let head = format!("{:.4} {name} ({})", r.result.score, r.result.path); + if r.result.snippet.is_empty() { + head + } else { + format!("{head}\n {}", r.result.snippet) + } + }) + .collect::>() + .join("\n"), + } +} + +pub fn cmd_similar(cli: &Cli, note: &str, limit: usize) -> Result<(), CliError> { + // Candidate generation reads the *stored* vectors (no query embedding), so + // like `neighbors` it needs no live model — a prior `reindex` supplies them. + // Open with the fake; it's a pure, instant local read. (The z grading keys + // on the vault's RECORDED model id, so opening with the fake for reading + // never turns it off.) + let vault = open_vault(cli.vault_or_cwd(), false)?; + let results = vault.similar(note, limit)?; + if cli.json { + print_json(&results)?; + } else if limit == 0 { + // An ask for nothing yields nothing, silently: the empty-state copy + // below reads the candidate set, and a zero limit proves nothing about + // it — printing "nothing to compare" here would be the one way this + // command could still make a claim it can't check (GH #197). + } else if results.is_empty() { + // Two honest empty states, and neither claims "nothing relates" + // (GH #197): at a nonzero ask, an empty list means the candidate set is + // genuinely empty — nothing unlinked with stored vectors to compare — + // or the vault's similarity isn't semantic yet. + let status = vault.embed_status()?; + if status.embedded > 0 { + println!("Nothing unlinked has stored vectors to compare."); + } else { + println!( + "No similar notes. (If you haven't yet, run `b2 init` then `b2 reindex` so similarity is semantic.)" + ); + } + } else { + for r in &results { + let name = display_name(r.title.as_deref(), &r.path); + println!("{:.4} {name} ({})", r.score, r.path); + if !r.evidence.is_empty() { + println!(" {}", r.evidence); + } + } + // Nudge toward the commit step, on stderr so stdout stays pure results. + eprintln!("Commit one with: b2 link {note} --type "); + } + Ok(()) +} + +/// Longest passage excerpt `similar --explain` prints per side, so a pair stays readable +/// in a terminal. `--json` carries the full text. +const EXPLAIN_EXCERPT_CHARS: usize = 200; + +pub fn cmd_explain_similar( + cli: &Cli, + note: &str, + other: &str, + limit: usize, +) -> Result<(), CliError> { + // A pure read over stored vectors, like `similar`: the fake is enough to open with. + let vault = open_vault(cli.vault_or_cwd(), false)?; + let ex = vault.explain_similar(note, other, limit)?; + if cli.json { + return print_json(&ex); + } + let anchor = display_name(ex.anchor.title.as_deref(), &ex.anchor.path); + let candidate = display_name(ex.candidate.title.as_deref(), &ex.candidate.path); + println!( + "{candidate} ({}) against {anchor} ({})", + ex.candidate.path, ex.anchor.path + ); + let z = ex.z.map(|z| format!(", z {z:.2}")).unwrap_or_default(); + let standing = match ex.standing { + SimilarStanding::SameNote => "That is the same note.".to_string(), + SimilarStanding::AnchorUnembedded => format!( + "{anchor} has no stored vectors yet, so there is nothing to compare from. Run `b2 reindex`." + ), + SimilarStanding::Linked => { + "Already linked, so Similar leaves it out: it shows what you haven't connected." + .to_string() + } + SimilarStanding::Unembedded => { + "Not embedded yet, so it can't be compared. Run `b2 reindex`.".to_string() + } + SimilarStanding::NotShortlisted { shortlist } => format!( + "Never scored: judged as a whole note it ranks #{}, past the first-pass shortlist of {shortlist}.", + ex.centroid_rank.unwrap_or(0) + ), + SimilarStanding::Ranked { + rank, + of, + served: true, + } => format!("Shown at #{rank} of {limit} ({of} notes scored{z})."), + SimilarStanding::Ranked { + rank, + of, + served: false, + } => format!("Ranked #{rank} of {of} scored{z}: past the {limit} shown."), + }; + println!("{standing}"); + if let (Some(c), SimilarStanding::Ranked { rank, .. }) = (ex.centroid_rank, ex.standing) { + println!("Judged as a whole note it ranks #{c}; by its best passage, #{rank}."); + } + if !ex.shared_neighbors.is_empty() { + let names: Vec = ex + .shared_neighbors + .iter() + .map(|n| format!("{} ({})", display_name(n.title.as_deref(), &n.path), n.path)) + .collect(); + println!("Both link with: {}", names.join(", ")); + } + if !ex.pairs.is_empty() { + println!("\nPassage pairs, nearest first:"); + // Pairs carry a grade only when the list does; say why they don't, once, rather + // than print bare pairs that read as unmeasured by accident. + if ex.pairs.iter().all(|p| p.z.is_none()) { + println!( + " Ungraded: too few notes to compare against, or a vault indexed without the real model." + ); + } + } + for (i, p) in ex.pairs.iter().enumerate() { + let grade = p.z.map(|z| format!(" z {z:.2}")).unwrap_or_default(); + let same = if p.identical { " identical text" } else { "" }; + println!("{:>3}.{grade}{same}", i + 1); + for (side, passage) in [("this", &p.anchor), ("that", &p.candidate)] { + let heading = passage + .heading_path + .as_deref() + .map(|h| format!("[{h}] ")) + .unwrap_or_default(); + println!( + " {side}: {heading}{}", + excerpt(&passage.text, EXPLAIN_EXCERPT_CHARS) + ); + } + } + Ok(()) +} + +/// A passage flattened to one line and cut at `max` characters. +fn excerpt(text: &str, max: usize) -> String { + let flat = text.split_whitespace().collect::>().join(" "); + if flat.chars().count() <= max { + flat + } else { + format!("{}…", flat.chars().take(max).collect::().trim_end()) + } +} + +/// The presentation rule for naming a note: its title when it has one, else its +/// vault-relative path. +fn display_name<'a>(title: Option<&'a str>, path: &'a str) -> &'a str { + title.unwrap_or(path) +} + +/// The direction glyph: `→` for an outbound edge (this note → other), `←` inbound. +fn arrow(direction: &str) -> &'static str { + if direction == "outbound" { + "→" + } else { + "←" + } +} + +/// Append a resource line's decorations: the `(embed)` marker and the quoted caption. +fn decorate(line: &mut String, embed: bool, caption: Option<&str>) { + use std::fmt::Write as _; + if embed { + line.push_str(" (embed)"); + } + if let Some(c) = caption { + let _ = write!(line, " — \"{c}\""); + } +} + +/// The one unit-tested corner of this adapter: ADR-0015's three verdict states as +/// rendered by [`search_report`] (GH #202). +/// +/// The integration suite spawns the binary under `B2_EMBEDDER=fake`, whose embedder has no +/// calibrated bar — so `vouched` there is always `None` and the two states that matter are +/// unreachable from it. That is a fact about the model seam, not a hole to leave: the +/// rendering is pure, so it is tested here directly rather than hidden behind an +/// `#[ignore]` the suite would keep reporting as present. +#[cfg(test)] +mod tests { + use super::*; + use b2_core::vault::{EvidencedResult, SearchResult}; + + fn view(vouched: Option, n: usize) -> SearchEvidenceView { + SearchEvidenceView { + results: (0..n) + .map(|i| EvidencedResult { + result: SearchResult { + path: format!("notes/n{i}.md"), + title: Some(format!("Note {i}")), + score: 0.5, + snippet: "a matched line".to_string(), + }, + bm25_rank: Some(i), + vector_rank: Some(i), + cos: Some(0.7), + }) + .collect(), + vouched, + chunk_total: 100, + terms: Vec::new(), + best_cos: Some(0.7), + } + } + + #[test] + fn an_unvouched_query_shows_the_empty_state_and_none_of_its_rows() { + // Strict (GH #202, decision 1): the engine served rows, and the surface + // shows none of them — no reveal, no count of what is being withheld, + // since either would put them forward as candidates after all. + let out = search_report(&view(Some(false), 10), "Fasdfadsf"); + assert!(out.starts_with("No matches."), "{out}"); + assert!(!out.contains("notes/n0.md"), "no row leaks: {out}"); + assert!(out.contains("Fasdfadsf"), "names the query back: {out}"); + } + + #[test] + fn a_vouched_query_serves_its_rows() { + let out = search_report(&view(Some(true), 2), "memory"); + assert!( + out.contains("notes/n0.md") && out.contains("notes/n1.md"), + "{out}" + ); + assert!(out.contains("a matched line"), "snippets ride along: {out}"); + } + + #[test] + fn no_calibrated_bar_serves_its_rows_exactly_as_before() { + // `None` is *no verdict*, never "no matches" (M2): the fake embedder and + // every uncalibrated model land here, and folding it into `false` would + // blank the app on a dev vault. + let out = search_report(&view(None, 2), "memory"); + assert!(out.contains("notes/n0.md"), "{out}"); + assert!(!out.contains("No matches"), "{out}"); + } + + #[test] + fn an_empty_list_reads_as_no_results_whatever_the_verdict() { + // The genuinely-empty list keeps its own older copy, which claims + // nothing about evidence — an unbuilt index is not a judgment. + for vouched in [Some(true), None] { + let out = search_report(&view(vouched, 0), "memory"); + assert_eq!(out, "No results.", "{vouched:?}"); + } + } +} diff --git a/crates/b2-cli/src/reindex.rs b/crates/b2-cli/src/reindex.rs new file mode 100644 index 0000000..7ac374e --- /dev/null +++ b/crates/b2-cli/src/reindex.rs @@ -0,0 +1,345 @@ +//! Building the index: `init` (the model it embeds with), `reindex` with its lock and +//! cross-process cancel, and `status`. + +use crate::args::Cli; +use crate::cancel::{cancel_flow, CANCEL}; +use crate::error::CliError; +use crate::print_json; +use crate::wiring::open_vault; +use b2_embed::{provision, EmbedConfig}; +use std::fs::{File, OpenOptions}; +use std::io::{IsTerminal, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; + +pub fn cmd_init(json: bool) -> Result<(), CliError> { + // Global, per-machine setup — no vault involved. + let config = EmbedConfig::load()?; + let report = provision(&config, |line| eprintln!("{line}"))?; + if json { + print_json(&report)?; + } else if report.already_present { + println!("Model '{}' is already installed.", report.model); + } else { + println!( + "Installed '{}' ({} dims). Run `b2 reindex` to embed your vault.", + report.model, report.dim + ); + } + Ok(()) +} + +pub fn cmd_reindex( + cli: &Cli, + vault: Option<&Path>, + force: bool, + dry_run: bool, + cancel: bool, +) -> Result<(), CliError> { + // Reindex writes an index → require an explicit vault (positional wins), + // never a silent cwd fallback. See `Cli::require_vault`. + let root = cli.require_vault(vault)?; + if cancel { + // Signals another process and returns; it never opens the vault (no + // model load, no index read) — the run being cancelled owns all of that. + return cancel_reindex(root, cli.json); + } + if dry_run { + // A dry-run neither embeds nor stamps → no model needed (open with + // the fake, like `neighbors`); it's a pure read, so there's no slow + // embed phase to show progress for. + let vault = open_vault(root, false)?; + let plan = vault.plan_reindex(force)?; + if cli.json { + print_json(&plan)?; + } else { + print_reindex_plan(&plan); + } + return Ok(()); + } + // Single-in-flight: take an advisory lock *before* the slow model load, so a second + // `b2 reindex` refuses cleanly instead of two processes writing the same index. + // Advisory, not a PID file: the OS frees it the instant the holder exits (crash, kill + // or Ctrl-C included), so nothing stale is left behind. Held until this fn ends. + let lock = open_reindex_lock(root)?; + match lock.try_lock() { + Ok(()) => {} + Err(std::fs::TryLockError::WouldBlock) => return Err(CliError::ReindexRunning), + Err(std::fs::TryLockError::Error(e)) => return Err(CliError::Io(e)), + } + // Now that the lock is ours, stamp who holds it: the address `b2 reindex + // --cancel` signals, and what `b2 status` prints so a manual `kill` stays + // available (GH #55). Best-effort — a failed write costs the cancel + // affordance, not the reindex. + let _ = record_reindex_pid(&lock); + // Reindex embeds every changed chunk → it needs the real model. + let vault = open_vault(root, true)?; + // Wire Ctrl-C to the cooperative-cancel flag now that the model is loaded and + // real embedding is next. (During the model load the default SIGINT still + // applies — nothing is written yet, so a hard stop there is safe.) Best-effort: + // if the handler can't be installed, Ctrl-C keeps its default (terminate), which + // still leaves a consistent index since edges + FTS land before any vectors. + let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); + // Embedding a large vault on CPU is slow; show a live progress line so it + // never looks frozen. Only on an interactive stderr (never in --json, and + // never when piped/captured) so machine output and tests stay clean. + let report = if cli.json || !std::io::stderr().is_terminal() { + vault.reindex_with_progress(force, &mut |_| cancel_flow())? + } else { + // Name the vault being indexed up front, then a live line that counts + // the notes actually (re)embedded — not every note, most of which an + // incremental run reuses untouched — with the current file + its chunks. + let shown = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + eprintln!("Indexing {}", shown.display()); + let mut progressed = false; + let mut on_progress = |p: b2_core::ingest::ReindexProgress| { + progressed = true; + // \x1b[K clears any tail of a previous, longer line (paths vary in + // length); safe here because this branch only runs on a real terminal. + eprint!( + "\r embedding {}/{} · {} ({} chunk{})\x1b[K", + p.notes_embedded, + p.notes_to_embed, + p.note_path, + p.note_chunks, + if p.note_chunks == 1 { "" } else { "s" }, + ); + let _ = std::io::stderr().flush(); + // Stop after this batch if Ctrl-C was pressed, + // else carry on. The batch is already written above, so a cancel here + // never tears a write. + cancel_flow() + }; + let report = vault.reindex_with_progress(force, &mut on_progress)?; + if progressed { + eprintln!(); // close the progress line + } + report + }; + if cli.json { + print_json(&report)?; + } else { + print_reindex_report(&report); + } + Ok(()) +} + +/// The human-readable `reindex --dry-run` preview (the `--json` sibling prints the +/// plan itself). +fn print_reindex_plan(plan: &b2_core::vault::ReindexPlan) { + println!( + "Dry run: would index {} note(s) — {} to embed. Nothing in your vault is written either way.", + plan.would_index, plan.would_embed + ); +} + +/// The human-readable reindex summary: the one stdout line, then the stderr +/// notices — skipped files and the cancelled line. +fn print_reindex_report(report: &b2_core::vault::ReindexReport) { + println!( + "Indexed {} notes ({} embedded{}) and {} resources{}", + report.indexed, + report.embedded, + if report.notes_pruned > 0 { + format!(", {} pruned", report.notes_pruned) + } else { + String::new() + }, + report.resources_indexed, + if report.resources_pruned > 0 { + format!(" ({} pruned)", report.resources_pruned) + } else { + String::new() + } + ); + // One unreadable file no longer aborts the reindex — it is skipped and + // named here (to stderr, so it never pollutes the machine-readable stdout + // line above) with a short, file-level reason. + if !report.skipped.is_empty() { + eprintln!("Skipped {} unreadable file(s):", report.skipped.len()); + for s in &report.skipped { + eprintln!(" - {} ({})", s.path, s.reason); + } + } + // The counts above already report the partial work truthfully; add the + // one line that tells the user it was interrupted and is safe to resume. + if report.cancelled { + eprintln!( + "Cancelled — the index is consistent but only partly embedded. Re-run `b2 reindex` to finish the rest." + ); + } +} + +pub fn cmd_status(cli: &Cli) -> Result<(), CliError> { + // Read-only coverage report: how much of the vault is embedded (semantic + // ranking live vs. keyword-only) and whether a background reindex is in + // flight — the companion to backgrounding a slow reindex with `b2 reindex &`. + // A pure model-free DB read (#26): open with the fake. + let root = cli.vault_or_cwd(); + let vault = open_vault(root, false)?; + let status = vault.embed_status()?; + let holder = reindex_holder(root); + if cli.json { + print_json(&serde_json::json!({ + "embedded": status.embedded, + "total": status.total, + "reindex_running": holder.is_some(), + // The running process's id — `null` when nothing is running (and + // on the sliver of a moment before a fresh holder stamps it). + "reindex_pid": holder.as_ref().and_then(|h| h.pid), + }))?; + } else { + if status.total == 0 { + println!("No notes indexed yet. Run `b2 reindex` to build the index."); + } else if status.embedded == 0 { + println!( + "Embedded 0/{} notes — keyword-only. Run `b2 reindex` for semantic ranking.", + status.total + ); + } else if status.embedded < status.total { + println!( + "Embedded {}/{} notes — semantic ranking partial ({} still keyword-only).", + status.embedded, + status.total, + status.total - status.embedded + ); + } else { + println!( + "Embedded {}/{} notes — semantic ranking fully live.", + status.embedded, status.total + ); + } + // Name the process, not just the fact: `--cancel` is the supported stop, + // and the pid keeps a plain `kill -INT` as the documented fallback. + match holder.as_ref().map(|h| h.pid) { + Some(Some(pid)) => println!( + "A reindex is currently running (pid {pid}). Stop it with `b2 reindex --cancel` (or `kill -INT {pid}`)." + ), + Some(None) => { + println!("A reindex is currently running. Stop it with `b2 reindex --cancel`.") + } + None => {} + } + } + Ok(()) +} + +/// Path to the single-in-flight advisory lock for `reindex`, under the disposable +/// index dir (`/.b2/reindex.lock`, gitignored along with the rest of `.b2/`). +fn reindex_lock_path(root: &Path) -> PathBuf { + root.join(".b2").join("reindex.lock") +} + +/// Open (creating if absent) the reindex lock file, ensuring `.b2/` exists first so +/// the lock can be taken *before* the index does on a first-ever reindex. The caller +/// takes the lock with [`File::try_lock`] and holds the returned handle for the run. +fn open_reindex_lock(root: &Path) -> Result { + std::fs::create_dir_all(root.join(".b2"))?; + Ok(OpenOptions::new() + .create(true) + // A lock file carries no contents we care about — never truncate it (that's a + // needless write, and could race a holder). Presence + the advisory lock is all. + .truncate(false) + .read(true) + .write(true) + .open(reindex_lock_path(root))?) +} + +/// Stamp this process's id into the reindex lock — **only ever called with the lock +/// held**, which is what makes truncate-then-write safe (`open_reindex_lock` deliberately +/// does not truncate, since that would race a holder). +/// +/// The pid outlives the run, harmlessly: [`reindex_holder`] reads it *only* when the lock +/// is contended, so the **lock**, never the file's contents, decides whether a run is in +/// flight. +fn record_reindex_pid(lock: &File) -> std::io::Result<()> { + let mut handle = lock; + handle.set_len(0)?; + handle.seek(SeekFrom::Start(0))?; + // A trailing newline so the file reads sanely under `cat`; parsing trims it. + writeln!(handle, "{}", std::process::id())?; + handle.flush() +} + +/// A reindex holding the lock, seen from another process. +#[derive(Debug)] +struct ReindexHolder { + /// The pid it stamped into the lock. `None` when the lock is held but no readable + /// pid is there — a holder that took the lock microseconds ago and hasn't written + /// yet, an older `b2` that stamped none, or an unreadable file. "Running" is the + /// lock's answer; the pid is the extra affordance that may be missing. + pid: Option, +} + +/// Peek at the reindex lock — the best-effort read behind `b2 status` and `b2 reindex +/// --cancel`: open the *existing* lock file and try to take it; a contended lock means +/// a run is in flight, and its pid is read from the file we could not lock. Never +/// creates the file (both callers are read-only on the vault), so a missing lock file +/// simply means no reindex has run; any I/O hiccup degrades to "not running". +fn reindex_holder(root: &Path) -> Option { + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(reindex_lock_path(root)) + .ok()?; + // If we *can* take it, nobody holds it; `file` drops here and releases at once. + if !matches!(file.try_lock(), Err(std::fs::TryLockError::WouldBlock)) { + return None; + } + let mut buf = String::new(); + let pid = file + .read_to_string(&mut buf) + .ok() + .and_then(|_| buf.trim().parse().ok()); + Some(ReindexHolder { pid }) +} + +/// `b2 reindex --cancel` (GH #55): stop a reindex on this vault by signalling the pid its +/// lock names. It adds **no cancellation machinery** — SIGINT is exactly what Ctrl-C +/// delivers, so the holder takes the shipped path to a consistent, re-runnable partial +/// index. That is the point: a run backgrounded with `b2 reindex &` has no controlling +/// terminal for Ctrl-C to reach. +/// +/// One window is inherent and accepted: a holder still loading the model hasn't installed +/// the handler yet, so SIGINT terminates it outright — the same as Ctrl-C there today, and +/// safe for the same reason (nothing is written until embedding starts). +fn cancel_reindex(root: &Path, json: bool) -> Result<(), CliError> { + let Some(holder) = reindex_holder(root) else { + return Err(CliError::NoReindexRunning); + }; + let Some(pid) = holder.pid else { + return Err(CliError::ReindexPidUnknown); + }; + signal_reindex(pid)?; + if json { + // `signalled`, not `cancelled`: the request landed; the run stops at its next + // batch boundary and reports the partial work itself (honest tense, like the + // dry-run's `would_*` keys). + print_json(&serde_json::json!({ + "signalled": true, + "pid": pid, + }))?; + } else { + println!( + "Cancelling the reindex on this vault (pid {pid}). It stops after the current batch, leaving a consistent index — re-run `b2 reindex` to finish." + ); + } + Ok(()) +} + +/// Send the cancel signal to a reindex holder. SIGINT, deliberately: the identical +/// signal a foreground Ctrl-C raises, so there is one cancel path, not two. +fn signal_reindex(pid: u32) -> Result<(), CliError> { + use nix::sys::signal::{kill, Signal}; + use nix::unistd::Pid; + // A pid that doesn't fit `pid_t` was never one we wrote — treat a garbled lock as + // nothing to cancel rather than signalling whatever the truncation would name. + let raw = i32::try_from(pid).map_err(|_| CliError::NoReindexRunning)?; + match kill(Pid::from_raw(raw), Signal::SIGINT) { + Ok(()) => Ok(()), + // The holder exited between the lock peek and the signal: there is nothing left + // to cancel — the outcome the user wanted, reported as such rather than as I/O. + Err(nix::errno::Errno::ESRCH) => Err(CliError::NoReindexRunning), + Err(e) => Err(CliError::Io(std::io::Error::from_raw_os_error(e as i32))), + } +} diff --git a/crates/b2-cli/src/wiring.rs b/crates/b2-cli/src/wiring.rs new file mode 100644 index 0000000..98dbc58 --- /dev/null +++ b/crates/b2-cli/src/wiring.rs @@ -0,0 +1,52 @@ +//! Model wiring: which embedder a vault opens with, and which chat provider answers. + +use crate::args::LlmArgs; +use crate::error::CliError; +use b2_core::embed::Embedder; +use b2_core::llm::{FakeLlm, LlmProvider}; +use b2_core::vault::Vault; +use b2_embed::{EmbedConfig, LocalEmbedder}; +use b2_llm::{LlmConfig, OpenAiCompatProvider}; +use std::path::Path; + +/// Pick + wire the chat provider — [`open_vault`]'s sibling for the second seam. +/// +/// `B2_LLM=fake` forces the deterministic [`FakeLlm`], the `B2_EMBEDDER=fake` sibling. +/// Otherwise the real client is built from the environment with this command's flags laid +/// over it, and **probed before anything else happens** — the `b2 init` posture applied to +/// chat: a stopped daemon is a sentence before the question, never a surprise after it. +pub fn open_llm(args: &LlmArgs) -> Result, CliError> { + if use_fake_llm() { + return Ok(Box::new(FakeLlm)); + } + let config = + LlmConfig::from_env().with_overrides(args.llm_url.as_deref(), args.llm_model.as_deref()); + let provider = OpenAiCompatProvider::new(config); + provider.probe()?; + Ok(Box::new(provider)) +} + +pub fn use_fake_llm() -> bool { + std::env::var_os("B2_LLM").is_some_and(|v| v == "fake") +} + +/// Open a vault with the appropriate embedder. `needs_semantic` commands (`reindex`, +/// `search`) load the real [`LocalEmbedder`] and **fail fast** with "run `b2 init`" if it +/// is absent; pure-graph commands pass `false` and use the fake, so no model is required +/// just to explore the graph. `B2_EMBEDDER=fake` forces the fake everywhere. +pub fn open_vault(root: &Path, needs_semantic: bool) -> Result { + if needs_semantic && !use_fake_embedder() { + let config = EmbedConfig::load()?; + let embedder = LocalEmbedder::load(&config)?; + Ok(Vault::open_with_embedder( + root, + Box::new(embedder) as Box, + )?) + } else { + Ok(Vault::open(root)?) + } +} + +pub fn use_fake_embedder() -> bool { + std::env::var_os("B2_EMBEDDER").is_some_and(|v| v == "fake") +} diff --git a/crates/b2-cli/src/write.rs b/crates/b2-cli/src/write.rs new file mode 100644 index 0000000..dfc4d91 --- /dev/null +++ b/crates/b2-cli/src/write.rs @@ -0,0 +1,197 @@ +//! The write commands: `add`, `write`, `mv`, `rm` and `link`. Each needs an explicit +//! vault, so none can touch the wrong directory. + +use crate::args::Cli; +use crate::error::CliError; +use crate::print_json; +use crate::wiring::open_vault; +use b2_core::resource::{doc_kind, DocKind}; +use std::io::{IsTerminal, Read}; +use std::path::Path; + +pub fn cmd_add( + cli: &Cli, + path: &str, + title: Option<&str>, + content: Option<&str>, +) -> Result<(), CliError> { + // Add writes a new note (and embeds its body) → require an explicit vault + // (no silent cwd), and it needs the real model like `reindex`/`mv`/`link`. + let vault = open_vault(cli.require_vault(None)?, true)?; + let report = vault.add_note(path, title, content)?; + if cli.json { + print_json(&report)?; + } else { + println!("Created {}.", report.path); + } + Ok(()) +} + +pub fn cmd_write(cli: &Cli, note: &str) -> Result<(), CliError> { + // A body splice + re-projection: **model-free** (like the desktop's save and + // `rm`), still an explicit vault like every write. Refuse an interactive + // terminal up front so the command never silently hangs waiting for + // hand-typed input — the new body is always *piped* (an agent, `cat file |`, …). + let stdin = std::io::stdin(); + if stdin.is_terminal() { + return Err(CliError::StdinRequired); + } + let vault = open_vault(cli.require_vault(None)?, false)?; + let mut body = String::new(); + stdin.lock().read_to_string(&mut body)?; + // Stateless one-shot: read the current on-disk revision and chain the write on + // it. A CLI holds no long-lived buffer, so there's no external-edit window to + // guard here — the content-hash guard exists for the desktop's in-memory + // buffer; for a one-shot, the contract is simply "the body becomes exactly + // this" (`Vault::write` still keeps the frontmatter bytes untouched). + let current = vault.read(note)?; + let report = vault.write(note, &body, ¤t.revision)?; + if cli.json { + print_json(&report)?; + } else { + println!("Wrote {} ({} bytes).", report.path, body.len()); + } + Ok(()) +} + +pub fn cmd_mv(cli: &Cli, from: &str, to: &str) -> Result<(), CliError> { + // A move rewrites files (and re-embeds them on re-projection) → require an + // explicit vault (no silent cwd), and it needs the real model the index was + // built with, like `reindex`/`add`/`link`. + let root = cli.require_vault(None)?; + let vault = open_vault(root, true)?; + // Kind dispatch (§9b #8): an existing directory moves as a folder + // (every file under it, one rename); otherwise the two file arms + // differ only in the report type they print. + // The human "Moved" line differs per arm, the rewrite tally is shared. + if is_dir_arg(root, from) { + let report = vault.move_dir(from, to)?; + if cli.json { + print_json(&report)?; + } else { + println!( + "Moved {}/ → {}/ ({} note(s), {} file(s))", + report.from, report.to, report.moved_notes, report.moved_resources + ); + print_rewrite_tally(report.links_rewritten, report.rewrote.len()); + } + } else if doc_kind(from) == DocKind::Resource { + let report = vault.move_resource(from, to)?; + if cli.json { + print_json(&report)?; + } else { + println!("Moved {} → {}", report.from, report.to); + print_rewrite_tally(report.links_rewritten, report.rewrote.len()); + } + } else { + let report = vault.move_note(from, to)?; + if cli.json { + print_json(&report)?; + } else { + println!("Moved {} → {}", report.from, report.to); + print_rewrite_tally(report.links_rewritten, report.rewrote.len()); + } + } + Ok(()) +} + +pub fn cmd_rm(cli: &Cli, target: &str, recursive: bool) -> Result<(), CliError> { + // A delete removes files and index rows but never rewrites a body + // (inbound links dangle, they aren't repaired) → **model-free**, like + // the desktop's delete; still an explicit vault, like every write. + let root = cli.require_vault(None)?; + let vault = open_vault(root, false)?; + // Kind dispatch (§9b #8), mirroring `mv`: an existing directory deletes + // as a folder — gated on --recursive, the CLI's stand-in for the + // desktop's confirm dialog — else the extension picks the file arm. + if is_dir_arg(root, target) { + if !recursive { + return Err(CliError::RecursiveRequired(target.to_string())); + } + let report = vault.delete_dir(target)?; + if cli.json { + print_json(&report)?; + } else { + println!( + "Deleted {}/ ({} note(s), {} file(s))", + report.dir, report.deleted_notes, report.deleted_resources + ); + print_dangled(&report.dangled); + } + } else if doc_kind(target) == DocKind::Resource { + let report = vault.delete_resource(target)?; + if cli.json { + print_json(&report)?; + } else { + println!("Deleted {}", report.path); + print_dangled(&report.dangled); + } + } else { + let report = vault.delete_note(target)?; + if cli.json { + print_json(&report)?; + } else { + println!("Deleted {}", report.path); + print_dangled(&report.dangled); + } + } + Ok(()) +} + +pub fn cmd_link( + cli: &Cli, + src: &str, + dst: &str, + edge_type: &str, + explanation: Option<&str>, +) -> Result<(), CliError> { + // Link writes the source note's frontmatter and re-projects it → require an + // explicit vault (no silent cwd), opening with the same real model the index + // was built with (like `add`/`mv`); a frontmatter-only edit won't re-embed. + let vault = open_vault(cli.require_vault(None)?, true)?; + let report = vault.link(src, dst, edge_type, explanation)?; + if cli.json { + print_json(&report)?; + } else if report.created { + println!( + "Linked {} —{}→ {}. Wrote the relation into the source note's frontmatter.", + report.src_path, report.relation, report.dst_path + ); + } else { + println!( + "Already linked {} —{}→ {}. Nothing changed.", + report.src_path, report.relation, report.dst_path + ); + } + Ok(()) +} + +/// Whether a `mv`/`rm` argument names an existing directory under `root` — the +/// kind-dispatch test that routes to the folder arm (a trailing `/` is tolerated). +fn is_dir_arg(root: &Path, arg: &str) -> bool { + root.join(arg.trim_end_matches('/')).is_dir() +} + +/// The `mv` tally, shared by its three arms: how many inbound link targets were +/// rewritten across how many files — or that nothing linked to the moved item. +fn print_rewrite_tally(links_rewritten: usize, rewrote_files: usize) { + if links_rewritten > 0 { + println!("Rewrote {links_rewritten} inbound link(s) across {rewrote_files} file(s)."); + } else { + println!("No inbound links to rewrite."); + } +} + +/// The `rm` tally, shared by its three arms: which surviving files' links now +/// dangle — or that none were affected. +fn print_dangled(dangled: &[String]) { + if dangled.is_empty() { + println!("No inbound links affected."); + } else { + println!( + "Links in {} file(s) now unresolved: {}", + dangled.len(), + dangled.join(", ") + ); + } +} From d7a57850fa4c919f1d54e8c4d4ce17a4d3055942 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:38:28 +0000 Subject: [PATCH 10/30] refactor(examples): share the eval harness's helpers in examples/common MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four harness examples each carried their own copy of the same plumbing. They now share one module, `crates/b2-embed/examples/common/mod.rs` (Cargo discovers only `examples/*.rs` and `examples/*/main.rs`, so it is a module, not an example). `groundedness` reuses it from b2-llm by `#[path]`; the module imports nothing b2-llm's dev-dependencies don't also carry. Shared: `truncate` (the saturating form, so `max == 0` no longer underflows in eval/calibrate/stability), `git_short_sha`, `append_result`, `cosine_of`, `pile_stats`, the strength-band landmarks with one `Band` classification, the harness's z restatement (`passage_z`; calibrate applies its replayed MIN_POPULATION floor first, as before), the flat-corpus throwaway vault (`ScratchVault`), the title-as-query probe, and stability's `reject_unknown_flags`/`has_flag`. The flag check also refuses a value on a switch (`--verbose=1` passed the name check and then matched no `has_flag`, silently running a different measurement — the case the check exists for). `calibrate --search` now reads term coverage off the engine's `QueryTermView::idf` instead of re-deriving the IDF by hand (same formula, same inputs). The deliberate restatements stay restatements: eval's labelled bake-off still recomputes coverage itself as the drift check, and the z recheck is still the harness's own arithmetic, never an engine call. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-embed/examples/calibrate.rs | 85 ++------- crates/b2-embed/examples/common/mod.rs | 245 +++++++++++++++++++++++++ crates/b2-embed/examples/eval.rs | 164 +++-------------- crates/b2-embed/examples/stability.rs | 53 +----- crates/b2-llm/examples/groundedness.rs | 55 +----- 5 files changed, 299 insertions(+), 303 deletions(-) create mode 100644 crates/b2-embed/examples/common/mod.rs diff --git a/crates/b2-embed/examples/calibrate.rs b/crates/b2-embed/examples/calibrate.rs index c019f8c..26b2237 100644 --- a/crates/b2-embed/examples/calibrate.rs +++ b/crates/b2-embed/examples/calibrate.rs @@ -39,8 +39,11 @@ //! reproduces GH #196's finding on its reporting vault: 16 of 17 anchors dark. It is a //! **simulation** — the shipped surface gates nothing. +mod common; + use b2_core::vault::{SimilarView, Vault}; use b2_embed::{EmbedConfig, LocalEmbedder}; +use common::{cosine_of, passage_z, pile_stats, term_coverage, title_query, truncate, Band}; use rusqlite::Connection; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; @@ -50,15 +53,10 @@ use std::path::PathBuf; /// prefix; anything a personal vault holds fits far under this. const SCAN_LIMIT: usize = 100_000; /// The replayed gate is inert under this population — `discover.rs`'s statistics -/// guard, restated here because the replay must match the rule it prices. +/// guard, restated here because the replay must match the rule it prices. (The +/// strength bands the pane paints are [`common::BAND_STRONG_Z`]'s, shared with +/// `make eval`'s calibration block, where their values are re-measured.) const MIN_POPULATION: usize = 12; -/// The strength-band landmarks the desktop paints (`ui/src/strength.ts`, GH #182): -/// `●●●` at or above the labelled-mate population's upper quartile, `●●○` at or -/// above the retired leader bar. Restated constants, not imports — the bands are -/// UI copy, and `make eval`'s calibration block is where their values are -/// re-measured. -const BAND_STRONG_Z: f64 = 2.52; -const BAND_CLEAR_Z: f64 = 1.96; /// The z existence gate being replayed — GH #197 retired it from the engine; the /// instrument keeps it (and any variant the flags name) priceable. @@ -81,14 +79,10 @@ struct AnchorReading { impl AnchorReading { fn from_candidates(path: &str, cands: &[SimilarView]) -> Self { let d2: Vec = cands.iter().map(|c| c.score * c.score).collect(); + // The harness's own z (never the engine's), gated at the replayed rule's + // population floor before the shared restatement's own inertness guard. let z = (d2.len() >= MIN_POPULATION) - .then(|| { - let n = d2.len() as f64; - let mean = d2.iter().sum::() / n; - let var = d2.iter().map(|d| (d - mean).powi(2)).sum::() / (n - 1.0); - let sd = var.sqrt(); - (sd > 0.0).then(|| d2.iter().map(|d| (mean - d) / sd).collect::>()) - }) + .then(|| passage_z(&d2)) .flatten(); Self { path: path.to_string(), @@ -130,12 +124,10 @@ impl AnchorReading { let z = self.z.as_ref()?; let (mut strong, mut clear, mut near) = (0, 0, 0); for &v in z.iter().take(limit) { - if v >= BAND_STRONG_Z { - strong += 1; - } else if v >= BAND_CLEAR_Z { - clear += 1; - } else { - near += 1; + match Band::of(v) { + Band::Strong => strong += 1, + Band::Clear => clear += 1, + Band::Near => near += 1, } } Some((strong, clear, near)) @@ -167,26 +159,6 @@ impl AnchorReading { } } -/// `similar` scores are negated L2 over unit vectors: `cos = 1 − d²/2`. -fn cosine_of(score: f64) -> f64 { - 1.0 - (score * score) / 2.0 -} - -fn pile_stats(pile: &[f64]) -> Option<(f64, f64, f64)> { - if pile.is_empty() { - return None; - } - let mut sorted = pile.to_vec(); - sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let mid = sorted.len() / 2; - let median = if sorted.len().is_multiple_of(2) { - (sorted[mid - 1] + sorted[mid]) / 2.0 - } else { - sorted[mid] - }; - Some((sorted[0], median, sorted[sorted.len() - 1])) -} - /// **Candidate 2** of GH #200's bake-off, replayed: an *authored-edge reference bar*. It /// calibrates "what related looks like **in this vault**" from the one labelled population /// every real vault carries — the score distribution of the human's committed edges — and @@ -572,19 +544,11 @@ fn read_search_transfer( own_path: Option| -> Result> { let view = vault.search_evidence(query, limit)?; - let idf = |df: usize| ((view.chunk_total as f64 + 1.0) / (df as f64 + 1.0)).ln(); - let total: f64 = view.terms.iter().map(|t| idf(t.df)).sum(); - let coverage = (total > f64::EPSILON).then(|| { - view.terms - .iter() - .filter(|t| t.df >= 1) - .map(|t| idf(t.df)) - .fold(0.0, |a, b| a + b) - / total - }); Ok(SearchProbe { query: query.to_string(), - coverage, + // The engine's own term weights — `make eval`'s labelled bake-off is + // where the formula is independently restated and drift-checked. + coverage: term_coverage(&view), best_cos: view.best_cos, // The engine's own verdict, not a restatement of it: this bench // prices what would actually ship. @@ -607,13 +571,7 @@ fn read_search_transfer( if titles.len() == MAX_TITLE_QUERIES { break; } - let title = note.title.clone().unwrap_or_else(|| { - std::path::Path::new(¬e.path) - .file_stem() - .map(|s| s.to_string_lossy().replace(['-', '_'], " ")) - .unwrap_or_default() - }); - if !title.trim().is_empty() { + if let Some(title) = title_query(¬e) { titles.push((title, note.path)); } } @@ -1341,12 +1299,3 @@ fn search_json(reading: &SearchReading) -> serde_json::Value { fn r4(x: f64) -> f64 { (x * 1e4).round() / 1e4 } - -fn truncate(s: &str, max: usize) -> String { - if s.chars().count() <= max { - s.to_string() - } else { - let cut: String = s.chars().take(max - 1).collect(); - format!("{cut}…") - } -} diff --git a/crates/b2-embed/examples/common/mod.rs b/crates/b2-embed/examples/common/mod.rs new file mode 100644 index 0000000..6743c10 --- /dev/null +++ b/crates/b2-embed/examples/common/mod.rs @@ -0,0 +1,245 @@ +//! Helpers the eval-harness examples share (`eval`, `calibrate`, `stability`, and +//! `b2-llm`'s `groundedness`, which pulls this file in by `#[path]`). +//! +//! Cargo discovers examples as `examples/*.rs` and `examples/*/main.rs`, so this +//! directory is a module, never an example of its own. Every example that says +//! `mod common;` compiles the whole file and uses a different subset of it, hence +//! the `dead_code` allowance — the same posture as `b2-core`'s `tests/common`. +//! +//! What lives here is plumbing and the harness's **own** restatements of engine +//! maths. The restatements stay restatements on purpose: they exist so an +//! instrument can cross-check the engine, and a call into the engine would turn +//! each check into a comparison of a number with itself. Nothing here may import +//! beyond what `b2-llm`'s dev-dependencies also carry (b2-core, b2-embed, +//! serde_json, tempfile). +#![allow(dead_code)] + +use b2_core::vault::{NoteSummary, SearchEvidenceView}; +use std::error::Error; +use std::path::{Path, PathBuf}; + +/// The strength-band landmarks the desktop paints (`ui/src/strength.ts`, GH #182): +/// `●●●` at or above the labelled-mate population's upper quartile, `●●○` at or +/// above the retired leader bar. Restated constants, not imports — the bands are +/// UI copy, and `make eval`'s calibration block is the instrument their values are +/// re-measured by (and what a loner anchor's always-served cards *claim*, GH #197's +/// A2 readout). +pub const BAND_STRONG_Z: f64 = 2.52; +pub const BAND_CLEAR_Z: f64 = 1.96; + +/// The strength band a z paints — see [`BAND_STRONG_Z`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Band { + Strong, + Clear, + Near, +} + +impl Band { + /// The band `z` falls in. A non-finite z compares false everywhere and reads + /// as [`Band::Near`], the band that claims least. + pub fn of(z: f64) -> Self { + if z >= BAND_STRONG_Z { + Band::Strong + } else if z >= BAND_CLEAR_Z { + Band::Clear + } else { + Band::Near + } + } + + pub fn glyph(self) -> &'static str { + match self { + Band::Strong => "●●●", + Band::Clear => "●●○", + Band::Near => "●○○", + } + } +} + +/// Z-score a population of squared distances, oriented nearer = higher — the +/// harness's own restatement of the arithmetic `discover::candidates` applies to +/// the stage-2 best-pair distances (GH #192), kept so the engine's z can be +/// cross-checked rather than merely trusted. `None` when no meaningful statistic +/// exists (under two values, or zero variance), mirroring the engine's own +/// inertness guard; a caller replaying a larger minimum population applies it +/// before calling. +pub fn passage_z(d2: &[f64]) -> Option> { + if d2.len() < 2 { + return None; + } + let n = d2.len() as f64; + let mean = d2.iter().sum::() / n; + let var = d2.iter().map(|d| (d - mean).powi(2)).sum::() / (n - 1.0); + let sd = var.sqrt(); + (sd > 0.0).then(|| d2.iter().map(|d| (mean - d) / sd).collect()) +} + +/// A `similar` score is negated L2 distance between L2-normalized vectors (the +/// real embedder normalizes every row), so it converts exactly: `cos = 1 − d²/2`. +/// Cosine is the unit the floor ruling is stated in and the unit that survives a +/// model swap comparison, so the piles are recorded in it. +pub fn cosine_of(score: f64) -> f64 { + 1.0 - (score * score) / 2.0 +} + +/// (min, median, max) of a pile, or `None` while it's empty. +pub fn pile_stats(pile: &[f64]) -> Option<(f64, f64, f64)> { + if pile.is_empty() { + return None; + } + let mut sorted = pile.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let mid = sorted.len() / 2; + let median = if sorted.len().is_multiple_of(2) { + (sorted[mid - 1] + sorted[mid]) / 2.0 + } else { + sorted[mid] + }; + Some((sorted[0], median, sorted[sorted.len() - 1])) +} + +/// A note's title as a search query: its frontmatter title, else its file slug with +/// `-`/`_` read as spaces (`drone-comb` → "drone comb"). `None` when that is blank. +/// +/// Titles are the label-free positives both search benches share (GH #201): a +/// note's own title names material the vault demonstrably holds, so nothing read +/// off one can be relabelled to clear it. +pub fn title_query(note: &NoteSummary) -> Option { + let title = note.title.clone().unwrap_or_else(|| { + Path::new(¬e.path) + .file_stem() + .map(|s| s.to_string_lossy().replace(['-', '_'], " ")) + .unwrap_or_default() + }); + (!title.trim().is_empty()).then_some(title) +} + +/// The share of a query's term IDF the vault carries, read off the view's own +/// per-term weights ([`b2_core::vault::QueryTermView::idf`]) rather than a second +/// copy of the formula. `None` when no term carries any weight (the lexical half +/// abstaining, not scoring zero). +/// +/// `make eval`'s labelled bake-off keeps its own restatement of this arithmetic +/// on purpose, as a drift check against the engine; one such check is the check, +/// so every other bench reads the engine's weights through here. +pub fn term_coverage(view: &SearchEvidenceView) -> Option { + let total: f64 = view.terms.iter().map(|t| t.idf).sum(); + (total > f64::EPSILON).then(|| { + view.terms + .iter() + .filter(|t| t.df >= 1) + .map(|t| t.idf) + .fold(0.0, |a, b| a + b) + / total + }) +} + +/// A throwaway vault holding a copy of one **flat** corpus directory — the eval +/// corpora are single-level, so only regular files are copied: `fs::copy` errors +/// on a directory, and a future subfolder (or any stray non-file) must not abort +/// the run. The temp dir lives exactly as long as this value, so declare the +/// `Vault` opened on [`Self::root`] after it (locals drop in reverse order). +pub struct ScratchVault { + _tmp: tempfile::TempDir, + root: PathBuf, +} + +impl ScratchVault { + pub fn copy_flat(corpus_dir: &Path) -> std::io::Result { + let tmp = tempfile::TempDir::new()?; + let root = tmp.path().join("vault"); + std::fs::create_dir_all(&root)?; + for entry in std::fs::read_dir(corpus_dir)? { + let entry = entry?; + if entry.file_type()?.is_file() { + std::fs::copy(entry.path(), root.join(entry.file_name()))?; + } + } + Ok(Self { _tmp: tmp, root }) + } + + pub fn root(&self) -> &Path { + &self.root + } +} + +/// Whether `name` was passed as a bare flag. +pub fn has_flag(args: &[String], name: &str) -> bool { + args.iter().any(|a| a == name) +} + +/// A typo'd flag must not run a *different* measurement than the one asked for. +/// +/// `known` lists every accepted flag; `valued` the ones among them that take a +/// value as the next argument (`--vault path`; the `--vault=path` form needs no +/// entry). Positional arguments are refused too — every flag-driven example here +/// takes none. +pub fn reject_unknown_flags( + args: &[String], + known: &[&str], + valued: &[&str], +) -> Result<(), Box> { + let mut skip_next = false; + for arg in args { + if skip_next { + skip_next = false; + continue; + } + if !arg.starts_with("--") { + return Err(format!("unexpected argument {arg:?}").into()); + } + let (name, inline_value) = match arg.split_once('=') { + Some((name, _)) => (name, true), + None => (arg.as_str(), false), + }; + if !known.contains(&name) { + return Err(format!("unknown flag {arg:?}; known: {}", known.join(" ")).into()); + } + // `--sweep=1` would pass the name check and then match no `has_flag` — + // the same silent wrong measurement as a typo, so a switch takes no value. + if inline_value && !valued.contains(&name) { + return Err(format!("flag {name} takes no value (got {arg:?})").into()); + } + skip_next = !inline_value && valued.contains(&name); + } + Ok(()) +} + +/// Append one row to a results log (creating it on first run). Append-only, so +/// runs accumulate into one dataset — the same convention as `B2_LOG_FILE`. +pub fn append_result(path: &Path, row: &serde_json::Value) -> Result<(), Box> { + use std::io::Write; + let mut f = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + writeln!(f, "{row}")?; + Ok(()) +} + +/// The repo's short commit hash, best-effort (`None` outside a git checkout). +pub fn git_short_sha() -> Option { + let out = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .ok()?; + if !out.status.success() { + return None; + } + Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) +} + +/// `s` cut to at most `max` chars, an ellipsis marking the cut. Counted in chars +/// and cut on a char boundary (byte-index arithmetic lands mid-codepoint on text +/// with em dashes or °C), and saturating, so `max == 0` yields a bare ellipsis +/// rather than an underflow panic. +pub fn truncate(s: &str, max: usize) -> String { + if s.chars().count() <= max { + s.to_string() + } else { + let cut: String = s.chars().take(max.saturating_sub(1)).collect(); + format!("{cut}…") + } +} diff --git a/crates/b2-embed/examples/eval.rs b/crates/b2-embed/examples/eval.rs index b9a15c3..8691b9c 100644 --- a/crates/b2-embed/examples/eval.rs +++ b/crates/b2-embed/examples/eval.rs @@ -58,12 +58,18 @@ // costs the thing the row is for: a reader seeing every recorded field at once. #![recursion_limit = "256"] +mod common; + use b2_core::chunk::ChunkConfig; use b2_core::db::FtsTokenizer; use b2_core::embed::Embedder; use b2_core::search::EvidenceBar; use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, Vault}; use b2_embed::{provision, EmbedConfig, LocalEmbedder}; +use common::{ + append_result, cosine_of, git_short_sha, passage_z, pile_stats, term_coverage, title_query, + truncate, Band, ScratchVault, +}; use serde::Deserialize; use std::collections::HashMap; use std::ops::ControlFlow; @@ -90,13 +96,6 @@ const FOLD_MUTUAL_K: [usize; 3] = [3, 5, 10]; /// as "mutually near", which is the rule going vacuous rather than a depth /// anyone would ship. const FOLD_K_SWEEP: usize = 15; -/// The strength-band landmarks the desktop paints (`ui/src/strength.ts`, -/// GH #182), restated for the negatives' band readout (GH #197's A2): what a -/// loner anchor's always-served cards *claim*. Restated rather than imported — -/// the bands are UI copy, and this block is the instrument their values are -/// re-measured by. -const BAND_STRONG_Z: f64 = 2.52; -const BAND_CLEAR_Z: f64 = 1.96; /// The soft reference floor on the default config's hybrid note hit@1. /// Untouched by the GH #197 re-derivation: retrieval never had a gate to /// retire. @@ -548,18 +547,9 @@ fn run() -> Result> { check_batch_matches_single(&embedder)?; // Build a throwaway vault from the corpus. - let tmp = tempfile::TempDir::new()?; - let vault_root = tmp.path().join("vault"); - std::fs::create_dir_all(&vault_root)?; - for entry in std::fs::read_dir(&corpus_dir)? { - let entry = entry?; - // Regular files only: `fs::copy` errors on a directory, so a future - // corpus/ subfolder (or any stray non-file) must not abort the run. - if entry.file_type()?.is_file() { - std::fs::copy(entry.path(), vault_root.join(entry.file_name()))?; - } - } - let mut vault = Vault::open_with_embedder(&vault_root, Box::new(embedder))?; + let scratch = ScratchVault::copy_flat(&corpus_dir)?; + let vault_root = scratch.root(); + let mut vault = Vault::open_with_embedder(vault_root, Box::new(embedder))?; // ---- Phase 1: projection only → the BM25-only baseline. ------------------ // The vector space does not exist yet, so `search`/`search_chunks` run @@ -599,7 +589,7 @@ fn run() -> Result> { let floor_z = score_floor_z(&vault, &sim_set)?; // The search evidence dump (invariants.md D2, GH #201) — the query-side // sibling of the z calibration, read over the same built vault. - let evidence = score_search_evidence(&vault_root, &vault, &positives, &negatives)?; + let evidence = score_search_evidence(vault_root, &vault, &positives, &negatives)?; eprintln!( "[eval] embedded {chunks} chunks in {embed_secs:.1}s ({} candidates per signal at K={K}, \ {} for the passage view)\n", @@ -620,7 +610,7 @@ fn run() -> Result> { let git = git_short_sha(); append_result( &results_path, - result_row( + &result_row( &git, &model_id, dim, @@ -649,7 +639,7 @@ fn run() -> Result> { let dense = score_dense(&evals_dir, &dense_set)?; print_dense_report(&dense); print_fold_bench(&dense.fold); - append_result(&results_path, dense_row(&git, &model_id, dim, &dense))?; + append_result(&results_path, &dense_row(&git, &model_id, dim, &dense))?; // The tail bake-off's cross-bench join (GH #206) — printable only here, // where both corpora's readings exist in one run. print_tail_join(&evidence, &tail, &dense.search.titles); @@ -701,7 +691,7 @@ fn run() -> Result> { ); append_result( &results_path, - result_row( + &result_row( &git, &model_id, dim, @@ -829,7 +819,7 @@ fn run() -> Result> { print_rank_moves(&positives, &hybrid, &pass); append_result( &results_path, - result_row( + &result_row( &git, &model_id, dim, @@ -1131,16 +1121,8 @@ fn score_dense( let config = EmbedConfig::load()?; let embedder = LocalEmbedder::load(&config)?; let model_id = embedder.model_id().to_string(); - let tmp = tempfile::TempDir::new()?; - let vault_root = tmp.path().join("vault"); - std::fs::create_dir_all(&vault_root)?; - for entry in std::fs::read_dir(&corpus_dir)? { - let entry = entry?; - if entry.file_type()?.is_file() { - std::fs::copy(entry.path(), vault_root.join(entry.file_name()))?; - } - } - let vault = Vault::open_with_embedder(&vault_root, Box::new(embedder))?; + let scratch = ScratchVault::copy_flat(&corpus_dir)?; + let vault = Vault::open_with_embedder(scratch.root(), Box::new(embedder))?; vault.project(false)?; let (chunks, embed_secs) = timed_embed(&vault)?; @@ -1273,26 +1255,19 @@ const DENSE_NONSENSE: [&str; 2] = ["shjfasd", "vrelqip zonktar wembleforth"]; /// Replay the shipped bar over the dense fixture (see [`DenseSearch`]). /// /// Coverage is read off [`b2_core::vault::QueryTermView::idf`] — the view's own -/// weights, not a second copy of the formula. The orthogonal corpus's bake-off -/// re-derives its arithmetic deliberately, as a drift check against the engine; -/// one such check is the check, and a second would only be two places to fix. +/// weights, not a second copy of the formula ([`term_coverage`]). The orthogonal +/// corpus's bake-off re-derives its arithmetic deliberately, as a drift check +/// against the engine; one such check is the check, and a second would only be two +/// places to fix. fn score_dense_search( vault: &Vault, model_id: &str, ) -> Result> { let read = |query: &str, keep: bool| -> Result> { let view = vault.search_evidence(query, K)?; - let total: f64 = view.terms.iter().map(|t| t.idf).sum(); Ok(SearchProbe { query: query.to_string(), - coverage: (total > f64::EPSILON).then(|| { - view.terms - .iter() - .filter(|t| t.df >= 1) - .map(|t| t.idf) - .fold(0.0, |a, b| a + b) - / total - }), + coverage: term_coverage(&view), best_cos: view.best_cos, vouched: view.vouched, rows: view @@ -1312,13 +1287,7 @@ fn score_dense_search( // The fixture's notes carry no frontmatter title, so the slug is the // query — `drone-comb` → "drone comb", which is the pair of words the // retired ceiling called stopwords. - let title = note.title.clone().unwrap_or_else(|| { - std::path::Path::new(¬e.path) - .file_stem() - .map(|s| s.to_string_lossy().replace(['-', '_'], " ")) - .unwrap_or_default() - }); - if !title.trim().is_empty() { + if let Some(title) = title_query(¬e) { titles.push(read(&title, true)?); } } @@ -3729,47 +3698,6 @@ fn print_tail_join(ev: &SearchEvidence, orth: &TailBench, titles: &[SearchProbe] } } -/// A `similar` score is negated L2 distance between L2-normalized vectors -/// (the real embedder normalizes every row), so it converts exactly: -/// `cos = 1 − d²/2`. Cosine is the unit the floor ruling is stated in and the -/// unit that survives a model swap comparison, so the piles are recorded in it. -fn cosine_of(score: f64) -> f64 { - 1.0 - (score * score) / 2.0 -} - -/// Z-score a population of squared distances, oriented nearer = higher — the -/// harness's own restatement of the arithmetic `discover::candidates` applies -/// to the stage-2 best-pair distances (GH #192), kept so the engine's z can be -/// cross-checked rather than merely trusted. `None` when no meaningful -/// statistic exists (under two values, or zero variance), mirroring the -/// engine's own inertness guard. -fn passage_z(d2: &[f64]) -> Option> { - if d2.len() < 2 { - return None; - } - let n = d2.len() as f64; - let mean = d2.iter().sum::() / n; - let var = d2.iter().map(|d| (d - mean).powi(2)).sum::() / (n - 1.0); - let sd = var.sqrt(); - (sd > 0.0).then(|| d2.iter().map(|d| (mean - d) / sd).collect()) -} - -/// (min, median, max) of a pile, or None while it's empty. -fn pile_stats(pile: &[f64]) -> Option<(f64, f64, f64)> { - if pile.is_empty() { - return None; - } - let mut sorted = pile.to_vec(); - sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let mid = sorted.len() / 2; - let median = if sorted.len().is_multiple_of(2) { - (sorted[mid - 1] + sorted[mid]) / 2.0 - } else { - sorted[mid] - }; - Some((sorted[0], median, sorted[sorted.len() - 1])) -} - /// `LocalEmbedder::embed_batch` must be a faithful map of `embed`: right-padding short rows /// to the batch's longest and masking them out has to leave each row's CLS vector unchanged. /// The reindex path batches freely, so a regression here would silently corrupt every stored @@ -4120,7 +4048,7 @@ fn print_floor_windows(z: &FloorZ) { a.anchor, c.path, c.z, - band_glyph(c.z) + Band::of(c.z).glyph() ), None => println!(" {} (no candidates)", a.anchor), } @@ -4135,18 +4063,6 @@ fn print_floor_windows(z: &FloorZ) { } } -/// The strength band a z paints (`ui/src/strength.ts`'s landmarks, restated — -/// see [`BAND_STRONG_Z`]). -fn band_glyph(z: f64) -> &'static str { - if z >= BAND_STRONG_Z { - "●●●" - } else if z >= BAND_CLEAR_Z { - "●●○" - } else { - "●○○" - } -} - /// The paired per-query diff between two fused passes — what an A/B is actually /// judged on. At this corpus's n, every aggregate delta is worth 1–2 queries, so /// "hit@1 +0.05" and "these two queries flipped, this one broke" are the same @@ -4485,31 +4401,6 @@ fn result_row( }) } -/// Append one row to the results log (creating it on first run). Append-only, so -/// runs accumulate into one dataset — the same convention as `B2_LOG_FILE`. -fn append_result(path: &Path, row: serde_json::Value) -> Result<(), Box> { - use std::io::Write; - let mut f = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(path)?; - writeln!(f, "{row}")?; - Ok(()) -} - -/// The repo's short commit hash, best-effort (None outside a git checkout). -fn git_short_sha() -> Option { - let out = std::process::Command::new("git") - .args(["rev-parse", "--short", "HEAD"]) - .current_dir(env!("CARGO_MANIFEST_DIR")) - .output() - .ok()?; - if !out.status.success() { - return None; - } - Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) -} - /// Corpus notes are copied flat into the vault, so a result path equals (or ends /// with) the labelled relevant path. fn paths_match(result_path: &str, relevant: &str) -> bool { @@ -4523,12 +4414,3 @@ fn rank_str(rank: Option) -> String { None => format!("✗>{K}"), } } - -fn truncate(s: &str, max: usize) -> String { - if s.chars().count() <= max { - s.to_string() - } else { - let cut: String = s.chars().take(max - 1).collect(); - format!("{cut}…") - } -} diff --git a/crates/b2-embed/examples/stability.rs b/crates/b2-embed/examples/stability.rs index 4d10782..8c98b06 100644 --- a/crates/b2-embed/examples/stability.rs +++ b/crates/b2-embed/examples/stability.rs @@ -47,9 +47,12 @@ //! an unlabelled corpus cannot say whether it improved. Exit status is 0 for any completed //! measurement. +mod common; + use b2_core::embed::Embedder; use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, ChunkSearchResult, Vault}; use b2_embed::{provision, EmbedConfig, LocalEmbedder}; +use common::{git_short_sha, has_flag, reject_unknown_flags, truncate}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::ops::ControlFlow; @@ -75,6 +78,8 @@ const KEY_CHARS: usize = 48; /// Report column widths: the probe text, then one column per compared view. const PROBE_COL: usize = 44; const CELL_COL: usize = 26; +/// Every flag this probe takes; anything else is refused rather than ignored. +const KNOWN_FLAGS: [&str; 4] = ["--bless", "--model", "--verbose", "--vault"]; /// The hand-authored probe set (`evals/stability.json`) — plain queries, no labels. #[derive(Deserialize)] @@ -128,7 +133,7 @@ fn run() -> Result<(), Box> { let real_model = has_flag(&args, "--model"); let verbose = has_flag(&args, "--verbose"); let vault_arg = flag_value(&args, "--vault")?; - reject_unknown_flags(&args)?; + reject_unknown_flags(&args, &KNOWN_FLAGS, &["--vault"])?; // A baseline is a *deterministic* artifact over a *committed* vault; blessing // one from a real-model run or a private vault would commit a number nobody @@ -628,10 +633,6 @@ fn copy_dir_all(from: &Path, to: &Path) -> std::io::Result<()> { Ok(()) } -fn has_flag(args: &[String], name: &str) -> bool { - args.iter().any(|a| a == name) -} - /// `--vault ` / `--vault=`, erroring on a flag given without a value /// rather than silently probing the default vault under a name the user did not mean. fn flag_value(args: &[String], name: &str) -> Result, Box> { @@ -649,27 +650,6 @@ fn flag_value(args: &[String], name: &str) -> Result, Box Result<(), Box> { - const KNOWN: [&str; 4] = ["--bless", "--model", "--verbose", "--vault"]; - let mut skip_next = false; - for arg in args { - if skip_next { - skip_next = false; - continue; - } - if !arg.starts_with("--") { - return Err(format!("unexpected argument {arg:?}").into()); - } - let name = arg.split('=').next().unwrap_or(arg); - if !KNOWN.contains(&name) { - return Err(format!("unknown flag {arg:?}; known: {}", KNOWN.join(" ")).into()); - } - skip_next = arg == "--vault"; - } - Ok(()) -} - /// Same directory on disk, canonicalized — so `--vault fixtures/test-vault` from the /// repo root is recognized as the committed vault the baseline belongs to. fn same_dir(a: &Path, b: &Path) -> bool { @@ -704,24 +684,3 @@ fn display_from_repo(path: &Path) -> String { None => path.display().to_string(), } } - -fn git_short_sha() -> Option { - let out = std::process::Command::new("git") - .args(["rev-parse", "--short", "HEAD"]) - .current_dir(env!("CARGO_MANIFEST_DIR")) - .output() - .ok()?; - if !out.status.success() { - return None; - } - Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) -} - -fn truncate(s: &str, max: usize) -> String { - if s.chars().count() <= max { - s.to_string() - } else { - let cut: String = s.chars().take(max - 1).collect(); - format!("{cut}…") - } -} diff --git a/crates/b2-llm/examples/groundedness.rs b/crates/b2-llm/examples/groundedness.rs index 356a84d..0afe084 100644 --- a/crates/b2-llm/examples/groundedness.rs +++ b/crates/b2-llm/examples/groundedness.rs @@ -32,12 +32,18 @@ //! Model quality is read off the numbers, not the exit code: a gate that fails on every small //! local model would be a gate nobody runs. +// The retrieval harness's shared helpers, reused rather than copied: everything the +// module imports is in this crate's (dev-)dependencies too. +#[path = "../../b2-embed/examples/common/mod.rs"] +mod common; + use b2_core::chat::{cited_markers, ASK_PASSAGES}; use b2_core::embed::Embedder; use b2_core::llm::{LlmProvider, NO_EVIDENCE_ANSWER}; use b2_core::vault::Vault; use b2_embed::{provision, EmbedConfig, LocalEmbedder}; use b2_llm::{LlmConfig, OpenAiCompatProvider}; +use common::{append_result, git_short_sha, truncate, ScratchVault}; use serde::Deserialize; use std::error::Error; use std::ops::ControlFlow; @@ -128,16 +134,8 @@ fn run() -> Result> { eprintln!("[eval] embedder = {embed_model}"); // A throwaway vault from the corpus — nothing here touches a real one. - let tmp = tempfile::TempDir::new()?; - let vault_root = tmp.path().join("vault"); - std::fs::create_dir_all(&vault_root)?; - for entry in std::fs::read_dir(&corpus_dir)? { - let entry = entry?; - if entry.file_type()?.is_file() { - std::fs::copy(entry.path(), vault_root.join(entry.file_name()))?; - } - } - let vault = Vault::open_with_embedder(&vault_root, Box::new(embedder))?; + let scratch = ScratchVault::copy_flat(&corpus_dir)?; + let vault = Vault::open_with_embedder(scratch.root(), Box::new(embedder))?; let report = vault.reindex()?; eprintln!( "[eval] indexed {} notes; asking {} questions\n", @@ -341,40 +339,3 @@ fn print_summary(row: &serde_json::Value) { row["mean_total_ms"].as_f64().unwrap_or_default() ); } - -/// Append one row to the results log (creating it on first run). Append-only, so -/// runs accumulate into one dataset — the retrieval eval's convention. -fn append_result(path: &Path, row: &serde_json::Value) -> Result<(), Box> { - use std::io::Write; - let mut f = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(path)?; - writeln!(f, "{row}")?; - Ok(()) -} - -/// The repo's short commit hash, best-effort (None outside a git checkout). -fn git_short_sha() -> Option { - let out = std::process::Command::new("git") - .args(["rev-parse", "--short", "HEAD"]) - .current_dir(env!("CARGO_MANIFEST_DIR")) - .output() - .ok()?; - if !out.status.success() { - return None; - } - Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) -} - -fn truncate(s: &str, max: usize) -> String { - // Counted in chars, cut on a char boundary: the byte-index arithmetic this - // used (`&s[..i - 1]`) lands mid-codepoint — and panics — whenever the char - // before the cut is multibyte, which question text (em dashes, °C) is. - if s.chars().count() <= max { - s.to_string() - } else { - let cut: String = s.chars().take(max.saturating_sub(1)).collect(); - format!("{cut}…") - } -} From 240c8772cccdad5730b54b0962a1d1faa43393b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:38:53 +0000 Subject: [PATCH 11/30] fix(eval): refuse unknown flags instead of silently ignoring them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `eval` read its flags with `args().any(|a| a == "--sweep")`, so a typo (`--swep`, `--stemmer=1`) ran the default measurement alone and printed a report the reader took for the A/B they asked for. It now goes through the shared `reject_unknown_flags` (the check `stability` already had) and exits 1 on anything but `--sweep`/`--stemmer`. A bare `--` is still dropped, as `stability` does, so a pasted `cargo run … -- --sweep` keeps working; every invocation in the Makefile and docs/evals.md is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-embed/examples/eval.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/b2-embed/examples/eval.rs b/crates/b2-embed/examples/eval.rs index 8691b9c..f5bbcb1 100644 --- a/crates/b2-embed/examples/eval.rs +++ b/crates/b2-embed/examples/eval.rs @@ -8,6 +8,9 @@ //! cargo run -p b2-embed --example eval -- --stemmer # + FTS tokenizer A/B (the #157 gate) //! ``` //! +//! Any other argument is refused (exit 1) rather than ignored: a typo'd `--swep` would +//! otherwise score only the default config while the reader believes the A/B ran. +//! //! **`docs/evals.md` is the notebook of record** — the corpus, what the exit code //! enforces, every verdict this harness has ruled, and the process rules. Read it before //! touching the corpus, the labels, or a constant here. What this comment carries is only @@ -67,8 +70,8 @@ use b2_core::search::EvidenceBar; use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, Vault}; use b2_embed::{provision, EmbedConfig, LocalEmbedder}; use common::{ - append_result, cosine_of, git_short_sha, passage_z, pile_stats, term_coverage, title_query, - truncate, Band, ScratchVault, + append_result, cosine_of, git_short_sha, has_flag, passage_z, pile_stats, reject_unknown_flags, + term_coverage, title_query, truncate, Band, ScratchVault, }; use serde::Deserialize; use std::collections::HashMap; @@ -498,8 +501,14 @@ fn main() { /// Returns whether the default-config hybrid pass cleared the reference floor. fn run() -> Result> { - let sweep = std::env::args().any(|a| a == "--sweep"); - let stemmer = std::env::args().any(|a| a == "--stemmer"); + // A bare `--` is dropped rather than refused (`make stability`'s posture: a pasted + // `cargo run … -- --sweep` must not fail on its own separator); anything else + // unrecognised is refused, since a typo'd flag would otherwise run the default + // measurement while the reader believes an A/B ran. + let args: Vec = std::env::args().skip(1).filter(|a| a != "--").collect(); + reject_unknown_flags(&args, &["--sweep", "--stemmer"], &[])?; + let sweep = has_flag(&args, "--sweep"); + let stemmer = has_flag(&args, "--stemmer"); let evals_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("evals"); let corpus_dir = evals_dir.join("corpus"); let results_path = evals_dir.join("results.jsonl"); From d4b1960ec641c760b52f6663ded20bf73f06473f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:44:45 +0000 Subject: [PATCH 12/30] refactor(ui): split render.ts by surface, and share its repeated markup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit render.ts (2,687 lines) keeps the tree, note pane, discovery column, menus and dialogs, and re-exports everything the app and the suite imported from it. The rest moves to modules named for their surface: - markdown.ts: renderMarkdown and the marked config — the one sanitizing Markdown path, now importable without the panes (livepreview.ts does). - chatview.ts, graphview.ts, explainview.ts, keysview.ts, settingsview.ts. - widgets.ts: markup more than one surface paints — sideTab, strengthHtml, segmentedHtml (theme and chat-mode controls), reindexMeterHtml (the top bar's meter and Settings → Index's). Also keysGroupsHtml (the reference and the ⌘ sheet), modelDetail in chat.ts (the setup card and the model picker), and tabDomId exported from settingstabs.ts for main.ts's two hand-built settings-tab ids. The module graph is acyclic. Every exported surface was snapshotted across a state matrix before and after the move (755 cases, identical). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/chat.test.ts | 8 + ui/src/chat.ts | 7 + ui/src/chatview.ts | 298 +++++++ ui/src/explainview.ts | 186 +++++ ui/src/graphview.ts | 306 +++++++ ui/src/keysview.ts | 188 +++++ ui/src/livepreview.ts | 2 +- ui/src/main.ts | 20 +- ui/src/markdown.ts | 136 ++++ ui/src/render.ts | 1717 +--------------------------------------- ui/src/settingstabs.ts | 6 + ui/src/settingsview.ts | 567 +++++++++++++ ui/src/widgets.ts | 69 ++ 13 files changed, 1814 insertions(+), 1696 deletions(-) create mode 100644 ui/src/chatview.ts create mode 100644 ui/src/explainview.ts create mode 100644 ui/src/graphview.ts create mode 100644 ui/src/keysview.ts create mode 100644 ui/src/markdown.ts create mode 100644 ui/src/settingsview.ts create mode 100644 ui/src/widgets.ts diff --git a/ui/src/chat.test.ts b/ui/src/chat.test.ts index 07f3d95..39ef560 100644 --- a/ui/src/chat.test.ts +++ b/ui/src/chat.test.ts @@ -12,6 +12,7 @@ import { citationRowKey, errorMessage, formatModelSize, + modelDetail, pullCommand, retrievalNote, toolCapInput, @@ -260,3 +261,10 @@ test("a model's size reads as an inventory line", () => { assert.equal(formatModelSize(500_000_000), "477 MB"); assert.equal(formatModelSize(0), ""); }); + +test("an installed model's detail is its parameters and size, whichever it has", () => { + assert.equal(modelDetail({ name: "m", size: 2_019_393_189, parameters: "3.2B" }), "3.2B · 1.9 GB"); + assert.equal(modelDetail({ name: "m", size: 2_019_393_189, parameters: null }), "1.9 GB"); + assert.equal(modelDetail({ name: "m", size: 0, parameters: "3.2B" }), "3.2B"); + assert.equal(modelDetail({ name: "m", size: 0, parameters: null }), ""); +}); diff --git a/ui/src/chat.ts b/ui/src/chat.ts index f70cd5e..4d1f5a8 100644 --- a/ui/src/chat.ts +++ b/ui/src/chat.ts @@ -36,6 +36,7 @@ import type { ChatSetup, ChatTurn, Citation, + OllamaModel, ToolCallCap, ToolUse, } from "./types"; @@ -331,6 +332,12 @@ export const OLLAMA_QUICKSTART_URL = "https://docs.ollama.com/quickstart"; * provider is the explicit act M5 is about, so this is a link, never a pre-filled URL. */ export const OLLAMA_CLOUD_URL = "https://docs.ollama.com/cloud"; +/** An installed model's one-line detail — its parameter label and size, whichever it + * has ("3.2B · 2.0 GB") — as both the setup card and the Settings picker print it. */ +export function modelDetail(m: OllamaModel): string { + return [m.parameters ?? "", formatModelSize(m.size)].filter(Boolean).join(" · "); +} + /** A model's on-disk size, for the installed list. Whole GB past a gigabyte, one decimal * below — an inventory line, not a measurement. */ export function formatModelSize(bytes: number): string { diff --git a/ui/src/chatview.ts b/ui/src/chatview.ts new file mode 100644 index 0000000..38a6ffc --- /dev/null +++ b/ui/src/chatview.ts @@ -0,0 +1,298 @@ +// The chat pane's paint (flow ④) and the setup card it shares with Settings → Chat. The +// pure logic is chat.ts; the streaming, cancellation and focus discipline are main.ts's. + +import { escapeHtml } from "./escape.ts"; +import type { AppState } from "./state.ts"; +import { displayKeys } from "./bindings.ts"; +import { + type ChatMessage, + OLLAMA_QUICKSTART_URL, + STREAMING_ROW_KEY, + chatReady, + chatStateOf, + citationRowKey, + modelDetail, + pullCommand, + retrievalNote, + toolsLine, + turnRowKey, +} from "./chat.ts"; +import type { ChatSetup } from "./types.ts"; +import { renderMarkdown } from "./markdown.ts"; +import { sideTab } from "./widgets.ts"; + +// --- chat (flow ④, GH #151/#153/#155) ----------------------------------------------- +// +// The right column's third mode: ask a question, watch the answer stream, click a +// citation to open the note behind it — *without* the conversation leaving the screen, +// which is why chat lives here rather than in the centre pane (chat.ts's header). +// +// Three rules this builder holds, all of them invariants rather than styling: +// +// • **E5 — model output is untrusted content.** An answer is a string from a model that +// was itself fed note content (which anyone can author), so it renders through the one +// sanitizing `renderMarkdown` seam like every other document. The *streaming* half is +// stronger still: it is written as `textContent` by `paintChatStream` (main.ts), so a +// half-arrived answer is never parsed as markup at all. +// • **A citation navigates in-app, never the webview.** Each one is a `data-open` button +// — the same delegation search results and discovery cards use — so the mouse and ⏎ +// share one activation path (K1), and no `href` ever exists to be followed. +// • **K1 — the pane is keyboard-complete.** Rows are `role="treeitem"` over chat.ts's +// row order (the same order the arrows walk), with a roving tabstop; every control has +// a stable `id` so `paintSide` can hand focus back across the repaint each token causes. +export function chatPaneHtml(state: AppState, roving: string | null): string { + const setup = state.chatSetup; + const streaming = state.chatStreaming !== null; + const model = setup + ? `${escapeHtml(setup.model)}` + : ""; + const head = `
+

Chat

+ ${model} + +
`; + // No composer until there is something to answer with: a disabled field under a card + // that says why is chrome with nothing behind it, and one more stop for a keyboard user + // to walk past on the way to the fix the card is pointing at. + return head + chatStageHtml(state, roving) + (chatReady(state) ? chatComposerHtml(state) : ""); +} + +/** The conversation, or the state that stands in for one (chat.ts's `chatEmptyState` + * makes that choice once, so the paint doesn't re-derive it branch by branch). */ +function chatStageHtml(state: AppState, roving: string | null): string { + switch (chatStateOf(state)) { + case "no-vault": + return `

Open a vault to chat with your notes.

`; + case "loading": + return `

Looking for a model…

`; + case "no-server": + case "no-model": + return chatSetupCardHtml(state.chatSetup, false); + case "ready": + return chatLogHtml(state, roving); + } +} + +/** The transcript. Empty until the first question — and then it is what the pane is. */ +function chatLogHtml(state: AppState, roving: string | null): string { + if (state.chatMessages.length === 0 && state.chatStreaming === null) { + const fake = state.chatSetup?.state === "fake"; + return `
+
+

Ask your notes a question.

+

Answers come only from passages B2 retrieves from this vault, cited by + [n]. Nothing here is written to your notes, and the conversation isn’t saved.

+ ${ + fake + ? `

The fake chat provider is in use (B2_LLM=fake) — answers are deterministic test scaffolding, not a model.

` + : "" + } +
+
`; + } + const turns = state.chatMessages + .map((m, i) => chatTurnHtml(m, i, roving)) + .join(""); + // The in-flight answer is a row of its own so the keyboard can sit on it while it + // fills. `#chat-stream` is the element `paintChatStream` writes tokens into — as text, + // never markup — which is what keeps a streaming answer off the full-render path. + const live = + state.chatStreaming === null + ? "" + : `
+
B2
+
${escapeHtml(state.chatStreaming)}
+
`; + return `
${turns}${live}
`; +} + +/** One turn: the question as typed, or the answer — rendered through the sanitizing + * Markdown seam (E5) — with its citations as rows beneath it. */ +function chatTurnHtml(m: ChatMessage, index: number, roving: string | null): string { + const key = turnRowKey(index); + const row = (cls: string, role: string, body: string): string => + `
+
${role}
+ ${body} +
`; + if (m.role === "user") { + return row("chat-question", "You", `
${escapeHtml(m.text)}
`); + } + if (m.error !== undefined) { + return row( + "chat-answer chat-failed", + "B2", + `
${escapeHtml(m.error)}
`, + ); + } + const stopped = m.cancelled + ? `

Stopped — this answer is partial.

` + : ""; + // Which B2 tools the answer was built from (a *Why?* turn). Tool names are the + // model's own words, so they are escaped like every other value in the chrome (E5). + const line = toolsLine(m.tools); + const used = line ? `

${escapeHtml(line)}

` : ""; + const cites = m.citations + .map( + (c) => ``, + ) + .join(""); + return ( + row( + "chat-answer", + "B2", + `
${renderMarkdown(m.text)}
${stopped}${used}`, + ) + cites + ); +} + +/** + * The composer. A ` +
+ ${ + streaming + ? `` + : `` + } + ${note ? `${escapeHtml(note)}` : ""} +
+ `; +} + +/** + * The setup card — deliberately **Ollama-native** (GH #151: guided setup is a per-runtime + * feature, and Ollama is the runtime B2 guides), shown both as the chat pane's empty state + * and inside Settings → Chat. + * + * Three cards, one builder, because they are the same facts in a different order: no + * server (start the daemon), no model (pull one — sized to this machine, illustrative and + * non-binding), and the Settings copy of both. A non-Ollama endpoint gets the message and + * nothing else: there is no honest instruction to give about pulling a model into LM + * Studio or a cloud provider. + */ +export function chatSetupCardHtml(setup: ChatSetup | null, inSettings: boolean): string { + if (!setup) return ""; + const ollama = setup.ollama; + const message = setup.message + ? `

${escapeHtml(setup.message)}

` + : ""; + // What's installed, when the daemon answered — so "no model" can offer what *is* there + // instead of only naming what isn't. + // + // The pane's card only. In Settings the same inventory *is* the Model field (a picker, + // `chatModelFieldHtml`), and two controls setting one value is two things to keep in + // step — the second of which would apply on click while the first waits for Save. + const installed = + !inSettings && ollama && ollama.running && ollama.installed.length > 0 + ? `
+
Installed models
+
    ${ollama.installed + .map( + (m) => + `
  • ${escapeHtml( + modelDetail(m), + )}
    • `, + ) + .join("")}
+
` + : ""; + // Pane-only for the same reason as the inventory: in Settings the **Local** section's + // own copy already carries `ollama pull` sized to this machine (`localNoteHtml`), and + // one command printed twice on one screen reads as two different instructions. + const suggestion = + !inSettings && ollama && ollama.suggested + ? `
+
Suggested for this machine
+

${ + ollama.ram_gb ? `${ollama.ram_gb} GB of memory` : "This machine" + } — a ${escapeHtml(ollama.suggested.size)} model. Illustrative, not a requirement.

+

${escapeHtml( + pullCommand(ollama.suggested.model), + )}

+
` + : ""; + const tiers = + ollama && ollama.tiers.length > 0 + ? `
Sizes by memory +
    ${ollama.tiers + .map( + (t) => + `
  • ${escapeHtml(t.ram)} → ${escapeHtml(t.size)} ${escapeHtml( + t.model, + )}
    • `, + ) + .join("")}
+
` + : ""; + const install = + ollama && !ollama.running + ? `

B2 talks to any OpenAI-compatible model server; Ollama is the + one it can walk you through. Start it with ollama serve — or, if it + isn’t installed yet, the + Ollama quickstart is the install and the + first pull, in that order.

` + : ""; + // A retry, in the pane only: the card's whole job is to be looked at while the user + // goes and fixes something (starts the daemon, pulls a model), so it must be able to + // notice that they did — without making them close and reopen the pane. Settings has + // its own re-probe, spelled *Save and test*. + const recheck = inSettings + ? "" + : `
`; + return `
+ ${inSettings ? "" : `
Chat isn’t ready yet
`} + ${message} + ${recheck} + ${install} + ${suggestion} + ${installed} + ${tiers} + ${ + inSettings + ? "" + : `

Everything else in B2 — search, discovery, editing — + works exactly as it does with chat off.

` + } +
`; +} diff --git a/ui/src/explainview.ts b/ui/src/explainview.ts new file mode 100644 index 0000000..07e112d --- /dev/null +++ b/ui/src/explainview.ts @@ -0,0 +1,186 @@ +// Explain's paint: the Compare view for one Similar card (GH #236). The words and the +// strip's geometry are explain.ts's. + +import { escapeHtml } from "./escape.ts"; +import type { AppState } from "./state.ts"; +import { STRENGTH_MIN_CANDIDATES, strengthBand } from "./strength.ts"; +import { + EXPLAIN_PAIRS_SHOWN, + explainSummary, + explainNames, + fieldCaption, + fieldHelp, + fieldStrip, + standingText, + wholeNoteText, +} from "./explain.ts"; +import { icon } from "./icons.ts"; +import type { PassageView } from "./types.ts"; +import { strengthHtml } from "./widgets.ts"; + +// --- Explain: the Compare view for one Similar card (GH #236) ------------------------ +// +// The centre pane's fourth mode: the open note against one of its Similar cards, read +// model-free from the same computation as the list (`Vault::explain_similar`), so its +// rank, grade and best passage are the card's. The words and the strip's geometry are +// explain.ts's; this is only paint. Passage text is note content, so it is escaped and +// shown as text (E5), never rendered as Markdown. Every control is a real button in tab +// order, and Escape backs out, as it does from the graph (K1). + +export function explainPaneHtml(state: AppState): string { + const ec = state.explainCard; + const bar = `
+ +
`; + if (!ec) return bar; + if (ec.error !== null) { + return `
${bar}

${escapeHtml(ec.error)}

`; + } + const v = ec.view; + if (v === null) { + return `
${bar}
`; + } + const name = (x: { path: string; title: string | null }) => escapeHtml(x.title ?? x.path); + const names = explainNames(v); + const band = strengthBand(v.z); + const summary = explainSummary(v); + const whole = wholeNoteText(v); + + const strip = fieldStrip(v.population, v.z); + const W = 600; + const H = 44; + const help = strip + ? `` + : ""; + const fieldHtml = strip + ? `
+

Where it sits ${help}

+ ${ + ec.help + ? `
${fieldHelp(v) + .map((t) => `

${escapeHtml(t)}

`) + .join("")}
` + : "" + } + + + + ${strip.ticks + .map( + (t) => + ``, + ) + .join("")} + ${[strip.clear, strip.strong] + .map((x) => ``) + .join("")} + ${strip.dots + .map((x) => ``) + .join("")} + ${ + strip.marker !== null + ? `` + : "" + } + + + +

${escapeHtml( + fieldCaption(v.population, names.anchor), + )} · σ from their average

+
` + : `

Ungraded: ranked by nearness only. Grading needs ${STRENGTH_MIN_CANDIDATES} or more notes to compare against.

`; + + const passage = (who: string, p: PassageView) => + `
+
${escapeHtml(who)}${ + p.heading_path ? ` · ${escapeHtml(p.heading_path)}` : "" + }
+
${escapeHtml(p.text)}
+
`; + const shown = ec.allPairs ? v.pairs : v.pairs.slice(0, EXPLAIN_PAIRS_SHOWN); + const pairsHtml = v.pairs.length + ? `
+

Passage pairs · each of its passages, with this note’s nearest

+
    ${shown + .map( + (p, i) => `
  1. +
    #${i + 1} ${strengthHtml(p.z ?? undefined)}${ + p.identical ? `identical text` : "" + }
    +
    ${passage("This note", p.anchor)}${passage( + "Suggested", + p.candidate, + )}
    +
    2. `, + ) + .join("")}
+ ${ + v.pairs.length > EXPLAIN_PAIRS_SHOWN + ? `` + : "" + } +
` + : ""; + + const links = v.shared_neighbors.length + ? `

Both link with ${v.shared_neighbors + .map( + (s) => + ``, + ) + .join(", ")}.

` + : `

They share no linked notes.

`; + + return `
${bar} +
+
+
Explain · suggested for ${name(v.anchor)}
+

${name(v.candidate)}

+
${escapeHtml(v.candidate.path)}
+

${band ? `${strengthHtml(v.z ?? undefined)} ` : ""}${escapeHtml( + standingText(v), + )}

+ ${whole ? `

${escapeHtml(whole)}

` : ""} +
+ ${ + summary + ? `
${escapeHtml(summary.label)}. ${escapeHtml( + summary.detail, + )}
` + : "" + } + ${fieldHtml} + ${pairsHtml} +

Links

${links}
+
+ + +
+
+
`; +} diff --git a/ui/src/graphview.ts b/ui/src/graphview.ts new file mode 100644 index 0000000..91b8fb8 --- /dev/null +++ b/ui/src/graphview.ts @@ -0,0 +1,306 @@ +// The connection graph's paint — the centre pane's third mode — and the two chips the +// reading bar shares with the graph bar (the graph toggle, Edit). Scene geometry is +// graph.ts's; this is only markup. + +import { escapeHtml } from "./escape.ts"; +import type { AppState } from "./state.ts"; +import { coverage } from "./coverage.ts"; +import { displayKeys } from "./bindings.ts"; +import { icon, resourceIcon, sceneIcon } from "./icons.ts"; +import type { NoteView } from "./types.ts"; +import { + buildScene, + NODE_R, + VIEW_H, + VIEW_W, + type Category, + type GraphEdge, + type GraphNode, + type GraphScene, +} from "./graph.ts"; + +// --- the anchored ghost graph (GH #22) ---------------------------------------------- +// +// The center pane's third mode: the open note's typed neighborhood as hand-rolled, +// deterministic SVG — scene geometry from `graph.ts` (pure, unit-tested), markup +// here, clicks delegated in main.ts. The reading key: color = edge category, solid = +// authored / dashed teal = latent (`similar`), disc = note / square = resource / +// dashed hollow = dangling. Everything renders from state the note-open already +// fetched, so entering the graph costs no IPC. + +/** The graph toggle chip, shared by the reading bar (off) and the graph bar (on). + * + * Its chord comes out of the live registry rather than being spelled here: ⌘G is + * rebindable (#121), and a tooltip naming the shipped default would be wrong for exactly + * the user who changed it. */ +export function graphToggleHtml(active: boolean): string { + const chord = escapeHtml(displayKeys(["graph.toggle"])); + return ``; +} + +/** The Edit chip, shared by the reading bar and the graph bar. Its chord comes out of the + * live registry for `graphToggleHtml`'s reason; `title` overrides the tooltip when the + * chip is disabled for a reason worth naming. */ +export function editToggleHtml(disabled: boolean, title?: string): string { + const hint = title ?? `Edit this note — ${displayKeys(["edit.toggle"])} (autosaves as you type)`; + return ``; +} + +/** Fixed-point SVG coordinate — keeps the markup compact and diff-stable. */ +function px(v: number): string { + return (Math.round(v * 10) / 10).toString(); +} + +/** One edge's path (a straight segment, or the parallel-separating quadratic). */ +function edgePathD(e: GraphEdge): string { + return e.cx === null || e.cy === null + ? `M ${px(e.x1)} ${px(e.y1)} L ${px(e.x2)} ${px(e.y2)}` + : `M ${px(e.x1)} ${px(e.y1)} Q ${px(e.cx)} ${px(e.cy)} ${px(e.x2)} ${px(e.y2)}`; +} + +function edgeHtml(e: GraphEdge): string { + if (e.ghost) { + return ``; + } + const verb = e.label.replace(/[^a-z0-9-]/gi, ""); + const marker = e.arrow ? ` marker-end="url(#garr-${e.category})"` : ""; + const label = `${escapeHtml(e.label)}`; + return `${label}`; +} + +/** A node's shape + glyph, by kind (labels are added by the group builder). */ +function nodeShapeHtml(n: GraphNode): string { + const x = px(n.x); + const y = px(n.y); + switch (n.kind) { + case "anchor": + return ` + + `; + case "resource": { + const s = NODE_R.resource - 2; + return ` + ${sceneIcon(resourceIcon(n.sub), n.x, n.y, 16, "gglyph")}`; + } + case "dangling": + return ` + ${sceneIcon("exclamation-triangle", n.x, n.y, 16, "gglyph")}`; + default: + return ``; + } +} + +/** The tooltip line(s) for a node — also the activation affordance's explanation. + * Phrased for both hands: an activatable node says "⏎", because the graph is + * reachable by Tab and arrow keys too (K1, GH #78), not by mouse alone. */ +function nodeTitle(n: GraphNode): string { + switch (n.kind) { + case "anchor": + return `${n.full} — the open note. Click or ⏎ to return to reading.`; + case "ghost": + // The strength figure when there is one; a bare "similar but not linked" + // otherwise. "similarity ?" claimed a measurement existed and was merely + // unavailable — an ungraded candidate was never measured at all. + return `${n.full} — similar but not linked${ + n.sub ? ` (${n.sub} above this note's other candidates)` : "" + }. Click or ⏎ to link it; right-click (or ${displayKeys(["menu.open"])}) for more.`; + case "dangling": + return `${n.full} resolves to no note or file — fix the link in the note.`; + case "resource": + return `${n.full} (${n.sub ?? "file"}) — click or ⏎ to open.`; + default: + return `${n.full} — click or ⏎ to open.`; + } +} + +/** The accessible name for a focusable node — what a screen reader announces, and + * what the node's own `` can't be (that one is the mouse tooltip prose). */ +function nodeAriaLabel(n: GraphNode): string { + switch (n.kind) { + case "anchor": + return `${n.full} — the open note; back to reading`; + case "ghost": + return `${n.full} — similar but unlinked; link it`; + case "resource": + return `${n.full} — open this file`; + default: + return `${n.full} — open this note`; + } +} + +/** + * One scene node as an interactive `<g>`, its incident edges inside it so a pure-CSS + * hover lights the node *and* its edges while the rest of the scene dims. The click + * affordance rides existing delegation: notes reuse `data-open`, resources + * `data-open-resource`; ghosts get `data-ghost-link` (→ the link palette) plus the + * `data-card-*` pair the right-click menu reads; the anchor toggles back to reading. + * + * An activatable node is also a **keyboard** control (K1, GH #78): `tabindex="0"` puts + * it in the Tab order and `role="button"` says what it is, so a graph is walkable and + * openable with no mouse. main.ts turns ⏎/Space on a focused node into the same click + * these attributes already answer — one activation path, not two. A `dangling` node + * stays inert: it opens nothing (there's nothing there), so it isn't a tab stop. + * + * It also carries `data-gnode` — the scene id (graph.ts), which is what the note pane's + * focus restoration re-finds it *by* after an `innerHTML` swap destroys the element + * itself (GH #91). The discovery row's `data-side-row` for the graph: an identity that + * outlives the repaint, not a pointer into it. + */ +function nodeGroupHtml(n: GraphNode, edges: GraphEdge[], order: number): string { + const attrs: string[] = [`class="gnode is-${n.kind}"`, `style="--i:${order}"`]; + if (n.kind === "note" && n.path) attrs.push(`data-open="${escapeHtml(n.path)}"`); + if (n.kind === "anchor") attrs.push(`data-toggle-graph="1"`); + if (n.kind === "resource" && n.path) attrs.push(`data-open-resource="${escapeHtml(n.path)}"`); + if (n.kind === "ghost" && n.path) { + attrs.push( + `data-ghost-link="${escapeHtml(n.path)}"`, + `data-card-path="${escapeHtml(n.path)}"`, + `data-card-title="${escapeHtml(n.title ?? "")}"`, + ); + } + if (n.kind !== "dangling") { + attrs.push( + `tabindex="0"`, + `role="button"`, + `aria-label="${escapeHtml(nodeAriaLabel(n))}"`, + `data-gnode="${escapeHtml(n.id)}"`, + ); + } + const r = NODE_R[n.kind]; + // Text goes on the side of the node facing *away* from the anchor (above for the + // upper half of the scene), so a label never sits in its own edge's path. + const above = n.kind !== "anchor" && n.y < VIEW_H / 2 - 20; + const label = `<text class="gnode-label" x="${px(n.x)}" y="${px( + above ? n.y - r - 14 : n.y + r + 18, + )}">${escapeHtml(n.label)}</text>`; + const sub = n.sub + ? `<text class="gnode-sub" x="${px(n.x)}" y="${px( + above ? n.y - r - 29 : n.y + r + 33, + )}">${escapeHtml(n.sub)}</text>` + : ""; + return `<g ${attrs.join(" ")}> + <title>${escapeHtml(nodeTitle(n))} + ${edges.map(edgeHtml).join("")} + ${nodeShapeHtml(n)} + ${label}${sub} + `; +} + +/** The honest ghost-halo caveat (coverage.ts's tiers, like `searchCaveat`, #26): why there are + * no ghosts right now, or null when there are (or when silence is the honest state). */ +function ghostHintHtml(state: AppState): string { + if (state.similar.length > 0) return ""; + if (state.discoveringSimilar) + return `
scanning for latent connections…
`; + const c = coverage(state); + if (!c.model) + return `
ghost connections need the semantic model — run b2 init, then Reindex
`; + if (c.embedded === "none" || c.embedded === "partial") + return `
ghosts appear once the vault is embedded — Reindex
`; + return ""; +} + +/** The centered guidance when there's nothing to draw (the anchor always shows). */ +function graphEmptyHtml(state: AppState, scene: GraphScene): string { + if (scene.edges.length > 0) return ""; + if (state.discoveringSimilar) return ""; + return `

No connections yet.

+

B2 floats similar-but-unlinked notes here as ghosts — click one to make the connection real.

`; +} + +/** The reading key, one quiet strip: verb colors, edge states, node shapes. */ +function graphLegendHtml(): string { + const cats: [Category, string][] = [ + ["references", "references"], + ["supports", "supports"], + ["contradicts", "contradicts"], + ]; + const dots = cats + .map(([c, label]) => `${label}`) + .join(""); + return ``; +} + +/** Arrowhead markers, one per category (an SVG marker can't inherit its edge's + * stroke everywhere yet). */ +function graphDefsHtml(): string { + const cats: Category[] = ["references", "supports", "contradicts", "other"]; + const arrow = (id: string, cls: string) => + ` + + `; + return `${cats.map((c) => arrow(`garr-${c}`, `garr cat-${c}`)).join("")}`; +} + +/** + * The graph pane — the note pane's third mode (Reading / Editing / Graph). Bar: + * the same action chips as reading; stage: the SVG scene (fills the pane, + * `viewBox`-scaled) with overlay hints; footer: the reading key. + */ +export function graphPaneHtml(state: AppState, n: NoteView): string { + const scene = buildScene({ + anchor: { path: n.path, title: n.title }, + connections: state.connections, + resources: state.resourceLinks, + unresolved: state.unresolved, + ghosts: state.similar, + }); + + // Edges live inside their node's group (hover affordance); the anchor renders + // last so it always paints on top of edge crossings. + const byNode = new Map(); + for (const e of scene.edges) { + const owner = e.from === "anchor" ? e.to : e.from; + const list = byNode.get(owner) ?? []; + list.push(e); + byNode.set(owner, list); + } + // Paint order: ghosts lowest (their long dashed spokes must pass *under* the + // authored orbit), authored above them, the anchor on top of everything. The + // stagger index is narrative, not paint, order: authored pops first, ghosts after. + const authoredNodes = scene.nodes.filter((node) => node.kind !== "anchor" && node.kind !== "ghost"); + const ghostNodes = scene.nodes.filter((node) => node.kind === "ghost"); + const anchor = scene.nodes.find((node) => node.kind === "anchor"); + const groups = [ + ...ghostNodes.map((node, i) => + nodeGroupHtml(node, byNode.get(node.id) ?? [], authoredNodes.length + 1 + i), + ), + ...authoredNodes.map((node, i) => nodeGroupHtml(node, byNode.get(node.id) ?? [], i + 1)), + ...(anchor ? [nodeGroupHtml(anchor, [], 0)] : []), + ].join(""); + + return `
+
+
+ ${graphToggleHtml(true)} + ${editToggleHtml(state.loading)} +
+
+
+ + ${graphDefsHtml()} + ${groups} + + ${graphEmptyHtml(state, scene)} + ${ghostHintHtml(state)} +
+ ${graphLegendHtml()} +
`; +} diff --git a/ui/src/keysview.ts b/ui/src/keysview.ts new file mode 100644 index 0000000..bd84b72 --- /dev/null +++ b/ui/src/keysview.ts @@ -0,0 +1,188 @@ +// The keyboard's two surfaces: Settings → Keyboard (the reference, and the recorder that +// rebinds from it) and the ⌘-hold sheet. The table is shortcuts.ts; the algebra keymap.ts. + +import { escapeHtml } from "./escape.ts"; +import type { AppState } from "./state.ts"; +import { type ShortcutGroup, type ShortcutKey, shortcuts } from "./shortcuts.ts"; +import { cmdShortcuts } from "./cmdhold.ts"; +import { + DEFAULT_BINDINGS, + activeBindings, + displayChord, + displayKeys, + findBinding, +} from "./bindings.ts"; +import { customized, refused } from "./keymap.ts"; + +// Keyboard — the discoverable half of invariant K1, and now its *editable* half too: one +// surface for the table, reached by `?` from anywhere or by walking the rail, where every +// chord B2 owns is a button that rebinds it (#121). The table itself is `shortcuts.ts` +// (GH #78); the algebra and the judgement are keymap.ts. +// +// The recorder is a strip at the top of the panel rather than a widget spliced into the +// row it edits. The grid is CSS multi-column, so an inline block would land wherever the +// column flow put it — and the panel is a page of table you scroll, so a control that +// appeared below the fold would be a control nobody saw. The chip being edited carries +// `.kbd-recording` instead, which is what ties the strip to its row. +export function keyboardPanelHtml(state: AppState): string { + const changed = customized(DEFAULT_BINDINGS, state.keyOverrides).length; + const resetAll = changed + ? `` + : ""; + // The ⌘-hold sheet is a *gesture*, not a chord, so it has no row in the table below — + // and a keyboard affordance documented nowhere is the exact failure K1 names. The + // reference is where someone looks for it, so the reference is where it is said. + return `
Keyboard shortcuts
+

B2 is fully operable from the keyboard — the mouse is an accelerator, never a requirement. Click a chord to change it. Hold ⌘ on its own anywhere in the app for a quick reminder of the ⌘ chords below.

+
${resetAll}
+ ${recorderHtml(state)} + ${shortcutsGridHtml(state)}`; +} + +/** The recorder: what is being rebound, what has been pressed, and what that would mean. + * + * Empty markup when nothing is recording, so the strip costs no vertical space until it + * is asked for. Every control carries a stable `id` — the settings builder's rule above + * — because a captured chord repaints the dialog under the keyboard that captured it. */ +function recorderHtml(state: AppState): string { + const rec = state.recorder; + if (!rec) return ""; + const b = findBinding(activeBindings(), rec.id); + if (!b) return ""; + const now = b.keys.map((k) => `${escapeHtml(displayChord(k))}`).join(" "); + const captured = rec.candidate + ? `${escapeHtml(displayChord(rec.candidate))}` + : `Press a chord…`; + const lines = [ + ...(rec.hint ? [{ tier: "warn" as const, message: rec.hint }] : []), + ...rec.problems, + ] + .map( + (p) => + `
  • ${escapeHtml(p.message)}
    • `, + ) + .join(""); + const blocked = refused(rec.problems); + const canSave = rec.candidate !== null && !blocked; + const isChanged = state.keyOverrides[rec.id] !== undefined; + // `tabindex="-1"`: focusable so main.ts can take the keyboard off whatever had it (a + // tree row, or CodeMirror, which would otherwise type the chord into the buffer behind + // the dialog), but not a Tab stop — the strip is a target to press keys at, not a + // control to land on. The id is what `captureModalFocus` hands focus back by, which + // matters here more than anywhere: every captured chord repaints this dialog. + return `
    +
    + ${escapeHtml(b.label)} + now ${now} +
    +
    ${captured}
    + ${lines ? `
      ${lines}
    ` : ""} +

    Esc cancels, ⏎ accepts — every other key is recorded. If a chord you press never appears above, macOS or another app took it before B2 could see it; B2 can only tell you about the chord you actually pressed.

    +
    + + + ${isChanged ? `` : ""} +
    +
    `; +} + +/** Every chord the app answers to, grouped, from the one table in shortcuts.ts. The app + * menu bar's chords are *not* here — macOS prints those beside their own menu items and + * nothing in this panel could move them (shortcuts.ts's header says why they left). + * + * A chip that names a command is a ``; + }; + return `
    ${keysGroupsHtml(shortcuts(), chip)}
    `; +} + +/** Groups of shortcut rows as the reference and the ⌘ sheet both lay them out; `chip` + * paints one chord (a rebind button in the reference, a plain `` on the sheet). */ +function keysGroupsHtml(groups: readonly ShortcutGroup[], chip: (k: ShortcutKey) => string): string { + return groups + .map( + (g) => `
    +

    ${escapeHtml(g.title)}

    +
    ${g.items + .map((s) => `
    ${s.keys.map(chip).join(" ")}
    ${escapeHtml(s.action)}
    `) + .join("")}
    +
    `, + ) + .join(""); +} + +/** + * The ⌘-hold sheet: what a held ⌘ can do, painted over the app while it is held + * (cmdhold.ts owns the machine and the projection; main.ts owns the timer). + * + * Empty markup when it isn't up, so the layer costs nothing the rest of the time — the + * same shape `recorderHtml` and `contextMenuHtml` use. Empty markup too when the + * projection has nothing in it, which is reachable: every ⌘ chord is rebindable, so a user + * can move the lot off ⌘ and would otherwise get an empty card for their trouble. + * + * Three things it deliberately isn't. It is not a dialog: nothing here is focusable, focus + * does not move to it and does not come back, because the whole gesture is one the user is + * already mid-way through — taking the keyboard would cancel the chord they are about to + * press. It is `pointer-events: none` in the stylesheet for the same reason on the other + * device: ⌘-click and ⌘-drag have to keep landing on the app underneath. And it is + * `aria-hidden`, which reads as a strange thing to say about a keyboard aid until you + * consider who it would be talking to — a screen-reader user has the whole reference in + * Settings → Keyboard, reachable by chord, and what an aria-live region would add here is + * a page of chords announced every time a modifier is held a beat too long. The reference + * is the accessible surface; this is a glance. + */ +export function cmdSheetHtml(state: AppState): string { + if (!state.cmdSheet) return ""; + const groups = cmdShortcuts(); + if (groups.length === 0) return ""; + const body = keysGroupsHtml(groups, (k) => `${escapeHtml(k.text)}`); + // The footer says both halves of the contract: how it goes away (the thing a user who + // did not mean to summon it needs first), and where the rest of the keyboard lives — + // this sheet is the ⌘ slice, and someone reading it is exactly the someone who would + // want the whole table. + return ``; +} diff --git a/ui/src/livepreview.ts b/ui/src/livepreview.ts index c6c7113..b2f57d8 100644 --- a/ui/src/livepreview.ts +++ b/ui/src/livepreview.ts @@ -46,7 +46,7 @@ import { WIKILINK_EXACT, type EmbedImages, } from "./embeds.ts"; -import { renderMarkdown } from "./render.ts"; +import { renderMarkdown } from "./markdown.ts"; // --- the wikilink tree extension (spec §4, insight §2.3) -------------------------- // diff --git a/ui/src/main.ts b/ui/src/main.ts index 7aca2f7..9f00170 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -65,7 +65,15 @@ import { userMessage, whyQuestion, } from "./chat"; -import { isSettingsTab, tabMove, tabNavFor, tabStep, type SettingsTabId } from "./settingstabs"; +import { + isSettingsTab, + tabDomId, + tabMove, + tabNavFor, + tabStep, + type SettingsTabId, +} from "./settingstabs"; +import { reindexMeterHtml } from "./widgets"; import { externalUrl, isInPageAnchor } from "./links"; import { embedImagesField, livePreview, setEmbedImages, wikilink } from "./livepreview"; import { @@ -2711,7 +2719,7 @@ async function openSettings(tab?: SettingsTabId): Promise { // `paintModal` restores focus to the tab that had it, which is no longer the // selected one, and a roving tabstop that disagrees with the highlight is worse // than no tabstop. The reads below are skipped too: nothing about the host changed. - if (tab) document.getElementById(`settings-tab-${tab}`)?.focus(); + if (tab) document.getElementById(tabDomId(tab))?.focus(); return; } try { @@ -2762,7 +2770,7 @@ function selectSettingsTab(tab: SettingsTabId, focusTab: boolean): void { if (state.settingsTab === tab && !focusTab) return; state.settingsTab = tab; render(); - if (focusTab) document.getElementById(`settings-tab-${tab}`)?.focus(); + if (focusTab) document.getElementById(tabDomId(tab))?.focus(); } /** @@ -4368,11 +4376,7 @@ function buildShell(): void { - + ${reindexMeterHtml({ hidden: true, indeterminate: false })}
    "); +} + +// The postprocess hook is where the trust boundary sits (E5, GH #77). Two properties come +// from putting it *here* rather than inside `renderMarkdown`: sanitizing is the last thing +// that happens to the HTML — nothing, not even B2's own table wrapper, is spliced in +// afterwards — and it holds for every `marked.parse` in the app, so a future second call +// site cannot render an unsanitized note by forgetting a step. +// +// `breaks: true` makes a single newline a `
    `, the Obsidian reading of a note. The +// editor shows the file one line per line, and the default CommonMark fold — one newline +// is a space, only a blank line ends a paragraph — had the reading view disagreeing with +// it about where the author's lines end (linebreaks.test.ts pins both halves: a break for +// one newline, a paragraph for two, and neither inside a fence or a list). +marked.use({ + extensions: [wikilink], + gfm: true, + breaks: true, + hooks: { postprocess: (html: string) => sanitizeHtml(wrapTables(html)) }, +}); + +/** + * Note body → the HTML the panes write into the DOM. Sanitized (see the hook above). + * + * `images` is what an `![[picture.png]]` embed draws with — the note's loaded pictures, + * keyed by vault-relative path (`state.embedImages`). Omitted, every embed reads as its + * link, which is both the honest state before the bytes arrive and what a caller with no + * pictures to offer wants. + */ +export function renderMarkdown(md: string, images: EmbedImages = NO_EMBED_IMAGES): string { + renderImages = images; + try { + return marked.parse(md, { async: false }) as string; + } finally { + renderImages = NO_EMBED_IMAGES; + } +} diff --git a/ui/src/render.ts b/ui/src/render.ts index b1fee99..b526eef 100644 --- a/ui/src/render.ts +++ b/ui/src/render.ts @@ -1,55 +1,32 @@ -// The view: Markdown → HTML (with clickable wikilinks) and the pane HTML builders. -// Pure functions of state — no IPC, no DOM mutation (main.ts writes the output in). +// The view: the pane HTML builders, and the view layer's one import surface. Pure +// functions of state — no IPC, no DOM mutation (main.ts writes the output in). +// +// This module paints the tree, the note pane, the discovery column, the menus and the +// dialogs; the bigger surfaces have modules of their own — chatview.ts, graphview.ts, +// explainview.ts, settingsview.ts, keysview.ts — and the small shared pieces are +// widgets.ts's. What the rest of the app imports is re-exported below, so a caller never +// has to know which module a surface lives in. // // Safety (invariant E5 — **note content is untrusted input**): authorship is not trust, a -// `.md` can come from anyone (a shared vault, a downloaded or clipped note), so this -// module treats every note body as hostile. Two rules hold together here: B2 HTML-escapes -// every value *it* interpolates (titles, paths, snippets) so UI chrome can't be broken by -// note content, and the one Markdown→HTML path sanitizes its output — `sanitize.ts`, wired -// below as `marked`'s `postprocess` hook, so *every* caller is covered by construction +// `.md` can come from anyone (a shared vault, a downloaded or clipped note), so every note +// body is hostile. Two rules hold together: B2 HTML-escapes every value *it* interpolates +// (titles, paths, snippets) so UI chrome can't be broken by note content, and the one +// Markdown→HTML path, `renderMarkdown` (markdown.ts, re-exported here), sanitizes its +// output as `marked`'s `postprocess` hook — so *every* caller is covered by construction // rather than by remembering. The webview CSP (`default-src 'self'`, no inline scripts) -// is a second, independent layer, never the only one -// (crates/b2-desktop/CLAUDE.md, GH #77). +// is a second, independent layer, never the only one (crates/b2-desktop/CLAUDE.md, GH #77). -import { marked, type Tokens, type TokenizerAndRendererExtension } from "marked"; // Relative imports carry their `.ts` here (the idiom highlight.ts and reconcile.ts already // use) because render.test.ts runs this module off the source under node's type-stripping, // which resolves by real filename — a bundler-style extensionless value import doesn't // resolve there. tsc rewrites nothing (noEmit). import { escapeHtml } from "./escape.ts"; -import { sanitizeHtml } from "./sanitize.ts"; -import { - embedWidth, - NO_EMBED_IMAGES, - WIKILINK_ANCHORED, - type EmbedImages, -} from "./embeds.ts"; import { RELATION_VERBS, openDocPath, type AppState, type SideSection } from "./state.ts"; import { allDirs, baseName, moveRefusal, renamePrefill } from "./move.ts"; import { shouldPromptEmbedInstall } from "./embedreminder.ts"; import { coverage } from "./coverage.ts"; import { STRENGTH_MIN_CANDIDATES, strengthBand } from "./strength.ts"; -import { - EXPLAIN_PAIRS_SHOWN, - explainSummary, - explainNames, - fieldCaption, - fieldHelp, - fieldStrip, - standingText, - wholeNoteText, -} from "./explain.ts"; -import { type ShortcutKey, shortcuts } from "./shortcuts.ts"; -import { cmdShortcuts } from "./cmdhold.ts"; -import { - DEFAULT_BINDINGS, - activeBindings, - displayChord, - displayKeys, - findBinding, -} from "./bindings.ts"; -import { customized, refused } from "./keymap.ts"; -import { SETTINGS_TABS, type SettingsTabId } from "./settingstabs.ts"; +import { displayKeys } from "./bindings.ts"; import { buildTree, rovingPath, @@ -64,163 +41,24 @@ import { folderIcon, icon, NOTE_ICON, - resourceIcon, - sceneIcon, type IconName, } from "./icons.ts"; -import { - cardKey, - cardRowKey, - rovingSideKey, - sectionRowKey, - sideRows, -} from "./sidenav.ts"; -import { - type ChatMessage, - OLLAMA_CLOUD_URL, - OLLAMA_QUICKSTART_URL, - PULL_PLACEHOLDER, - LOCAL_CHAT_ENDPOINT, - STREAMING_ROW_KEY, - chatReady, - chatStateOf, - citationRowKey, - formatModelSize, - pullCommand, - retrievalNote, - toolsLine, - turnRowKey, -} from "./chat.ts"; -import type { ChatSetup, NoteView, PassageView, ResourceExplainView } from "./types.ts"; -import { - buildScene, - NODE_R, - VIEW_H, - VIEW_W, - type Category, - type GraphEdge, - type GraphNode, - type GraphScene, -} from "./graph.ts"; - -// Re-exported so this module stays the view layer's single import surface (main.ts -// reaches for it here); the definition lives in escape.ts. +import { cardKey, cardRowKey, rovingSideKey, sectionRowKey, sideRows } from "./sidenav.ts"; +import type { NoteView, ResourceExplainView } from "./types.ts"; +import { renderMarkdown } from "./markdown.ts"; +import { sideTab, strengthHtml } from "./widgets.ts"; +import { chatPaneHtml } from "./chatview.ts"; +import { editToggleHtml, graphPaneHtml, graphToggleHtml } from "./graphview.ts"; +import { explainPaneHtml } from "./explainview.ts"; +import { cmdSheetHtml } from "./keysview.ts"; +import { reindexDisabled, reindexLabel, settingsScreenHtml } from "./settingsview.ts"; + +// Re-exported so this module stays the view layer's single import surface (main.ts and +// the suite reach for them here); the definitions live in the modules named. export { escapeHtml }; - -// A `[[target]]` / `[[target|label]]` wikilink becomes an in-app anchor carrying the -// raw target; main.ts delegates a click on `.wikilink` to open that note. This is the -// MVP's in-app navigation (spec §4) — the buffer stays byte-honest Markdown. -// -// `![[target]]` is the **embed** form of the same link — the core reads the `!` as the -// embed marker and records it on the edge (`link.rs`). Where the target is a picture the -// note has loaded, the embed *shows* it: the anchor stays (so the image is still the -// link, and clicking it still opens the resource card), and the `` becomes its -// label. Everything else about an embed is unchanged — with no picture in hand (not an -// image, not indexed, too large, or simply not read yet) it reads as its link, which is -// what B2 showed before there was a viewer. -// -// The marker is *grammar* either way, so it is consumed rather than left beside the -// anchor as a stray `!`. The `|`-part changes meaning with it: on a plain wikilink it is -// the display **label**, on an embed a display **width** (`![[shot.png|500]]`, aspect -// ratio kept — `embedWidth` in embeds.ts). So an embed never labels itself "500"; it -// draws itself 500px wide, or falls back to naming its target. -const wikilink: TokenizerAndRendererExtension = { - name: "wikilink", - level: "inline", - start(src: string) { - const i = src.indexOf("[["); - if (i < 0) return undefined; - // Back up over the embed marker: this offset is where marked stops emitting plain - // text, so anchoring it at the `[[` hands the reader the `!` before we ever tokenize. - return i > 0 && src[i - 1] === "!" ? i - 1 : i; - }, - tokenizer(src: string) { - const m = WIKILINK_ANCHORED.exec(src); - if (!m) return undefined; - const embed = m[1] === "!"; - const target = m[2].trim(); - return { - type: "wikilink", - raw: m[0], - embed, - target, - width: embed ? embedWidth(m[3]) : null, - label: embed ? target : (m[3] ?? m[2]).trim(), - } as Tokens.Generic; - }, - renderer(token: Tokens.Generic) { - const target = String(token.target); - const anchor = ``; - const src = token.embed ? renderImages.get(target) : undefined; - if (!src) return `${anchor}${escapeHtml(String(token.label))}`; - // The alt text is the filename: it is all B2 knows about the picture, and it is what - // the embed would have read as had the bytes not arrived (the resource card's viewer - // makes the same choice, for the same reason). - const name = baseName(target); - const width = typeof token.width === "number" ? ` width="${token.width}"` : ""; - return `${anchor}${escapeHtml( - name, - )}`; - }, -}; - -// The pictures the *current* `renderMarkdown` call may draw. -// -// A module-local rather than a parameter because `marked`'s extensions are registered -// once, globally (the `marked.use` below), so the renderer above has no way to be handed -// per-call data. It is safe to hold it here for exactly one reason, and the reason is -// worth stating: `marked.parse(…, { async: false })` runs to completion synchronously, -// so between the assignment and the `finally` no other render can interleave. Every -// caller still passes its images as an argument — the seam is `renderMarkdown`, and this -// variable never outlives one of its calls. -let renderImages: EmbedImages = NO_EMBED_IMAGES; - -// Wrap each table in a scroll box so a wide one scrolls *within* its column instead of -// stretching the pane. The table itself must stay a real `display: table` (the wrapper -// is what's `display: block; overflow-x: auto`) — a `display: block` table splits -// marked's whitespace-separated ``/`` into two anonymous tables, so -// `border-collapse` can't join the header row onto the body (the gap bug). marked -// escapes cell content, so these are the only literal `` tags in the output. -function wrapTables(html: string): string { - return html - .replace(/
    /g, '
    ') - .replace(/<\/table>/g, "
    "); -} - -// The postprocess hook is where the trust boundary sits (E5, GH #77). Two properties come -// from putting it *here* rather than inside `renderMarkdown`: sanitizing is the last thing -// that happens to the HTML — nothing, not even B2's own table wrapper, is spliced in -// afterwards — and it holds for every `marked.parse` in the app, so a future second call -// site cannot render an unsanitized note by forgetting a step. -// -// `breaks: true` makes a single newline a `
    `, the Obsidian reading of a note. The -// editor shows the file one line per line, and the default CommonMark fold — one newline -// is a space, only a blank line ends a paragraph — had the reading view disagreeing with -// it about where the author's lines end (linebreaks.test.ts pins both halves: a break for -// one newline, a paragraph for two, and neither inside a fence or a list). -marked.use({ - extensions: [wikilink], - gfm: true, - breaks: true, - hooks: { postprocess: (html: string) => sanitizeHtml(wrapTables(html)) }, -}); - -/** - * Note body → the HTML the panes write into the DOM. Sanitized (see the hook above). - * - * `images` is what an `![[picture.png]]` embed draws with — the note's loaded pictures, - * keyed by vault-relative path (`state.embedImages`). Omitted, every embed reads as its - * link, which is both the honest state before the bytes arrive and what a caller with no - * pictures to offer wants. - */ -export function renderMarkdown(md: string, images: EmbedImages = NO_EMBED_IMAGES): string { - renderImages = images; - try { - return marked.parse(md, { async: false }) as string; - } finally { - renderImages = NO_EMBED_IMAGES; - } -} +export { renderMarkdown }; +export { cmdSheetHtml }; +export { reindexDisabled, reindexLabel }; // --- file tree -------------------------------------------------------------------- // @@ -694,11 +532,6 @@ export function sidePaneHtml(state: AppState): string { : discoverySectionHtml(state, roving); } -/** A row's slot in the roving tabstop: exactly one row is tabbable, the rest are -1. */ -function sideTab(key: string, roving: string | null): string { - return ` tabindex="${key === roving ? "0" : "-1"}"`; -} - // The honest search-ranking caveat (#26). Search always answers over the keyword (BM25) // index; this says how much *semantic* ranking is mixed in, so a projected-but-unembedded // vault never silently under-ranks: @@ -858,26 +691,6 @@ function cardFold(key: string, collapsed: boolean): string { `; } -// The discovery card's strength cell: a banded read of the candidate's z -// (`strength.ts`), replacing the raw negated-L2 the card used to print (GH #150). -// No z → no cell: a statistic that wasn't computed isn't claimed (raw mode, tiny -// pools). The glyph is decorative; the band name is the accessible content. -function strengthHtml(z: number | undefined): string { - const band = strengthBand(z); - if (!band) return ""; - // The figure rides in the markup and CSS reveals it on the selected/hovered card, so - // the number is one keystroke away rather than pointer-only (`title=` alone was a hole - // in K1). The accessible name carries *both* halves the eye gets — the band and the - // figure — because naming only the band would leave a screen reader with "clear match" - // and no way to reach the 2.5σ behind it. The figure's own span stays `aria-hidden`, so - // it is announced once (as part of this name) rather than twice. - return `${band.glyph}`; -} - /** A caveat *about the list you are looking at* — not an empty state and not an error. * The install banner already settled the tone for this class of message: accent tones, * never `--danger`, because "B2 didn't grade these" is a nudge about what the numbers @@ -1085,1476 +898,6 @@ function unresolvedCardsHtml(state: AppState, roving: string | null): string { .join(""); } -// --- chat (flow ④, GH #151/#153/#155) ----------------------------------------------- -// -// The right column's third mode: ask a question, watch the answer stream, click a -// citation to open the note behind it — *without* the conversation leaving the screen, -// which is why chat lives here rather than in the centre pane (chat.ts's header). -// -// Three rules this builder holds, all of them invariants rather than styling: -// -// • **E5 — model output is untrusted content.** An answer is a string from a model that -// was itself fed note content (which anyone can author), so it renders through the one -// sanitizing `renderMarkdown` seam like every other document. The *streaming* half is -// stronger still: it is written as `textContent` by `paintChatStream` (main.ts), so a -// half-arrived answer is never parsed as markup at all. -// • **A citation navigates in-app, never the webview.** Each one is a `data-open` button -// — the same delegation search results and discovery cards use — so the mouse and ⏎ -// share one activation path (K1), and no `href` ever exists to be followed. -// • **K1 — the pane is keyboard-complete.** Rows are `role="treeitem"` over chat.ts's -// row order (the same order the arrows walk), with a roving tabstop; every control has -// a stable `id` so `paintSide` can hand focus back across the repaint each token causes. -function chatPaneHtml(state: AppState, roving: string | null): string { - const setup = state.chatSetup; - const streaming = state.chatStreaming !== null; - const model = setup - ? `${escapeHtml(setup.model)}` - : ""; - const head = `
    -

    Chat

    - ${model} - -
    `; - // No composer until there is something to answer with: a disabled field under a card - // that says why is chrome with nothing behind it, and one more stop for a keyboard user - // to walk past on the way to the fix the card is pointing at. - return head + chatStageHtml(state, roving) + (chatReady(state) ? chatComposerHtml(state) : ""); -} - -/** The conversation, or the state that stands in for one (chat.ts's `chatEmptyState` - * makes that choice once, so the paint doesn't re-derive it branch by branch). */ -function chatStageHtml(state: AppState, roving: string | null): string { - switch (chatStateOf(state)) { - case "no-vault": - return `

    Open a vault to chat with your notes.

    `; - case "loading": - return `

    Looking for a model…

    `; - case "no-server": - case "no-model": - return chatSetupCardHtml(state.chatSetup, false); - case "ready": - return chatLogHtml(state, roving); - } -} - -/** The transcript. Empty until the first question — and then it is what the pane is. */ -function chatLogHtml(state: AppState, roving: string | null): string { - if (state.chatMessages.length === 0 && state.chatStreaming === null) { - const fake = state.chatSetup?.state === "fake"; - return `
    -
    -

    Ask your notes a question.

    -

    Answers come only from passages B2 retrieves from this vault, cited by - [n]. Nothing here is written to your notes, and the conversation isn’t saved.

    - ${ - fake - ? `

    The fake chat provider is in use (B2_LLM=fake) — answers are deterministic test scaffolding, not a model.

    ` - : "" - } -
    -
    `; - } - const turns = state.chatMessages - .map((m, i) => chatTurnHtml(m, i, roving)) - .join(""); - // The in-flight answer is a row of its own so the keyboard can sit on it while it - // fills. `#chat-stream` is the element `paintChatStream` writes tokens into — as text, - // never markup — which is what keeps a streaming answer off the full-render path. - const live = - state.chatStreaming === null - ? "" - : `
    -
    B2
    -
    ${escapeHtml(state.chatStreaming)}
    -
    `; - return `
    ${turns}${live}
    `; -} - -/** One turn: the question as typed, or the answer — rendered through the sanitizing - * Markdown seam (E5) — with its citations as rows beneath it. */ -function chatTurnHtml(m: ChatMessage, index: number, roving: string | null): string { - const key = turnRowKey(index); - const row = (cls: string, role: string, body: string): string => - `
    -
    ${role}
    - ${body} -
    `; - if (m.role === "user") { - return row("chat-question", "You", `
    ${escapeHtml(m.text)}
    `); - } - if (m.error !== undefined) { - return row( - "chat-answer chat-failed", - "B2", - `
    ${escapeHtml(m.error)}
    `, - ); - } - const stopped = m.cancelled - ? `

    Stopped — this answer is partial.

    ` - : ""; - // Which B2 tools the answer was built from (a *Why?* turn). Tool names are the - // model's own words, so they are escaped like every other value in the chrome (E5). - const line = toolsLine(m.tools); - const used = line ? `

    ${escapeHtml(line)}

    ` : ""; - const cites = m.citations - .map( - (c) => ``, - ) - .join(""); - return ( - row( - "chat-answer", - "B2", - `
    ${renderMarkdown(m.text)}
    ${stopped}${used}`, - ) + cites - ); -} - -/** - * The composer. A ` -
    - ${ - streaming - ? `` - : `` - } - ${note ? `${escapeHtml(note)}` : ""} -
    - `; -} - -/** - * The setup card — deliberately **Ollama-native** (GH #151: guided setup is a per-runtime - * feature, and Ollama is the runtime B2 guides), shown both as the chat pane's empty state - * and inside Settings → Chat. - * - * Three cards, one builder, because they are the same facts in a different order: no - * server (start the daemon), no model (pull one — sized to this machine, illustrative and - * non-binding), and the Settings copy of both. A non-Ollama endpoint gets the message and - * nothing else: there is no honest instruction to give about pulling a model into LM - * Studio or a cloud provider. - */ -function chatSetupCardHtml(setup: ChatSetup | null, inSettings: boolean): string { - if (!setup) return ""; - const ollama = setup.ollama; - const message = setup.message - ? `

    ${escapeHtml(setup.message)}

    ` - : ""; - // What's installed, when the daemon answered — so "no model" can offer what *is* there - // instead of only naming what isn't. - // - // The pane's card only. In Settings the same inventory *is* the Model field (a picker, - // `chatModelFieldHtml`), and two controls setting one value is two things to keep in - // step — the second of which would apply on click while the first waits for Save. - const installed = - !inSettings && ollama && ollama.running && ollama.installed.length > 0 - ? `
    -
    Installed models
    -
      ${ollama.installed - .map( - (m) => - `
    • ${escapeHtml( - [m.parameters ?? "", formatModelSize(m.size)].filter(Boolean).join(" · "), - )}
      • `, - ) - .join("")}
    -
    ` - : ""; - // Pane-only for the same reason as the inventory: in Settings the **Local** section's - // own copy already carries `ollama pull` sized to this machine (`localNoteHtml`), and - // one command printed twice on one screen reads as two different instructions. - const suggestion = - !inSettings && ollama && ollama.suggested - ? `
    -
    Suggested for this machine
    -

    ${ - ollama.ram_gb ? `${ollama.ram_gb} GB of memory` : "This machine" - } — a ${escapeHtml(ollama.suggested.size)} model. Illustrative, not a requirement.

    -

    ${escapeHtml( - pullCommand(ollama.suggested.model), - )}

    -
    ` - : ""; - const tiers = - ollama && ollama.tiers.length > 0 - ? `
    Sizes by memory -
      ${ollama.tiers - .map( - (t) => - `
    • ${escapeHtml(t.ram)} → ${escapeHtml(t.size)} ${escapeHtml( - t.model, - )}
      • `, - ) - .join("")}
    -
    ` - : ""; - const install = - ollama && !ollama.running - ? `

    B2 talks to any OpenAI-compatible model server; Ollama is the - one it can walk you through. Start it with ollama serve — or, if it - isn’t installed yet, the - Ollama quickstart is the install and the - first pull, in that order.

    ` - : ""; - // A retry, in the pane only: the card's whole job is to be looked at while the user - // goes and fixes something (starts the daemon, pulls a model), so it must be able to - // notice that they did — without making them close and reopen the pane. Settings has - // its own re-probe, spelled *Save and test*. - const recheck = inSettings - ? "" - : `
    `; - return `
    - ${inSettings ? "" : `
    Chat isn’t ready yet
    `} - ${message} - ${recheck} - ${install} - ${suggestion} - ${installed} - ${tiers} - ${ - inSettings - ? "" - : `

    Everything else in B2 — search, discovery, editing — - works exactly as it does with chat off.

    ` - } -
    `; -} - -// --- the anchored ghost graph (GH #22) ---------------------------------------------- -// -// The center pane's third mode: the open note's typed neighborhood as hand-rolled, -// deterministic SVG — scene geometry from `graph.ts` (pure, unit-tested), markup -// here, clicks delegated in main.ts. The reading key: color = edge category, solid = -// authored / dashed teal = latent (`similar`), disc = note / square = resource / -// dashed hollow = dangling. Everything renders from state the note-open already -// fetched, so entering the graph costs no IPC. - -/** The graph toggle chip, shared by the reading bar (off) and the graph bar (on). - * - * Its chord comes out of the live registry rather than being spelled here: ⌘G is - * rebindable (#121), and a tooltip naming the shipped default would be wrong for exactly - * the user who changed it. */ -function graphToggleHtml(active: boolean): string { - const chord = escapeHtml(displayKeys(["graph.toggle"])); - return ``; -} - -/** The Edit chip, shared by the reading bar and the graph bar. Its chord comes out of the - * live registry for `graphToggleHtml`'s reason; `title` overrides the tooltip when the - * chip is disabled for a reason worth naming. */ -function editToggleHtml(disabled: boolean, title?: string): string { - const hint = title ?? `Edit this note — ${displayKeys(["edit.toggle"])} (autosaves as you type)`; - return ``; -} - -/** Fixed-point SVG coordinate — keeps the markup compact and diff-stable. */ -function px(v: number): string { - return (Math.round(v * 10) / 10).toString(); -} - -/** One edge's path (a straight segment, or the parallel-separating quadratic). */ -function edgePathD(e: GraphEdge): string { - return e.cx === null || e.cy === null - ? `M ${px(e.x1)} ${px(e.y1)} L ${px(e.x2)} ${px(e.y2)}` - : `M ${px(e.x1)} ${px(e.y1)} Q ${px(e.cx)} ${px(e.cy)} ${px(e.x2)} ${px(e.y2)}`; -} - -function edgeHtml(e: GraphEdge): string { - if (e.ghost) { - return ``; - } - const verb = e.label.replace(/[^a-z0-9-]/gi, ""); - const marker = e.arrow ? ` marker-end="url(#garr-${e.category})"` : ""; - const label = `${escapeHtml(e.label)}`; - return `${label}`; -} - -/** A node's shape + glyph, by kind (labels are added by the group builder). */ -function nodeShapeHtml(n: GraphNode): string { - const x = px(n.x); - const y = px(n.y); - switch (n.kind) { - case "anchor": - return ` - - `; - case "resource": { - const s = NODE_R.resource - 2; - return ` - ${sceneIcon(resourceIcon(n.sub), n.x, n.y, 16, "gglyph")}`; - } - case "dangling": - return ` - ${sceneIcon("exclamation-triangle", n.x, n.y, 16, "gglyph")}`; - default: - return ``; - } -} - -/** The tooltip line(s) for a node — also the activation affordance's explanation. - * Phrased for both hands: an activatable node says "⏎", because the graph is - * reachable by Tab and arrow keys too (K1, GH #78), not by mouse alone. */ -function nodeTitle(n: GraphNode): string { - switch (n.kind) { - case "anchor": - return `${n.full} — the open note. Click or ⏎ to return to reading.`; - case "ghost": - // The strength figure when there is one; a bare "similar but not linked" - // otherwise. "similarity ?" claimed a measurement existed and was merely - // unavailable — an ungraded candidate was never measured at all. - return `${n.full} — similar but not linked${ - n.sub ? ` (${n.sub} above this note's other candidates)` : "" - }. Click or ⏎ to link it; right-click (or ${displayKeys(["menu.open"])}) for more.`; - case "dangling": - return `${n.full} resolves to no note or file — fix the link in the note.`; - case "resource": - return `${n.full} (${n.sub ?? "file"}) — click or ⏎ to open.`; - default: - return `${n.full} — click or ⏎ to open.`; - } -} - -/** The accessible name for a focusable node — what a screen reader announces, and - * what the node's own `` can't be (that one is the mouse tooltip prose). */ -function nodeAriaLabel(n: GraphNode): string { - switch (n.kind) { - case "anchor": - return `${n.full} — the open note; back to reading`; - case "ghost": - return `${n.full} — similar but unlinked; link it`; - case "resource": - return `${n.full} — open this file`; - default: - return `${n.full} — open this note`; - } -} - -/** - * One scene node as an interactive `<g>`, its incident edges inside it so a pure-CSS - * hover lights the node *and* its edges while the rest of the scene dims. The click - * affordance rides existing delegation: notes reuse `data-open`, resources - * `data-open-resource`; ghosts get `data-ghost-link` (→ the link palette) plus the - * `data-card-*` pair the right-click menu reads; the anchor toggles back to reading. - * - * An activatable node is also a **keyboard** control (K1, GH #78): `tabindex="0"` puts - * it in the Tab order and `role="button"` says what it is, so a graph is walkable and - * openable with no mouse. main.ts turns ⏎/Space on a focused node into the same click - * these attributes already answer — one activation path, not two. A `dangling` node - * stays inert: it opens nothing (there's nothing there), so it isn't a tab stop. - * - * It also carries `data-gnode` — the scene id (graph.ts), which is what the note pane's - * focus restoration re-finds it *by* after an `innerHTML` swap destroys the element - * itself (GH #91). The discovery row's `data-side-row` for the graph: an identity that - * outlives the repaint, not a pointer into it. - */ -function nodeGroupHtml(n: GraphNode, edges: GraphEdge[], order: number): string { - const attrs: string[] = [`class="gnode is-${n.kind}"`, `style="--i:${order}"`]; - if (n.kind === "note" && n.path) attrs.push(`data-open="${escapeHtml(n.path)}"`); - if (n.kind === "anchor") attrs.push(`data-toggle-graph="1"`); - if (n.kind === "resource" && n.path) attrs.push(`data-open-resource="${escapeHtml(n.path)}"`); - if (n.kind === "ghost" && n.path) { - attrs.push( - `data-ghost-link="${escapeHtml(n.path)}"`, - `data-card-path="${escapeHtml(n.path)}"`, - `data-card-title="${escapeHtml(n.title ?? "")}"`, - ); - } - if (n.kind !== "dangling") { - attrs.push( - `tabindex="0"`, - `role="button"`, - `aria-label="${escapeHtml(nodeAriaLabel(n))}"`, - `data-gnode="${escapeHtml(n.id)}"`, - ); - } - const r = NODE_R[n.kind]; - // Text goes on the side of the node facing *away* from the anchor (above for the - // upper half of the scene), so a label never sits in its own edge's path. - const above = n.kind !== "anchor" && n.y < VIEW_H / 2 - 20; - const label = `<text class="gnode-label" x="${px(n.x)}" y="${px( - above ? n.y - r - 14 : n.y + r + 18, - )}">${escapeHtml(n.label)}</text>`; - const sub = n.sub - ? `<text class="gnode-sub" x="${px(n.x)}" y="${px( - above ? n.y - r - 29 : n.y + r + 33, - )}">${escapeHtml(n.sub)}</text>` - : ""; - return `<g ${attrs.join(" ")}> - <title>${escapeHtml(nodeTitle(n))} - ${edges.map(edgeHtml).join("")} - ${nodeShapeHtml(n)} - ${label}${sub} - `; -} - -/** The honest ghost-halo caveat (coverage.ts's tiers, like `searchCaveat`, #26): why there are - * no ghosts right now, or null when there are (or when silence is the honest state). */ -function ghostHintHtml(state: AppState): string { - if (state.similar.length > 0) return ""; - if (state.discoveringSimilar) - return `
    scanning for latent connections…
    `; - const c = coverage(state); - if (!c.model) - return `
    ghost connections need the semantic model — run b2 init, then Reindex
    `; - if (c.embedded === "none" || c.embedded === "partial") - return `
    ghosts appear once the vault is embedded — Reindex
    `; - return ""; -} - -/** The centered guidance when there's nothing to draw (the anchor always shows). */ -function graphEmptyHtml(state: AppState, scene: GraphScene): string { - if (scene.edges.length > 0) return ""; - if (state.discoveringSimilar) return ""; - return `

    No connections yet.

    -

    B2 floats similar-but-unlinked notes here as ghosts — click one to make the connection real.

    `; -} - -/** The reading key, one quiet strip: verb colors, edge states, node shapes. */ -function graphLegendHtml(): string { - const cats: [Category, string][] = [ - ["references", "references"], - ["supports", "supports"], - ["contradicts", "contradicts"], - ]; - const dots = cats - .map(([c, label]) => `${label}`) - .join(""); - return ``; -} - -/** Arrowhead markers, one per category (an SVG marker can't inherit its edge's - * stroke everywhere yet). */ -function graphDefsHtml(): string { - const cats: Category[] = ["references", "supports", "contradicts", "other"]; - const arrow = (id: string, cls: string) => - ` - - `; - return `${cats.map((c) => arrow(`garr-${c}`, `garr cat-${c}`)).join("")}`; -} - -/** - * The graph pane — the note pane's third mode (Reading / Editing / Graph). Bar: - * the same action chips as reading; stage: the SVG scene (fills the pane, - * `viewBox`-scaled) with overlay hints; footer: the reading key. - */ -function graphPaneHtml(state: AppState, n: NoteView): string { - const scene = buildScene({ - anchor: { path: n.path, title: n.title }, - connections: state.connections, - resources: state.resourceLinks, - unresolved: state.unresolved, - ghosts: state.similar, - }); - - // Edges live inside their node's group (hover affordance); the anchor renders - // last so it always paints on top of edge crossings. - const byNode = new Map(); - for (const e of scene.edges) { - const owner = e.from === "anchor" ? e.to : e.from; - const list = byNode.get(owner) ?? []; - list.push(e); - byNode.set(owner, list); - } - // Paint order: ghosts lowest (their long dashed spokes must pass *under* the - // authored orbit), authored above them, the anchor on top of everything. The - // stagger index is narrative, not paint, order: authored pops first, ghosts after. - const authoredNodes = scene.nodes.filter((node) => node.kind !== "anchor" && node.kind !== "ghost"); - const ghostNodes = scene.nodes.filter((node) => node.kind === "ghost"); - const anchor = scene.nodes.find((node) => node.kind === "anchor"); - const groups = [ - ...ghostNodes.map((node, i) => - nodeGroupHtml(node, byNode.get(node.id) ?? [], authoredNodes.length + 1 + i), - ), - ...authoredNodes.map((node, i) => nodeGroupHtml(node, byNode.get(node.id) ?? [], i + 1)), - ...(anchor ? [nodeGroupHtml(anchor, [], 0)] : []), - ].join(""); - - return `
    -
    -
    - ${graphToggleHtml(true)} - ${editToggleHtml(state.loading)} -
    -
    -
    - - ${graphDefsHtml()} - ${groups} - - ${graphEmptyHtml(state, scene)} - ${ghostHintHtml(state)} -
    - ${graphLegendHtml()} -
    `; -} - -// --- Explain: the Compare view for one Similar card (GH #236) ------------------------ -// -// The centre pane's fourth mode: the open note against one of its Similar cards, read -// model-free from the same computation as the list (`Vault::explain_similar`), so its -// rank, grade and best passage are the card's. The words and the strip's geometry are -// explain.ts's; this is only paint. Passage text is note content, so it is escaped and -// shown as text (E5), never rendered as Markdown. Every control is a real button in tab -// order, and Escape backs out, as it does from the graph (K1). - -function explainPaneHtml(state: AppState): string { - const ec = state.explainCard; - const bar = `
    - -
    `; - if (!ec) return bar; - if (ec.error !== null) { - return `
    ${bar}

    ${escapeHtml(ec.error)}

    `; - } - const v = ec.view; - if (v === null) { - return `
    ${bar}
    `; - } - const name = (x: { path: string; title: string | null }) => escapeHtml(x.title ?? x.path); - const names = explainNames(v); - const band = strengthBand(v.z); - const summary = explainSummary(v); - const whole = wholeNoteText(v); - - const strip = fieldStrip(v.population, v.z); - const W = 600; - const H = 44; - const help = strip - ? `` - : ""; - const fieldHtml = strip - ? `
    -

    Where it sits ${help}

    - ${ - ec.help - ? `
    ${fieldHelp(v) - .map((t) => `

    ${escapeHtml(t)}

    `) - .join("")}
    ` - : "" - } - - - - ${strip.ticks - .map( - (t) => - ``, - ) - .join("")} - ${[strip.clear, strip.strong] - .map((x) => ``) - .join("")} - ${strip.dots - .map((x) => ``) - .join("")} - ${ - strip.marker !== null - ? `` - : "" - } - - - -

    ${escapeHtml( - fieldCaption(v.population, names.anchor), - )} · σ from their average

    -
    ` - : `

    Ungraded: ranked by nearness only. Grading needs ${STRENGTH_MIN_CANDIDATES} or more notes to compare against.

    `; - - const passage = (who: string, p: PassageView) => - `
    -
    ${escapeHtml(who)}${ - p.heading_path ? ` · ${escapeHtml(p.heading_path)}` : "" - }
    -
    ${escapeHtml(p.text)}
    -
    `; - const shown = ec.allPairs ? v.pairs : v.pairs.slice(0, EXPLAIN_PAIRS_SHOWN); - const pairsHtml = v.pairs.length - ? `
    -

    Passage pairs · each of its passages, with this note’s nearest

    -
      ${shown - .map( - (p, i) => `
    1. -
      #${i + 1} ${strengthHtml(p.z ?? undefined)}${ - p.identical ? `identical text` : "" - }
      -
      ${passage("This note", p.anchor)}${passage( - "Suggested", - p.candidate, - )}
      -
      2. `, - ) - .join("")}
    - ${ - v.pairs.length > EXPLAIN_PAIRS_SHOWN - ? `` - : "" - } -
    ` - : ""; - - const links = v.shared_neighbors.length - ? `

    Both link with ${v.shared_neighbors - .map( - (s) => - ``, - ) - .join(", ")}.

    ` - : `

    They share no linked notes.

    `; - - return `
    ${bar} -
    -
    -
    Explain · suggested for ${name(v.anchor)}
    -

    ${name(v.candidate)}

    -
    ${escapeHtml(v.candidate.path)}
    -

    ${band ? `${strengthHtml(v.z ?? undefined)} ` : ""}${escapeHtml( - standingText(v), - )}

    - ${whole ? `

    ${escapeHtml(whole)}

    ` : ""} -
    - ${ - summary - ? `
    ${escapeHtml(summary.label)}. ${escapeHtml( - summary.detail, - )}
    ` - : "" - } - ${fieldHtml} - ${pairsHtml} -

    Links

    ${links}
    -
    - - -
    -
    -
    `; -} - -/** A cumulative-duration label from milliseconds: "3h 25m", "12m 04s", "45s", "0s". */ -function formatDuration(ms: number): string { - const totalSec = Math.round(ms / 1000); - const h = Math.floor(totalSec / 3600); - const m = Math.floor((totalSec % 3600) / 60); - const s = totalSec % 60; - if (h > 0) return `${h}h ${String(m).padStart(2, "0")}m`; - if (m > 0) return `${m}m ${String(s).padStart(2, "0")}s`; - return `${s}s`; -} - -// The per-model embedding-time ledger (b2-desktop stats.rs): a running total per model, -// summed across every reindex since you selected it, so a model swap can be judged on -// real speed. Switching to a model restarts its total (the swap re-embeds the whole -// corpus), so each row covers only that model's current stint — the copy says so. One row -// per model that has history: total time, chunks, and derived throughput, current marked. -function embedStatsHtml(state: AppState): string { - const byModel = new Map(state.embedStats.map((s) => [s.model, s])); - // Order by the picker so rows are stable; only models with recorded time appear. - const rows = state.models - .map((m) => ({ model: m, stat: byModel.get(m.id) })) - .filter((r) => r.stat && r.stat.chunks > 0); - const head = - `
    Embedding time
    ` + - `

    Running total per model, summed across every reindex since you selected it. Switching models restarts the total.

    `; - if (rows.length === 0) { - return ( - head + - `

    No embedding runs recorded yet — Reindex to start measuring.

    ` - ); - } - const list = rows - .map(({ model, stat }) => { - const s = stat!; - const perSec = s.total_ms > 0 ? (s.chunks / (s.total_ms / 1000)).toFixed(1) : "—"; - const marker = model.current ? ` current` : ""; - return `
    - ${escapeHtml(model.label)}${marker} - ${formatDuration(s.total_ms)} · ${s.chunks.toLocaleString()} chunks · ${perSec} chunks/sec -
    `; - }) - .join(""); - return head + `
    ${list}
    `; -} - -// --- Settings (⌘,) -------------------------------------------------------------- -// -// A tabbed surface over a rail (settingstabs.ts) — General, Index, Embedding, Chat, -// Keyboard — rather than the one scrolling column it grew out of, and since it outgrew a -// floating box too it takes the whole window (`settingsScreenHtml` below). It keeps the -// link modal's `.field` chrome, so a section is written as a form and nothing about the -// surface it lands on is a section's business. -// -// **Every control in here carries a stable `id`**, and that is load-bearing, not tidy: -// `#modal-root` is swapped wholesale on a repaint, so main.ts's `captureModalFocus` can -// only put the keyboard back on what it was on by re-finding it by id after the swap -// (crates/b2-desktop/CLAUDE.md, "Two things that bite"). A settings control with no id -// is a control that ejects the keyboard to `` the moment it's used. - -/** The panel for one section. Split per tab rather than one long builder so a new - * section is a `case` plus a builder, and the others can't shift under it. */ -function settingsPanelHtml(state: AppState): string { - switch (state.settingsTab) { - case "general": - return generalPanelHtml(state); - case "index": - return indexPanelHtml(state); - case "embedding": - return embeddingPanelHtml(state); - case "chat": - return chatPanelHtml(state); - case "keyboard": - return keyboardPanelHtml(state); - } -} - -// Chat — which model answers your questions, and where it runs. The spec's two named -// configurations (GH #151), and they are one setting rather than two: **Local** is a -// localhost endpoint (Ollama's, unless pointed elsewhere) and **Cloud models** is a -// provider's, so the segmented control below is a *view* of the URL, not a second piece -// of state to keep in step with it. -// -// The privacy copy sits beside the Cloud fields deliberately: **the consent moment is the -// configuration moment** (invariant M5 — note content never leaves the machine unbidden), -// informed where the decision is made rather than by a popup later. -// -// None of this is vault or index state. Changing the chat model costs no reindex — the -// contrast with M2 that makes "change models at any time" true by construction. -function chatPanelHtml(state: AppState): string { - const setup = state.chatSetup; - const cloud = state.chatCloud; - const modes: { id: "local" | "cloud"; label: string }[] = [ - { id: "local", label: "Local" }, - { id: "cloud", label: "Cloud models" }, - ]; - const segments = modes - .map((m) => { - const on = cloud === (m.id === "cloud"); - return ``; - }) - .join(""); - // The status line, in the setup card's own words when there's a problem — the same - // sentence the chat pane shows, from the same probe. - const status = ((): string => { - if (!setup) return `

    Checking the model server…

    `; - if (setup.state === "ready") - return `

    Connected · ${escapeHtml(setup.model)}

    `; - if (setup.state === "fake") - return `

    ${escapeHtml(setup.message ?? "")}

    `; - return chatSetupCardHtml(setup, true); - })(); - const key = cloud ? cloudKeyHtml(setup) : localNoteHtml(setup); - return `
    Chat model
    -

    Grounded chat answers only from passages B2 retrieves - from this vault. Changing the model costs no reindex — nothing about chat is stored.

    -
    - Where it runs -
    ${segments}
    -
    - - ${chatModelFieldHtml(state, setup)} - ${key} - ${toolCapFieldHtml(setup)} -
    - -
    - ${status}`; -} - -/** - * **Tool calls per reply** — the cap on what one model reply may ask B2 to run - * (`LlmConfig::max_tool_calls`). A safety bound, not a tuning knob, and the copy says so: - * a reply past it is stopped with an error, and the only reason to raise it is a model - * that really does ask for that many. Painted from the host's own numbers (`tool_calls`), - * saved with the rest of the panel by *Save and test*, validated by chat.ts's - * `toolCapInput`. - * - * `type="text"` with a numeric keypad rather than `type="number"`: `captureModalFocus` - * re-selects the focused field's caret across a repaint, and a number input throws on - * `setSelectionRange`. - */ -function toolCapFieldHtml(setup: ChatSetup | null): string { - if (!setup) return ""; - const cap = setup.tool_calls; - return ` -

    The most B2 tools the chat model may call in one - reply when it explains a suggestion. A reply that asks for more is stopped with an - error instead of being run. Default ${cap.default}, up to ${cap.ceiling}; clear the - field to go back to the default. Overrides B2_LLM_MAX_TOOL_CALLS.

    `; -} - -/** - * The **Model** field — a picker over what the daemon actually has, or a text box. - * - * Two shapes for one value, chosen by whether there is an inventory to pick from. A - * typed model name is the commonest local-setup mistake there is (the daemon is up, the - * name is just not one it has), and the fix was previously to read it off a card *after* - * getting it wrong. When `/api/tags` answered, the list of installed models is simply - * what the field offers. - * - * The text box stays reachable on purpose, and is not a fallback: a list of *installed* - * models structurally cannot contain the one you are pulling right now, and naming it - * before the pull finishes is a real thing to do. So the picker carries a way out - * (`chatModelTyped`), and the way back is beside the box. - * - * Both shapes carry the **same id**, because the id is the contract: `saveChatConfig` - * reads `.value` off it (a `` agree on that), and - * `captureModalFocus` puts the keyboard back on it by id after the repaint. - */ -function chatModelFieldHtml(state: AppState, setup: ChatSetup | null): string { - const ollama = setup?.ollama; - // A **Local** control by definition: `/api/tags` is one daemon's inventory, which - // says nothing about what a cloud provider serves. `chatCloud` and not `setup.cloud` - // because the view flag turns over the instant *Cloud models* is pressed, while the - // setup is whatever the last probe found — and nothing re-probes on that press (the - // URL field is deliberately cleared, there being no default provider). Reading the - // stale answer would leave a picker of this machine's models under an empty cloud - // endpoint until Save and test, with *Type a model name* standing between the user - // and the field they came to fill in. - const installed = !state.chatCloud && ollama?.running ? ollama.installed : []; - const current = setup?.model ?? ""; - if (state.chatModelTyped || installed.length === 0) { - // The way back, offered only when there is something to go back *to*. - const pick = - installed.length > 0 - ? `
    ` - : ""; - return ` - ${pick}`; - } - // The configured model leads the list when the daemon doesn't have it — dropping it - // would silently re-point the configuration at whatever happened to be first, as a side - // effect of *looking* at the field. - const missing = - current !== "" && !installed.some((m) => m.name === current) - ? `` - : ""; - const options = installed - .map((m) => { - const detail = [m.parameters ?? "", formatModelSize(m.size)].filter(Boolean).join(" · "); - return ``; - }) - .join(""); - return ` -
    - For one you haven’t pulled yet.
    `; -} - -// The **Local** configuration's whole note: nothing leaves, so there is no key field and -// no privacy warning to give — only the fact that makes the difference legible. -// -// Plus the one command that changes what the picker above can offer. Spelled with the -// suggested model when this machine's memory could be read and as a `` shape -// when it couldn't — either way beside the quickstart, which is the page carrying both -// the install and the pull for someone who has neither. -function localNoteHtml(setup: ChatSetup | null): string { - const suggested = setup?.ollama?.suggested?.model; - return `

    Local models keep everything on this machine — your - question and the retrieved passages never leave it. B2 talks to any - OpenAI-compatible server; Ollama is the one it can walk you through.

    -

    Add a model with - ${escapeHtml(pullCommand(PULL_PLACEHOLDER))}${ - suggested ? ` — e.g. ${escapeHtml(pullCommand(suggested))}` : "" - }, then pick it above. The Ollama quickstart - has the whole sequence.

    `; -} - -// The **Cloud models** key field, and the sentence saying where that key lives. -// -// Four states, because a user has to be told *before* they wonder (GH #176). B2 remembers -// a key in the macOS Keychain — encrypted at rest, and there next launch — but two of the -// four are cases where what they just did isn't quite what they'd assume: -// -// - `environment` — `B2_LLM_API_KEY` overrides anything saved here, so a key typed into -// this field is stored and *not used*. Saying so is the difference between a documented -// precedence and a field that silently does nothing. -// - `session` — the Keychain refused, so the key works now and is gone at quit. The -// degrade is deliberate (chat must not break on a locked keychain) but it is not -// something to discover at the next launch. -// -// The field itself always paints empty: a password input that echoed its secret back would -// be a worse idea than not showing it at all. Which is what makes an empty save mean -// "keep", and leaves Remove as the only way back to a keyless configuration — without it a -// key could never be removed, and repointing the endpoint would send the old provider's -// token to the new one. -function cloudKeyHtml(setup: ChatSetup | null): string { - const source = setup?.api_key_source ?? "none"; - const placeholder = - source === "none" - ? "sk-…" - : source === "environment" - ? "•••••••• (from your environment)" - : source === "stored" - ? "•••••••• (saved in your Keychain)" - : "•••••••• (this session only)"; - const where = { - none: "", - environment: `

    B2_LLM_API_KEY is set in your - environment, and that is the key in force — it overrides any key saved here. - Unset it in your shell to go back to the one B2 remembers.

    `, - stored: `

    Saved in your macOS Keychain — encrypted at rest, - and here the next time you open B2.

    `, - session: `

    Kept for this session only: B2 couldn’t save it to - your Keychain, so it will be gone when you quit. Saving again will retry.

    `, - }[source]; - // The closing sentence is where the key *lives*, and it has to agree with `where` above. - // Under `session` it cannot be the general "B2 saves it in your Keychain": this key is - // precisely the one B2 could not save, and a paragraph that says both is worse than - // either — a user reading "couldn't save it" and then "B2 saves the key" has no way to - // know which sentence is about them. - const storage = - source === "session" - ? `This key was not saved — B2 normally keeps it in your macOS Keychain, - never in a plain file. Set B2_LLM_API_KEY in your environment to have one - that persists regardless.` - : `B2 saves the key in your macOS Keychain, never in a plain file — set - B2_LLM_API_KEY in your environment to override it.`; - // Offered whenever there is a key to remove. Under `environment` it still has work to - // do — it clears the one B2 remembers — but it cannot touch a variable the app doesn't - // own, so the label says which key it means. - const remove = - source === "none" - ? "" - : `
    - Removes the key B2 saved. A key set in B2_LLM_API_KEY - is your environment's, and stays.
    `; - // Where to *get* an endpoint, since B2 ships no default cloud provider and never will - // (picking one is the explicit act M5 is about — see `setChatMode`). Ollama's hosted - // models are named because they are the one provider B2 already knows how to talk to - // without a second thought: the same `/v1` surface, the same model names as the local - // configuration. A link, though, not a pre-filled URL. - const whereToGet = `

    Any OpenAI-compatible provider works — - put its /v1 URL above. Ollama’s hosted models are one: - Ollama cloud.

    `; - return `${whereToGet} - - ${where} - ${remove} -

    - Cloud models send your question and the retrieved note passages to the - configured provider. Nothing else leaves your machine, and B2 still writes - nothing to your notes. ${storage} -

    `; -} - -// General — app-wide preferences that belong to no subsystem. Appearance is the only one -// today; this is the tab a vault or editor preference lands in rather than being wedged -// beside the embedding model, which it has nothing to do with. -function generalPanelHtml(state: AppState): string { - // Appearance: System (follow the OS) / Light / Dark. A segmented control rather than a - // ${options} - - ${detail} - ${deviceRow} - ${provisionRow} -

    Changing the model re-embeds the whole vault on the next - Reindex. A newly-chosen model is downloaded with the button above.

    - ${embedStatsHtml(state)} - ${ - state.modelsDir - ? `
    Model files
    -

    ${escapeHtml( - state.modelsDir, - )}

    ` - : "" - }`; -} - -// Keyboard — the discoverable half of invariant K1, and now its *editable* half too: one -// surface for the table, reached by `?` from anywhere or by walking the rail, where every -// chord B2 owns is a button that rebinds it (#121). The table itself is `shortcuts.ts` -// (GH #78); the algebra and the judgement are keymap.ts. -// -// The recorder is a strip at the top of the panel rather than a widget spliced into the -// row it edits. The grid is CSS multi-column, so an inline block would land wherever the -// column flow put it — and the panel is a page of table you scroll, so a control that -// appeared below the fold would be a control nobody saw. The chip being edited carries -// `.kbd-recording` instead, which is what ties the strip to its row. -function keyboardPanelHtml(state: AppState): string { - const changed = customized(DEFAULT_BINDINGS, state.keyOverrides).length; - const resetAll = changed - ? `` - : ""; - // The ⌘-hold sheet is a *gesture*, not a chord, so it has no row in the table below — - // and a keyboard affordance documented nowhere is the exact failure K1 names. The - // reference is where someone looks for it, so the reference is where it is said. - return `
    Keyboard shortcuts
    -

    B2 is fully operable from the keyboard — the mouse is an accelerator, never a requirement. Click a chord to change it. Hold ⌘ on its own anywhere in the app for a quick reminder of the ⌘ chords below.

    -
    ${resetAll}
    - ${recorderHtml(state)} - ${shortcutsGridHtml(state)}`; -} - -/** The recorder: what is being rebound, what has been pressed, and what that would mean. - * - * Empty markup when nothing is recording, so the strip costs no vertical space until it - * is asked for. Every control carries a stable `id` — the settings builder's rule above - * — because a captured chord repaints the dialog under the keyboard that captured it. */ -function recorderHtml(state: AppState): string { - const rec = state.recorder; - if (!rec) return ""; - const b = findBinding(activeBindings(), rec.id); - if (!b) return ""; - const now = b.keys.map((k) => `${escapeHtml(displayChord(k))}`).join(" "); - const captured = rec.candidate - ? `${escapeHtml(displayChord(rec.candidate))}` - : `Press a chord…`; - const lines = [ - ...(rec.hint ? [{ tier: "warn" as const, message: rec.hint }] : []), - ...rec.problems, - ] - .map( - (p) => - `
  • ${escapeHtml(p.message)}
    • `, - ) - .join(""); - const blocked = refused(rec.problems); - const canSave = rec.candidate !== null && !blocked; - const isChanged = state.keyOverrides[rec.id] !== undefined; - // `tabindex="-1"`: focusable so main.ts can take the keyboard off whatever had it (a - // tree row, or CodeMirror, which would otherwise type the chord into the buffer behind - // the dialog), but not a Tab stop — the strip is a target to press keys at, not a - // control to land on. The id is what `captureModalFocus` hands focus back by, which - // matters here more than anywhere: every captured chord repaints this dialog. - return `
    -
    - ${escapeHtml(b.label)} - now ${now} -
    -
    ${captured}
    - ${lines ? `
      ${lines}
    ` : ""} -

    Esc cancels, ⏎ accepts — every other key is recorded. If a chord you press never appears above, macOS or another app took it before B2 could see it; B2 can only tell you about the chord you actually pressed.

    -
    - - - ${isChanged ? `` : ""} -
    -
    `; -} - -/** Every chord the app answers to, grouped, from the one table in shortcuts.ts. The app - * menu bar's chords are *not* here — macOS prints those beside their own menu items and - * nothing in this panel could move them (shortcuts.ts's header says why they left). - * - * A chip that names a command is a ``; - }; - const groups = shortcuts() - .map( - (g) => `
    -

    ${escapeHtml(g.title)}

    -
    ${g.items - .map((s) => `
    ${s.keys.map(chip).join(" ")}
    ${escapeHtml(s.action)}
    `) - .join("")}
    -
    `, - ) - .join(""); - return `
    ${groups}
    `; -} - -/** - * The ⌘-hold sheet: what a held ⌘ can do, painted over the app while it is held - * (cmdhold.ts owns the machine and the projection; main.ts owns the timer). - * - * Empty markup when it isn't up, so the layer costs nothing the rest of the time — the - * same shape `recorderHtml` and `contextMenuHtml` use. Empty markup too when the - * projection has nothing in it, which is reachable: every ⌘ chord is rebindable, so a user - * can move the lot off ⌘ and would otherwise get an empty card for their trouble. - * - * Three things it deliberately isn't. It is not a dialog: nothing here is focusable, focus - * does not move to it and does not come back, because the whole gesture is one the user is - * already mid-way through — taking the keyboard would cancel the chord they are about to - * press. It is `pointer-events: none` in the stylesheet for the same reason on the other - * device: ⌘-click and ⌘-drag have to keep landing on the app underneath. And it is - * `aria-hidden`, which reads as a strange thing to say about a keyboard aid until you - * consider who it would be talking to — a screen-reader user has the whole reference in - * Settings → Keyboard, reachable by chord, and what an aria-live region would add here is - * a page of chords announced every time a modifier is held a beat too long. The reference - * is the accessible surface; this is a glance. - */ -export function cmdSheetHtml(state: AppState): string { - if (!state.cmdSheet) return ""; - const groups = cmdShortcuts(); - if (groups.length === 0) return ""; - const body = groups - .map( - (g) => `
    -

    ${escapeHtml(g.title)}

    -
    ${g.items - .map( - (s) => - `
    ${s.keys.map((k) => `${escapeHtml(k.text)}`).join(" ")}
    ` + - `
    ${escapeHtml(s.action)}
    `, - ) - .join("")}
    -
    `, - ) - .join(""); - // The footer says both halves of the contract: how it goes away (the thing a user who - // did not mean to summon it needs first), and where the rest of the keyboard lives — - // this sheet is the ⌘ slice, and someone reading it is exactly the someone who would - // want the whole table. - return ``; -} - -/** The tab id's element id — the one both the rail and `aria-labelledby` derive from, - * and what main.ts re-focuses a tab by after the modal repaints. */ -function tabDomId(id: SettingsTabId): string { - return `settings-tab-${id}`; -} - -/** - * Settings: a vertical rail of sections beside the active panel, taking **the whole - * window** rather than floating in a box. - * - * It was a floating dialog until it stopped fitting in one. Five sections, and the two - * ends of the range don't want the same rectangle: Chat is a provider configuration with - * a setup card and a page of privacy copy, Keyboard is forty rows of chord table, and - * General is a three-button theme switch. A fixed box sized for the long ones leaves the - * short ones mostly empty and *still* puts the rest below the fold. A surface that big - * has stopped being an interruption you dismiss and become a place you go, so it says - * so — it covers the app, and the panel gets the whole remaining rectangle. - * - * Modal semantics are unchanged (`role="dialog"` + `aria-modal`, the ⇥ trap, Escape, the - * focus return in main.ts): the app is still underneath, and Done is still where you came - * from. What goes with the box is the **backdrop** — there is no "outside" left to click, - * so the ways out are Done and Escape, and main.ts's click handler dropped that branch to - * match. The three rows are fixed header / scrolling panel / fixed footer, which is what - * keeps Done and the key hints on screen no matter how long a section runs. - * - * DOM order is load-bearing: rail, then panel, then Done. `focusIntoOverlay` opens - * Settings on `overlayFocusables()[0]` and documents that as "the selected tab", which is - * true only while nothing focusable precedes the rail — hence a header that carries the - * title alone and a Done button that stays in the footer. - * - * The rail is the ARIA `tabs` pattern (settingstabs.ts owns the moves): `role="tablist"`, - * one `role="tab"` per section, and a **roving `tabindex`** so the whole rail is a single - * Tab stop — a settings surface whose Tab sequence starts with N section buttons is one - * you Tab *past*, not through. The panel carries `tabindex="0"` on purpose even when it - * holds its own controls: it is the scroll container, and a region you can't focus is a - * region you can't scroll without the mouse (the Keyboard section is a page of table and - * nothing else, so this is the only way to read past the fold). - */ -function settingsScreenHtml(state: AppState): string { - const active = state.settingsTab; - const tabs = SETTINGS_TABS.map((t) => { - const on = t.id === active; - return ``; - }).join(""); - // `.settings-measure` caps the line length inside a panel that is now as wide as the - // window: prose set across 1600px is prose nobody reads back to the start of. Keyboard - // is the one section that isn't prose — a two-column reference read in columns — so it - // takes the wider measure, and the choice is here rather than in the panel builder - // because it is a fact about the *surface*, not about what the section says. - const measure = - active === "keyboard" ? "settings-measure settings-measure-wide" : "settings-measure"; - return ``; -} - /** One menu row. `chord` names the direct shortcut where one exists — a menu is where * a keyboard user *learns* the chord that lets them skip the menu next time — so it * comes out of the live registry (`displayKeys`), never spelled here. */ diff --git a/ui/src/settingstabs.ts b/ui/src/settingstabs.ts index 27574cb..9ca3567 100644 --- a/ui/src/settingstabs.ts +++ b/ui/src/settingstabs.ts @@ -69,6 +69,12 @@ export const SETTINGS_TABS: SettingsTab[] = [ /** The default section — what ⌘, opens on the first time in a session. */ export const DEFAULT_SETTINGS_TAB: SettingsTabId = "general"; +/** A tab's element id — the one the rail paints, `aria-labelledby` points at, and + * main.ts re-focuses a tab by after the surface repaints. */ +export function tabDomId(id: SettingsTabId): string { + return `settings-tab-${id}`; +} + /** Whether an untrusted string (a `data-settings-tab` attribute, a stored preference) * names a real section. */ export function isSettingsTab(value: unknown): value is SettingsTabId { diff --git a/ui/src/settingsview.ts b/ui/src/settingsview.ts new file mode 100644 index 0000000..d162c9c --- /dev/null +++ b/ui/src/settingsview.ts @@ -0,0 +1,567 @@ +// Settings: the full-window tabbed surface and its panels (General, Index, Embedding, +// Chat, Keyboard). The rail's moves are settingstabs.ts's; the keyboard panel is +// keysview.ts's, the chat setup card chatview.ts's. + +import { escapeHtml } from "./escape.ts"; +import type { AppState } from "./state.ts"; +import { coverage } from "./coverage.ts"; +import { displayKeys } from "./bindings.ts"; +import { SETTINGS_TABS, tabDomId } from "./settingstabs.ts"; +import { + LOCAL_CHAT_ENDPOINT, + OLLAMA_CLOUD_URL, + OLLAMA_QUICKSTART_URL, + PULL_PLACEHOLDER, + modelDetail, + pullCommand, +} from "./chat.ts"; +import type { ChatSetup } from "./types.ts"; +import { reindexMeterHtml, segmentedHtml } from "./widgets.ts"; +import { chatSetupCardHtml } from "./chatview.ts"; +import { keyboardPanelHtml } from "./keysview.ts"; + +/** A cumulative-duration label from milliseconds: "3h 25m", "12m 04s", "45s", "0s". */ +function formatDuration(ms: number): string { + const totalSec = Math.round(ms / 1000); + const h = Math.floor(totalSec / 3600); + const m = Math.floor((totalSec % 3600) / 60); + const s = totalSec % 60; + if (h > 0) return `${h}h ${String(m).padStart(2, "0")}m`; + if (m > 0) return `${m}m ${String(s).padStart(2, "0")}s`; + return `${s}s`; +} + +// The per-model embedding-time ledger (b2-desktop stats.rs): a running total per model, +// summed across every reindex since you selected it, so a model swap can be judged on +// real speed. Switching to a model restarts its total (the swap re-embeds the whole +// corpus), so each row covers only that model's current stint — the copy says so. One row +// per model that has history: total time, chunks, and derived throughput, current marked. +function embedStatsHtml(state: AppState): string { + const byModel = new Map(state.embedStats.map((s) => [s.model, s])); + // Order by the picker so rows are stable; only models with recorded time appear. + const rows = state.models + .map((m) => ({ model: m, stat: byModel.get(m.id) })) + .filter((r) => r.stat && r.stat.chunks > 0); + const head = + `
    Embedding time
    ` + + `

    Running total per model, summed across every reindex since you selected it. Switching models restarts the total.

    `; + if (rows.length === 0) { + return ( + head + + `

    No embedding runs recorded yet — Reindex to start measuring.

    ` + ); + } + const list = rows + .map(({ model, stat }) => { + const s = stat!; + const perSec = s.total_ms > 0 ? (s.chunks / (s.total_ms / 1000)).toFixed(1) : "—"; + const marker = model.current ? ` current` : ""; + return `
    + ${escapeHtml(model.label)}${marker} + ${formatDuration(s.total_ms)} · ${s.chunks.toLocaleString()} chunks · ${perSec} chunks/sec +
    `; + }) + .join(""); + return head + `
    ${list}
    `; +} + +// --- Settings (⌘,) -------------------------------------------------------------- +// +// A tabbed surface over a rail (settingstabs.ts) — General, Index, Embedding, Chat, +// Keyboard — rather than the one scrolling column it grew out of, and since it outgrew a +// floating box too it takes the whole window (`settingsScreenHtml` below). It keeps the +// link modal's `.field` chrome, so a section is written as a form and nothing about the +// surface it lands on is a section's business. +// +// **Every control in here carries a stable `id`**, and that is load-bearing, not tidy: +// `#modal-root` is swapped wholesale on a repaint, so main.ts's `captureModalFocus` can +// only put the keyboard back on what it was on by re-finding it by id after the swap +// (crates/b2-desktop/CLAUDE.md, "Two things that bite"). A settings control with no id +// is a control that ejects the keyboard to `` the moment it's used. + +/** The panel for one section. Split per tab rather than one long builder so a new + * section is a `case` plus a builder, and the others can't shift under it. */ +function settingsPanelHtml(state: AppState): string { + switch (state.settingsTab) { + case "general": + return generalPanelHtml(state); + case "index": + return indexPanelHtml(state); + case "embedding": + return embeddingPanelHtml(state); + case "chat": + return chatPanelHtml(state); + case "keyboard": + return keyboardPanelHtml(state); + } +} + +// Chat — which model answers your questions, and where it runs. The spec's two named +// configurations (GH #151), and they are one setting rather than two: **Local** is a +// localhost endpoint (Ollama's, unless pointed elsewhere) and **Cloud models** is a +// provider's, so the segmented control below is a *view* of the URL, not a second piece +// of state to keep in step with it. +// +// The privacy copy sits beside the Cloud fields deliberately: **the consent moment is the +// configuration moment** (invariant M5 — note content never leaves the machine unbidden), +// informed where the decision is made rather than by a popup later. +// +// None of this is vault or index state. Changing the chat model costs no reindex — the +// contrast with M2 that makes "change models at any time" true by construction. +function chatPanelHtml(state: AppState): string { + const setup = state.chatSetup; + const cloud = state.chatCloud; + const segments = segmentedHtml( + "Chat model location", + "settings-chat-", + "data-chat-mode", + [ + { id: "local", label: "Local" }, + { id: "cloud", label: "Cloud models" }, + ], + cloud ? "cloud" : "local", + ); + // The status line, in the setup card's own words when there's a problem — the same + // sentence the chat pane shows, from the same probe. + const status = ((): string => { + if (!setup) return `

    Checking the model server…

    `; + if (setup.state === "ready") + return `

    Connected · ${escapeHtml(setup.model)}

    `; + if (setup.state === "fake") + return `

    ${escapeHtml(setup.message ?? "")}

    `; + return chatSetupCardHtml(setup, true); + })(); + const key = cloud ? cloudKeyHtml(setup) : localNoteHtml(setup); + return `
    Chat model
    +

    Grounded chat answers only from passages B2 retrieves + from this vault. Changing the model costs no reindex — nothing about chat is stored.

    +
    + Where it runs + ${segments} +
    + + ${chatModelFieldHtml(state, setup)} + ${key} + ${toolCapFieldHtml(setup)} +
    + +
    + ${status}`; +} + +/** + * **Tool calls per reply** — the cap on what one model reply may ask B2 to run + * (`LlmConfig::max_tool_calls`). A safety bound, not a tuning knob, and the copy says so: + * a reply past it is stopped with an error, and the only reason to raise it is a model + * that really does ask for that many. Painted from the host's own numbers (`tool_calls`), + * saved with the rest of the panel by *Save and test*, validated by chat.ts's + * `toolCapInput`. + * + * `type="text"` with a numeric keypad rather than `type="number"`: `captureModalFocus` + * re-selects the focused field's caret across a repaint, and a number input throws on + * `setSelectionRange`. + */ +function toolCapFieldHtml(setup: ChatSetup | null): string { + if (!setup) return ""; + const cap = setup.tool_calls; + return ` +

    The most B2 tools the chat model may call in one + reply when it explains a suggestion. A reply that asks for more is stopped with an + error instead of being run. Default ${cap.default}, up to ${cap.ceiling}; clear the + field to go back to the default. Overrides B2_LLM_MAX_TOOL_CALLS.

    `; +} + +/** + * The **Model** field — a picker over what the daemon actually has, or a text box. + * + * Two shapes for one value, chosen by whether there is an inventory to pick from. A + * typed model name is the commonest local-setup mistake there is (the daemon is up, the + * name is just not one it has), and the fix was previously to read it off a card *after* + * getting it wrong. When `/api/tags` answered, the list of installed models is simply + * what the field offers. + * + * The text box stays reachable on purpose, and is not a fallback: a list of *installed* + * models structurally cannot contain the one you are pulling right now, and naming it + * before the pull finishes is a real thing to do. So the picker carries a way out + * (`chatModelTyped`), and the way back is beside the box. + * + * Both shapes carry the **same id**, because the id is the contract: `saveChatConfig` + * reads `.value` off it (a `` agree on that), and + * `captureModalFocus` puts the keyboard back on it by id after the repaint. + */ +function chatModelFieldHtml(state: AppState, setup: ChatSetup | null): string { + const ollama = setup?.ollama; + // A **Local** control by definition: `/api/tags` is one daemon's inventory, which + // says nothing about what a cloud provider serves. `chatCloud` and not `setup.cloud` + // because the view flag turns over the instant *Cloud models* is pressed, while the + // setup is whatever the last probe found — and nothing re-probes on that press (the + // URL field is deliberately cleared, there being no default provider). Reading the + // stale answer would leave a picker of this machine's models under an empty cloud + // endpoint until Save and test, with *Type a model name* standing between the user + // and the field they came to fill in. + const installed = !state.chatCloud && ollama?.running ? ollama.installed : []; + const current = setup?.model ?? ""; + if (state.chatModelTyped || installed.length === 0) { + // The way back, offered only when there is something to go back *to*. + const pick = + installed.length > 0 + ? `
    ` + : ""; + return ` + ${pick}`; + } + // The configured model leads the list when the daemon doesn't have it — dropping it + // would silently re-point the configuration at whatever happened to be first, as a side + // effect of *looking* at the field. + const missing = + current !== "" && !installed.some((m) => m.name === current) + ? `` + : ""; + const options = installed + .map((m) => { + const detail = modelDetail(m); + return ``; + }) + .join(""); + return ` +
    + For one you haven’t pulled yet.
    `; +} + +// The **Local** configuration's whole note: nothing leaves, so there is no key field and +// no privacy warning to give — only the fact that makes the difference legible. +// +// Plus the one command that changes what the picker above can offer. Spelled with the +// suggested model when this machine's memory could be read and as a `` shape +// when it couldn't — either way beside the quickstart, which is the page carrying both +// the install and the pull for someone who has neither. +function localNoteHtml(setup: ChatSetup | null): string { + const suggested = setup?.ollama?.suggested?.model; + return `

    Local models keep everything on this machine — your + question and the retrieved passages never leave it. B2 talks to any + OpenAI-compatible server; Ollama is the one it can walk you through.

    +

    Add a model with + ${escapeHtml(pullCommand(PULL_PLACEHOLDER))}${ + suggested ? ` — e.g. ${escapeHtml(pullCommand(suggested))}` : "" + }, then pick it above. The Ollama quickstart + has the whole sequence.

    `; +} + +// The **Cloud models** key field, and the sentence saying where that key lives. +// +// Four states, because a user has to be told *before* they wonder (GH #176). B2 remembers +// a key in the macOS Keychain — encrypted at rest, and there next launch — but two of the +// four are cases where what they just did isn't quite what they'd assume: +// +// - `environment` — `B2_LLM_API_KEY` overrides anything saved here, so a key typed into +// this field is stored and *not used*. Saying so is the difference between a documented +// precedence and a field that silently does nothing. +// - `session` — the Keychain refused, so the key works now and is gone at quit. The +// degrade is deliberate (chat must not break on a locked keychain) but it is not +// something to discover at the next launch. +// +// The field itself always paints empty: a password input that echoed its secret back would +// be a worse idea than not showing it at all. Which is what makes an empty save mean +// "keep", and leaves Remove as the only way back to a keyless configuration — without it a +// key could never be removed, and repointing the endpoint would send the old provider's +// token to the new one. +function cloudKeyHtml(setup: ChatSetup | null): string { + const source = setup?.api_key_source ?? "none"; + const placeholder = + source === "none" + ? "sk-…" + : source === "environment" + ? "•••••••• (from your environment)" + : source === "stored" + ? "•••••••• (saved in your Keychain)" + : "•••••••• (this session only)"; + const where = { + none: "", + environment: `

    B2_LLM_API_KEY is set in your + environment, and that is the key in force — it overrides any key saved here. + Unset it in your shell to go back to the one B2 remembers.

    `, + stored: `

    Saved in your macOS Keychain — encrypted at rest, + and here the next time you open B2.

    `, + session: `

    Kept for this session only: B2 couldn’t save it to + your Keychain, so it will be gone when you quit. Saving again will retry.

    `, + }[source]; + // The closing sentence is where the key *lives*, and it has to agree with `where` above. + // Under `session` it cannot be the general "B2 saves it in your Keychain": this key is + // precisely the one B2 could not save, and a paragraph that says both is worse than + // either — a user reading "couldn't save it" and then "B2 saves the key" has no way to + // know which sentence is about them. + const storage = + source === "session" + ? `This key was not saved — B2 normally keeps it in your macOS Keychain, + never in a plain file. Set B2_LLM_API_KEY in your environment to have one + that persists regardless.` + : `B2 saves the key in your macOS Keychain, never in a plain file — set + B2_LLM_API_KEY in your environment to override it.`; + // Offered whenever there is a key to remove. Under `environment` it still has work to + // do — it clears the one B2 remembers — but it cannot touch a variable the app doesn't + // own, so the label says which key it means. + const remove = + source === "none" + ? "" + : `
    + Removes the key B2 saved. A key set in B2_LLM_API_KEY + is your environment's, and stays.
    `; + // Where to *get* an endpoint, since B2 ships no default cloud provider and never will + // (picking one is the explicit act M5 is about — see `setChatMode`). Ollama's hosted + // models are named because they are the one provider B2 already knows how to talk to + // without a second thought: the same `/v1` surface, the same model names as the local + // configuration. A link, though, not a pre-filled URL. + const whereToGet = `

    Any OpenAI-compatible provider works — + put its /v1 URL above. Ollama’s hosted models are one: + Ollama cloud.

    `; + return `${whereToGet} + + ${where} + ${remove} +

    + Cloud models send your question and the retrieved note passages to the + configured provider. Nothing else leaves your machine, and B2 still writes + nothing to your notes. ${storage} +

    `; +} + +// General — app-wide preferences that belong to no subsystem. Appearance is the only one +// today; this is the tab a vault or editor preference lands in rather than being wedged +// beside the embedding model, which it has nothing to do with. +function generalPanelHtml(state: AppState): string { + // Appearance: System (follow the OS) / Light / Dark. A segmented control rather than a + // ${options} + + ${detail} + ${deviceRow} + ${provisionRow} +

    Changing the model re-embeds the whole vault on the next + Reindex. A newly-chosen model is downloaded with the button above.

    + ${embedStatsHtml(state)} + ${ + state.modelsDir + ? `
    Model files
    +

    ${escapeHtml( + state.modelsDir, + )}

    ` + : "" + }`; +} + +/** + * Settings: a vertical rail of sections beside the active panel, taking **the whole + * window** rather than floating in a box. + * + * It was a floating dialog until it stopped fitting in one. Five sections, and the two + * ends of the range don't want the same rectangle: Chat is a provider configuration with + * a setup card and a page of privacy copy, Keyboard is forty rows of chord table, and + * General is a three-button theme switch. A fixed box sized for the long ones leaves the + * short ones mostly empty and *still* puts the rest below the fold. A surface that big + * has stopped being an interruption you dismiss and become a place you go, so it says + * so — it covers the app, and the panel gets the whole remaining rectangle. + * + * Modal semantics are unchanged (`role="dialog"` + `aria-modal`, the ⇥ trap, Escape, the + * focus return in main.ts): the app is still underneath, and Done is still where you came + * from. What goes with the box is the **backdrop** — there is no "outside" left to click, + * so the ways out are Done and Escape, and main.ts's click handler dropped that branch to + * match. The three rows are fixed header / scrolling panel / fixed footer, which is what + * keeps Done and the key hints on screen no matter how long a section runs. + * + * DOM order is load-bearing: rail, then panel, then Done. `focusIntoOverlay` opens + * Settings on `overlayFocusables()[0]` and documents that as "the selected tab", which is + * true only while nothing focusable precedes the rail — hence a header that carries the + * title alone and a Done button that stays in the footer. + * + * The rail is the ARIA `tabs` pattern (settingstabs.ts owns the moves): `role="tablist"`, + * one `role="tab"` per section, and a **roving `tabindex`** so the whole rail is a single + * Tab stop — a settings surface whose Tab sequence starts with N section buttons is one + * you Tab *past*, not through. The panel carries `tabindex="0"` on purpose even when it + * holds its own controls: it is the scroll container, and a region you can't focus is a + * region you can't scroll without the mouse (the Keyboard section is a page of table and + * nothing else, so this is the only way to read past the fold). + */ +export function settingsScreenHtml(state: AppState): string { + const active = state.settingsTab; + const tabs = SETTINGS_TABS.map((t) => { + const on = t.id === active; + return ``; + }).join(""); + // `.settings-measure` caps the line length inside a panel that is now as wide as the + // window: prose set across 1600px is prose nobody reads back to the start of. Keyboard + // is the one section that isn't prose — a two-column reference read in columns — so it + // takes the wider measure, and the choice is here rather than in the panel builder + // because it is a fact about the *surface*, not about what the section says. + const measure = + active === "keyboard" ? "settings-measure settings-measure-wide" : "settings-measure"; + return ``; +} diff --git a/ui/src/widgets.ts b/ui/src/widgets.ts new file mode 100644 index 0000000..a0dc3ce --- /dev/null +++ b/ui/src/widgets.ts @@ -0,0 +1,69 @@ +// Small pieces of markup more than one surface paints — kept in one place so the surfaces +// that share them (render.ts's panes, chatview.ts, explainview.ts, settingsview.ts, and +// main.ts's shell) can't drift apart on how they look or what they announce. + +import { escapeHtml } from "./escape.ts"; +import { strengthBand } from "./strength.ts"; + +/** A row's slot in the roving tabstop: exactly one row is tabbable, the rest are -1. */ +export function sideTab(key: string, roving: string | null): string { + return ` tabindex="${key === roving ? "0" : "-1"}"`; +} + +// The discovery card's strength cell: a banded read of the candidate's z +// (`strength.ts`), replacing the raw negated-L2 the card used to print (GH #150). +// No z → no cell: a statistic that wasn't computed isn't claimed (raw mode, tiny +// pools). The glyph is decorative; the band name is the accessible content. +export function strengthHtml(z: number | undefined): string { + const band = strengthBand(z); + if (!band) return ""; + // The figure rides in the markup and CSS reveals it on the selected/hovered card, so + // the number is one keystroke away rather than pointer-only (`title=` alone was a hole + // in K1). The accessible name carries *both* halves the eye gets — the band and the + // figure — because naming only the band would leave a screen reader with "clear match" + // and no way to reach the 2.5σ behind it. The figure's own span stays `aria-hidden`, so + // it is announced once (as part of this name) rather than twice. + return `${band.glyph}`; +} + +/** A segmented control: mutually exclusive choices that read at a glance (the theme, + * Local / Cloud models). Each segment is a real button carrying `idPrefix + id` — the + * stable id the settings surface re-focuses by after a repaint — and `attr="id"` for the + * click delegation. */ +export function segmentedHtml( + label: string, + idPrefix: string, + attr: string, + choices: readonly { id: string; label: string }[], + selected: string, +): string { + const buttons = choices + .map((c) => { + const on = selected === c.id; + return ``; + }) + .join(""); + return `
    ${buttons}
    `; +} + +/** + * The index-run meter: track, label and Cancel. Two are painted — the top bar's, and + * Settings → Index's while a run is live (Settings covers the bar) — and `paintReindex` + * (main.ts) writes the same values into every `.reindex-progress` on screen, so the two + * can't disagree about a run. Classes, not ids, for that reason; the one id is the + * Settings Cancel's, which that surface re-focuses by after a repaint. + */ +export function reindexMeterHtml(o: { hidden: boolean; indeterminate: boolean; cancelId?: string }): string { + const id = o.cancelId ? `id="${o.cancelId}" ` : ""; + return `
    +
    + + +
    `; +} From ffd865a5b9dc799cb85ef66bfee3db33cd7394ce Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:45:33 +0000 Subject: [PATCH 13/30] refactor(cli): wire through the shared rules, say every error precisely, render once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wiring: open_vault/open_llm shrink to b2_embed::embedder_for and b2_llm::probed_provider over the CLI's flags; the fake switches and the fake-chat notice are the libraries' (the notice now prints as b2-llm's sentence, without the "note:" prefix). Errors: - Unreachable / model-missing use b2-llm's sentences and append only the CLI's flag hint (--llm-url, --llm-model with the served list). - The InvalidRelation verb list is read from b2_core::relation::CORE. - The `_ =>` catch-all is gone: EmbedError::Load/Config/Io/UnknownModel get their own messages (Config names the config.toml path), so `b2 init` no longer says "check the vault path"; the internal remainder is spelled out so a new CliError variant fails to compile. - is_ollama is no longer re-exported by b2-llm (no caller left). Rendering: - `emit(json, &value, human)` replaces the per-command `if cli.json { print_json } else { … }` blocks; explain and similar --explain get named human printers. - `install_cancel_on_sigint(exit_when_idle)` is the one Ctrl-C handler (the REPL passes true), and `while_answering` owns the per-turn CANCEL/ANSWERING bookkeeping, so the flags are private to cancel.rs. - `status --json` and `reindex --cancel --json` serialize typed StatusView/CancelView structs with the same keys. Comments: the orphan is_ollama block, cmd_mv's doubled comment, the VaultRequired doc (every write command, not just reindex) and the write-command list in Cli.vault are fixed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-cli/src/args.rs | 9 +- crates/b2-cli/src/cancel.rs | 42 ++++++- crates/b2-cli/src/chat.rs | 42 +++---- crates/b2-cli/src/error.rs | 164 +++++++++++++++++++++------ crates/b2-cli/src/main.rs | 27 ++++- crates/b2-cli/src/read.rs | 208 ++++++++++++++++++----------------- crates/b2-cli/src/reindex.rs | 117 ++++++++++---------- crates/b2-cli/src/wiring.rs | 51 +++------ crates/b2-cli/src/write.rs | 96 ++++++---------- crates/b2-llm/src/lib.rs | 6 +- crates/b2-llm/src/setup.rs | 2 +- 11 files changed, 426 insertions(+), 338 deletions(-) diff --git a/crates/b2-cli/src/args.rs b/crates/b2-cli/src/args.rs index 5f8c8cf..63f62a1 100644 --- a/crates/b2-cli/src/args.rs +++ b/crates/b2-cli/src/args.rs @@ -39,8 +39,9 @@ For agents (--json): pub struct Cli { /// Vault root (the folder of Markdown). The index lives in `/.b2/`. /// Set it with `-C ` or `$B2_VAULT_PATH` (the flag wins). Read-only commands - /// fall back to the current dir; commands that write (`reindex`/`add`/`mv`/`rm`/ - /// `link`) require it explicitly, so they can never silently touch the wrong directory. + /// fall back to the current dir; commands that write (`reindex`/`add`/`write`/`mv`/ + /// `rm`/`link`) require it explicitly, so they can never silently touch the wrong + /// directory. #[arg(short = 'C', long = "vault", global = true, env = "B2_VAULT_PATH")] pub vault: Option, @@ -243,8 +244,8 @@ impl Cli { self.vault.as_deref().unwrap_or_else(|| Path::new(".")) } - /// The vault root for commands that **write** (`reindex`, `add`, `mv`, `rm`, `link`): - /// `positional` wins, then `-C`/`$B2_VAULT_PATH`; with none, error rather than + /// The vault root for commands that **write** (`reindex`, `add`, `write`, `mv`, `rm`, + /// `link`): `positional` wins, then `-C`/`$B2_VAULT_PATH`; with none, error rather than /// silently mutating the current directory, where a stale binary or mistyped var would /// otherwise leave a stray `.b2/`. The write-side counterpart to /// [`vault_or_cwd`](Self::vault_or_cwd). diff --git a/crates/b2-cli/src/cancel.rs b/crates/b2-cli/src/cancel.rs index ca82f88..e17b7c1 100644 --- a/crates/b2-cli/src/cancel.rs +++ b/crates/b2-cli/src/cancel.rs @@ -1,5 +1,5 @@ -//! The cooperative-cancel flags a SIGINT handler flips, read by the two long loops a -//! command can stop part-way: the reindex embed loop and the chat token stream. +//! Ctrl-C as a cooperative cancel: the flags a SIGINT handler flips, read by the two long +//! loops a command can stop part-way — the reindex embed loop and the chat token stream. use std::ops::ControlFlow; use std::sync::atomic::{AtomicBool, Ordering}; @@ -12,13 +12,45 @@ use std::sync::atomic::{AtomicBool, Ordering}; /// every batch boundary, stopping *after* the current batch so a cancel leaves a /// consistent, re-runnable index; and the chat surfaces' token callback at every token, /// where stopping renders the partial answer honestly. `chat` clears it before each turn. -pub static CANCEL: AtomicBool = AtomicBool::new(false); +static CANCEL: AtomicBool = AtomicBool::new(false); /// Whether `b2 chat` is streaming an answer right now — which is what decides /// what Ctrl-C *means* in the REPL: cancel this answer, or leave. Only `chat` -/// maintains it (see its handler); every other command's SIGINT meaning is +/// maintains it ([`while_answering`]); every other command's SIGINT meaning is /// unconditional. -pub static ANSWERING: AtomicBool = AtomicBool::new(false); +static ANSWERING: AtomicBool = AtomicBool::new(false); + +/// Route Ctrl-C to [`CANCEL`] instead of killing the process, so a long loop stops at its +/// next checkpoint and what it already did stands. +/// +/// `exit_when_idle` is for a REPL, where Ctrl-C means two things and a handler that only +/// ever meant one would trap the user: mid-answer it cancels the stream, but at an idle +/// prompt it must still be the way out, since swallowing it would leave `/exit` and +/// Ctrl-D as the only exits. `ctrlc` runs the handler on its own thread. +/// +/// Best-effort: if the handler can't be installed, Ctrl-C keeps its default (terminate), +/// which is still safe — a reindex writes edges and FTS before any vectors, and a chat +/// stores nothing. +pub fn install_cancel_on_sigint(exit_when_idle: bool) { + let _ = ctrlc::set_handler(move || { + if exit_when_idle && !ANSWERING.load(Ordering::SeqCst) { + // 128 + SIGINT, the shell's own convention for "interrupted". + std::process::exit(130); + } + CANCEL.store(true, Ordering::SeqCst); + }); +} + +/// Run one REPL answer with Ctrl-C meaning "stop this answer". A fresh cancel budget per +/// turn: the Ctrl-C that stopped the *last* answer must not cancel this one before its +/// first token. +pub fn while_answering(answer: impl FnOnce() -> T) -> T { + CANCEL.store(false, Ordering::SeqCst); + ANSWERING.store(true, Ordering::SeqCst); + let result = answer(); + ANSWERING.store(false, Ordering::SeqCst); + result +} /// Map the Ctrl-C flag onto the cooperative-cancel signal the engine's loops read — /// [`ControlFlow::Break`] once a cancel has been requested, else diff --git a/crates/b2-cli/src/chat.rs b/crates/b2-cli/src/chat.rs index fd9a35d..7d50c13 100644 --- a/crates/b2-cli/src/chat.rs +++ b/crates/b2-cli/src/chat.rs @@ -2,14 +2,13 @@ //! they share. use crate::args::{Cli, LlmArgs}; -use crate::cancel::{cancel_flow, ANSWERING, CANCEL}; +use crate::cancel::{cancel_flow, install_cancel_on_sigint, while_answering}; use crate::error::{user_message, CliError}; -use crate::wiring::{open_llm, open_vault, use_fake_llm}; +use crate::wiring::{open_llm, open_vault}; use b2_core::llm::{ChatTurn, LlmProvider}; use b2_core::vault::{AnswerView, Vault}; use std::io::{IsTerminal, Write}; use std::ops::ControlFlow; -use std::sync::atomic::Ordering; pub fn cmd_ask(cli: &Cli, question: &str, llm_args: &LlmArgs) -> Result<(), CliError> { // The provider first, deliberately: it is the cheap check, and loading the real @@ -20,7 +19,7 @@ pub fn cmd_ask(cli: &Cli, question: &str, llm_args: &LlmArgs) -> Result<(), CliE let vault = open_vault(cli.vault_or_cwd(), true)?; // Ctrl-C cancels the answer at the next token rather than killing the process, so // what already streamed stays on screen and is reported as partial. - let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); + install_cancel_on_sigint(false); ask_streamed(&vault, llm.as_ref(), question, &[], cli.json)?; if !cli.json { note_fake_llm(); @@ -39,7 +38,7 @@ pub fn cmd_why( // A pure read over stored vectors, like `similar`: nothing embeds a query, so the // real model is never loaded. let vault = open_vault(cli.vault_or_cwd(), false)?; - let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); + install_cancel_on_sigint(false); let json = cli.json; let answer = vault.why_similar(llm.as_ref(), note, candidate, limit, &mut |token| { stream_token(token, json) @@ -54,18 +53,9 @@ pub fn cmd_why( pub fn cmd_chat(cli: &Cli, llm_args: &LlmArgs) -> Result<(), CliError> { let llm = open_llm(llm_args)?; let vault = open_vault(cli.vault_or_cwd(), true)?; - // Ctrl-C means two different things in a REPL, and a handler that only ever meant one - // would trap the user: mid-answer it cancels the stream (the partial text stands), but - // at an idle prompt it must still be the way out, since swallowing it would leave - // `/exit` and Ctrl-D as the only exits. `ctrlc` runs this on its own thread. - let _ = ctrlc::set_handler(|| { - if ANSWERING.load(Ordering::SeqCst) { - CANCEL.store(true, Ordering::SeqCst); - } else { - // 128 + SIGINT, the shell's own convention for "interrupted". - std::process::exit(130); - } - }); + // Mid-answer, Ctrl-C cancels the stream (the partial text stands); at an idle prompt + // it leaves, as it would anywhere else in a terminal. + install_cancel_on_sigint(true); // Prompts and the banner are chrome for a human at a terminal: on stderr so // stdout stays answers, and only when there's a terminal there to read them // (the `reindex` progress-line rule). @@ -105,12 +95,8 @@ pub fn cmd_chat(cli: &Cli, llm_args: &LlmArgs) -> Result<(), CliError> { if matches!(question, "/exit" | "/quit") { break; } - // A fresh cancel budget per turn: the Ctrl-C that stopped the *last* answer - // must not cancel this one before its first token. - CANCEL.store(false, Ordering::SeqCst); - ANSWERING.store(true, Ordering::SeqCst); - let turn = ask_streamed(&vault, llm.as_ref(), question, &history, cli.json); - ANSWERING.store(false, Ordering::SeqCst); + let turn = + while_answering(|| ask_streamed(&vault, llm.as_ref(), question, &history, cli.json)); match turn { Ok(answer) => { // A cancelled answer goes into the history too: the human saw that @@ -221,12 +207,10 @@ fn print_answer_tail(answer: &AnswerView) { /// Never overstate what answered (the `search` fake-embedder caveat, applied to /// the chat seam): under `B2_LLM=fake` the "answer" is deterministic scaffolding, -/// not a model. On stderr, so stdout stays the answer. +/// not a model. On stderr, so stdout stays the answer; the sentence is b2-llm's, so the +/// desktop's setup card says the same thing. fn note_fake_llm() { - if use_fake_llm() { - eprintln!( - "note: the fake chat provider is in use (B2_LLM=fake) — answers are \ - deterministic test scaffolding, not a model." - ); + if b2_llm::fake_requested() { + eprintln!("{}", b2_llm::FAKE_NOTICE); } } diff --git a/crates/b2-cli/src/error.rs b/crates/b2-cli/src/error.rs index 96cf6f9..06ceddf 100644 --- a/crates/b2-cli/src/error.rs +++ b/crates/b2-cli/src/error.rs @@ -1,7 +1,7 @@ //! The CLI's error type and its one translation into a user-facing sentence. -use b2_embed::EmbedError; -use b2_llm::{is_ollama, refusal_message, LlmError, OLLAMA_INSTALL_URL}; +use b2_embed::{EmbedConfig, EmbedError}; +use b2_llm::{model_missing_message, refusal_message, unreachable_message, LlmError}; /// The CLI's error, composing the two crates it drives. Kept internal; `user_message` /// turns it into a generic, actionable, no-internals-leaked line (logging policy). @@ -23,8 +23,9 @@ pub enum CliError { /// A filesystem error creating `.b2/` or opening the reindex lock file. #[error(transparent)] Io(#[from] std::io::Error), - /// `reindex` was run with no vault at all (no positional, no `-C`, no - /// `$B2_VAULT_PATH`) — refuse rather than silently index the current directory. + /// A command that writes (`reindex`, `add`, `write`, `mv`, `rm`, `link`) was run with + /// no vault at all (no positional, no `-C`, no `$B2_VAULT_PATH`) — refuse rather than + /// silently write into the current directory. #[error("no vault specified")] VaultRequired, /// Another `reindex` already holds the single-in-flight lock on this vault. @@ -34,7 +35,7 @@ pub enum CliError { #[error("no reindex is running")] NoReindexRunning, /// `reindex --cancel` found a run in flight whose lock names no readable pid, so - /// there is no address to signal (see [`ReindexHolder::pid`]). + /// there is no address to signal (a holder that hasn't stamped its pid yet). #[error("the running reindex recorded no pid")] ReindexPidUnknown, /// `rm` was pointed at a folder without `--recursive` — refuse rather than @@ -48,12 +49,6 @@ pub enum CliError { StdinRequired, } -// `is_ollama` — "does this endpoint look like the Ollama daemon" — decides whether -// Ollama's own commands belong in an error message (`--llm-url` also points at LM Studio, -// llama.cpp, vLLM and cloud endpoints, where "run `ollama serve`" is advice about the -// wrong program). The rule lives in `b2-llm` (GH #155), where guided setup needs the same -// answer for a bigger decision, so one rule with two callers cannot drift. - /// The head of a model server's own model list, for "…or pick one it already /// serves". Bounded: a local runtime holds a handful, but a cloud endpoint lists /// hundreds, and a hundred-model line is not a hint. @@ -70,6 +65,22 @@ fn model_hint(available: &[String]) -> Option { }) } +/// The core relation verbs, as a message lists them — read from `b2_core::relation::CORE` +/// so a verb added there is offered here without an edit. +fn core_verbs() -> String { + b2_core::relation::CORE + .iter() + .map(|c| c.verb) + .collect::>() + .join(", ") +} + +/// The embedder config file, as a message names it. +fn config_file() -> String { + EmbedConfig::config_path() + .map_or_else(|| "config.toml".to_string(), |p| p.display().to_string()) +} + /// Translate an internal error into a generic, actionable, user-facing message — /// never leaking sqlite/io/serde internals. Set `B2_DEBUG` to also print the detail. pub fn user_message(err: &CliError) -> String { @@ -89,6 +100,29 @@ pub fn user_message(err: &CliError) -> String { CliError::Embed(EmbedError::Download(_)) => { "Could not download the embedding model. Check your network and try `b2 init` again.".to_string() } + CliError::Embed(EmbedError::Load(_)) => { + "The embedding model's files failed to load. Run `b2 init` to fetch them again.".to_string() + } + // `config.toml` didn't parse, or its `source` names a folder missing a model file. + // Either way the fix is in that one file, so name it. + CliError::Embed(EmbedError::Config(_)) => format!( + "B2 couldn't use its embedder settings. Check the [embedder] table in {}, then try again.", + config_file() + ), + CliError::Embed(EmbedError::Io(_)) => { + "B2 couldn't read or write the embedding model's files. Check that the model cache is readable and writable, then run `b2 init` again.".to_string() + } + // Only a settings picker names a model to switch to; the CLI never does, so this + // arm exists for exhaustiveness and says what the fix would be. + CliError::Embed(EmbedError::UnknownModel(m)) => format!( + "'{m}' isn't an embedding model B2 offers. Set `model` in {} to one of: {}.", + config_file(), + b2_embed::AVAILABLE_MODELS + .iter() + .map(|m| m.id) + .collect::>() + .join(", ") + ), CliError::Core(b2_core::Error::MoveTargetExists(p)) => format!( "Can't move: a file already exists at '{p}'. Choose a different destination." ), @@ -109,7 +143,8 @@ pub fn user_message(err: &CliError) -> String { "That note path isn't valid. Give a vault-relative path like `notes/new-name.md`.".to_string() } CliError::Core(b2_core::Error::InvalidRelation(v)) => format!( - "'{v}' isn't a known relation type. Use one of: references, supports, contradicts." + "'{v}' isn't a known relation type. Use one of: {}.", + core_verbs() ), CliError::Core(b2_core::Error::WriteConflict(_)) => { "This note changed on disk since it was opened. Reload the note, then reapply your edit.".to_string() @@ -141,16 +176,13 @@ pub fn user_message(err: &CliError) -> String { CliError::StdinRequired => { "No body piped on stdin. Pipe the new note body in, e.g. `cat new-body.md | b2 write notes/foo`.".to_string() } - // The E4 case chat is most likely to hit: nothing is serving the endpoint. - // Named by the endpoint the user actually configured — and *advised* by it - // too: `ollama serve` is the fix for Ollama and no help at all for LM - // Studio, llama.cpp, vLLM, or a cloud provider, all of which `--llm-url` - // supports. - CliError::Llm(LlmError::Unreachable { endpoint, .. }) if is_ollama(endpoint) => format!( - "Can't reach the model server at {endpoint} — is Ollama running? (`ollama serve`, or install: {OLLAMA_INSTALL_URL})" - ), + // The E4 case chat is most likely to hit: nothing is serving the endpoint. The + // sentence is b2-llm's, advised by the endpoint the user actually configured + // (`ollama serve` only when it is Ollama's); the CLI adds only its own way of + // pointing somewhere else. CliError::Llm(LlmError::Unreachable { endpoint, .. }) => format!( - "Can't reach the model server at {endpoint}. Start it, or point --llm-url (or B2_LLM_URL) at one that's running." + "{} Or point --llm-url (or B2_LLM_URL) at a different one.", + unreachable_message(endpoint) ), // Something answered the probe and refused it — a *different* mistake from // nothing listening, and one this used to report as success. The sentence is @@ -159,18 +191,15 @@ pub fn user_message(err: &CliError) -> String { CliError::Llm(LlmError::Refused { endpoint, status, message }) => { refusal_message(endpoint, *status, message, None) } - CliError::Llm(LlmError::ModelMissing { model, endpoint, available }) => format!( - "Model '{model}' isn't available at {endpoint}. {}{}", - if is_ollama(endpoint) { - format!("Pull it with `ollama pull {model}`") - } else { - "Load it there".to_string() - }, + // b2-llm's sentence again, plus the CLI's flag — and the server's own list, since + // "pick one it already serves" is only advice when it names them. + CliError::Llm(LlmError::ModelMissing { model, endpoint, available }) => { + let msg = model_missing_message(model, endpoint); match model_hint(available) { - Some(list) => format!(", or point --llm-model at one it already serves ({list})."), - None => ".".to_string(), + Some(list) => format!("{msg} Or point --llm-model at one it already serves ({list})."), + None => msg, } - ), + } // Not the generic chat failure below: the server is up and the model is there, // so that advice would mislead. The cap is the one fix on this side of the wire. CliError::Core(b2_core::Error::ToolCallLimit { limit }) => format!( @@ -182,7 +211,12 @@ pub fn user_message(err: &CliError) -> String { CliError::Llm(_) | CliError::Core(b2_core::Error::Llm(_)) => { "The model server couldn't answer. Check that it's running and that the model is installed (`ollama list`), then try again.".to_string() } - _ => "Something went wrong. Please check the vault path and try again.".to_string(), + // Everything else in the composed crates is an internal (sqlite/io/serde/…) this + // message must never show. Spelled out rather than `_`, so a new `CliError` + // variant fails to compile here instead of silently landing in the catch-all. + CliError::Core(_) | CliError::Serde(_) | CliError::Io(_) => { + "Something went wrong. Please check the vault path and try again.".to_string() + } }; if std::env::var_os("B2_DEBUG").is_some() { // Every wrapper variant is `#[error(transparent)]` and every local variant @@ -210,6 +244,72 @@ mod tests { ); } + #[test] + fn an_invalid_relation_lists_every_core_verb() { + let msg = user_message(&CliError::Core(b2_core::Error::InvalidRelation( + "refutes".into(), + ))); + assert!(msg.contains("'refutes'"), "{msg}"); + for verb in b2_core::relation::CORE { + assert!(msg.contains(verb.verb), "{msg}"); + } + } + + /// `b2 init` meets these two, and they used to fall through to "check the vault path" + /// — advice about a vault the command never opened. + #[test] + fn embedder_setup_failures_say_what_to_fix_not_to_check_the_vault() { + for err in [ + EmbedError::Load("weights: truncated".into()), + EmbedError::Config("config.toml: expected `=`".into()), + ] { + let msg = user_message(&CliError::Embed(err)); + assert!(!msg.contains("vault path"), "{msg}"); + assert!( + !msg.contains("truncated") && !msg.contains("expected"), + "the detail stays internal: {msg}" + ); + } + let load = user_message(&CliError::Embed(EmbedError::Load(String::new()))); + assert!(load.contains("b2 init"), "{load}"); + let config = user_message(&CliError::Embed(EmbedError::Config(String::new()))); + assert!(config.contains("[embedder]"), "{config}"); + } + + /// The sentences are b2-llm's, shared with the desktop; the CLI adds only its flags. + #[test] + fn chat_setup_failures_use_the_shared_sentence_plus_the_cli_flag() { + let endpoint = "http://localhost:1234/v1"; + let unreachable = user_message(&CliError::Llm(LlmError::Unreachable { + endpoint: endpoint.into(), + detail: "connection refused".into(), + })); + assert!( + unreachable.starts_with(&unreachable_message(endpoint)), + "{unreachable}" + ); + assert!(unreachable.contains("--llm-url"), "{unreachable}"); + + let missing = |available: Vec| { + user_message(&CliError::Llm(LlmError::ModelMissing { + model: "llama3.2".into(), + endpoint: "http://localhost:11434/v1".into(), + available, + })) + }; + let bare = missing(Vec::new()); + assert_eq!( + bare, + model_missing_message("llama3.2", "http://localhost:11434/v1") + ); + assert!(bare.contains("ollama pull llama3.2"), "{bare}"); + let listed = missing(vec!["qwen2.5:latest".into()]); + assert!( + listed.contains("--llm-model") && listed.contains("qwen2.5:latest"), + "{listed}" + ); + } + #[test] fn a_move_that_could_not_be_undone_names_the_files_to_check() { let msg = user_message(&CliError::Core(b2_core::Error::MoveIncomplete { diff --git a/crates/b2-cli/src/main.rs b/crates/b2-cli/src/main.rs index a20ebdb..debfe3a 100644 --- a/crates/b2-cli/src/main.rs +++ b/crates/b2-cli/src/main.rs @@ -1,9 +1,10 @@ //! `b2` — one of the two dumb adapters over the `b2-core` typed API (ADR-0012), //! headless-first: "the CLI is the UI before the UI". It holds **no engine logic** — it -//! parses args, injects the embedder and chat provider, calls the [`Vault`] façade, and -//! prints (human-readable, or `--json` for agents). +//! parses args, injects the embedder and chat provider, calls the +//! [`Vault`](b2_core::vault::Vault) façade, and prints (human-readable, or `--json` for +//! agents). `dispatch` below routes each subcommand to its domain module. //! -//! The embedder is the real candle-backed [`LocalEmbedder`] by default. It is **not +//! The embedder is the real candle-backed [`b2_embed::LocalEmbedder`] by default. It is **not //! bundled**: `b2 init` downloads it into a shared XDG cache, and `reindex`/`search` fail //! fast with "run `b2 init`" if it is absent, never a surprise mid-command download //! (ADR-0020). `B2_EMBEDDER=fake` forces the deterministic fake — an offline/dev mode, @@ -40,8 +41,8 @@ fn main() -> ExitCode { } } -/// The thin router: each subcommand's whole behavior lives in its `cmd_*` fn below; -/// this match only destructures the parsed args and forwards them. +/// The thin router: each subcommand's whole behavior lives in its `cmd_*` fn, in the +/// module for its domain; this match only destructures the parsed args and forwards them. fn dispatch(cli: &Cli) -> Result<(), CliError> { match &cli.command { Command::Init => cmd_init(cli.json), @@ -100,3 +101,19 @@ pub fn print_json(value: &T) -> Result<(), CliErro println!("{}", serde_json::to_string_pretty(value)?); Ok(()) } + +/// A command's one result, rendered for whoever asked: the value itself as JSON under +/// `--json`, else `human`'s text. Every single-result command ends here, so the two +/// renderings can't come from different values. +pub fn emit( + json: bool, + value: &T, + human: impl FnOnce(&T), +) -> Result<(), CliError> { + if json { + print_json(value) + } else { + human(value); + Ok(()) + } +} diff --git a/crates/b2-cli/src/read.rs b/crates/b2-cli/src/read.rs index 750f4bc..98b5f1d 100644 --- a/crates/b2-cli/src/read.rs +++ b/crates/b2-cli/src/read.rs @@ -3,10 +3,12 @@ use crate::args::Cli; use crate::error::CliError; -use crate::print_json; -use crate::wiring::{open_vault, use_fake_embedder}; +use crate::wiring::open_vault; +use crate::{emit, print_json}; use b2_core::resource::{doc_kind, DocKind}; -use b2_core::vault::{SearchEvidenceView, SimilarStanding}; +use b2_core::vault::{ + ExplainView, ResourceExplainView, SearchEvidenceView, SimilarExplainView, SimilarStanding, +}; pub fn cmd_neighbors(cli: &Cli, note: &str) -> Result<(), CliError> { // Neighbors is a pure graph query — it never embeds, so don't require @@ -18,12 +20,12 @@ pub fn cmd_neighbors(cli: &Cli, note: &str) -> Result<(), CliError> { // resolved-neighbors array contract; the full structured picture, // including these, is `b2 explain --json`. let unresolved = vault.unresolved_links(note)?; - if cli.json { - print_json(&neighbors)?; - } else if neighbors.is_empty() && unresolved.is_empty() { - println!("No neighbors."); - } else { - for n in &neighbors { + emit(cli.json, &neighbors, |neighbors| { + if neighbors.is_empty() && unresolved.is_empty() { + println!("No neighbors."); + return; + } + for n in neighbors { let arrow = arrow(&n.direction); let name = display_name(n.title.as_deref(), &n.path); let explanation = n @@ -39,8 +41,7 @@ pub fn cmd_neighbors(cli: &Cli, note: &str) -> Result<(), CliError> { u.relation, u.target ); } - } - Ok(()) + }) } pub fn cmd_explain(cli: &Cli, note: &str) -> Result<(), CliError> { @@ -50,87 +51,90 @@ pub fn cmd_explain(cli: &Cli, note: &str) -> Result<(), CliError> { // Kind dispatch by the argument's own shape (core's one rule, §9b #8): // a resource arg gets the fallback card's view — metadata + backlinks. if doc_kind(note) == DocKind::Resource { - let view = vault.explain_resource(note)?; - if cli.json { - print_json(&view)?; - } else { - println!("{} ({}, {} bytes)", view.path, view.class, view.size); - if view.backlinks.is_empty() { - println!("No backlinks yet."); - } else { - println!("Backlinks:"); - for b in &view.backlinks { - let name = display_name(b.title.as_deref(), &b.path); - let mut line = format!(" ← {name} ({}) {}", b.path, b.r#type); - decorate(&mut line, b.embed, b.caption.as_deref()); - println!("{line}"); - } - } - } - return Ok(()); + return emit( + cli.json, + &vault.explain_resource(note)?, + print_resource_card, + ); } - let view = vault.explain(note)?; - if cli.json { - print_json(&view)?; + emit(cli.json, &vault.explain(note)?, print_explanation) +} + +/// `explain` on a resource, for a human: the fallback card's metadata and backlinks. +fn print_resource_card(view: &ResourceExplainView) { + println!("{} ({}, {} bytes)", view.path, view.class, view.size); + if view.backlinks.is_empty() { + println!("No backlinks yet."); } else { - let name = display_name(view.title.as_deref(), &view.path); - println!("{name} ({})", view.path); - if view.connections.is_empty() && view.resources.is_empty() && view.unresolved.is_empty() { - // Zero connections at all — nothing links to it and it links to - // nothing (an orphan; the kernel only surfaces, never archives). - println!("No connections yet."); - } else if !view.connections.is_empty() { - println!("Connections:"); - for c in &view.connections { - let arrow = arrow(&c.direction); - let target = display_name(c.title.as_deref(), &c.path); - println!( - " {arrow} {} {target} ({}) [{}]", - c.label, c.path, c.origin - ); - if let Some(why) = &c.explanation { - println!(" why: {why}"); - } - } - // If nothing points *at* the note, it's an orphan — surfaced, not - // acted on (invariants.md; files are only touched when asked). - if !view.connections.iter().any(|c| c.direction == "inbound") { - println!("No inbound links — this note is an orphan."); + println!("Backlinks:"); + for b in &view.backlinks { + let name = display_name(b.title.as_deref(), &b.path); + let mut line = format!(" ← {name} ({}) {}", b.path, b.r#type); + decorate(&mut line, b.embed, b.caption.as_deref()); + println!("{line}"); + } + } +} + +/// `explain` on a note, for a human: every connection with its "why", then the links +/// at resources and the ones that resolve to nothing. +fn print_explanation(view: &ExplainView) { + let name = display_name(view.title.as_deref(), &view.path); + println!("{name} ({})", view.path); + if view.connections.is_empty() && view.resources.is_empty() && view.unresolved.is_empty() { + // Zero connections at all — nothing links to it and it links to + // nothing (an orphan; the kernel only surfaces, never archives). + println!("No connections yet."); + } else if !view.connections.is_empty() { + println!("Connections:"); + for c in &view.connections { + let arrow = arrow(&c.direction); + let target = display_name(c.title.as_deref(), &c.path); + println!( + " {arrow} {} {target} ({}) [{}]", + c.label, c.path, c.origin + ); + if let Some(why) = &c.explanation { + println!(" why: {why}"); } } - // Outbound links at resources (images, PDFs, …) — the third target - // kind an edge can have, shown from the note's side (GH #22). - if !view.resources.is_empty() { - println!("Resource links:"); - for r in &view.resources { - let mut line = format!( - " → {} {} ({}) [{}]", - r.relation, r.path, r.class, r.origin - ); - decorate(&mut line, r.embed, r.caption.as_deref()); - println!("{line}"); - if let Some(why) = &r.explanation { - println!(" why: {why}"); - } + // If nothing points *at* the note, it's an orphan — surfaced, not + // acted on (invariants.md; files are only touched when asked). + if !view.connections.iter().any(|c| c.direction == "inbound") { + println!("No inbound links — this note is an orphan."); + } + } + // Outbound links at resources (images, PDFs, …) — the third target + // kind an edge can have, shown from the note's side (GH #22). + if !view.resources.is_empty() { + println!("Resource links:"); + for r in &view.resources { + let mut line = format!( + " → {} {} ({}) [{}]", + r.relation, r.path, r.class, r.origin + ); + decorate(&mut line, r.embed, r.caption.as_deref()); + println!("{line}"); + if let Some(why) = &r.explanation { + println!(" why: {why}"); } } - // Dangling outbound links (a `[[folder]]` or a typo): a note is one - // `.md` file, so these resolve to nothing — shown as broken rather - // than silently dropped (GH #12). - if !view.unresolved.is_empty() { - println!("Unresolved links:"); - for u in &view.unresolved { - println!( - " ⚠ {} [[{}]] (no matching note or file) [{}]", - u.relation, u.target, u.origin - ); - if let Some(why) = &u.explanation { - println!(" why: {why}"); - } + } + // Dangling outbound links (a `[[folder]]` or a typo): a note is one + // `.md` file, so these resolve to nothing — shown as broken rather + // than silently dropped (GH #12). + if !view.unresolved.is_empty() { + println!("Unresolved links:"); + for u in &view.unresolved { + println!( + " ⚠ {} [[{}]] (no matching note or file) [{}]", + u.relation, u.target, u.origin + ); + if let Some(why) = &u.explanation { + println!(" why: {why}"); } } } - Ok(()) } pub fn cmd_search( @@ -147,27 +151,24 @@ pub fn cmd_search( // `limit` confident-looking results for a query the vault holds nothing for. // `--exclude` paths are the caller's subtraction, never the verdict's. let view = vault.search_evidence_excluding(query, limit, exclude)?; - if cli.json { - // The whole view, verdict included. This is an OBJECT where `--json` used to be - // an array — a deliberate break (GH #202), because a query-level verdict has - // nowhere to live in a list of rows; the rows themselves stay additive. - // - // The JSON serves the rows even at `vouched: false`, where the human surface below - // shows none: an agent handed the rows *plus* an explicit verdict can be honest - // about them, where a human handed rows alone cannot. - print_json(&view)?; - } else { - println!("{}", search_report(&view, query)); + // `--json` is the whole view, verdict included. This is an OBJECT where `--json` used + // to be an array — a deliberate break (GH #202), because a query-level verdict has + // nowhere to live in a list of rows; the rows themselves stay additive. + // + // The JSON serves the rows even at `vouched: false`, where the human surface shows + // none: an agent handed the rows *plus* an explicit verdict can be honest about them, + // where a human handed rows alone cannot. + emit(cli.json, &view, |view| { + println!("{}", search_report(view, query)); // Honesty (never overstate): with the fake embedder the vector half // isn't semantic. Under the real model it is, so no caveat. Kept on // stderr so stdout stays pure results. - if use_fake_embedder() { + if b2_embed::fake_requested() { eprintln!( "note: keyword (BM25) ranking is live; semantic ranking is off (fake embedder)." ); } - } - Ok(()) + }) } /// The human-mode rendering of a search — ADR-0015's three verdict states as one pure @@ -264,9 +265,12 @@ pub fn cmd_explain_similar( // A pure read over stored vectors, like `similar`: the fake is enough to open with. let vault = open_vault(cli.vault_or_cwd(), false)?; let ex = vault.explain_similar(note, other, limit)?; - if cli.json { - return print_json(&ex); - } + emit(cli.json, &ex, |ex| print_similar_explanation(ex, limit)) +} + +/// `similar --explain`, for a human: where the candidate stands against the list shown +/// at `limit` (and why, if it is not a card), then the passage pairs behind it. +fn print_similar_explanation(ex: &SimilarExplainView, limit: usize) { let anchor = display_name(ex.anchor.title.as_deref(), &ex.anchor.path); let candidate = display_name(ex.candidate.title.as_deref(), &ex.candidate.path); println!( @@ -339,7 +343,6 @@ pub fn cmd_explain_similar( ); } } - Ok(()) } /// A passage flattened to one line and cut at `max` characters. @@ -378,8 +381,7 @@ fn decorate(line: &mut String, embed: bool, caption: Option<&str>) { } } -/// The one unit-tested corner of this adapter: ADR-0015's three verdict states as -/// rendered by [`search_report`] (GH #202). +/// ADR-0015's three verdict states as rendered by [`search_report`] (GH #202). /// /// The integration suite spawns the binary under `B2_EMBEDDER=fake`, whose embedder has no /// calibrated bar — so `vouched` there is always `None` and the two states that matter are diff --git a/crates/b2-cli/src/reindex.rs b/crates/b2-cli/src/reindex.rs index 7ac374e..a57c900 100644 --- a/crates/b2-cli/src/reindex.rs +++ b/crates/b2-cli/src/reindex.rs @@ -2,31 +2,30 @@ //! cross-process cancel, and `status`. use crate::args::Cli; -use crate::cancel::{cancel_flow, CANCEL}; +use crate::cancel::{cancel_flow, install_cancel_on_sigint}; +use crate::emit; use crate::error::CliError; -use crate::print_json; use crate::wiring::open_vault; use b2_embed::{provision, EmbedConfig}; +use serde::Serialize; use std::fs::{File, OpenOptions}; use std::io::{IsTerminal, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; -use std::sync::atomic::Ordering; pub fn cmd_init(json: bool) -> Result<(), CliError> { // Global, per-machine setup — no vault involved. let config = EmbedConfig::load()?; let report = provision(&config, |line| eprintln!("{line}"))?; - if json { - print_json(&report)?; - } else if report.already_present { - println!("Model '{}' is already installed.", report.model); - } else { - println!( - "Installed '{}' ({} dims). Run `b2 reindex` to embed your vault.", - report.model, report.dim - ); - } - Ok(()) + emit(json, &report, |report| { + if report.already_present { + println!("Model '{}' is already installed.", report.model); + } else { + println!( + "Installed '{}' ({} dims). Run `b2 reindex` to embed your vault.", + report.model, report.dim + ); + } + }) } pub fn cmd_reindex( @@ -49,13 +48,7 @@ pub fn cmd_reindex( // the fake, like `neighbors`); it's a pure read, so there's no slow // embed phase to show progress for. let vault = open_vault(root, false)?; - let plan = vault.plan_reindex(force)?; - if cli.json { - print_json(&plan)?; - } else { - print_reindex_plan(&plan); - } - return Ok(()); + return emit(cli.json, &vault.plan_reindex(force)?, print_reindex_plan); } // Single-in-flight: take an advisory lock *before* the slow model load, so a second // `b2 reindex` refuses cleanly instead of two processes writing the same index. @@ -76,10 +69,8 @@ pub fn cmd_reindex( let vault = open_vault(root, true)?; // Wire Ctrl-C to the cooperative-cancel flag now that the model is loaded and // real embedding is next. (During the model load the default SIGINT still - // applies — nothing is written yet, so a hard stop there is safe.) Best-effort: - // if the handler can't be installed, Ctrl-C keeps its default (terminate), which - // still leaves a consistent index since edges + FTS land before any vectors. - let _ = ctrlc::set_handler(|| CANCEL.store(true, Ordering::SeqCst)); + // applies — nothing is written yet, so a hard stop there is safe.) + install_cancel_on_sigint(false); // Embedding a large vault on CPU is slow; show a live progress line so it // never looks frozen. Only on an interactive stderr (never in --json, and // never when piped/captured) so machine output and tests stay clean. @@ -116,12 +107,7 @@ pub fn cmd_reindex( } report }; - if cli.json { - print_json(&report)?; - } else { - print_reindex_report(&report); - } - Ok(()) + emit(cli.json, &report, print_reindex_report) } /// The human-readable `reindex --dry-run` preview (the `--json` sibling prints the @@ -179,16 +165,13 @@ pub fn cmd_status(cli: &Cli) -> Result<(), CliError> { let vault = open_vault(root, false)?; let status = vault.embed_status()?; let holder = reindex_holder(root); - if cli.json { - print_json(&serde_json::json!({ - "embedded": status.embedded, - "total": status.total, - "reindex_running": holder.is_some(), - // The running process's id — `null` when nothing is running (and - // on the sliver of a moment before a fresh holder stamps it). - "reindex_pid": holder.as_ref().and_then(|h| h.pid), - }))?; - } else { + let view = StatusView { + embedded: status.embedded, + total: status.total, + reindex_running: holder.is_some(), + reindex_pid: holder.and_then(|h| h.pid), + }; + emit(cli.json, &view, |status| { if status.total == 0 { println!("No notes indexed yet. Run `b2 reindex` to build the index."); } else if status.embedded == 0 { @@ -211,17 +194,37 @@ pub fn cmd_status(cli: &Cli) -> Result<(), CliError> { } // Name the process, not just the fact: `--cancel` is the supported stop, // and the pid keeps a plain `kill -INT` as the documented fallback. - match holder.as_ref().map(|h| h.pid) { - Some(Some(pid)) => println!( + match (status.reindex_running, status.reindex_pid) { + (true, Some(pid)) => println!( "A reindex is currently running (pid {pid}). Stop it with `b2 reindex --cancel` (or `kill -INT {pid}`)." ), - Some(None) => { + (true, None) => { println!("A reindex is currently running. Stop it with `b2 reindex --cancel`.") } - None => {} + (false, _) => {} } - } - Ok(()) + }) +} + +/// `b2 status`: embedding coverage, and whether a reindex holds the lock. The `--json` +/// keys are a contract agents read (`tests/cli.rs` pins them). +#[derive(Debug, Serialize)] +struct StatusView { + embedded: usize, + total: usize, + reindex_running: bool, + /// The running process's id — `null` when nothing is running (and on the sliver + /// of a moment before a fresh holder stamps it). + reindex_pid: Option, +} + +/// `b2 reindex --cancel`'s result. `signalled`, not `cancelled`: the request landed; +/// the run stops at its next batch boundary and reports the partial work itself (honest +/// tense, like the dry-run's `would_*` keys). +#[derive(Debug, Serialize)] +struct CancelView { + signalled: bool, + pid: u32, } /// Path to the single-in-flight advisory lock for `reindex`, under the disposable @@ -311,20 +314,16 @@ fn cancel_reindex(root: &Path, json: bool) -> Result<(), CliError> { return Err(CliError::ReindexPidUnknown); }; signal_reindex(pid)?; - if json { - // `signalled`, not `cancelled`: the request landed; the run stops at its next - // batch boundary and reports the partial work itself (honest tense, like the - // dry-run's `would_*` keys). - print_json(&serde_json::json!({ - "signalled": true, - "pid": pid, - }))?; - } else { + let view = CancelView { + signalled: true, + pid, + }; + emit(json, &view, |view| { println!( - "Cancelling the reindex on this vault (pid {pid}). It stops after the current batch, leaving a consistent index — re-run `b2 reindex` to finish." + "Cancelling the reindex on this vault (pid {}). It stops after the current batch, leaving a consistent index — re-run `b2 reindex` to finish.", + view.pid ); - } - Ok(()) + }) } /// Send the cancel signal to a reindex holder. SIGINT, deliberately: the identical diff --git a/crates/b2-cli/src/wiring.rs b/crates/b2-cli/src/wiring.rs index 98dbc58..a667c9d 100644 --- a/crates/b2-cli/src/wiring.rs +++ b/crates/b2-cli/src/wiring.rs @@ -1,52 +1,33 @@ //! Model wiring: which embedder a vault opens with, and which chat provider answers. +//! The rules themselves live in `b2-embed` and `b2-llm`, shared with the desktop; this +//! module only lays the CLI's flags over them. use crate::args::LlmArgs; use crate::error::CliError; -use b2_core::embed::Embedder; -use b2_core::llm::{FakeLlm, LlmProvider}; +use b2_core::llm::LlmProvider; use b2_core::vault::Vault; -use b2_embed::{EmbedConfig, LocalEmbedder}; -use b2_llm::{LlmConfig, OpenAiCompatProvider}; +use b2_llm::LlmConfig; use std::path::Path; /// Pick + wire the chat provider — [`open_vault`]'s sibling for the second seam. /// -/// `B2_LLM=fake` forces the deterministic [`FakeLlm`], the `B2_EMBEDDER=fake` sibling. -/// Otherwise the real client is built from the environment with this command's flags laid -/// over it, and **probed before anything else happens** — the `b2 init` posture applied to -/// chat: a stopped daemon is a sentence before the question, never a surprise after it. +/// `B2_LLM=fake` forces the deterministic fake, the `B2_EMBEDDER=fake` sibling. Otherwise +/// the real client is built from the environment with this command's flags laid over it, +/// and **probed before anything else happens** — the `b2 init` posture applied to chat: a +/// stopped daemon is a sentence before the question, never a surprise after it. pub fn open_llm(args: &LlmArgs) -> Result, CliError> { - if use_fake_llm() { - return Ok(Box::new(FakeLlm)); - } let config = LlmConfig::from_env().with_overrides(args.llm_url.as_deref(), args.llm_model.as_deref()); - let provider = OpenAiCompatProvider::new(config); - provider.probe()?; - Ok(Box::new(provider)) -} - -pub fn use_fake_llm() -> bool { - std::env::var_os("B2_LLM").is_some_and(|v| v == "fake") + Ok(b2_llm::probed_provider(config)?) } /// Open a vault with the appropriate embedder. `needs_semantic` commands (`reindex`, -/// `search`) load the real [`LocalEmbedder`] and **fail fast** with "run `b2 init`" if it -/// is absent; pure-graph commands pass `false` and use the fake, so no model is required -/// just to explore the graph. `B2_EMBEDDER=fake` forces the fake everywhere. +/// `search`) load the real model and **fail fast** with "run `b2 init`" if it is absent; +/// pure-graph commands pass `false` and use the fake, so no model is required just to +/// explore the graph. `B2_EMBEDDER=fake` forces the fake everywhere. pub fn open_vault(root: &Path, needs_semantic: bool) -> Result { - if needs_semantic && !use_fake_embedder() { - let config = EmbedConfig::load()?; - let embedder = LocalEmbedder::load(&config)?; - Ok(Vault::open_with_embedder( - root, - Box::new(embedder) as Box, - )?) - } else { - Ok(Vault::open(root)?) - } -} - -pub fn use_fake_embedder() -> bool { - std::env::var_os("B2_EMBEDDER").is_some_and(|v| v == "fake") + Ok(match b2_embed::embedder_for(needs_semantic)? { + Some(embedder) => Vault::open_with_embedder(root, Box::new(embedder))?, + None => Vault::open(root)?, + }) } diff --git a/crates/b2-cli/src/write.rs b/crates/b2-cli/src/write.rs index dfc4d91..06d3171 100644 --- a/crates/b2-cli/src/write.rs +++ b/crates/b2-cli/src/write.rs @@ -2,8 +2,8 @@ //! vault, so none can touch the wrong directory. use crate::args::Cli; +use crate::emit; use crate::error::CliError; -use crate::print_json; use crate::wiring::open_vault; use b2_core::resource::{doc_kind, DocKind}; use std::io::{IsTerminal, Read}; @@ -19,12 +19,9 @@ pub fn cmd_add( // (no silent cwd), and it needs the real model like `reindex`/`mv`/`link`. let vault = open_vault(cli.require_vault(None)?, true)?; let report = vault.add_note(path, title, content)?; - if cli.json { - print_json(&report)?; - } else { - println!("Created {}.", report.path); - } - Ok(()) + emit(cli.json, &report, |report| { + println!("Created {}.", report.path) + }) } pub fn cmd_write(cli: &Cli, note: &str) -> Result<(), CliError> { @@ -46,12 +43,9 @@ pub fn cmd_write(cli: &Cli, note: &str) -> Result<(), CliError> { // this" (`Vault::write` still keeps the frontmatter bytes untouched). let current = vault.read(note)?; let report = vault.write(note, &body, ¤t.revision)?; - if cli.json { - print_json(&report)?; - } else { + emit(cli.json, &report, |report| { println!("Wrote {} ({} bytes).", report.path, body.len()); - } - Ok(()) + }) } pub fn cmd_mv(cli: &Cli, from: &str, to: &str) -> Result<(), CliError> { @@ -60,39 +54,28 @@ pub fn cmd_mv(cli: &Cli, from: &str, to: &str) -> Result<(), CliError> { // built with, like `reindex`/`add`/`link`. let root = cli.require_vault(None)?; let vault = open_vault(root, true)?; - // Kind dispatch (§9b #8): an existing directory moves as a folder - // (every file under it, one rename); otherwise the two file arms - // differ only in the report type they print. - // The human "Moved" line differs per arm, the rewrite tally is shared. + // Kind dispatch (§9b #8): an existing directory moves as a folder (every file + // under it, one rename); otherwise the extension picks the file arm. The human + // "Moved" line differs per arm; the rewrite tally is shared. if is_dir_arg(root, from) { - let report = vault.move_dir(from, to)?; - if cli.json { - print_json(&report)?; - } else { + emit(cli.json, &vault.move_dir(from, to)?, |report| { println!( "Moved {}/ → {}/ ({} note(s), {} file(s))", report.from, report.to, report.moved_notes, report.moved_resources ); print_rewrite_tally(report.links_rewritten, report.rewrote.len()); - } + }) } else if doc_kind(from) == DocKind::Resource { - let report = vault.move_resource(from, to)?; - if cli.json { - print_json(&report)?; - } else { + emit(cli.json, &vault.move_resource(from, to)?, |report| { println!("Moved {} → {}", report.from, report.to); print_rewrite_tally(report.links_rewritten, report.rewrote.len()); - } + }) } else { - let report = vault.move_note(from, to)?; - if cli.json { - print_json(&report)?; - } else { + emit(cli.json, &vault.move_note(from, to)?, |report| { println!("Moved {} → {}", report.from, report.to); print_rewrite_tally(report.links_rewritten, report.rewrote.len()); - } + }) } - Ok(()) } pub fn cmd_rm(cli: &Cli, target: &str, recursive: bool) -> Result<(), CliError> { @@ -108,34 +91,24 @@ pub fn cmd_rm(cli: &Cli, target: &str, recursive: bool) -> Result<(), CliError> if !recursive { return Err(CliError::RecursiveRequired(target.to_string())); } - let report = vault.delete_dir(target)?; - if cli.json { - print_json(&report)?; - } else { + emit(cli.json, &vault.delete_dir(target)?, |report| { println!( "Deleted {}/ ({} note(s), {} file(s))", report.dir, report.deleted_notes, report.deleted_resources ); print_dangled(&report.dangled); - } + }) } else if doc_kind(target) == DocKind::Resource { - let report = vault.delete_resource(target)?; - if cli.json { - print_json(&report)?; - } else { + emit(cli.json, &vault.delete_resource(target)?, |report| { println!("Deleted {}", report.path); print_dangled(&report.dangled); - } + }) } else { - let report = vault.delete_note(target)?; - if cli.json { - print_json(&report)?; - } else { + emit(cli.json, &vault.delete_note(target)?, |report| { println!("Deleted {}", report.path); print_dangled(&report.dangled); - } + }) } - Ok(()) } pub fn cmd_link( @@ -150,20 +123,19 @@ pub fn cmd_link( // was built with (like `add`/`mv`); a frontmatter-only edit won't re-embed. let vault = open_vault(cli.require_vault(None)?, true)?; let report = vault.link(src, dst, edge_type, explanation)?; - if cli.json { - print_json(&report)?; - } else if report.created { - println!( - "Linked {} —{}→ {}. Wrote the relation into the source note's frontmatter.", - report.src_path, report.relation, report.dst_path - ); - } else { - println!( - "Already linked {} —{}→ {}. Nothing changed.", - report.src_path, report.relation, report.dst_path - ); - } - Ok(()) + emit(cli.json, &report, |report| { + if report.created { + println!( + "Linked {} —{}→ {}. Wrote the relation into the source note's frontmatter.", + report.src_path, report.relation, report.dst_path + ); + } else { + println!( + "Already linked {} —{}→ {}. Nothing changed.", + report.src_path, report.relation, report.dst_path + ); + } + }) } /// Whether a `mv`/`rm` argument names an existing directory under `root` — the diff --git a/crates/b2-llm/src/lib.rs b/crates/b2-llm/src/lib.rs index b2c2745..97c3bd8 100644 --- a/crates/b2-llm/src/lib.rs +++ b/crates/b2-llm/src/lib.rs @@ -28,9 +28,9 @@ use serde::Serialize; pub use provider::OpenAiCompatProvider; pub use setup::{ - is_ollama, model_missing_message, probe_setup, pull_command, refusal_message, - unreachable_message, ChatSetup, ChatState, ModelTier, OllamaModel, OllamaSetup, ToolCallCap, - MODEL_TIERS, OLLAMA_INSTALL_URL, + model_missing_message, probe_setup, pull_command, refusal_message, unreachable_message, + ChatSetup, ChatState, ModelTier, OllamaModel, OllamaSetup, ToolCallCap, MODEL_TIERS, + OLLAMA_INSTALL_URL, }; /// The environment variable that swaps in the deterministic [`FakeLlm`]: `B2_LLM=fake`, diff --git a/crates/b2-llm/src/setup.rs b/crates/b2-llm/src/setup.rs index 6776327..3553c5c 100644 --- a/crates/b2-llm/src/setup.rs +++ b/crates/b2-llm/src/setup.rs @@ -39,7 +39,7 @@ const TAGS_TIMEOUT: Duration = Duration::from_secs(5); /// decides two things: whether Ollama's own commands belong in an error message, and /// whether [`probe_setup`] asks `/api/tags` at all. Deliberately a *guess about the /// runtime*, not a security check: being wrong costs one refused request. -pub fn is_ollama(base_url: &str) -> bool { +fn is_ollama(base_url: &str) -> bool { let endpoint = base_url.to_ascii_lowercase(); endpoint.contains(OLLAMA_PORT) || endpoint.contains("ollama") } From 3e41f45ec258d9c1bd2e0d95666f4cd63d6bdc34 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:44:03 +0000 Subject: [PATCH 14/30] refactor(core): drop ingest results nothing reads, share discovery's kernel and search's fusion tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ingest (flow ①): - ingest_file / project_file / ingest_vault return () — every caller discarded Ingested / Projected. plan_reindex returns two counts, not a per-note Vec. ingest_vault_with_progress and IngestOutcome go: Vault::reindex composes project_vault + embed_vault itself. - project_note_and_chunks takes a Rechunk enum instead of a (force, consult_vectors) bool pair whose fourth combination never occurred. - One walk_vault (walk + sort), one body_hash shared by the projection and the dry-run forecast, one skipped() constructor. Discovery: stage 2 and passage_pairs ran two hand-copied max-sim loops that had to agree for the card and its explanation to agree. They now share nearest_pairs, so they agree by construction. Population's unread n field goes and the type is private. Search: keyword-only and hybrid share one fuse() tail and Retrieval::empty(). Float sorts use total_cmp (same order for every reachable value, a defined place for NaN). db: embedding_space_exists is table_exists; embedding_space_matches reads recorded_embedder instead of re-parsing meta. Error::Serde was never constructed. Missing Debug derives added; docs citing retired plans ("slice 1", "research §9b", migrate()) corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/db.rs | 49 +++--- crates/b2-core/src/discover.rs | 104 ++++++------ crates/b2-core/src/error.rs | 12 +- crates/b2-core/src/ingest.rs | 293 ++++++++++++++------------------- crates/b2-core/src/search.rs | 59 ++++--- crates/b2-core/src/vault.rs | 31 ++-- crates/b2-core/tests/cancel.rs | 11 +- crates/b2-core/tests/embed.rs | 6 +- 8 files changed, 265 insertions(+), 300 deletions(-) diff --git a/crates/b2-core/src/db.rs b/crates/b2-core/src/db.rs index d1e4125..9acfe8b 100644 --- a/crates/b2-core/src/db.rs +++ b/crates/b2-core/src/db.rs @@ -501,6 +501,7 @@ fn apply_schema(conn: &Connection) -> Result<()> { /// One note's projection into `notes` (+ its `aliases`). Borrowed view so callers /// pass slices of an already-parsed note without extra allocation. +#[derive(Debug)] pub struct NoteRow<'a> { pub path: &'a str, pub r#type: &'a str, @@ -583,11 +584,12 @@ pub fn prune_notes_except(conn: &Connection, seen: &HashSet<&str>) -> Result { pub path: &'a str, pub class: &'a str, @@ -646,7 +648,7 @@ pub struct ResourceDetail { } /// Every inventoried resource — [`ResourceListing`] rows, path-ordered — the -/// file tree's resource half (`Vault::list_resources`, research §9b #10). +/// file tree's resource half (`Vault::list_resources`). pub fn list_resources(conn: &Connection) -> Result> { let mut stmt = conn.prepare("SELECT path, class, size, mtime FROM resources ORDER BY path")?; let rows = stmt.query_map([], |r| { @@ -788,7 +790,7 @@ pub fn prune_resources_except(conn: &Connection, seen: &HashSet) -> Resu } // --------------------------------------------------------------------------- -// chunks (FTS kept in lockstep by the triggers in migrate()) +// chunks (FTS kept in lockstep by the triggers in apply_schema()) // --------------------------------------------------------------------------- /// The content address of one chunk's embed input: blake3 of the chunk text. The text @@ -929,7 +931,7 @@ pub fn rebuild_fts(conn: &Connection, tokenizer: FtsTokenizer) -> Result<()> { } // --------------------------------------------------------------------------- -// embeddings — the vector tables are created at embed time, not in migrate(): +// embeddings — the vector tables are created at embed time, not in apply_schema(): // their *existence* is the "this vault has an embedding space" signal the // projected-but-unembedded fallbacks key on (ADR-0006). // --------------------------------------------------------------------------- @@ -1202,6 +1204,15 @@ pub struct PendingChunk { pub text_hash: String, } +/// The row mapper both pending-set queries share: `SELECT c.note_path, c.text, c.text_hash`. +fn pending_chunk(r: &rusqlite::Row) -> rusqlite::Result { + Ok(PendingChunk { + note_path: r.get(0)?, + text: r.get(1)?, + text_hash: r.get(2)?, + }) +} + /// Every chunk still lacking a stored vector, in `(path, seq)` order — the /// **DB-derived pending set** the embed pass fills. Deriving it here is what decouples /// projection from embedding: nothing is handed between the passes in memory, so any @@ -1221,13 +1232,7 @@ pub fn chunks_missing_vectors(conn: &Connection) -> Result> { WHERE v.text_hash IS NULL ORDER BY c.note_path, c.seq", )?; - let rows = stmt.query_map([], |r| { - Ok(PendingChunk { - note_path: r.get(0)?, - text: r.get(1)?, - text_hash: r.get(2)?, - }) - })?; + let rows = stmt.query_map([], pending_chunk)?; Ok(rows.collect::>>()?) } @@ -1246,13 +1251,7 @@ pub fn note_chunks_missing_vectors( WHERE c.note_path = ?1 AND v.text_hash IS NULL ORDER BY c.seq", )?; - let rows = stmt.query_map([note_path], |r| { - Ok(PendingChunk { - note_path: r.get(0)?, - text: r.get(1)?, - text_hash: r.get(2)?, - }) - })?; + let rows = stmt.query_map([note_path], pending_chunk)?; Ok(rows.collect::>>()?) } @@ -1439,11 +1438,7 @@ fn scan_vector_distances(conn: &Connection, query: &[f32]) -> Result Result, /// An embed form (`![alt](…)` / `![[…]]`) — display nicety, not a verb. pub embed: bool, - /// The authored alt/link/alias text — an image's index text (slice 3). + /// The authored alt/link/alias text. pub caption: Option, pub occurrence_index: i64, } /// Replace a note's edges. Every edge is authored (body links ∪ frontmatter /// `b2_relations:`), so this deletes the note's edges and re-inserts them from the -/// current Markdown (Flow ①) — the whole graph is a projection of Markdown, with no -/// suggestion rows to preserve. +/// current Markdown (Flow ①) — the whole graph is a projection of Markdown (G1). pub fn replace_authored_edges(conn: &Connection, src_path: &str, edges: &[EdgeRow]) -> Result<()> { conn.execute("DELETE FROM edges WHERE src_path = ?1", [src_path])?; for e in edges { @@ -1629,7 +1624,7 @@ pub fn resolve_link_target(conn: &Connection, link_path: &str) -> Result Result> { Ok(conn diff --git a/crates/b2-core/src/discover.rs b/crates/b2-core/src/discover.rs index 3db701a..e656536 100644 --- a/crates/b2-core/src/discover.rs +++ b/crates/b2-core/src/discover.rs @@ -88,16 +88,14 @@ pub struct CandidateNote { /// (ADR-0014). Any distance in the same space can be read against it, which is how a /// passage pair in the explain view is graded on the same yardstick as the cards. #[derive(Debug, Clone, Copy, PartialEq)] -pub struct Population { - /// How many notes were scored: the size of the population the z is read against. - pub n: usize, +struct Population { mean: f64, sd: f64, } impl Population { /// The z of one squared distance, oriented so nearer is higher. - pub fn z(&self, dist_sq: f32) -> f64 { + fn z(&self, dist_sq: f32) -> f64 { (self.mean - dist_sq as f64) / self.sd } } @@ -132,10 +130,8 @@ fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result> = db::note_chunk_vectors(conn, anchor)? - .into_iter() - .map(|(_, v)| v) - .collect(); + let (anchor_ids, anchor_vecs): (Vec, Vec>) = + db::note_chunk_vectors(conn, anchor)?.into_iter().unzip(); let Some(anchor_centroid) = centroid_of(&anchor_vecs) else { return Ok(None); }; @@ -155,11 +151,7 @@ fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result = coarse.into_iter().map(|(_, note)| note).collect(); // Stage 1 ends here, and nothing judges it: the shortlist is a recall device, never @@ -171,33 +163,28 @@ fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result = Vec::new(); for note_path in coarse.iter().take(shortlist) { - let mut best: Option<(f32, i64)> = None; - for (chunk_id, v) in db::note_chunk_vectors(conn, note_path)? { - for a in &anchor_vecs { - let dist_sq = l2_sq(a, &v); - if best.is_none_or(|(cur, _)| dist_sq < cur) { - best = Some((dist_sq, chunk_id)); - } - } - } - if let Some((dist_sq, evidence_chunk_id)) = best { + let pairs = nearest_pairs( + &anchor_ids, + &anchor_vecs, + db::note_chunk_vectors(conn, note_path)?, + ); + let best = pairs + .into_iter() + .reduce(|best, p| if p.0 < best.0 { p } else { best }); + if let Some((dist_sq, _, evidence_chunk_id)) = best { scored.push((dist_sq, note_path.clone(), evidence_chunk_id)); } } // Nearest-first, ties by path: the served order, and — because z below is affine in // this squared distance — also descending z. One sort key serves the row order and // the strength band, so the two can never disagree. - scored.sort_by(|a, b| { - a.0.partial_cmp(&b.0) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| a.1.cmp(&b.1)) - }); + scored.sort_by(|a, b| a.0.total_cmp(&b.0).then_with(|| a.1.cmp(&b.1))); // The statistic, computed AFTER stage 2 on the best-passage distances (GH #192) and // gating nothing (ADR-0014): the population is every scored shortlist note, which on @@ -215,11 +202,7 @@ fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result 0.0 { - population = Some(Population { - n: scored.len(), - mean, - sd, - }); + population = Some(Population { mean, sd }); } } @@ -431,23 +414,16 @@ fn passage_pairs_sq( if limit == 0 || !db::embedding_space_exists(conn)? { return Ok(Vec::new()); } - let anchor_vecs = db::note_chunk_vectors(conn, anchor)?; - let mut pairs: Vec<(f32, i64, i64)> = Vec::new(); - for (candidate_chunk_id, v) in db::note_chunk_vectors(conn, candidate)? { - let mut best: Option<(f32, i64)> = None; - for (anchor_chunk_id, a) in &anchor_vecs { - let dist_sq = l2_sq(a, &v); - if best.is_none_or(|(cur, _)| dist_sq < cur) { - best = Some((dist_sq, *anchor_chunk_id)); - } - } - if let Some((dist_sq, anchor_chunk_id)) = best { - pairs.push((dist_sq, anchor_chunk_id, candidate_chunk_id)); - } - } + let (anchor_ids, anchor_vecs): (Vec, Vec>) = + db::note_chunk_vectors(conn, anchor)?.into_iter().unzip(); + let mut pairs = nearest_pairs( + &anchor_ids, + &anchor_vecs, + db::note_chunk_vectors(conn, candidate)?, + ); // Stable, so equal distances keep the candidate's chunk order — the earliest chunk // wins a tie, as it does in `candidates`. - pairs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + pairs.sort_by(|a, b| a.0.total_cmp(&b.0)); Ok(pairs .into_iter() .take(limit) @@ -463,3 +439,27 @@ fn passage_pairs_sq( }) .collect()) } + +/// Discovery's max-sim kernel: for each candidate chunk, in order, the anchor chunk +/// nearest to it, as `(squared distance, anchor chunk, candidate chunk)`. Strictly-less +/// keeps the earliest anchor chunk on a tie. Stage 2 takes the minimum of this and the +/// explanation sorts it, so a card and its explanation read one computation. Squared L2 +/// is the ranking key without the per-comparison `sqrt`; the readers take it once per +/// surfaced pair. Empty when either side has no vectors. +fn nearest_pairs( + anchor_ids: &[i64], + anchor_vecs: &[Vec], + candidate: Vec<(i64, Vec)>, +) -> Vec<(f32, i64, i64)> { + candidate + .into_iter() + .filter_map(|(candidate_chunk_id, v)| { + anchor_ids + .iter() + .zip(anchor_vecs) + .map(|(id, a)| (l2_sq(a, &v), *id)) + .reduce(|best, p| if p.0 < best.0 { p } else { best }) + .map(|(dist_sq, anchor_chunk_id)| (dist_sq, anchor_chunk_id, candidate_chunk_id)) + }) + .collect() +} diff --git a/crates/b2-core/src/error.rs b/crates/b2-core/src/error.rs index 059b116..9797107 100644 --- a/crates/b2-core/src/error.rs +++ b/crates/b2-core/src/error.rs @@ -2,7 +2,7 @@ //! variants to choose a user-facing message. /// Errors surfaced by the index engine. Kept internal/structured — user-facing -/// surfaces (CLI, future GUI) translate these into generic, actionable messages. +/// surfaces (the CLI, the desktop app) translate these into generic, actionable messages. #[derive(thiserror::Error, Debug)] pub enum Error { #[error("sqlite error: {0}")] @@ -11,9 +11,6 @@ pub enum Error { #[error("io error: {0}")] Io(#[from] std::io::Error), - #[error("serialization error: {0}")] - Serde(#[from] serde_json::Error), - #[error("frontmatter edit unsupported: {0}")] Frontmatter(String), @@ -146,14 +143,13 @@ pub enum Error { IndexTooNew { found: i64, supported: i64 }, /// A resource reference (vault-relative path) did not resolve to any - /// inventoried resource — the resource sibling of [`Error::NoteNotFound`] - /// (file-type support slice 1). + /// inventoried resource — the resource sibling of [`Error::NoteNotFound`]. #[error("resource not found: {0}")] ResourceNotFound(String), /// The operation exists for notes but not (yet) for resources — e.g. - /// `b2 similar ` before slice 3 gives resources chunks and - /// centroids. Distinct from [`Error::ResourceNotFound`] so the adapters can + /// `b2 similar `, until resources have chunks and centroids of their + /// own. Distinct from [`Error::ResourceNotFound`] so the adapters can /// say "not yet" rather than "no such file". #[error("not supported for resources yet: {0}")] ResourceUnsupported(String), diff --git a/crates/b2-core/src/ingest.rs b/crates/b2-core/src/ingest.rs index f6f406f..8aa9112 100644 --- a/crates/b2-core/src/ingest.rs +++ b/crates/b2-core/src/ingest.rs @@ -7,8 +7,8 @@ //! order (phase 1 projects every note + its chunks, phase 2 derives edges against the //! now-complete resolver); and [`embed_vault`], the model-bound one, which fills //! whatever chunks still lack a vector — a pending set **derived from the DB**, never -//! handed over in memory. [`ingest_vault_with_progress`] is their composition; -//! [`ingest_file`] re-projects a single note inline against an already-built index. +//! handed over in memory. `Vault::reindex` is their composition; [`ingest_file`] +//! re-projects a single note inline against an already-built index. use crate::chunk::{chunk_body, ChunkConfig}; use crate::db::{self, EdgeRow, NoteRow}; @@ -82,16 +82,6 @@ pub(crate) fn unix_mtime(meta: &fs::Metadata) -> Option { Some(since_epoch.as_secs() as i64) } -/// Outcome of ingesting one file. -#[derive(Debug, Clone)] -pub struct Ingested { - /// The note's vault-relative path — its identity (L1). - pub path: String, - /// Whether this note was (re)embedded this run — `false` when an unchanged body - /// let the incremental path reuse its existing vectors. - pub embedded: bool, -} - /// What [`project_note_and_chunks`] returns for one note: the material later phases /// need — the body for edge derivation, and the `(text_hash, text)` pairs still /// needing a vector for the embed step. @@ -101,16 +91,47 @@ struct ProjectedNote { pending: Vec<(String, String)>, } -/// One note's entry in a [`plan_reindex`] preview (`reindex --dry-run`): what a real -/// reindex *would* do to this file, decided read-only. Only one question is left to -/// preview — a run that writes nothing (ADR-0004) has nothing to warn about beyond the -/// work it will do. -#[derive(Debug, Clone)] -pub struct PlannedNote { - /// Vault-relative path of the note. - pub path: String, - /// A real reindex would (re)embed this note's body (changed, fresh, or forced). - pub would_embed: bool, +/// A [`plan_reindex`] preview (`reindex --dry-run`): what a real reindex *would* do, +/// decided read-only. Only one question is left to preview — a run that writes nothing +/// (ADR-0004) has nothing to warn about beyond the work it will do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Plan { + /// Notes a real reindex would project (every readable `.md` file the walk collects). + pub notes: usize, + /// …of which this many would be (re)embedded (changed, fresh, or forced). + pub would_embed: usize, +} + +/// When [`project_note_and_chunks`] re-cuts a note's chunks. Three callers, three +/// questions — which is why this is not a pair of booleans. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Rechunk { + /// The projection pass: re-chunk a new or body-changed note, reading only `notes`, + /// never vector state ("unchanged body but missing vectors" is [`embed_vault`]'s + /// job, which keeps [`project_vault`] free of the vector tables). + IfBodyChanged, + /// A forced projection (`reindex --force`): re-chunk every note. + Always, + /// The inline path ([`ingest_file`]), which embeds what it re-cuts: also re-chunk a + /// note left mid-embed ([`would_reembed`]'s vector-state check), and hand back the + /// chunks still missing a vector. + IfBodyChangedOrUnembedded, +} + +impl Rechunk { + fn forced(force: bool) -> Self { + if force { + Self::Always + } else { + Self::IfBodyChanged + } + } +} + +/// A note body's content hash — the incremental pass's "did the body change?" key, +/// shared by the real projection and the dry-run so the forecast cannot drift from it. +fn body_hash(body: &str) -> String { + blake3::hash(body.as_bytes()).to_hex().to_string() } /// Progress during the embed phase, reported **per batch** so a large vault never @@ -141,21 +162,15 @@ pub struct ReindexProgress { /// frontmatter relations, and the `(text_hash, text)` pairs still needing a vector; /// embedding is deferred. No embedder here, and **no write to the vault**. /// -/// **Incremental:** unless `force`, a note whose body hash is unchanged is left -/// untouched and `pending` comes back empty; frontmatter-only edits still re-project -/// the note row and its edges. The invariant `incremental ≡ full rebuild` holds because -/// the re-used rows are byte-for-byte what a fresh projection would produce. -/// -/// `consult_vectors` selects the re-chunk predicate. The full-vault pass passes `false` -/// — it reads only `notes`, because "unchanged body but missing vectors" is -/// [`embed_vault`]'s job, and that is what keeps [`project_vault`] free of the vector -/// tables. [`ingest_file`] passes `true` (it embeds inline), so a note left mid-embed -/// is healed by [`would_reembed`]'s vector-state check. +/// **Incremental:** unless [`Rechunk::Always`], a note whose body hash is unchanged is +/// left untouched and `pending` comes back empty; frontmatter-only edits still +/// re-project the note row and its edges. The invariant `incremental ≡ full rebuild` +/// holds because the re-used rows are byte-for-byte what a fresh projection would +/// produce. `pending` is only ever filled for [`Rechunk::IfBodyChangedOrUnembedded`]. fn project_note_and_chunks( ctx: ProjectionCtx, rel_path: &str, - force: bool, - consult_vectors: bool, + when: Rechunk, ) -> Result { let ProjectionCtx { conn, root, cfg } = ctx; let abs = root.join(rel_path); @@ -163,16 +178,20 @@ fn project_note_and_chunks( let parsed = note::parse(&raw); let body = parsed.body().to_string(); - let body_hash = blake3::hash(body.as_bytes()).to_hex().to_string(); + let body_hash = body_hash(&body); let mtime = fs::metadata(&abs).ok().as_ref().and_then(unix_mtime); // Decide the re-chunk BEFORE the upsert overwrites `body_hash`. The inline path // also reads vector state (its caller ensured the space, hence // `space_exists = true`); the projection pass reads only `notes`. - let rechunk = if consult_vectors { - would_reembed(conn, rel_path, &body_hash, force, true)? - } else { - force || db::note_body_hash(conn, rel_path)?.as_deref() != Some(body_hash.as_str()) + let rechunk = match when { + Rechunk::Always => true, + Rechunk::IfBodyChanged => { + db::note_body_hash(conn, rel_path)?.as_deref() != Some(body_hash.as_str()) + } + Rechunk::IfBodyChangedOrUnembedded => { + would_reembed(conn, rel_path, &body_hash, false, true)? + } }; let fields = parsed.fields(); @@ -196,7 +215,7 @@ fn project_note_and_chunks( }, )?; - let relations = parsed.fields().relations.clone(); + let relations = fields.relations.clone(); // Incremental fast path: an unchanged body means identical chunks — reuse them and // return no pending work. A re-chunk hands back pairs only for chunks with **no @@ -205,7 +224,7 @@ fn project_note_and_chunks( let pending = if rechunk { let chunks = chunk_body(&body, cfg); db::replace_chunks(conn, rel_path, &chunks)?; - if consult_vectors { + if when == Rechunk::IfBodyChangedOrUnembedded { pending_for_note(conn, rel_path)? } else { // The projection pass never reads vector state; the embed pass derives @@ -474,66 +493,31 @@ fn derive_edge_id(src_path: &str, target_key: &str, edge_type: &str, occurrence: /// (the incremental path). Projects note + chunks + edges. The context's chunking /// policy is the same one every other path uses, so a single-note re-projection cuts /// identically to a full rebuild. -pub fn ingest_file(ctx: EmbedCtx, rel_path: &str) -> Result { +pub fn ingest_file(ctx: EmbedCtx, rel_path: &str) -> Result<()> { let EmbedCtx { proj, embedder } = ctx; let conn = proj.conn; db::ensure_embedding_space(conn, embedder.model_id(), embedder.dim())?; // Incremental (force=false): a frontmatter-only edit leaves the body unchanged, so // this re-projects note + edges without re-embedding. Vector state IS consulted — // this path embeds inline, so a note left mid-embed re-chunks and re-embeds here. - let p = project_note_and_chunks(proj, rel_path, false, true)?; - let embedded = !p.pending.is_empty(); + let p = project_note_and_chunks(proj, rel_path, Rechunk::IfBodyChangedOrUnembedded)?; // A single-note re-projection is never cancelled — always run to completion. embed_pending(conn, embedder, rel_path, &p.pending, |_| { ControlFlow::Continue(()) })?; project_edges(conn, rel_path, &p.body, &p.relations)?; - Ok(Ingested { - path: rel_path.to_string(), - embedded, - }) -} - -/// The result of a (possibly cancelled) full ingest: every projected note, plus -/// whether the embed phase was cut short. A cancelled run is still **consistent** — -/// every note has chunks + FTS + edges, only a *prefix* has vectors — so `notes` -/// describes the partial work truthfully and a re-run embeds the remainder. Vectors are -/// tracked **per note**: a note interrupted mid-embed is not fully embedded, so its -/// resume re-embeds it in full — at most one note's worth of redo. -pub struct IngestOutcome { - pub notes: Vec, - /// The embed phase stopped early because `on_progress` returned - /// [`ControlFlow::Break`]. Always `false` for a run that was never cancelled. - pub cancelled: bool, - /// Files the projection pass skipped as unreadable (see [`SkippedNote`]); empty on - /// a clean vault. A whole-vault reindex reports these rather than failing on them. - pub skipped: Vec, - /// Ghost rows pruned by the projection pass (see [`ProjectOutcome`], #31). - pub notes_pruned: usize, - /// The resource inventory's counts (see [`ProjectOutcome`]). - pub resources_indexed: usize, - pub resources_pruned: usize, + Ok(()) } -/// Ingest every `.md` file under `vault_root` (two-phase, deterministic order), +/// Reindex every `.md` file under `vault_root` — [`project_vault`] then [`embed_vault`], /// incrementally and with no progress reporting; dot-folders are skipped. A convenience /// wrapper the test suite drives: it builds the [`EmbedCtx`] itself around the default /// [`ChunkConfig`], which is why it takes loose arguments. -pub fn ingest_vault( - conn: &Connection, - vault_root: &Path, - embedder: &dyn Embedder, -) -> Result> { +pub fn ingest_vault(conn: &Connection, vault_root: &Path, embedder: &dyn Embedder) -> Result<()> { let cfg = ChunkConfig::default(); - let ctx = EmbedCtx::new(ProjectionCtx::new(conn, vault_root, &cfg), embedder); - Ok(ingest_vault_with_progress(ctx, false, &mut |_| ControlFlow::Continue(()))?.notes) -} - -/// One note's projection outcome: its vault-relative path, which is its identity -/// (L1) and everything a caller can learn from a pass that writes nothing. -#[derive(Debug, Clone)] -pub struct Projected { - pub path: String, + project_vault(ProjectionCtx::new(conn, vault_root, &cfg), false)?; + embed_vault(conn, embedder, &mut |_| ControlFlow::Continue(()))?; + Ok(()) } /// Re-project a single note **model-free** — the single-note sibling of @@ -541,12 +525,9 @@ pub struct Projected { /// body re-chunks, and the chunks join the DB-derived pending set for any later embed /// pass, so the save path needs no embedder. Contrast [`ingest_file`], which embeds /// inline for `add`/`link`/`mv` — ops that already require the model. -pub fn project_file(ctx: ProjectionCtx, rel_path: &str) -> Result { - let p = project_note_and_chunks(ctx, rel_path, false, false)?; - project_edges(ctx.conn, rel_path, &p.body, &p.relations)?; - Ok(Projected { - path: rel_path.to_string(), - }) +pub fn project_file(ctx: ProjectionCtx, rel_path: &str) -> Result<()> { + let p = project_note_and_chunks(ctx, rel_path, Rechunk::IfBodyChanged)?; + project_edges(ctx.conn, rel_path, &p.body, &p.relations) } /// A vault file (note **or** resource) the projection pass could not read, and @@ -560,7 +541,15 @@ pub struct SkippedNote { pub reason: String, } -/// Classify an I/O error hit while reading/stamping one note into a short, clean, +/// A file the walk met but could not read, as the pass reports it. +fn skipped(path: &str, err: &std::io::Error) -> SkippedNote { + SkippedNote { + path: path.to_string(), + reason: skip_reason(err), + } +} + +/// Classify an I/O error hit while reading one file into a short, clean, /// user-appropriate reason — the file's problem stated plainly, with no raw OS jargon /// (e.g. "stream did not contain valid UTF-8") and no B2 internal. Anything unusual /// falls back to a generic "could not be read". @@ -575,17 +564,17 @@ fn skip_reason(err: &std::io::Error) -> String { } /// The result of the model-free **projection pass** over the whole vault -/// ([`project_vault`]): every projected note, in the deterministic (sorted-path) +/// ([`project_vault`]): every projected note's path, in the deterministic (sorted-path) /// walk order, plus any files skipped as unreadable (empty on a clean vault). #[derive(Debug, Clone)] pub struct ProjectOutcome { - pub notes: Vec, + pub notes: Vec, pub skipped: Vec, /// Ghost rows pruned this pass (#31) — notes whose files were deleted outside /// b2 with no replacement. Zero on a vault with no out-of-band deletions. pub notes_pruned: usize, /// Resources inventoried this pass (unchanged ones included), and stale - /// inventory rows pruned — the slice-1 resource pass (spec §2). + /// inventory rows pruned. pub resources_indexed: usize, pub resources_pruned: usize, } @@ -611,11 +600,10 @@ pub struct EmbedOutcome { /// rebuild. It writes nothing to the vault (ADR-0004). pub fn project_vault(ctx: ProjectionCtx, force: bool) -> Result { let ProjectionCtx { conn, root, .. } = ctx; - let mut rel_paths = Vec::new(); - let mut resource_files = Vec::new(); - collect_vault_files(root, root, &mut rel_paths, &mut resource_files)?; - rel_paths.sort(); - resource_files.sort_by(|a, b| a.0.cmp(&b.0)); // paths are unique — a total order + let VaultWalk { + notes: rel_paths, + resources: resource_files, + } = walk_vault(root)?; // Phase 1: project every note + its chunks, which fills the resolver so phase 2 // never depends on file order. No pending pairs come back — the embed pass derives @@ -623,16 +611,13 @@ pub fn project_vault(ctx: ProjectionCtx, force: bool) -> Result let mut staged = Vec::with_capacity(rel_paths.len()); let mut skipped = Vec::new(); for rel in &rel_paths { - match project_note_and_chunks(ctx, rel, force, false) { + match project_note_and_chunks(ctx, rel, Rechunk::forced(force)) { Ok(p) => staged.push((rel.clone(), p.body, p.relations)), // A note we cannot read is *skipped*, not fatal: one bad file must never // abort a whole-vault reindex. Only filesystem failures reading THIS note // land here — the DB layer's `Error::Sqlite` is systemic and still aborts. // The read fails before any upsert, so no partial row is written. - Err(Error::Io(e)) => skipped.push(SkippedNote { - path: rel.clone(), - reason: skip_reason(&e), - }), + Err(Error::Io(e)) => skipped.push(self::skipped(rel, &e)), Err(other) => return Err(other), } } @@ -669,7 +654,7 @@ pub fn project_vault(ctx: ProjectionCtx, force: bool) -> Result let mut notes = Vec::with_capacity(staged.len()); for (path, body, relations) in staged { project_edges(conn, &path, &body, &relations)?; - notes.push(Projected { path }); + notes.push(path); } tracing::debug!( target: "b2::ingest", @@ -783,52 +768,8 @@ pub fn embed_vault( }) } -/// [`ingest_vault`] with `force` and an `on_progress` callback, so a slow full reindex -/// never looks frozen — **and can be cooperatively cancelled**: returning -/// [`ControlFlow::Break`] stops the embed pass at that batch boundary. Projection -/// completes before embedding starts, so a cancelled index is consistent: keyword search -/// and the graph are complete, only a prefix of notes has vectors. -/// -/// A thin composition of [`project_vault`] then [`embed_vault`]. From a clean index the -/// composed run is byte-identical to the old fused one; the sole intentional divergence -/// is a resume-after-partial run, where projection leaves an unchanged note's chunks in -/// place rather than regenerating their rowids — observably identical. -pub fn ingest_vault_with_progress( - ctx: EmbedCtx, - force: bool, - on_progress: &mut dyn FnMut(ReindexProgress) -> ControlFlow<()>, -) -> Result { - let ProjectOutcome { - notes: projected_notes, - skipped, - notes_pruned, - resources_indexed, - resources_pruned, - } = project_vault(ctx.proj, force)?; - let embed = embed_vault(ctx.proj.conn, ctx.embedder, on_progress)?; - - // Merge the two outcomes into the per-note report shape `reindex` has always - // returned: a note "embedded this run" iff the embed pass fully filled it. - let embedded: HashSet<&str> = embed.embedded.iter().map(String::as_str).collect(); - let notes = projected_notes - .into_iter() - .map(|p| Ingested { - embedded: embedded.contains(p.path.as_str()), - path: p.path, - }) - .collect(); - Ok(IngestOutcome { - notes, - cancelled: embed.cancelled, - skipped, - notes_pruned, - resources_indexed, - resources_pruned, - }) -} - /// A **read-only** preview of a reindex (`reindex --dry-run`). Walks every `.md` file -/// in the same order as [`ingest_vault`] and decides, per note, whether a real run would +/// in the same order as [`project_vault`] and decides, per note, whether a real run would /// (re)embed its body. That is the *whole* preview, and the shrinkage is the feature: /// the dry-run existed largely because a real run wrote to the vault, and one that /// writes nothing (ADR-0004) has only work to forecast. @@ -836,16 +777,15 @@ pub fn ingest_vault_with_progress( /// The embed decision reads the *currently stored* vectors, so it previews an /// incremental run under the embedder the index was built with; it does **not** detect a /// pending model swap, which would need the real model a dry-run avoids loading. -pub fn plan_reindex(conn: &Connection, vault_root: &Path, force: bool) -> Result> { +pub fn plan_reindex(conn: &Connection, vault_root: &Path, force: bool) -> Result { let space_exists = db::embedding_space_exists(conn)?; - let mut rel_paths = Vec::new(); // The dry-run previews *notes* (the embed decision); the resource inventory has - // no per-file decisions to preview, so its walk output is unused here. - let mut resource_files = Vec::new(); - collect_vault_files(vault_root, vault_root, &mut rel_paths, &mut resource_files)?; - rel_paths.sort(); - - let mut out = Vec::with_capacity(rel_paths.len()); + // no per-file decisions to preview, so its half of the walk is unused here. + let rel_paths = walk_vault(vault_root)?.notes; + let mut plan = Plan { + notes: 0, + would_embed: 0, + }; for rel in rel_paths { // Skip an unreadable file rather than abort — a real reindex would skip it too, // so the dry-run must not be the one place it still crashes the run. @@ -853,18 +793,32 @@ pub fn plan_reindex(conn: &Connection, vault_root: &Path, force: bool) -> Result Ok(raw) => raw, Err(_) => continue, }; - let body_hash = blake3::hash(note::parse(&raw).body().as_bytes()) - .to_hex() - .to_string(); + let body_hash = body_hash(note::parse(&raw).body()); // An unindexed path has no stored body hash, so this reads `true` for a note // new to the index — exactly as the real run decides it. - let would_embed = would_reembed(conn, &rel, &body_hash, force, space_exists)?; - out.push(PlannedNote { - path: rel, - would_embed, - }); + plan.notes += 1; + if would_reembed(conn, &rel, &body_hash, force, space_exists)? { + plan.would_embed += 1; + } } - Ok(out) + Ok(plan) +} + +/// What one vault walk found: every note path and every resource with its class, each +/// list sorted so the passes that share the walk run in one deterministic order. +struct VaultWalk { + notes: Vec, + resources: Vec<(String, ResourceClass)>, +} + +/// The vault walk both whole-vault passes share. +fn walk_vault(root: &Path) -> Result { + let mut notes = Vec::new(); + let mut resources = Vec::new(); + collect_vault_files(root, root, &mut notes, &mut resources)?; + notes.sort(); + resources.sort_by(|a, b| a.0.cmp(&b.0)); // paths are unique — a total order + Ok(VaultWalk { notes, resources }) } /// Walk the vault once, routing every file: `.md` (case-insensitive) to `notes`, @@ -928,10 +882,7 @@ fn project_resources( Ok(()) => indexed += 1, // Only a *filesystem* failure on this one file is recoverable — anything // else (SQLite, …) is systemic and aborts the pass, as everywhere else. - Err(Error::Io(e)) => skipped.push(SkippedNote { - path: rel.clone(), - reason: skip_reason(&e), - }), + Err(Error::Io(e)) => skipped.push(self::skipped(rel, &e)), Err(e) => return Err(e), } } diff --git a/crates/b2-core/src/search.rs b/crates/b2-core/src/search.rs index ba6f45d..e8f61a7 100644 --- a/crates/b2-core/src/search.rs +++ b/crates/b2-core/src/search.rs @@ -91,8 +91,7 @@ pub fn rrf_fuse(ranked_lists: &[Vec], k: usize) -> Vec<(i64, f64)> { let rank_of = |id: i64| tiebreak.get(&id).copied().unwrap_or(usize::MAX); let mut out: Vec<(i64, f64)> = scores.into_iter().collect(); out.sort_by(|a, b| { - b.1.partial_cmp(&a.1) - .unwrap_or(std::cmp::Ordering::Equal) + b.1.total_cmp(&a.1) .then(rank_of(a.0).cmp(&rank_of(b.0))) .then(a.0.cmp(&b.0)) }); @@ -428,6 +427,16 @@ pub struct Retrieval { pub best_cos: Option, } +impl Retrieval { + /// No hits and no dense reading: what a zero `limit` asks for. + fn empty() -> Self { + Self { + hits: Vec::new(), + best_cos: None, + } + } +} + /// How wide a pool to pull from each signal before fusing (qmd keeps ~30). /// /// `pub(crate)` because a *measurement* needs it: the façade's two candidate-pool @@ -456,10 +465,7 @@ pub fn keyword_only_search( limit: usize, ) -> Result { if limit == 0 { - return Ok(Retrieval { - hits: Vec::new(), - best_cos: None, - }); + return Ok(Retrieval::empty()); } let pool = pool_size(limit); let bm25 = keyword_search(conn, query, pool)?; @@ -469,11 +475,7 @@ pub fn keyword_only_search( pool, "keyword-only retrieval (no embedding space yet)" ); - let provenance = provenance_of(&bm25, &[]); - Ok(Retrieval { - hits: resolve_hits(conn, rrf_fuse(&[bm25], RRF_K), &provenance, limit)?, - best_cos: None, - }) + fuse(conn, bm25, &[], limit) } /// Hybrid search: BM25 ⊕ vector(query) -> RRF -> top `limit`, resolved to notes @@ -487,30 +489,41 @@ pub fn hybrid_search( limit: usize, ) -> Result { if limit == 0 { - return Ok(Retrieval { - hits: Vec::new(), - best_cos: None, - }); + return Ok(Retrieval::empty()); } let pool = pool_size(limit); let bm25 = keyword_search(conn, query, pool)?; let dense = db::vector_search(conn, &embedder.embed_query(query)?, pool)?; - let vector: Vec = dense.iter().map(|&(id, _)| id).collect(); - // The dense list is nearest-first, so its head *is* the best cosine — the - // absolute reading RRF is about to reduce to "rank 0". - let best_cos = dense.first().map(|&(_, d)| cosine_of_distance(d)); tracing::debug!( target: "b2::search", bm25_hits = bm25.len(), - vector_hits = vector.len(), - best_cos, + vector_hits = dense.len(), pool, "hybrid retrieval fusing BM25 ⊕ vector via RRF" ); + fuse(conn, bm25, &dense, limit) +} - let provenance = provenance_of(&bm25, &dense); +/// The shared tail of both retrievals: fuse the BM25 list with the dense one (if any) +/// by RRF, keep each chunk's provenance, resolve the first `limit` to notes, and carry +/// the dense half's absolute reading. With no dense list this is the single-list RRF +/// the keyword-only fallback has always served, and `best_cos` is `None`. +fn fuse( + conn: &rusqlite::Connection, + bm25: Vec, + dense: &[(i64, f32)], + limit: usize, +) -> Result { + let provenance = provenance_of(&bm25, dense); + // The dense list is nearest-first, so its head *is* the best cosine — the absolute + // reading RRF is about to reduce to "rank 0". + let best_cos = dense.first().map(|&(_, d)| cosine_of_distance(d)); + let mut lists = vec![bm25]; + if !dense.is_empty() { + lists.push(dense.iter().map(|&(id, _)| id).collect()); + } Ok(Retrieval { - hits: resolve_hits(conn, rrf_fuse(&[bm25, vector], RRF_K), &provenance, limit)?, + hits: resolve_hits(conn, rrf_fuse(&lists, RRF_K), &provenance, limit)?, best_cos, }) } diff --git a/crates/b2-core/src/vault.rs b/crates/b2-core/src/vault.rs index cfbff7f..289b37e 100644 --- a/crates/b2-core/src/vault.rs +++ b/crates/b2-core/src/vault.rs @@ -206,15 +206,24 @@ impl Vault { on_progress: &mut dyn FnMut(ingest::ReindexProgress) -> ControlFlow<()>, ) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "reindex", force).entered(); - let ingested = ingest::ingest_vault_with_progress(self.embed_ctx(), force, on_progress)?; + // Projection completes before embedding starts, so a cancelled run is still + // consistent: keyword search and the graph are complete, a prefix is embedded. + let projected = ingest::project_vault(self.ctx(), force)?; + let embed = ingest::embed_vault(&self.conn, self.embedder.as_ref(), on_progress)?; + // A note counts as embedded this run iff the embed pass fully filled it. + let filled: BTreeSet<&str> = embed.embedded.iter().map(String::as_str).collect(); Ok(ReindexReport { - indexed: ingested.notes.len(), - embedded: ingested.notes.iter().filter(|i| i.embedded).count(), - cancelled: ingested.cancelled, - skipped: ingested.skipped, - notes_pruned: ingested.notes_pruned, - resources_indexed: ingested.resources_indexed, - resources_pruned: ingested.resources_pruned, + indexed: projected.notes.len(), + embedded: projected + .notes + .iter() + .filter(|p| filled.contains(p.as_str())) + .count(), + cancelled: embed.cancelled, + skipped: projected.skipped, + notes_pruned: projected.notes_pruned, + resources_indexed: projected.resources_indexed, + resources_pruned: projected.resources_pruned, }) } @@ -262,10 +271,10 @@ impl Vault { /// note. A pure read, so it needs no model. pub fn plan_reindex(&self, force: bool) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "plan_reindex", force).entered(); - let planned = ingest::plan_reindex(&self.conn, &self.root, force)?; + let plan = ingest::plan_reindex(&self.conn, &self.root, force)?; Ok(ReindexPlan { - would_index: planned.len(), - would_embed: planned.iter().filter(|p| p.would_embed).count(), + would_index: plan.notes, + would_embed: plan.would_embed, }) } diff --git a/crates/b2-core/tests/cancel.rs b/crates/b2-core/tests/cancel.rs index 24ea96a..bb744bf 100644 --- a/crates/b2-core/tests/cancel.rs +++ b/crates/b2-core/tests/cancel.rs @@ -9,7 +9,7 @@ mod common; use b2_core::chunk::ChunkConfig; use b2_core::embed::FakeEmbedder; -use b2_core::ingest::{ingest_vault_with_progress, EmbedCtx, ProjectionCtx}; +use b2_core::ingest::{embed_vault, project_vault, ProjectionCtx}; use b2_core::open; use b2_core::vault::Vault; use common::{count, golden_vault_copy}; @@ -26,8 +26,9 @@ fn cancel_after_first_batch_leaves_a_consistent_resumable_index() { // Break at the very first embed batch. The golden vault's notes are small (one // batch each), so this embeds the first note only. let cfg = ChunkConfig::default(); - let ctx = EmbedCtx::new(ProjectionCtx::new(&conn, &vault, &cfg), &embedder); - let outcome = ingest_vault_with_progress(ctx, false, &mut |_| ControlFlow::Break(())).unwrap(); + let ctx = ProjectionCtx::new(&conn, &vault, &cfg); + project_vault(ctx, false).unwrap(); + let outcome = embed_vault(&conn, &embedder, &mut |_| ControlFlow::Break(())).unwrap(); assert!(outcome.cancelled, "the run reports itself cancelled"); // §5.1 — keyword + graph are COMPLETE at the cancel point: every note has chunks, @@ -53,8 +54,8 @@ fn cancel_after_first_batch_leaves_a_consistent_resumable_index() { // §5.2 — resume: an ordinary (uncancelled) reindex embeds exactly the remainder and // finishes. No corruption, no double-work. - let resumed = - ingest_vault_with_progress(ctx, false, &mut |_| ControlFlow::Continue(())).unwrap(); + project_vault(ctx, false).unwrap(); + let resumed = embed_vault(&conn, &embedder, &mut |_| ControlFlow::Continue(())).unwrap(); assert!(!resumed.cancelled); assert_eq!( count(&conn, "embeddings"), diff --git a/crates/b2-core/tests/embed.rs b/crates/b2-core/tests/embed.rs index af762ef..5c6af10 100644 --- a/crates/b2-core/tests/embed.rs +++ b/crates/b2-core/tests/embed.rs @@ -53,7 +53,7 @@ fn embed_batch_matches_embed_per_element() { #[test] fn reindex_with_progress_reports_cumulative_and_fully_embeds() { - use b2_core::ingest::{ingest_vault_with_progress, EmbedCtx, ProjectionCtx, ReindexProgress}; + use b2_core::ingest::{embed_vault, project_vault, ProjectionCtx, ReindexProgress}; let tmp = tempfile::TempDir::new().unwrap(); let vault = tmp.path().join("vault"); @@ -63,8 +63,8 @@ fn reindex_with_progress_reports_cumulative_and_fully_embeds() { let mut events: Vec = Vec::new(); let cfg = b2_core::chunk::ChunkConfig::default(); let embedder = FakeEmbedder::new(64); - let ctx = EmbedCtx::new(ProjectionCtx::new(&conn, &vault, &cfg), &embedder); - ingest_vault_with_progress(ctx, false, &mut |p| { + project_vault(ProjectionCtx::new(&conn, &vault, &cfg), false).unwrap(); + embed_vault(&conn, &embedder, &mut |p| { events.push(p); ControlFlow::Continue(()) }) From 03da7396300e676e01e202712aa450f74f89bae9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:44:40 +0000 Subject: [PATCH 15/30] refactor(core): one link grammar, one move pipeline, one delete pipeline link.rs exposes link_spans, a per-line span scanner that parse_links builds on, and a move now rewrites targets from those spans instead of its own whole-file scan. A stray [[ on one line no longer pairs with a ]] on a later one, so a link ingest projected is a link a move repairs. ParsedLink loses alias and typed (read only by link.rs's own tests); render_relation is the inverse of parse_relation. move_note, move_resource and move_dir now only validate and build a MoveSet; one executor plans the rewrites, commits them with the rename, re-keys every member and re-projects every inbound source. Behaviour made consistent: a self-link is reported under the note's new path, a resource move re-projects every inbound source (not only the rewritten ones), and a wikilink keeps its #heading fragment. A resource can no longer be moved to a .md path (MoveDestination), which a rebuild would have indexed as a note. A resource row missing from the index is inventoried at its new path instead of failing after the rename. delete_note, delete_resource and delete_dir run one delete pipeline over db::inbound_sources. db gains delete_note_row, delete_resource_row and repoint_resource (moved from mv), and the note/resource twin helpers share private implementations. pathspec gains the vault-relative path algebra (file_name, parent_dir, extension, is_md, is_under, rebase, relativize), and normalize_rel_md no longer turns Foo.MD into Foo.MD.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/db.rs | 216 ++++++-- crates/b2-core/src/link.rs | 372 +++++++++----- crates/b2-core/src/mv.rs | 910 ++++++++++++++++----------------- crates/b2-core/src/pathspec.rs | 137 ++++- crates/b2-core/src/resource.rs | 19 +- crates/b2-core/src/rm.rs | 164 +++--- crates/b2-core/tests/mv.rs | 203 ++++++++ 7 files changed, 1259 insertions(+), 762 deletions(-) diff --git a/crates/b2-core/src/db.rs b/crates/b2-core/src/db.rs index 220d6b6..1581ff9 100644 --- a/crates/b2-core/src/db.rs +++ b/crates/b2-core/src/db.rs @@ -22,7 +22,7 @@ use rusqlite::trace::{TraceEvent, TraceEventCodes}; use rusqlite::{ params, Connection, OptionalExtension, StatementStatus, Transaction, TransactionBehavior, }; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; use std::sync::OnceLock; use std::time::Duration; @@ -569,19 +569,71 @@ pub fn upsert_note(conn: &Connection, row: &NoteRow) -> Result<()> { /// be free to be NULL — the dangling case), so this must run *before* edge derivation, /// which then re-dangles the links that pointed here. pub fn prune_notes_except(conn: &Connection, seen: &HashSet<&str>) -> Result { - let mut stmt = conn.prepare("SELECT path FROM notes")?; + prune_members_except(conn, Members::Notes, |path| seen.contains(path)) +} + +/// Drop the `notes` row at `path` — the index half of deleting a note — and return how +/// many rows went (0 or 1). Everything keyed to the note cascades with the row, as for +/// [`prune_notes_except`]; inbound edges are the caller's to re-project. +pub fn delete_note_row(conn: &Connection, path: &str) -> Result { + delete_member(conn, Members::Notes, path) +} + +/// The two path-keyed member tables: a vault path names a note row or a resource row +/// (L1, L3), and the row-level housekeeping below is the same for both. +#[derive(Debug, Clone, Copy)] +enum Members { + Notes, + Resources, +} + +impl Members { + fn table(self) -> &'static str { + match self { + Members::Notes => "notes", + Members::Resources => "resources", + } + } +} + +/// Delete one member row by path; returns the rows deleted (0 or 1). +fn delete_member(conn: &Connection, members: Members, path: &str) -> Result { + let sql = format!("DELETE FROM {} WHERE path = ?1", members.table()); + Ok(conn.execute(&sql, [path])?) +} + +/// Delete every member row whose path `keep` rejects; returns how many went. +fn prune_members_except( + conn: &Connection, + members: Members, + keep: impl Fn(&str) -> bool, +) -> Result { + let mut stmt = conn.prepare(&format!("SELECT path FROM {}", members.table()))?; let stored = stmt .query_map([], |r| r.get::<_, String>(0))? .collect::, _>>()?; let mut pruned = 0; - for path in stored { - if !seen.contains(path.as_str()) { - pruned += conn.execute("DELETE FROM notes WHERE path = ?1", [&path])?; - } + for path in stored.iter().filter(|p| !keep(p)) { + pruned += delete_member(conn, members, path)?; } Ok(pruned) } +/// Every member path under the folder `dir` (vault-relative, no trailing slash), +/// path-ordered. Prefix-matched with `substr` (not `LIKE`) so a folder name containing +/// `%`/`_` never wildcards. +fn members_under_dir(conn: &Connection, members: Members, dir: &str) -> Result> { + let prefix = format!("{dir}/"); + let mut stmt = conn.prepare(&format!( + "SELECT path FROM {} + WHERE substr(path, 1, length(?1)) = ?1 + ORDER BY path", + members.table() + ))?; + let rows = stmt.query_map([&prefix], |r| r.get(0))?; + Ok(rows.collect::>>()?) +} + // --------------------------------------------------------------------------- // resources (file-type support slice 1 — data-model.md §10) // --------------------------------------------------------------------------- @@ -754,19 +806,7 @@ pub fn outbound_resource_edges(conn: &Connection, note_path: &str) -> Result Result> { - let mut stmt = conn.prepare( - "SELECT e.src_path, e.dst_path_raw - FROM edges e - WHERE e.dst_resource_path = ?1 - ORDER BY e.src_path, e.dst_path_raw", - )?; - let rows = stmt.query_map([path], |r| { - Ok(InboundEdge { - src_path: r.get(0)?, - dst_raw: r.get(1)?, - }) - })?; - Ok(rows.collect::>>()?) + inbound_edges_on(conn, "dst_resource_path", path) } /// Delete every `resources` row whose path is not in `seen` (the walk's survivors) @@ -774,17 +814,47 @@ pub fn inbound_resource_edge_targets(conn: &Connection, path: &str) -> Result) -> Result { - let mut stmt = conn.prepare("SELECT path FROM resources")?; - let stored = stmt - .query_map([], |r| r.get::<_, String>(0))? - .collect::, _>>()?; - let mut pruned = 0; - for path in stored { - if !seen.contains(&path) { - pruned += conn.execute("DELETE FROM resources WHERE path = ?1", [&path])?; - } - } - Ok(pruned) + prune_members_except(conn, Members::Resources, |path| seen.contains(path)) +} + +/// Drop the inventory row at `path` — the index half of deleting a resource — and +/// return how many rows went (0 or 1). Inbound edges re-dangle as for +/// [`prune_resources_except`]. +pub fn delete_resource_row(conn: &Connection, path: &str) -> Result { + delete_member(conn, Members::Resources, path) +} + +/// Re-key the inventory row at `old_path` to `new_path` — the resource sibling of +/// [`repoint_note_path`], and the index half of moving one. The bytes are the same, so +/// `size` and `content_hash` carry over; `class` (from the new extension) and `mtime` +/// (the moved file's) are the caller's. Returns whether `old_path` was inventoried — +/// `false` changes nothing. +/// +/// An upsert of the new row then a delete of the old, not an `UPDATE`: +/// `edges.dst_resource_path` has no `ON UPDATE CASCADE`, so the old row's inbound edges +/// re-dangle (`ON DELETE SET NULL`) until the caller re-projects their sources. +pub fn repoint_resource( + conn: &Connection, + old_path: &str, + new_path: &str, + class: &'static str, + mtime: Option, +) -> Result { + let Some(detail) = resource_detail(conn, old_path)? else { + return Ok(false); + }; + upsert_resource( + conn, + &ResourceRow { + path: new_path, + class, + size: detail.size, + mtime, + content_hash: &detail.content_hash, + }, + )?; + delete_resource_row(conn, old_path)?; + Ok(true) } // --------------------------------------------------------------------------- @@ -1525,13 +1595,67 @@ pub struct InboundEdge { /// materialized graph names the files to touch, so a move never scans the vault /// (index-engine.md §8). Ordered for deterministic rewriting. pub fn inbound_edge_targets(conn: &Connection, dst_path: &str) -> Result> { - let mut stmt = conn.prepare( + inbound_edges_on(conn, "dst_path", dst_path) +} + +/// Which member of a set an [`inbound_edges_of`] edge points at: an index into the +/// note paths or the resource paths the set was read from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InboundTarget { + Note(usize), + Resource(usize), +} + +/// Every edge pointing at one of a set of members — the notes at `notes`, the +/// resources at `resources` — tagged with the member it targets: the notes' edges +/// first, then the resources', each member's in [`inbound_edge_targets`] order. The one +/// graph read a move (which rewrites these links) and a delete (which dangles them) +/// both start from, bounded by the inbound count rather than a vault scan +/// (index-engine.md §8). +pub fn inbound_edges_of( + conn: &Connection, + notes: &[&str], + resources: &[&str], +) -> Result> { + let mut out = Vec::new(); + for (i, path) in notes.iter().enumerate() { + let edges = inbound_edge_targets(conn, path)?; + out.extend(edges.into_iter().map(|e| (InboundTarget::Note(i), e))); + } + for (i, path) in resources.iter().enumerate() { + let edges = inbound_resource_edge_targets(conn, path)?; + out.extend(edges.into_iter().map(|e| (InboundTarget::Resource(i), e))); + } + Ok(out) +} + +/// The notes linking into a set of members ([`inbound_edges_of`]), sorted and deduped — +/// the files whose edges must re-project once the set moves or goes. +pub fn inbound_sources( + conn: &Connection, + notes: &[&str], + resources: &[&str], +) -> Result> { + Ok(inbound_edges_of(conn, notes, resources)? + .into_iter() + .map(|(_, e)| e.src_path) + .collect()) +} + +/// The edges whose `column` (`dst_path` or `dst_resource_path`) names `path`, as +/// [`InboundEdge`] rows ordered by source then authored text. +fn inbound_edges_on( + conn: &Connection, + column: &'static str, + path: &str, +) -> Result> { + let mut stmt = conn.prepare(&format!( "SELECT e.src_path, e.dst_path_raw FROM edges e - WHERE e.dst_path = ?1 - ORDER BY e.src_path, e.dst_path_raw", - )?; - let rows = stmt.query_map([dst_path], |r| { + WHERE e.{column} = ?1 + ORDER BY e.src_path, e.dst_path_raw" + ))?; + let rows = stmt.query_map([path], |r| { Ok(InboundEdge { src_path: r.get(0)?, dst_raw: r.get(1)?, @@ -1541,31 +1665,15 @@ pub fn inbound_edge_targets(conn: &Connection, dst_path: &str) -> Result Result> { - let prefix = format!("{dir}/"); - let mut stmt = conn.prepare( - "SELECT path FROM notes - WHERE substr(path, 1, length(?1)) = ?1 - ORDER BY path", - )?; - let rows = stmt.query_map([&prefix], |r| r.get(0))?; - Ok(rows.collect::>>()?) + members_under_dir(conn, Members::Notes, dir) } /// Every inventoried resource path under the directory `dir` — the resource half /// of [`notes_under_dir`], same prefix semantics, path-ordered. pub fn resources_under_dir(conn: &Connection, dir: &str) -> Result> { - let prefix = format!("{dir}/"); - let mut stmt = conn.prepare( - "SELECT path FROM resources - WHERE substr(path, 1, length(?1)) = ?1 - ORDER BY path", - )?; - let rows = stmt.query_map([&prefix], |r| r.get(0))?; - Ok(rows.collect::>>()?) + members_under_dir(conn, Members::Resources, dir) } /// Re-key a note from `old_path` to `new_path` — **the** index-side move (ADR-0003). diff --git a/crates/b2-core/src/link.rs b/crates/b2-core/src/link.rs index 489f1b8..3c23ae4 100644 --- a/crates/b2-core/src/link.rs +++ b/crates/b2-core/src/link.rs @@ -1,4 +1,4 @@ -//! Parse a note body into the links that become edges. +//! Parse a note body into the links that become edges — B2's **one link grammar**. //! //! **The body carries no B2 syntax** (ADR-0010): every body construct is ordinary //! Obsidian Markdown and yields an untyped `references` edge — prose around a link is just @@ -6,7 +6,12 @@ //! `![alt](path)` (relative vault targets only, the `!` marking an **embed** and the //! text/alt captured as the edge's **caption**); and the `![[file.ext|alias]]` embed. A //! *typed* edge — ` [[path|alias]] — explanation` — exists only as a frontmatter -//! `b2_relations:` entry, parsed by [`parse_relation`]. +//! `b2_relations:` entry, parsed by [`parse_relation`] and written by [`render_relation`]. +//! +//! Everything that reads link text goes through [`link_spans`]: ingest projects edges from +//! it ([`parse_links`]) and a move rewrites targets in place from it (`mv.rs`), so what a +//! move repairs is exactly what ingest projected. The scan is **per line**: a stray `[[` +//! never pairs with a `]]` on a later line. //! //! Hand-rolled and deliberately minimal. Known simplifications, to revisit when queries //! need them: a typed frontmatter entry yields exactly one edge (extra wikilinks in its @@ -14,6 +19,8 @@ //! only `—`/`:` introduce an explanation; a Markdown link's text stops at the first `]` //! and its target at the first `)`. +use std::ops::Range; + /// A link found in a body, ready to be resolved + projected into `edges`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ParsedLink { @@ -23,33 +30,81 @@ pub struct ParsedLink { /// parenthesized target (becomes `dst_path_raw`; a `#fragment` suffix is /// stripped at *resolution*, never here). pub target_path: String, - /// The `|alias` display text, if present (wikilink forms only). - pub alias: Option, /// Trailing text after `—`/`:` on a typed frontmatter entry. pub explanation: Option, - /// True when this came from a ` [[..]]` frontmatter entry. - pub typed: bool, /// True for an embed form (`![alt](path)` / `![[file]]`) — recorded on the - /// edge as a display nicety, never a distinct verb (spec §3). + /// edge as a display nicety, never a distinct verb (data-model.md §3). pub embed: bool, /// True when the target came from a Markdown-form link (`[…](target)`). /// Resolution treats these with standard Markdown semantics — note-relative - /// first, then vault-root — while wikilink targets stay vault-root (spec §3). + /// first, then vault-root — while wikilink targets stay vault-root. pub md_form: bool, /// The authored display text — `![alt](…)`'s alt, `[text](…)`'s text, or a - /// wikilink's alias. Captured on the edge; it becomes an image's index text - /// (slice 3). + /// wikilink's alias. Stored as the edge's `caption` (data-model.md §3). pub caption: Option, } +/// Which syntax wrote a link. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LinkForm { + /// `[[path|alias]]` / `![[path|alias]]` — a vault-root target. + Wiki, + /// `[text](target)` / `![alt](target)` — note-relative first, then vault-root. + Markdown, +} + +/// Where one link sits in the scanned text, as byte ranges into it — so the one grammar +/// serves both reading a link ([`parse_links`]) and splicing a new target into it in +/// place (the move's rewrite). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LinkSpan { + pub form: LinkForm, + /// A `!` directly before the opening bracket. + pub embed: bool, + /// The target token, trimmed: `concepts/memory` in `[[ concepts/memory | M ]]`, + /// `img.png` in `![a]( img.png )`. Never empty. + pub target: Range, + /// The display text, trimmed: a wikilink's alias (present — possibly empty — whenever + /// the link has a `|`), or a Markdown link's text (absent when blank). + pub caption: Option>, +} + +/// Every vault link in `text`, in document order, scanned line by line. A Markdown-form +/// link at an external target (a scheme, an absolute path, a fragment-only anchor) is +/// not a vault link and yields no span; nor does a wikilink with an empty target. +pub(crate) fn link_spans(text: &str) -> Vec { + let mut spans = Vec::new(); + let mut line_start = 0; + for line in text.split_inclusive('\n') { + let content = match line.strip_suffix('\n') { + Some(l) => l.strip_suffix('\r').unwrap_or(l), + None => line, + }; + scan_line(content, line_start, &mut spans); + line_start += line.len(); + } + spans +} + /// Parse every link in `body`, in document order — all untyped `references` /// edges: the body carries no typed syntax (data-model §2). pub fn parse_links(body: &str) -> Vec { - let mut links = Vec::new(); - for line in body.lines() { - scan_inline_links(line, &mut links); + link_spans(body) + .iter() + .map(|span| reference(body, span)) + .collect() +} + +/// The untyped `references` edge a span authors. +fn reference(text: &str, span: &LinkSpan) -> ParsedLink { + ParsedLink { + edge_type: "references".to_string(), + target_path: text[span.target.clone()].to_string(), + explanation: None, + embed: span.embed, + md_form: span.form == LinkForm::Markdown, + caption: span.caption.clone().map(|c| text[c].to_string()), } - links } /// Parse a typed spec ` [[path|alias]] [— explanation]` — the shape of a @@ -74,17 +129,15 @@ fn parse_typed_spec(rest: &str) -> Option { let inner = &inner_and_rest[..close]; let tail = &inner_and_rest[close + 2..]; - let (target_path, alias) = split_target(inner); - if target_path.is_empty() { + let (target, alias) = split_wiki_inner(inner, 0); + if target.is_empty() { return None; } Some(ParsedLink { edge_type: verb.to_string(), - target_path, - caption: alias.clone(), - alias, + target_path: inner[target].to_string(), + caption: alias.map(|a| inner[a].to_string()), explanation: extract_explanation(tail), - typed: true, embed: false, md_form: false, }) @@ -100,18 +153,30 @@ pub fn parse_relation(spec: &str) -> Option { if let Some(link) = parse_typed_spec(spec) { return Some(link); } - // bare wikilink fallback → references - let mut tmp = Vec::new(); - scan_inline_links(spec, &mut tmp); - tmp.into_iter().next() + // bare link fallback → references (the entry is one YAML value, scanned whole) + let mut spans = Vec::new(); + scan_line(spec, 0, &mut spans); + spans.first().map(|span| reference(spec, span)) } -/// Collect every inline link in `line` as a `references` edge, in written order: +/// Render a typed frontmatter `b2_relations:` entry — the inverse of [`parse_relation`] +/// for a typed spec: ` [[target_path]]`, then ` — explanation` when one is given +/// (data-model §2). `target_path` is written verbatim as the link text, so the caller +/// chooses the convention (a note's `.md` dropped, as Obsidian writes links). A blank +/// explanation is omitted and a given one trimmed, exactly as parsing reads it back. +pub fn render_relation(verb: &str, target_path: &str, explanation: Option<&str>) -> String { + match explanation.map(str::trim).filter(|e| !e.is_empty()) { + Some(e) => format!("{verb} [[{target_path}]] — {e}"), + None => format!("{verb} [[{target_path}]]"), + } +} + +/// Collect every link in `line` as a [`LinkSpan`] offset by `base`, in written order: /// `[[path|alias]]` and `![[path|alias]]` (wikilink + embed), and Markdown's own /// `[text](target)` / `![alt](target)` — the latter only for **vault** targets /// (a scheme, an absolute path, or a fragment-only target is not a vault member -/// and yields nothing; spec §3). -fn scan_inline_links(line: &str, out: &mut Vec) { +/// and yields nothing; data-model.md §10). +fn scan_line(line: &str, base: usize, out: &mut Vec) { let mut i = 0; while i < line.len() { let rest = &line[i..]; @@ -126,6 +191,7 @@ fn scan_inline_links(line: &str, out: &mut Vec) { } }; let marker = usize::from(embed); // the '!' byte, when present + let open = base + i + marker; // the first '[' byte // Wikilink (plain or embed): `[[path|alias]]`. if let Some(inner_rest) = bracketed.strip_prefix("[[") { @@ -133,17 +199,13 @@ fn scan_inline_links(line: &str, out: &mut Vec) { i += marker + 1; continue; }; - let (target_path, alias) = split_target(&inner_rest[..close]); - if !target_path.is_empty() { - out.push(ParsedLink { - edge_type: "references".to_string(), - target_path, - caption: alias.clone(), - alias, - explanation: None, - typed: false, + let (target, alias) = split_wiki_inner(&inner_rest[..close], open + 2); + if !target.is_empty() { + out.push(LinkSpan { + form: LinkForm::Wiki, embed, - md_form: false, + target, + caption: alias, }); } i += marker + 2 + close + 2; @@ -151,21 +213,17 @@ fn scan_inline_links(line: &str, out: &mut Vec) { } // Markdown form: `[text](target)`. - if let Some((text, target, consumed)) = parse_md_link(bracketed) { - if !is_external_target(&target) { - let text = text.trim(); - out.push(ParsedLink { - edge_type: "references".to_string(), - target_path: target, - alias: None, - caption: (!text.is_empty()).then(|| text.to_string()), - explanation: None, - typed: false, + if let Some(md) = parse_md_link(bracketed) { + let target = shift(&md.target, open); + if !is_external_target(&line[target.start - base..target.end - base]) { + out.push(LinkSpan { + form: LinkForm::Markdown, embed, - md_form: true, + target, + caption: (!md.text.is_empty()).then(|| shift(&md.text, open)), }); } - i += marker + consumed; + i += marker + md.consumed; continue; } @@ -173,21 +231,57 @@ fn scan_inline_links(line: &str, out: &mut Vec) { } } -/// Parse a leading `[text](target)`; returns `(text, target, bytes consumed)`. -/// Minimal by design (module doc): text stops at the first `]`, the target at the -/// first `)`, and `](` must be adjacent. -fn parse_md_link(s: &str) -> Option<(String, String, usize)> { +/// A leading Markdown link's parts, as ranges into the string it was parsed from. +struct MdLink { + /// The link text, trimmed (possibly empty). + text: Range, + /// The target, trimmed (never empty). + target: Range, + /// Bytes from the `[` through the closing `)`. + consumed: usize, +} + +/// Parse a leading `[text](target)`. Minimal by design (module doc): text stops at the +/// first `]`, the target at the first `)`, and `](` must be adjacent. +fn parse_md_link(s: &str) -> Option { let inner = s.strip_prefix('[')?; let close = inner.find(']')?; - let text = &inner[..close]; let target_rest = inner[close + 1..].strip_prefix('(')?; let end = target_rest.find(')')?; - let target = target_rest[..end].trim(); + let target_start = close + 3; // '[' + text + ']' + '(' + let target = trimmed(&target_rest[..end], target_start); if target.is_empty() { return None; } - // '[' + text + ']' + '(' + target + ')' - Some((text.to_string(), target.to_string(), close + end + 4)) + Some(MdLink { + text: trimmed(&inner[..close], 1), + target, + // '[' + text + ']' + '(' + target + ')' + consumed: close + end + 4, + }) +} + +/// A wikilink's inner text (between `[[` and `]]`, starting at byte `base`) split at the +/// first `|` into the trimmed target and, when a `|` is present, the trimmed alias. +fn split_wiki_inner(inner: &str, base: usize) -> (Range, Option>) { + match inner.split_once('|') { + Some((path, alias)) => ( + trimmed(path, base), + Some(trimmed(alias, base + path.len() + 1)), + ), + None => (trimmed(inner, base), None), + } +} + +/// The range of `s.trim()` within the text `s` starts at byte `base` of. +fn trimmed(s: &str, base: usize) -> Range { + let start = base + (s.len() - s.trim_start().len()); + start..start + s.trim().len() +} + +/// `r` moved `by` bytes to the right. +fn shift(r: &Range, by: usize) -> Range { + r.start + by..r.end + by } /// A Markdown-form target that is **not** a vault member: any scheme @@ -204,14 +298,6 @@ fn is_external_target(target: &str) -> bool { } } -/// Split `path|alias` (alias optional) into trimmed parts. -fn split_target(inner: &str) -> (String, Option) { - match inner.split_once('|') { - Some((path, alias)) => (path.trim().to_string(), Some(alias.trim().to_string())), - None => (inner.trim().to_string(), None), - } -} - /// Read the explanation after a typed link: trailing text introduced by an /// em-dash or a colon (data-model §2). Anything else means no explanation. fn extract_explanation(tail: &str) -> Option { @@ -225,14 +311,14 @@ fn extract_explanation(tail: &str) -> Option { mod tests { use super::*; - /// One expected link, as the case tables spell it: `(type, target, caption, embed, typed)`. - type Expected<'a> = (&'a str, &'a str, Option<&'a str>, bool, bool); + /// One expected link, as the case tables spell it: `(type, target, caption, embed)`. + type Expected<'a> = (&'a str, &'a str, Option<&'a str>, bool); - /// Shorthand: parse one line, return `(type, target, caption, embed, typed)`. - fn parsed(line: &str) -> Vec<(String, String, Option, bool, bool)> { + /// Shorthand: parse one line, return `(type, target, caption, embed)`. + fn parsed(line: &str) -> Vec<(String, String, Option, bool)> { parse_links(line) .into_iter() - .map(|l| (l.edge_type, l.target_path, l.caption, l.embed, l.typed)) + .map(|l| (l.edge_type, l.target_path, l.caption, l.embed)) .collect() } @@ -242,13 +328,7 @@ mod tests { // ![alt](path) — embed with caption ( "See ![a sailboat](img/IMG_2041.jpg) here.", - &[( - "references", - "img/IMG_2041.jpg", - Some("a sailboat"), - true, - false, - )], + &[("references", "img/IMG_2041.jpg", Some("a sailboat"), true)], ), // [text](path) — plain link with caption ( @@ -258,39 +338,23 @@ mod tests { "papers/attention.pdf", Some("the paper"), false, - false, )], ), // empty alt: still an edge, no caption - ( - "![](img/x.png)", - &[("references", "img/x.png", None, true, false)], - ), + ("![](img/x.png)", &[("references", "img/x.png", None, true)]), // ![[file.ext]] embed, with and without alias ( "![[img/photo.png]]", - &[("references", "img/photo.png", None, true, false)], + &[("references", "img/photo.png", None, true)], ), ( "![[img/photo.png|hero shot]]", - &[( - "references", - "img/photo.png", - Some("hero shot"), - true, - false, - )], + &[("references", "img/photo.png", Some("hero shot"), true)], ), // bare wikilink to a resource: alias doubles as caption ( "[[papers/x.pdf|the paper]]", - &[( - "references", - "papers/x.pdf", - Some("the paper"), - false, - false, - )], + &[("references", "papers/x.pdf", Some("the paper"), false)], ), // a .md markdown link is parsed too — resolution dispatches by extension ( @@ -300,27 +364,20 @@ mod tests { "concepts/memory.md", Some("background"), false, - false, )], ), // fragment kept raw (stripped at resolution, not here) ( "[sec](notes/a.md#history)", - &[( - "references", - "notes/a.md#history", - Some("sec"), - false, - false, - )], + &[("references", "notes/a.md#history", Some("sec"), false)], ), // document order across mixed forms on one line ( "[[a.md]] then ![x](b.png) then [y](c.txt)", &[ - ("references", "a.md", None, false, false), - ("references", "b.png", Some("x"), true, false), - ("references", "c.txt", Some("y"), false, false), + ("references", "a.md", None, false), + ("references", "b.png", Some("x"), true), + ("references", "c.txt", Some("y"), false), ], ), ]; @@ -328,9 +385,7 @@ mod tests { let got = parsed(line); let want: Vec<_> = want .iter() - .map(|(t, p, c, e, ty)| { - (t.to_string(), p.to_string(), c.map(str::to_string), *e, *ty) - }) + .map(|(t, p, c, e)| (t.to_string(), p.to_string(), c.map(str::to_string), *e)) .collect(); assert_eq!(got, want, "line: {line}"); } @@ -357,7 +412,6 @@ mod tests { let links = parse_links("- supports [[papers/x.pdf|the paper]] — key evidence"); assert_eq!(links.len(), 1); let l = &links[0]; - assert!(!l.typed); assert_eq!(l.edge_type, "references"); assert_eq!(l.target_path, "papers/x.pdf"); assert_eq!(l.caption.as_deref(), Some("the paper")); @@ -374,43 +428,41 @@ mod tests { let links = parse_links("- see [[concepts/memory|Human memory]] for the mechanism\n"); assert_eq!(links.len(), 1); assert_eq!(links[0].edge_type, "references"); - assert!(!links[0].typed); } /// A prose link and a verb-led list item are the same thing to the parser: two - /// `references` edges in document order, neither typed and neither carrying an - /// explanation — no body shape is ever "special". + /// `references` edges in document order, neither carrying an explanation — no + /// body shape is ever "special". #[test] fn body_links_never_gain_a_type_from_surrounding_prose() { let body = "Spaced repetition exploits the [[concepts/memory|Human memory]] retrieval curve.\n\n## Relations\n- supports [[concepts/memory|Human memory]] — applies the forgetting curve\n"; let links = parse_links(body); assert_eq!(links.len(), 2); assert!(links.iter().all(|l| l.edge_type == "references")); - assert!(links.iter().all(|l| !l.typed && l.explanation.is_none())); + assert!(links.iter().all(|l| l.explanation.is_none())); } - /// `alias` is the wikilink's own `|`-part, distinct from the display `caption` - /// the other cases assert: absent means `None`, never an empty string. + /// A wikilink's caption is its own `|`-part: absent means `None`, never an empty + /// string. #[test] - fn a_wikilink_without_an_alias_keeps_a_none_alias() { + fn a_wikilink_without_an_alias_has_no_caption() { let links = parse_links("Refer to [[concepts/memory]].\n"); assert_eq!(links[0].target_path, "concepts/memory"); - assert_eq!(links[0].alias, None); + assert_eq!(links[0].caption, None); } #[test] fn parse_relation_reads_verb_link_and_explanation() { let l = parse_relation("supports [[papers/x.pdf|the paper]] — key evidence").unwrap(); - assert!(l.typed); assert_eq!(l.edge_type, "supports"); assert_eq!(l.target_path, "papers/x.pdf"); - assert_eq!(l.alias.as_deref(), Some("the paper")); + assert_eq!(l.caption.as_deref(), Some("the paper")); assert_eq!(l.explanation.as_deref(), Some("key evidence")); // A bare entry (no verb) falls back to an untyped reference. let bare = parse_relation("[[notes/a|A]]").unwrap(); - assert!(!bare.typed); assert_eq!(bare.edge_type, "references"); + assert_eq!(bare.target_path, "notes/a"); // No wikilink at all ⇒ no edge. assert!(parse_relation("just some words").is_none()); @@ -427,11 +479,45 @@ mod tests { assert_eq!(colon.explanation.as_deref(), Some("replaced after Q2")); let tail = parse_relation("inspired-by [[notes/x|X]]").unwrap(); - assert!(tail.typed); assert_eq!(tail.edge_type, "inspired-by"); assert_eq!(tail.explanation, None); } + /// What `b2 link` writes is what ingest reads back: rendering then parsing a + /// relation returns the verb, target and explanation it was rendered from — and a + /// blank explanation renders as none at all. + #[test] + fn a_rendered_relation_parses_back_to_what_it_was_rendered_from() { + let cases: &[(&str, &str, Option<&str>)] = &[ + ( + "supports", + "concepts/memory", + Some("applies the forgetting curve"), + ), + ("contradicts", "notes/a.md", None), + ("references", "papers/x.pdf", Some("the key table")), + ("inspired-by", "deep/nested/note", Some("a: colon inside")), + ]; + for &(verb, target, explanation) in cases { + let spec = render_relation(verb, target, explanation); + let got = parse_relation(&spec).unwrap(); + assert_eq!( + (got.edge_type.as_str(), got.target_path.as_str()), + (verb, target), + "{spec}" + ); + assert_eq!(got.explanation.as_deref(), explanation, "{spec}"); + } + assert_eq!( + render_relation("supports", "a", Some(" ")), + "supports [[a]]" + ); + assert_eq!( + render_relation("supports", "a", Some(" why ")), + "supports [[a]] — why" + ); + } + #[test] fn malformed_forms_do_not_derail_the_scan() { // an unclosed wikilink on one line never hides the next line's links @@ -441,7 +527,6 @@ mod tests { "references".into(), "a.png".into(), Some("ok".into()), - false, false )] ); @@ -455,4 +540,41 @@ mod tests { // a lone bang is prose assert_eq!(parsed("hey! [x](a.png)").len(), 1); } + + /// The spans are what a move splices into, so their ranges must name exactly the + /// trimmed target (and caption) bytes, offset across lines — CRLF endings included. + #[test] + fn spans_address_the_trimmed_target_bytes_across_lines() { + let text = "a [[ x/y | Why ]] b\r\n![alt]( img.png ) and [t](https://e.x)\n[[z]]"; + let spans = link_spans(text); + let slices: Vec<(LinkForm, bool, &str, Option<&str>)> = spans + .iter() + .map(|s| { + ( + s.form, + s.embed, + &text[s.target.clone()], + s.caption.clone().map(|c| &text[c]), + ) + }) + .collect(); + assert_eq!( + slices, + vec![ + (LinkForm::Wiki, false, "x/y", Some("Why")), + (LinkForm::Markdown, true, "img.png", Some("alt")), + (LinkForm::Wiki, false, "z", None), + ] + ); + } + + /// A stray `[[` never pairs with a `]]` on a later line, so the later line's link + /// is found — by ingest and, through the same spans, by a move's rewrite. + #[test] + fn a_stray_open_bracket_never_swallows_the_next_lines_link() { + let text = "broken [[x\nsee [[old]]\n"; + let spans = link_spans(text); + assert_eq!(spans.len(), 1); + assert_eq!(&text[spans[0].target.clone()], "old"); + } } diff --git a/crates/b2-core/src/mv.rs b/crates/b2-core/src/mv.rs index f3511c5..603558b 100644 --- a/crates/b2-core/src/mv.rs +++ b/crates/b2-core/src/mv.rs @@ -1,19 +1,24 @@ -//! Move / rename a note and repair inbound links. +//! Move / rename a note, a resource or a folder, and repair inbound links. //! //! **A move is where "rename keeps every backlink resolving" is earned.** Identity is -//! the vault-relative path (ADR-0003), so a move changes the moved note's identity, and -//! this module makes that a re-key rather than a break. Two halves, both bounded by the -//! moved note's backlink count: the human-facing copy — the inline `[[oldpath|alias]]` -//! text in every file linking *at* it, rewritten in place — and the index, one -//! [`db::repoint_note_path`] whose `ON UPDATE CASCADE` FKs carry chunks, aliases, -//! centroid and outbound edges atomically, then a re-projection of the inbound sources -//! so their `edges.dst_path` (no FK — it must be free to dangle) points at the new path. -//! The moved note's **vectors are not touched at all**: content-addressed (ADR-0006), -//! they belong to the chunk text, which a move does not change. +//! the vault-relative path (ADR-0003, L1/L3), so a move changes the moved member's +//! identity, and this module makes that a re-key rather than a break. Two halves, both +//! bounded by the moved set's backlink count: the human-facing copy — the link text in +//! every file linking *at* a moved member, rewritten in place — and the index: each moved +//! note re-keyed by one [`db::repoint_note_path`] whose `ON UPDATE CASCADE` FKs carry +//! chunks, aliases, centroid and outbound edges atomically, each moved resource by one +//! [`db::repoint_resource`], then a re-projection of the inbound sources so their edges +//! (`edges.dst_path` has no FK — it must be free to dangle) point at the new paths. A +//! moved note's **vectors are not touched at all**: content-addressed (ADR-0006), they +//! belong to the chunk text, which a move does not change. //! -//! It is **Markdown-first**: rewrite the inbound text, move the file, *then* re-project -//! from the now-current Markdown. And bounded, not a scan: [`db::inbound_edge_targets`] -//! names exactly the files to touch, so the cost is O(inbound links). +//! **One pipeline for every move.** `move_note`, `move_resource` and `move_dir` only +//! validate and build a [`MoveSet`] — the one rename, and the indexed members travelling +//! with it — and [`execute`] runs it. It is **Markdown-first**: rewrite the inbound text, +//! rename on disk, *then* re-project from the now-current Markdown. And bounded, not a +//! scan: [`db::inbound_edges_of`] names exactly the files to touch, so the cost is +//! O(inbound links). The link text is found by [`crate::link`]'s own scanner, so a move +//! rewrites exactly the links ingest projected. //! //! **The vault half is all or nothing** (GH #230). Reindexing can't repair rewritten //! link text (it projects whatever the Markdown says), so a failed move must leave every @@ -29,10 +34,12 @@ //! is still at its source. Re-running the same move finishes it (the rewrites are //! already done, so only the rename is left). -use crate::db; +use crate::db::{self, InboundTarget}; use crate::error::{Error, Result}; use crate::ingest::{self, EmbedCtx}; -use rusqlite::Connection; +use crate::link::{self, LinkForm}; +use crate::pathspec; +use crate::resource::ResourceClass; use serde::Serialize; use std::collections::{BTreeMap, BTreeSet}; use std::fs; @@ -46,8 +53,9 @@ use std::path::{Path, PathBuf}; pub struct MoveReport { pub from: String, pub to: String, - /// Vault-relative paths of the inbound files whose link text was rewritten - /// (sorted, deduped). Empty when nothing linked to the moved note. + /// Post-move vault-relative paths of the files whose link text was rewritten + /// (sorted, deduped) — the moved note itself under `to`, when it links to itself. + /// Empty when nothing linked to the moved note. pub rewrote: Vec, /// Total individual `[[…]]` link targets rewritten across `rewrote`. pub links_rewritten: usize, @@ -62,77 +70,27 @@ pub struct MoveReport { /// must open the vault with the embedder the index was built with. The *moved* note /// re-embeds nothing (ADR-0006). Errors with [`Error::MoveDestination`] for an invalid /// destination and [`Error::MoveTargetExists`] rather than clobber. +/// +/// Only the `[[…]]` form is rewritten at a note. A Markdown-form link at one +/// (`[x](notes/a.md)`) keeps its text, but its source is re-projected like every inbound +/// source, so the edge dangles exactly as a rebuild would project it rather than keep +/// naming a path that no longer exists. pub fn move_note(ctx: EmbedCtx, old_rel: &str, new_rel_input: &str) -> Result { - let (conn, root) = (ctx.proj.conn, ctx.proj.root); - let new_rel = normalize_dest(new_rel_input)?; + let new_rel = pathspec::normalize_rel_md(new_rel_input).map_err(Error::MoveDestination)?; refuse_same_path(&new_rel, old_rel, "note")?; - let old_abs = root.join(old_rel); - let new_abs = root.join(&new_rel); - refuse_occupied(&old_abs, &new_abs, &new_rel)?; - refuse_file_ancestor(root, &new_rel)?; - - // The graph names the bounded inbound set: for each active edge pointing at the - // moved note, its source file and the exact link text (`dst_path_raw`) written - // there. Group by file into a target→replacement map, each link keeping its own - // `.md`-or-not convention ([`wiki_replacement`]). - let mut by_file = ByFile::new(); - for e in db::inbound_edge_targets(conn, old_rel)? { - let replacement = wiki_replacement(&new_rel, &e.dst_raw); - by_file - .entry(e.src_path) - .or_default() - .insert(e.dst_raw, replacement); - } - // Every inbound source is re-projected below, whether or not its *text* changed: - // its `edges.dst_path` names the old path and carries no FK to cascade (it must - // be free to be NULL — the dangling case, G5). The two sets differ for a - // Markdown-form link at a note (`[x](notes/a.md)`), which resolves as an edge but - // is not a form `move_note` rewrites — its edge must still re-resolve, or it - // would keep naming a path that no longer exists. - let inbound: Vec = by_file.keys().cloned().collect(); - - // 1. Markdown first: rewrite inbound link text in place — the `[[…]]` form, which - // is the one a note move repairs (the `[…](…)` pass is the resource move's) — - // then move the file on disk, all or nothing. A self-link (the moved note links - // to itself) is rewritten at its old path, before the rename. - let plan = plan_inbound(root, &by_file, &ByFile::new())?; - commit(root, &plan.rewrites, &old_abs, &new_abs)?; - let (rewrote, links_rewritten) = (plan.rewrote(), plan.links_rewritten); - - // 2. Re-key the index before anything re-projects, so path-based link resolution - // is independent of re-projection order (the same reason full ingest is - // two-phase). One statement; the FK cascades carry the note's chunks, - // aliases, centroid and outbound edges with it. - db::repoint_note_path(conn, old_rel, &new_rel)?; - - // 3. Re-project from the now-current Markdown: the moved note (refreshing its - // filename-derived title and mtime), then every inbound source so its edges - // re-resolve at the new path. - ingest::ingest_file(ctx, &new_rel)?; - for src_path in &inbound { - if src_path == old_rel { - continue; // the moved note itself (a self-link) — already re-projected - } - ingest::ingest_file(ctx, src_path)?; - } - + let set = MoveSet { + notes: vec![(old_rel.to_string(), new_rel.clone())], + ..MoveSet::rename(old_rel, &new_rel) + }; + let moved = execute(ctx, &set)?; Ok(MoveReport { from: old_rel.to_string(), to: new_rel, - rewrote, - links_rewritten, + rewrote: moved.rewrote, + links_rewritten: moved.links_rewritten, }) } -/// Normalize + validate a move destination into a vault-relative `.md` path, -/// mapping any rejection ([`crate::pathspec::normalize_rel_md`] — empty, absolute, -/// or vault-escaping) onto [`Error::MoveDestination`]. The "onto its current path" -/// and "onto an existing file" checks stay in [`move_note`], which alone knows the -/// source path and can read the disk. -fn normalize_dest(input: &str) -> Result { - crate::pathspec::normalize_rel_md(input).map_err(Error::MoveDestination) -} - /// What [`move_resource`] did — the resource sibling of [`MoveReport`]. Since /// GH #170 the two carry the same fields, both arms being path-keyed /// (data-model.md §10); they stay separate types because the reports are @@ -151,79 +109,288 @@ pub struct ResourceMoveReport { /// Move the resource at `old_rel` to `new_rel_input` — the note move minus the identity /// step: rewrite every inbound link's authored text (both syntaxes, each keeping its own /// relative-vs-root convention), move the file, update the inventory, re-project the -/// touched notes. B2 never touches the resource's bytes; the move is path-only. Errors -/// mirror [`move_note`] (the façade owns [`Error::ResourceNotFound`]). +/// inbound notes. B2 never touches the resource's bytes; the move is path-only. Errors +/// mirror [`move_note`] (the façade owns [`Error::ResourceNotFound`]), plus +/// [`Error::MoveDestination`] for a `.md` destination: that path names a note, so the +/// file would stop being this resource (and a rebuild would index it as a note). pub fn move_resource( ctx: EmbedCtx, old_rel: &str, new_rel_input: &str, ) -> Result { - let (conn, root) = (ctx.proj.conn, ctx.proj.root); - let new_rel = crate::pathspec::normalize_rel(new_rel_input).map_err(Error::MoveDestination)?; + let new_rel = pathspec::normalize_rel(new_rel_input).map_err(Error::MoveDestination)?; refuse_same_path(&new_rel, old_rel, "resource")?; - let old_abs = root.join(old_rel); - let new_abs = root.join(&new_rel); - refuse_occupied(&old_abs, &new_abs, &new_rel)?; - refuse_file_ancestor(root, &new_rel)?; - - // The graph names the bounded inbound set; each authored target is rewritten in - // its own convention, fragment intact ([`resource_replacement`]). - let mut by_file = ByFile::new(); - for e in db::inbound_resource_edge_targets(conn, old_rel)? { - let replacement = - resource_replacement(&e.dst_raw, old_rel, &new_rel, &parent_dir(&e.src_path)); - by_file - .entry(e.src_path) - .or_default() - .insert(e.dst_raw, replacement); + let Some(class) = ResourceClass::of_path(&new_rel) else { + return Err(Error::MoveDestination(format!( + "{new_rel} is a note path; a resource keeps a non-.md extension" + ))); + }; + let set = MoveSet { + resources: vec![ResourceMove { + from: old_rel.to_string(), + to: new_rel.clone(), + class, + }], + ..MoveSet::rename(old_rel, &new_rel) + }; + let moved = execute(ctx, &set)?; + Ok(ResourceMoveReport { + from: old_rel.to_string(), + to: new_rel, + rewrote: moved.rewrote, + links_rewritten: moved.links_rewritten, + }) +} + +/// What [`move_dir`] did: the folder's old and new vault-relative paths, how many +/// **indexed** notes/resources travelled (unindexed files travel too — the whole +/// directory is renamed — but only indexed rows are counted), the files whose +/// link text was rewritten (reported at their **post-move** paths, sorted), and +/// the total link targets repaired. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DirMoveReport { + pub from: String, + pub to: String, + pub moved_notes: usize, + pub moved_resources: usize, + /// Post-move vault-relative paths of the files whose link text was rewritten + /// (sorted, deduped). Empty when no links referenced the moved set. + pub rewrote: Vec, + /// Total individual link targets rewritten across `rewrote`. + pub links_rewritten: usize, +} + +/// Move/rename the whole directory `from_input` to `to_input`. One `fs::rename` moves +/// the directory — so **unindexed** files inside travel too — after every inbound link +/// at the moved set is rewritten, exactly as the per-file moves do: +/// +/// - wikilinks are vault-root-anchored, so links *between* co-moved notes are rewritten +/// just like links from outside the set; +/// - note-relative Markdown targets between co-moved files survive unchanged (a computed +/// replacement equal to the authored text is skipped); +/// - after the rename every moved note's `notes.path` is repointed **first**, then each +/// moved file re-projects — so path-based link resolution never depends on +/// re-projection order, the same reason full ingest is two-phase. +/// +/// Re-projection re-embeds only genuinely rewritten bodies, but still requires the real +/// embedder. Errors: [`Error::DirNotFound`], [`Error::MoveDestination`] (including a +/// destination inside the moved folder), [`Error::MoveTargetExists`] rather than merge +/// (with the case-only-rename carve-out on case-insensitive filesystems). +pub fn move_dir(ctx: EmbedCtx, from_input: &str, to_input: &str) -> Result { + let conn = ctx.proj.conn; + let from = pathspec::normalize_rel_dir(from_input).map_err(Error::MoveDestination)?; + let to = pathspec::normalize_rel_dir(to_input).map_err(Error::MoveDestination)?; + refuse_same_path(&to, &from, "folder")?; + if pathspec::is_under(&to, &from) { + return Err(Error::MoveDestination(format!( + "{to} is inside the folder being moved" + ))); + } + if !ctx.proj.root.join(&from).is_dir() { + return Err(Error::DirNotFound(from)); } - // 1. Markdown first: rewrite inbound link text in place, both syntaxes — a - // resource is linked as `![[img.png]]` *or* `![](img.png)`, so the one map - // feeds both passes — then move the file on disk, all or nothing. - let plan = plan_inbound(root, &by_file, &by_file)?; + // Every indexed member under the folder travels to the same place under `to`. A + // resource keeps its file name, so its class (from the new path) is its old one. + let rebased = |old: String| pathspec::rebase(&old, &from, &to).map(|new| (old, new)); + let notes: Vec<(String, String)> = db::notes_under_dir(conn, &from)? + .into_iter() + .filter_map(&rebased) + .collect(); + let resources: Vec = db::resources_under_dir(conn, &from)? + .into_iter() + .filter_map(&rebased) + .filter_map(|(from, to)| { + let class = ResourceClass::of_path(&to)?; + Some(ResourceMove { from, to, class }) + }) + .collect(); + let set = MoveSet { + notes, + resources, + ..MoveSet::rename(&from, &to) + }; + let moved = execute(ctx, &set)?; + + Ok(DirMoveReport { + moved_notes: set.notes.len(), + moved_resources: set.resources.len(), + from, + to, + rewrote: moved.rewrote, + links_rewritten: moved.links_rewritten, + }) +} + +// --- the one move pipeline (GH #134) --------------------------------------------- +// +// A move is always the same steps — refuse a bad destination, rewrite the inbound files' +// link text, rename on disk, re-key, re-project — whatever travels. Each op's own +// refusals (the destination's shape, a folder moved into itself) run before `execute`, +// whose shared refusals (an occupied destination, a file where a folder must be) follow: +// that precedence is part of each op's contract. + +/// Everything one move carries: the one rename that moves it on disk (a file, or the +/// folder holding every member), and the **indexed** members travelling with it. +#[derive(Debug, Default)] +struct MoveSet { + /// The rename's vault-relative source and destination. + from: String, + to: String, + /// Each moved note's `(old path, new path)`, sorted by old path. + notes: Vec<(String, String)>, + /// Each moved resource, with its class at the new path. + resources: Vec, +} + +/// One resource travelling with a [`MoveSet`]. +#[derive(Debug)] +struct ResourceMove { + from: String, + to: String, + class: ResourceClass, +} + +impl MoveSet { + /// A set renaming `from` → `to` with no members yet. + fn rename(from: &str, to: &str) -> Self { + Self { + from: from.to_string(), + to: to.to_string(), + ..Self::default() + } + } + + /// Where the note at `path` is after the move — its new path if it travels, + /// otherwise `path` itself. + fn after<'a>(&'a self, path: &'a str) -> &'a str { + match self + .notes + .binary_search_by(|(old, _)| old.as_str().cmp(path)) + { + Ok(i) => &self.notes[i].1, + Err(_) => path, + } + } + + /// Whether the note at `path` travels with this set. + fn moves_note(&self, path: &str) -> bool { + self.notes + .binary_search_by(|(old, _)| old.as_str().cmp(path)) + .is_ok() + } +} + +/// What [`execute`] did: the rewritten files at their post-move paths (sorted), and +/// how many link targets it rewrote across them. +#[derive(Debug)] +struct Moved { + rewrote: Vec, + links_rewritten: usize, +} + +/// Run one move: refuse an occupied or unreachable destination, plan every inbound +/// rewrite, commit the rewrites and the rename (all or nothing), re-key every moved +/// member, then re-project from the now-current Markdown. +fn execute(ctx: EmbedCtx, set: &MoveSet) -> Result { + let (conn, root) = (ctx.proj.conn, ctx.proj.root); + let old_abs = root.join(&set.from); + let new_abs = root.join(&set.to); + refuse_occupied(&old_abs, &new_abs, &set.to)?; + refuse_file_ancestor(root, &set.to)?; + + // The graph names the bounded inbound set: for each edge pointing at a moved + // member, its source file and the exact link text (`dst_path_raw`) written there. + // Group by file into per-syntax target→replacement maps. A note is rewritten in + // its `[[…]]` form only, each link keeping its own `.md`-or-not convention; a + // resource is linked as `![[img.png]]` *or* `![](img.png)`, so its replacement + // feeds both, re-relativized against the source's **post-move** folder so a + // relative link between two co-moved files computes to itself and is skipped. + let note_paths: Vec<&str> = set.notes.iter().map(|(old, _)| old.as_str()).collect(); + let resource_paths: Vec<&str> = set.resources.iter().map(|r| r.from.as_str()).collect(); + let (mut wiki, mut md) = (ByFile::new(), ByFile::new()); + // Every inbound source is re-projected below, whether or not its *text* changes: + // its edges name paths that are about to change, and `edges.dst_path` carries no FK + // to cascade (it must be free to be NULL — the dangling case, G5). + let mut sources = BTreeSet::new(); + for (target, e) in db::inbound_edges_of(conn, ¬e_paths, &resource_paths)? { + let (replacement, both_forms) = match target { + InboundTarget::Note(i) => (wiki_replacement(&set.notes[i].1, &e.dst_raw), false), + InboundTarget::Resource(i) => { + let r = &set.resources[i]; + let src_dir = pathspec::parent_dir(set.after(&e.src_path)); + let replacement = resource_replacement(&e.dst_raw, &r.from, &r.to, src_dir); + (replacement, true) + } + }; + if replacement != e.dst_raw { + if both_forms { + md.entry(e.src_path.clone()) + .or_default() + .insert(e.dst_raw.clone(), replacement.clone()); + } + wiki.entry(e.src_path.clone()) + .or_default() + .insert(e.dst_raw, replacement); + } + sources.insert(e.src_path); + } + + // 1. Markdown first: rewrite inbound link text in place at the pre-move paths (a + // self-link or a co-moved linker included), then one rename moves the note, + // resource or folder — unindexed files in a folder travel for free — all or + // nothing. + let plan = plan_inbound(root, &wiki, &md)?; commit(root, &plan.rewrites, &old_abs, &new_abs)?; - let (rewrote, links_rewritten) = (plan.rewrote(), plan.links_rewritten); - - // 2. Update the inventory: same bytes at a new path (the hash is untouched; - // class re-derives from the new extension), then drop the old row — its - // inbound edges re-dangle (ON DELETE SET NULL) until the re-projection - // below re-resolves them at the new path. - repoint_resource_row(conn, old_rel, &new_rel, &new_abs)?; - - // 3. Re-project the rewritten notes from the now-current Markdown (their - // changed chunks re-embed inline, exactly like a note move's inbound set). - for src_path in &rewrote { - ingest::ingest_file(ctx, src_path)?; + + // 2. Re-key the index before anything re-projects, so path-based link resolution + // is independent of re-projection order (the same reason full ingest is + // two-phase). Old and new paths are disjoint — the destination didn't exist — so + // the UNIQUE(path) constraints can't trip. + for (old, new) in &set.notes { + db::repoint_note_path(conn, old, new)?; + } + for r in &set.resources { + let mtime = fs::metadata(root.join(&r.to)) + .ok() + .as_ref() + .and_then(ingest::unix_mtime); + if !db::repoint_resource(conn, &r.from, &r.to, r.class.as_str(), mtime)? { + // Not inventoried after all (an out-of-band change the index hasn't seen): + // inventory the file where it now is, as a reindex would. + ingest::project_resource_file(conn, root, &r.to, r.class, true)?; + } } - Ok(ResourceMoveReport { - from: old_rel.to_string(), - to: new_rel, + // 3. Re-project from the now-current Markdown: every moved note at its new path + // (refreshing its filename-derived title, mtime and outbound edges — an + // unchanged body reuses its vectors), then every inbound source that stayed put. + for (_, new) in &set.notes { + ingest::ingest_file(ctx, new)?; + } + for src in sources.iter().filter(|src| !set.moves_note(src)) { + ingest::ingest_file(ctx, src)?; + } + + let mut rewrote: Vec = plan + .rewrites + .iter() + .map(|r| set.after(&r.rel).to_string()) + .collect(); + rewrote.sort(); + Ok(Moved { rewrote, - links_rewritten, + links_rewritten: plan.links_rewritten, }) } -// --- the mechanics every move shares (GH #134) -------------------------------- -// -// A move is always the same four steps — refuse a bad destination, rewrite the inbound -// files' link text, rename on disk, re-project — differing only in what counts as a -// replacement. Composed rather than folded into one `preflight` because the refusal -// *order* is not uniform, and that precedence is part of each op's contract. - /// Each inbound file's authored-target → replacement map, keyed by the file's /// vault-relative path. `BTreeMap` throughout, so the rewrite order — and the /// `rewrote` list every report carries — is sorted and deterministic. -type ByFile = BTreeMap>; - -/// A vault-relative file's directory (`""` at the vault root) — the base a -/// note-relative Markdown target is resolved and re-relativized against. -fn parent_dir(path: &str) -> String { - path.rsplit_once('/') - .map(|(dir, _)| dir.to_string()) - .unwrap_or_default() -} +type ByFile = BTreeMap; + +/// One file's authored-target → replacement map for one link syntax. +type Targets = BTreeMap; /// Refuse a destination equal to the source. `subject` names the thing being /// moved, so the message reads in the user's own nouns ("… is the note's current @@ -287,36 +454,31 @@ struct Plan { links_rewritten: usize, } -impl Plan { - /// The rewritten files' vault-relative paths, sorted — every report's `rewrote`. - fn rewrote(&self) -> Vec { - self.rewrites.iter().map(|r| r.rel.clone()).collect() - } -} - /// Markdown first, for every move: read each inbound file and compute its rewritten /// link text, writing nothing. `wiki` holds each file's `[[…]]` replacements and `md` -/// its `[…](…)` ones; a file whose passes change nothing is left out of the plan, which -/// is how a relative link between two co-moved files stays a no-op. A file that can't -/// be read fails the move here, while the vault is still untouched. +/// its `[…](…)` ones; a file whose rewrite changes nothing is left out of the plan. A +/// file that can't be read fails the move here, while the vault is still untouched. fn plan_inbound(vault_root: &Path, wiki: &ByFile, md: &ByFile) -> Result { - let none = BTreeMap::new(); + let none = Targets::new(); let mut rewrites = Vec::new(); let mut links_rewritten = 0usize; let touched: BTreeSet<&str> = wiki.keys().chain(md.keys()).map(String::as_str).collect(); for src_path in touched { let abs = vault_root.join(src_path); let original = fs::read_to_string(&abs)?; - let (pass1, n1) = rewrite_links(&original, wiki.get(src_path).unwrap_or(&none)); - let (rewritten, n2) = rewrite_md_targets(&pass1, md.get(src_path).unwrap_or(&none)); - if n1 + n2 > 0 { + let (rewritten, n) = rewrite_targets( + &original, + wiki.get(src_path).unwrap_or(&none), + md.get(src_path).unwrap_or(&none), + ); + if n > 0 { rewrites.push(Rewrite { rel: src_path.to_string(), abs, original, rewritten, }); - links_rewritten += n1 + n2; + links_rewritten += n; } } Ok(Plan { @@ -325,6 +487,33 @@ fn plan_inbound(vault_root: &Path, wiki: &ByFile, md: &ByFile) -> Result { }) } +/// Rewrite every link in `raw` whose trimmed target is a key of its syntax's map — +/// `wiki` for `[[…]]`/`![[…]]`, `md` for `[…](…)`/`![…](…)` — to that key's replacement. +/// Only the target token changes: every other byte (the brackets, the `|alias`, the +/// link text, whitespace around the target) is preserved, and a target merely sharing a +/// prefix with a key is never touched. The links are the ones [`link::link_spans`] +/// finds — the scanner ingest projects edges from — so a move rewrites exactly the links +/// that point at it. Returns the rewritten text and the count of targets replaced. +fn rewrite_targets(raw: &str, wiki: &Targets, md: &Targets) -> (String, usize) { + let mut out = String::with_capacity(raw.len()); + let (mut copied, mut count) = (0usize, 0usize); + for span in link::link_spans(raw) { + let targets = match span.form { + LinkForm::Wiki => wiki, + LinkForm::Markdown => md, + }; + let Some(replacement) = targets.get(&raw[span.target.clone()]) else { + continue; + }; + out.push_str(&raw[copied..span.target.start]); + out.push_str(replacement); + copied = span.target.end; + count += 1; + } + out.push_str(&raw[copied..]); + (out, count) +} + /// One vault change [`commit`] made, and so must undo if a later step fails. #[derive(Debug)] enum Done { @@ -414,16 +603,35 @@ fn undo(vault_root: &Path, rewrites: &[Rewrite], done: &[Done]) -> Vec { unrestored } -/// The replacement for a wikilink at a note now living at `new_path`, preserving -/// the link's own `.md`-or-not convention (Obsidian omits the extension; an -/// authored `.md` is kept). Shared by the note move and the folder move, which -/// differ only in where `new_path` comes from. +/// Split an authored link target at its first `#` into the path and the fragment — +/// which addresses a place *inside* the target, so a move carries it through verbatim. +fn split_fragment(authored: &str) -> (&str, Option<&str>) { + match authored.split_once('#') { + Some((base, fragment)) => (base, Some(fragment)), + None => (authored, None), + } +} + +/// `base` with the authored `#fragment` (if any) put back on. +fn with_fragment(base: &str, fragment: Option<&str>) -> String { + match fragment { + Some(f) => format!("{base}#{f}"), + None => base.to_string(), + } +} + +/// The replacement for a wikilink at a note now living at `new_path`, preserving the +/// link's own `.md`-or-not convention (Obsidian omits the extension; an authored `.md` +/// is kept) and any `#heading` fragment. Only a lowercase `.md` is dropped: that is the +/// suffix the resolver's `+ ".md"` ladder adds back, so any other spelling stays whole. fn wiki_replacement(new_path: &str, authored: &str) -> String { - if authored.ends_with(".md") { - new_path.to_string() + let (base, fragment) = split_fragment(authored); + let new_base = if base.ends_with(".md") { + new_path } else { - new_path.strip_suffix(".md").unwrap_or(new_path).to_string() - } + new_path.strip_suffix(".md").unwrap_or(new_path) + }; + with_fragment(new_base, fragment) } /// The replacement for a link at a resource moving `old_path` → `new_path`, as @@ -431,21 +639,15 @@ fn wiki_replacement(new_path: &str, authored: &str) -> String { /// between two co-moved files computes to itself and is skipped). Two things /// survive: the authored convention — a vault-root target stays vault-root, /// anything else is re-relativized against `src_dir` — and a `#fragment` suffix, -/// carried through untouched. Shared by the resource move and the folder move. +/// carried through untouched. fn resource_replacement(authored: &str, old_path: &str, new_path: &str, src_dir: &str) -> String { - let (base, fragment) = match authored.split_once('#') { - Some((b, f)) => (b, Some(f)), - None => (authored, None), - }; + let (base, fragment) = split_fragment(authored); let new_base = if base.trim() == old_path { new_path.to_string() // authored vault-root — keep it vault-root } else { - relativize(src_dir, new_path) // authored note-relative — keep it relative + pathspec::relativize(src_dir, new_path) // authored note-relative — keep it relative }; - match fragment { - Some(f) => format!("{new_base}#{f}"), - None => new_base, - } + with_fragment(&new_base, fragment) } /// Whether `a` and `b` name the **same directory entry** on disk — true only on @@ -461,319 +663,22 @@ fn is_same_dirent(a: &Path, b: &Path) -> bool { } } -/// Repoint one inventory row from `old_rel` to `new_rel` (whose file now sits at -/// `new_abs`): upsert the new path with the same bytes' hash (class re-derives -/// from the new extension), then drop the old row — its inbound edges re-dangle -/// (ON DELETE SET NULL) until the caller's re-projection re-resolves them. -/// Shared by [`move_resource`] and [`move_dir`]. -fn repoint_resource_row( - conn: &Connection, - old_rel: &str, - new_rel: &str, - new_abs: &Path, -) -> Result<()> { - let detail = db::resource_detail(conn, old_rel)? - .ok_or_else(|| Error::ResourceNotFound(old_rel.to_string()))?; - let mtime = fs::metadata(new_abs) - .ok() - .as_ref() - .and_then(ingest::unix_mtime); - let class = crate::resource::ResourceClass::of_path(new_rel) - .map(|c| c.as_str().to_string()) - .unwrap_or_else(|| "binary".to_string()); - db::upsert_resource( - conn, - &db::ResourceRow { - path: new_rel, - class: &class, - size: detail.size, - mtime, - content_hash: &detail.content_hash, - }, - )?; - conn.execute("DELETE FROM resources WHERE path = ?1", [old_rel])?; - Ok(()) -} - -/// What [`move_dir`] did: the folder's old and new vault-relative paths, how many -/// **indexed** notes/resources travelled (unindexed files travel too — the whole -/// directory is renamed — but only indexed rows are counted), the files whose -/// link text was rewritten (reported at their **post-move** paths, sorted), and -/// the total link targets repaired. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct DirMoveReport { - pub from: String, - pub to: String, - pub moved_notes: usize, - pub moved_resources: usize, - /// Post-move vault-relative paths of the files whose link text was rewritten - /// (sorted, deduped). Empty when no links referenced the moved set. - pub rewrote: Vec, - /// Total individual link targets rewritten across `rewrote`. - pub links_rewritten: usize, -} - -/// Map `path` through the `from/ → to/` prefix, or return it unchanged when it -/// is outside the moved subtree. -fn remap_prefix(path: &str, from: &str, to: &str) -> String { - match path.strip_prefix(from).and_then(|r| r.strip_prefix('/')) { - Some(rest) => format!("{to}/{rest}"), - None => path.to_string(), - } -} - -/// Move/rename the whole directory `from_input` to `to_input`. One `fs::rename` moves -/// the directory — so **unindexed** files inside travel too — after every inbound link -/// at the moved set is rewritten, exactly as the per-file moves do: -/// -/// - wikilinks are vault-root-anchored, so links *between* co-moved notes are rewritten -/// just like links from outside the set; -/// - note-relative Markdown targets between co-moved files survive unchanged (a computed -/// replacement equal to the authored text is skipped); -/// - after the rename every moved note's `notes.path` is repointed **first**, then each -/// moved file re-projects — so path-based link resolution never depends on -/// re-projection order, the same reason full ingest is two-phase. -/// -/// Re-projection re-embeds only genuinely rewritten bodies, but still requires the real -/// embedder. Errors: [`Error::DirNotFound`], [`Error::MoveDestination`] (including a -/// destination inside the moved folder), [`Error::MoveTargetExists`] rather than merge -/// (with the case-only-rename carve-out on case-insensitive filesystems). -pub fn move_dir(ctx: EmbedCtx, from_input: &str, to_input: &str) -> Result { - let (conn, root) = (ctx.proj.conn, ctx.proj.root); - let from = crate::pathspec::normalize_rel_dir(from_input).map_err(Error::MoveDestination)?; - let to = crate::pathspec::normalize_rel_dir(to_input).map_err(Error::MoveDestination)?; - refuse_same_path(&to, &from, "folder")?; - if to.strip_prefix(&from).is_some_and(|r| r.starts_with('/')) { - return Err(Error::MoveDestination(format!( - "{to} is inside the folder being moved" - ))); - } - let old_abs = root.join(&from); - let new_abs = root.join(&to); - if !old_abs.is_dir() { - return Err(Error::DirNotFound(from)); - } - refuse_occupied(&old_abs, &new_abs, &to)?; - refuse_file_ancestor(root, &to)?; - - let moved_notes = db::notes_under_dir(conn, &from)?; - let moved_resources = db::resources_under_dir(conn, &from)?; - - // Build each inbound file's target->replacement maps. Wikilink note targets are - // vault-root-anchored; Markdown resource targets are convention-preserving, - // relativized against the source's **post-move** directory so inside-to-inside - // relative links become no-ops. Two maps per file because the two syntaxes rewrite - // through different passes. - let mut wiki_by_file = ByFile::new(); - let mut md_by_file = ByFile::new(); - - // Every inbound source of a moved note re-projects, rewritten or not — same - // reason as `move_note`: its `edges.dst_path` names a path that is about to - // change and no FK cascades it. - let mut inbound_notes: BTreeSet = BTreeSet::new(); - for old_path in &moved_notes { - let new_path = remap_prefix(old_path, &from, &to); - for e in db::inbound_edge_targets(conn, old_path)? { - inbound_notes.insert(e.src_path.clone()); - let replacement = wiki_replacement(&new_path, &e.dst_raw); - if replacement != e.dst_raw { - wiki_by_file - .entry(e.src_path) - .or_default() - .insert(e.dst_raw, replacement); - } - } - } - for old_path in &moved_resources { - let new_path = remap_prefix(old_path, &from, &to); - for e in db::inbound_resource_edge_targets(conn, old_path)? { - // The source's directory *after* the move — sources inside the moved - // set remap; outside sources keep their dir. - let src_dir_after = parent_dir(&remap_prefix(&e.src_path, &from, &to)); - let replacement = resource_replacement(&e.dst_raw, old_path, &new_path, &src_dir_after); - if replacement != e.dst_raw { - wiki_by_file - .entry(e.src_path.clone()) - .or_default() - .insert(e.dst_raw.clone(), replacement.clone()); - md_by_file - .entry(e.src_path) - .or_default() - .insert(e.dst_raw, replacement); - } - } - } - - // 1. Markdown first: rewrite each inbound file in place at its pre-move path, then - // one rename moves the whole directory (unindexed files travel for free), all - // or nothing. - let plan = plan_inbound(root, &wiki_by_file, &md_by_file)?; - commit(root, &plan.rewrites, &old_abs, &new_abs)?; - let (rewrote_old_paths, links_rewritten) = (plan.rewrote(), plan.links_rewritten); - - // 2. Repoint the resolver before any re-projection: every moved note's path - // (old and new sets are disjoint — the destination didn't exist — so the - // UNIQUE(path) constraint can't trip), then every moved resource's - // inventory row (so resource links resolve at their new paths too). - for old_path in &moved_notes { - db::repoint_note_path(conn, old_path, &remap_prefix(old_path, &from, &to))?; - } - for old_path in &moved_resources { - let new_path = remap_prefix(old_path, &from, &to); - repoint_resource_row(conn, old_path, &new_path, &root.join(&new_path))?; - } - - // 3. Re-project from the now-current Markdown: every moved note (refreshes - // the filename-derived title, mtime, and its outbound edges — an unchanged - // body reuses its vectors), then every touched file outside the moved - // set (moved ones were just re-projected at their new paths). "Touched" is - // the union of the rewritten files and the inbound sources of moved notes, - // which differ for a Markdown-form link at a note (see `move_note`). - let moved_note_old_paths: BTreeSet<&str> = moved_notes.iter().map(String::as_str).collect(); - for old_path in &moved_notes { - let new_path = remap_prefix(old_path, &from, &to); - ingest::ingest_file(ctx, &new_path)?; - } - let touched: BTreeSet<&str> = rewrote_old_paths - .iter() - .map(String::as_str) - .chain(inbound_notes.iter().map(String::as_str)) - .collect(); - for src_path in touched { - if moved_note_old_paths.contains(src_path) { - continue; - } - ingest::ingest_file(ctx, src_path)?; - } - - let mut rewrote: Vec = rewrote_old_paths - .iter() - .map(|p| remap_prefix(p, &from, &to)) - .collect(); - rewrote.sort(); - - Ok(DirMoveReport { - from, - to, - moved_notes: moved_notes.len(), - moved_resources: moved_resources.len(), - rewrote, - links_rewritten, - }) -} - -/// The relative path from `base_dir` (a vault-relative directory, `""` = root) -/// to `to_path` (a vault-relative file): shared prefix dropped, one `..` per -/// remaining base segment — the inverse of resolution's note-relative join. -fn relativize(base_dir: &str, to_path: &str) -> String { - let base: Vec<&str> = if base_dir.is_empty() { - Vec::new() - } else { - base_dir.split('/').collect() - }; - let to: Vec<&str> = to_path.split('/').collect(); - let shared = base - .iter() - .zip(to.iter()) - .take_while(|(a, b)| a == b) - .count(); - let mut out: Vec<&str> = Vec::with_capacity(base.len() - shared + to.len() - shared); - out.extend(std::iter::repeat_n("..", base.len() - shared)); - out.extend(&to[shared..]); - out.join("/") -} - -/// Rewrite every Markdown-form target (`[text](target)` / `![alt](target)`) -/// whose *trimmed* target is a key in `targets` — the `[…](…)` sibling of -/// [`rewrite_links`], same contract: only the target token changes, every other -/// byte (text, whitespace, the `](` frame) is preserved. -fn rewrite_md_targets(raw: &str, targets: &BTreeMap) -> (String, usize) { - let mut out = String::with_capacity(raw.len()); - let mut count = 0usize; - let mut rest = raw; - while let Some(open) = rest.find("](") { - out.push_str(&rest[..open + 2]); - let after = &rest[open + 2..]; - let Some(close) = after.find(')') else { - out.push_str(after); - return (out, count); - }; - let inner = &after[..close]; - match targets.get(inner.trim()) { - Some(replacement) => { - let lead = inner.len() - inner.trim_start().len(); - let trail = inner.len() - inner.trim_end().len(); - out.push_str(&inner[..lead]); - out.push_str(replacement); - out.push_str(&inner[inner.len() - trail..]); - count += 1; - } - None => out.push_str(inner), - } - out.push(')'); - rest = &after[close + 1..]; - } - out.push_str(rest); - (out, count) -} - -/// Rewrite every wikilink whose *trimmed* target is a key in `targets` to that -/// key's replacement, preserving all other bytes — surrounding whitespace inside -/// the brackets, the `|alias`, and the `[[`/`]]` themselves. The match is bounded -/// to the target token (up to `|` or `]]`), so moving `foo` never touches a -/// `[[foo-bar]]` that merely shares its prefix. Returns the rewritten text and the -/// count of targets replaced. -fn rewrite_links(raw: &str, targets: &BTreeMap) -> (String, usize) { - let mut out = String::with_capacity(raw.len()); - let mut count = 0usize; - let mut rest = raw; - while let Some(open) = rest.find("[[") { - out.push_str(&rest[..open + 2]); - let after = &rest[open + 2..]; - let Some(close) = after.find("]]") else { - out.push_str(after); - return (out, count); - }; - let inner = &after[..close]; - // Split off the display alias (kept verbatim, including its leading `|`). - let (path_part, alias_part) = match inner.find('|') { - Some(i) => (&inner[..i], Some(&inner[i..])), - None => (inner, None), - }; - match targets.get(path_part.trim()) { - Some(replacement) => { - // Preserve the path's own surrounding whitespace; swap only the - // trimmed target so every other byte is identical. - let lead = path_part.len() - path_part.trim_start().len(); - let trail = path_part.len() - path_part.trim_end().len(); - out.push_str(&path_part[..lead]); - out.push_str(replacement); - out.push_str(&path_part[path_part.len() - trail..]); - count += 1; - } - None => out.push_str(path_part), - } - if let Some(alias) = alias_part { - out.push_str(alias); - } - out.push_str("]]"); - rest = &after[close + 2..]; - } - out.push_str(rest); - (out, count) -} - #[cfg(test)] mod tests { use super::*; - fn targets(pairs: &[(&str, &str)]) -> BTreeMap { + fn targets(pairs: &[(&str, &str)]) -> Targets { pairs .iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect() } + /// The note move's rewrite: `[[…]]` targets only. + fn rewrite_links(raw: &str, t: &Targets) -> (String, usize) { + rewrite_targets(raw, t, &Targets::new()) + } + #[test] fn rewrites_the_target_and_keeps_the_alias() { let t = targets(&[("concepts/memory", "concepts/human-memory")]); @@ -831,11 +736,38 @@ mod tests { assert_eq!(n, 2); } + /// Each syntax is rewritten from its own map, and a Markdown target keeps the + /// whitespace and link text around it. + #[test] + fn each_syntax_is_rewritten_from_its_own_map() { + let wiki = targets(&[("img.png", "media/img.png")]); + let md = targets(&[("img.png", "../media/img.png")]); + let (out, n) = rewrite_targets("![[img.png|cap]] and ![alt]( img.png )\n", &wiki, &md); + assert_eq!( + out, + "![[media/img.png|cap]] and ![alt]( ../media/img.png )\n" + ); + assert_eq!(n, 2); + // A map for one syntax never touches the other's links. + let (out, n) = rewrite_targets("![alt](img.png)", &wiki, &Targets::new()); + assert_eq!((out.as_str(), n), ("![alt](img.png)", 0)); + } + + /// The divergence one grammar closes: a stray `[[` on one line used to pair with + /// the next line's `]]` in the move's whole-file scan, hiding a link ingest had + /// projected — so the backlink dangled after the move. + #[test] + fn a_stray_open_bracket_does_not_hide_the_next_lines_link() { + let t = targets(&[("old", "new")]); + let (out, n) = rewrite_links("broken [[x\nsee [[old]]\n", &t); + assert_eq!(out, "broken [[x\nsee [[new]]\n"); + assert_eq!(n, 1); + } + // --- the replacement rules, direct (GH #134) ------------------------------ // - // Both were computed inline in two places each before the extraction, so the - // only cover they had was through a whole-vault move; the `#fragment` rule had - // none at all. They are pure functions, so they get pinned here. + // Pure functions, so they get pinned here rather than only through a whole-vault + // move. #[test] fn a_wikilink_keeps_its_own_md_convention() { @@ -851,6 +783,20 @@ mod tests { ); } + /// A `[[note#heading]]` link resolves to the note (the fragment is stripped for + /// the lookup only), so its move keeps the heading it addresses. + #[test] + fn a_wikilink_keeps_its_heading_fragment() { + assert_eq!( + wiki_replacement("archive/memory.md", "concepts/memory#Recall"), + "archive/memory#Recall" + ); + assert_eq!( + wiki_replacement("archive/memory.md", "concepts/memory.md#Recall"), + "archive/memory.md#Recall" + ); + } + #[test] fn a_resource_link_keeps_its_convention_and_its_fragment() { // Authored vault-root (the target equals the resource's vault path) → stays @@ -900,13 +846,27 @@ mod tests { fn a_relative_link_between_co_moved_files_computes_to_itself() { // The folder move passes the source's **post-move** directory, so a link from // `dir/note.md` to `dir/img.png` is unchanged by moving `dir/` — and an - // unchanged replacement is what `move_dir` skips, leaving the file untouched. + // unchanged replacement is what the pipeline skips, leaving the file untouched. assert_eq!( resource_replacement("img.png", "dir/img.png", "moved/img.png", "moved"), "img.png" ); } + #[test] + fn a_set_maps_its_moved_notes_and_leaves_every_other_path() { + let set = MoveSet { + notes: vec![ + ("docs/a.md".to_string(), "media/a.md".to_string()), + ("docs/b.md".to_string(), "media/b.md".to_string()), + ], + ..MoveSet::rename("docs", "media") + }; + assert_eq!(set.after("docs/b.md"), "media/b.md"); + assert_eq!(set.after("hub.md"), "hub.md"); + assert!(set.moves_note("docs/a.md") && !set.moves_note("media/a.md")); + } + // --- the all-or-nothing commit (GH #230) ------------------------------------ // // The façade suite (tests/mv.rs) drives a failed rename end to end. These pin the diff --git a/crates/b2-core/src/pathspec.rs b/crates/b2-core/src/pathspec.rs index 801afeb..d973cd3 100644 --- a/crates/b2-core/src/pathspec.rs +++ b/crates/b2-core/src/pathspec.rs @@ -1,7 +1,13 @@ -//! Shared validation for a user-supplied vault-relative destination — the one concern -//! `b2 mv` and `b2 add` have in common. Error-type-free (it returns `Err(reason)` as a -//! plain string) so each authoring op maps the reason onto its own [`crate::Error`] -//! variant without the two coupling. Also home to the shared hidden-path predicate. +//! Vault-relative path algebra — the one place B2 spells out what a vault path is and how +//! paths relate. A vault path is `/`-separated and relative to the vault root; it is the +//! identity of the note or resource at it (L1, L3), so every op that validates, compares +//! or rewrites one goes through here rather than re-deriving the rules. +//! +//! Three concerns: validating a user-supplied destination ([`normalize_rel`] and its +//! folder/note variants — error-type-free, returning `Err(reason)` so each op maps it onto +//! its own [`crate::Error`] variant); the hidden-path predicate ([`is_hidden`]); and the +//! pure string algebra over paths that are already vault-relative ([`file_name`], +//! [`parent_dir`], [`extension`], [`is_under`], [`rebase`], [`relativize`]). /// Whether this entry's *name* is dot-prefixed — the hidden-path predicate shared by the /// folder walk, the ingest walk, and [`normalize_rel`]. **Hidden means hidden** (GH #136): @@ -57,11 +63,65 @@ pub(crate) fn normalize_rel_dir(input: &str) -> Result { /// variant of [`normalize_rel`]: same checks, plus `.md` appended if omitted. pub(crate) fn normalize_rel_md(input: &str) -> Result { let s = normalize_rel(input)?; - Ok(if s.ends_with(".md") { - s + Ok(if is_md(&s) { s } else { format!("{s}.md") }) +} + +/// A vault path's last segment — the file (or folder) name. +pub(crate) fn file_name(path: &str) -> &str { + path.rsplit('/').next().unwrap_or(path) +} + +/// A vault path's folder: everything before the last `/`, and `""` at the vault root — +/// the base a note-relative Markdown target is resolved and re-relativized against. +pub(crate) fn parent_dir(path: &str) -> &str { + path.rsplit_once('/').map_or("", |(dir, _)| dir) +} + +/// The extension of a vault path's file name, as written (case kept): the text after +/// the name's last `.`. `None` when there is none — no `.`, a trailing `.`, or only a +/// leading one (a dotfile's name is all stem). +pub(crate) fn extension(path: &str) -> Option<&str> { + match file_name(path).rsplit_once('.') { + Some((stem, ext)) if !stem.is_empty() && !ext.is_empty() => Some(ext), + _ => None, + } +} + +/// Whether a vault path names a note by its shape: its file name's extension is `md`, +/// in any case (`Foo.MD` is a note, as the walk classifies it). +pub(crate) fn is_md(path: &str) -> bool { + extension(path).is_some_and(|e| e.eq_ignore_ascii_case("md")) +} + +/// Whether `path` lies strictly inside the folder `dir` (at any depth). A folder is not +/// under itself, and a prefix-sharing sibling (`docs2` for `docs`) is not under it. +pub(crate) fn is_under(path: &str, dir: &str) -> bool { + path.strip_prefix(dir) + .is_some_and(|rest| rest.starts_with('/')) +} + +/// `path` carried by moving the folder `from` to `to`: the same place under `to`, or +/// `None` when `path` is not under `from` ([`is_under`]). +pub(crate) fn rebase(path: &str, from: &str, to: &str) -> Option { + let rest = path.strip_prefix(from)?.strip_prefix('/')?; + Some(format!("{to}/{rest}")) +} + +/// The relative path from the folder `base_dir` (`""` = the vault root) to the vault +/// path `to_path`: the shared leading folders dropped, one `..` per remaining `base_dir` +/// segment — the inverse of resolving a note-relative Markdown target. +pub(crate) fn relativize(base_dir: &str, to_path: &str) -> String { + let base: Vec<&str> = if base_dir.is_empty() { + Vec::new() } else { - format!("{s}.md") - }) + base_dir.split('/').collect() + }; + let to: Vec<&str> = to_path.split('/').collect(); + let shared = base.iter().zip(&to).take_while(|(a, b)| a == b).count(); + std::iter::repeat_n("..", base.len() - shared) + .chain(to[shared..].iter().copied()) + .collect::>() + .join("/") } #[cfg(test)] @@ -74,6 +134,67 @@ mod tests { assert_eq!(normalize_rel_md("notes/foo.md").unwrap(), "notes/foo.md"); } + /// An authored `.MD` is already a note path: the check is on the file name's + /// extension in any case, so no second `.md` is stacked on. + #[test] + fn an_uppercase_md_extension_is_kept_not_doubled() { + assert_eq!(normalize_rel_md("Foo.MD").unwrap(), "Foo.MD"); + assert_eq!(normalize_rel_md("a/b.Md").unwrap(), "a/b.Md"); + // A folder named like a note does not make its contents notes. + assert_eq!(normalize_rel_md("x.md/y").unwrap(), "x.md/y.md"); + } + + #[test] + fn names_folders_and_extensions_split_on_the_last_separator() { + assert_eq!(file_name("a/b/c.md"), "c.md"); + assert_eq!(file_name("c.md"), "c.md"); + assert_eq!(parent_dir("a/b/c.md"), "a/b"); + assert_eq!(parent_dir("c.md"), ""); + assert_eq!(extension("a/b.tar.gz"), Some("gz")); + assert_eq!(extension("a/Photo.PNG"), Some("PNG")); + assert_eq!( + extension("dir.v2/Makefile"), + None, + "a dotted folder lends no extension" + ); + assert_eq!( + extension("a/.gitignore"), + None, + "a dotfile's name is all stem" + ); + assert_eq!(extension("a/trailing."), None); + assert!(is_md("a/B.MD") && is_md("b.md")); + assert!(!is_md("a.md/b") && !is_md("a/.md") && !is_md("a.mdx")); + } + + #[test] + fn under_means_strictly_inside_and_rebase_follows_it() { + assert!(is_under("docs/a.md", "docs")); + assert!(is_under("docs/x/y/a.md", "docs")); + assert!(!is_under("docs", "docs"), "a folder is not under itself"); + assert!( + !is_under("docs2/a.md", "docs"), + "a prefix-sharing sibling is not under it" + ); + assert!(!is_under("a.md", "docs")); + + assert_eq!( + rebase("docs/x/a.md", "docs", "media").as_deref(), + Some("media/x/a.md") + ); + assert_eq!(rebase("docs2/a.md", "docs", "media"), None); + assert_eq!(rebase("docs", "docs", "media"), None); + } + + #[test] + fn relativize_walks_up_to_the_shared_folder_then_down() { + assert_eq!(relativize("", "a/b.png"), "a/b.png"); + assert_eq!(relativize("a", "a/b.png"), "b.png"); + assert_eq!(relativize("notes", "docs/plan.pdf"), "../docs/plan.pdf"); + assert_eq!(relativize("a/b/c", "a/x.png"), "../../x.png"); + assert_eq!(relativize("a/b", "c.png"), "../../c.png"); + } + #[test] fn trims_and_normalizes_separators() { assert_eq!(normalize_rel_md(" a\\b ").unwrap(), "a/b.md"); diff --git a/crates/b2-core/src/resource.rs b/crates/b2-core/src/resource.rs index 6247ac1..8480264 100644 --- a/crates/b2-core/src/resource.rs +++ b/crates/b2-core/src/resource.rs @@ -4,7 +4,7 @@ //! mislabeled file degrades gracefully. The table is closed with //! [`ResourceClass::Binary`] as the total fallback, so *every* file classifies. -/// The closed class table (research §3). Everything that is not a note maps to +/// The closed class table (data-model.md §10). Everything that is not a note maps to /// exactly one of these; `Binary` catches all the rest. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ResourceClass { @@ -31,11 +31,11 @@ impl ResourceClass { /// Classify a vault-relative path. `None` means the file is a **note** /// (`.md`) and belongs to the note pipeline, not the resource inventory. - /// Extensions are case-insensitive; no extension → `Binary`. + /// Extensions are case-insensitive and read off the file name (a dotted folder + /// lends none); no extension → `Binary`. pub fn of_path(path: &str) -> Option { - let ext = path - .rsplit_once('.') - .map(|(_, e)| e.to_ascii_lowercase()) + let ext = crate::pathspec::extension(path) + .map(str::to_ascii_lowercase) .unwrap_or_default(); Some(match ext.as_str() { "md" => return None, @@ -71,13 +71,8 @@ pub enum DocKind { /// Known limit, accepted: an extensionless *file* (`Makefile`) dispatches as a note ref — /// it is still walked, inventoried, and reachable through surfaces that know its kind. pub fn doc_kind(arg: &str) -> DocKind { - let name = arg.rsplit('/').next().unwrap_or(arg); - match name.rsplit_once('.') { - Some((stem, ext)) - if !stem.is_empty() && !ext.is_empty() && !ext.eq_ignore_ascii_case("md") => - { - DocKind::Resource - } + match crate::pathspec::extension(arg) { + Some(ext) if !ext.eq_ignore_ascii_case("md") => DocKind::Resource, _ => DocKind::Note, } } diff --git a/crates/b2-core/src/rm.rs b/crates/b2-core/src/rm.rs index cafe3dd..d059a93 100644 --- a/crates/b2-core/src/rm.rs +++ b/crates/b2-core/src/rm.rs @@ -5,16 +5,19 @@ //! nothing to rewrite them to — exactly the state an external `rm` plus a full reindex //! produces. That equivalence is the correctness bar (S3). //! -//! The single-note projection paths never prune, so each op drops its rows directly, then -//! re-projects the **surviving** inbound files: their edges re-derive against the pruned -//! tables and re-key to the dangling edge id a rebuild would derive. Bodies are untouched, -//! so re-projection re-chunks nothing and the ops are **model-free**. +//! The single-note projection paths never prune, so every delete runs one pipeline +//! ([`delete_set`]): read the inbound linkers off the graph, remove from disk, drop the +//! rows directly, then re-project the **surviving** linkers — their edges re-derive +//! against the pruned tables and re-key to the dangling edge id a rebuild would derive. +//! Bodies are untouched, so re-projection re-chunks nothing and the ops are +//! **model-free**. use crate::db; use crate::error::{Error, Result}; use crate::ingest::{self, ProjectionCtx}; +use crate::pathspec; use serde::Serialize; -use std::collections::BTreeSet; +use std::collections::HashSet; use std::fs; use std::io::ErrorKind; use std::path::Path; @@ -54,72 +57,34 @@ pub struct DirDeleteReport { pub dangled: Vec, } -/// Remove one file, tolerating a file already gone: an external delete that raced -/// us leaves exactly the state we are reconciling toward, so the projection -/// cleanup must still run rather than abort. -fn remove_file_if_present(abs: &Path) -> Result<()> { - match fs::remove_file(abs) { - Ok(()) => Ok(()), - Err(e) if e.kind() == ErrorKind::NotFound => Ok(()), - Err(e) => Err(e.into()), - } -} - -/// Re-project each surviving inbound file (bodies unchanged, so nothing -/// re-chunks and no vector is touched): their edges re-derive against the -/// pruned tables, re-dangling the links that pointed at the deleted target with -/// the raw-path-keyed edge ids a full rebuild would derive. -fn reproject_dangled(ctx: ProjectionCtx, dangled: &BTreeSet) -> Result<()> { - for src in dangled { - ingest::project_file(ctx, src)?; - } - Ok(()) -} - /// Delete the note at `rel`: file off disk, projection rows off the index /// (chunks/FTS/centroid/aliases/outbound edges cascade with the `notes` row), then /// re-project the inbound linkers so their edges re-dangle. The façade resolved the ref. /// The note's chunk **vectors** deliberately do not cascade — content-addressed and /// possibly shared (ADR-0006), they are collected by the whole-vault pass. pub fn delete_note(ctx: ProjectionCtx, rel: &str) -> Result { - let (conn, root) = (ctx.conn, ctx.root); - // The graph names the bounded inbound set before the rows go. A self-link's - // source is the note itself — it dies with the file, so it is not re-projected. - let dangled: BTreeSet = db::inbound_edge_targets(conn, rel)? - .into_iter() - .map(|e| e.src_path) - .filter(|p| p != rel) - .collect(); - - remove_file_if_present(&root.join(rel))?; - conn.execute("DELETE FROM notes WHERE path = ?1", [rel])?; - reproject_dangled(ctx, &dangled)?; - + let dangled = delete_set(ctx, &[rel], &[], || { + remove_file_if_present(&ctx.root.join(rel)) + })?; Ok(DeleteReport { path: rel.to_string(), - dangled: dangled.into_iter().collect(), + dangled, }) } /// Delete the resource at `rel` — the note delete minus the identity step: file /// off disk, inventory row off the index (inbound edges' `dst_resource_path` is /// `ON DELETE SET NULL`), then re-project the inbound linkers so their edges -/// re-key to the raw-path (dangling) ids a rebuild derives. +/// re-key to the raw-path (dangling) ids a rebuild derives. Errors with +/// [`Error::ResourceNotFound`] for a path not in the inventory. pub fn delete_resource(ctx: ProjectionCtx, rel: &str) -> Result { - let (conn, root) = (ctx.conn, ctx.root); - db::resource_detail(conn, rel)?.ok_or_else(|| Error::ResourceNotFound(rel.to_string()))?; - let dangled: BTreeSet = db::inbound_resource_edge_targets(conn, rel)? - .into_iter() - .map(|e| e.src_path) - .collect(); - - remove_file_if_present(&root.join(rel))?; - conn.execute("DELETE FROM resources WHERE path = ?1", [rel])?; - reproject_dangled(ctx, &dangled)?; - + db::resource_detail(ctx.conn, rel)?.ok_or_else(|| Error::ResourceNotFound(rel.to_string()))?; + let dangled = delete_set(ctx, &[], &[rel], || { + remove_file_if_present(&ctx.root.join(rel)) + })?; Ok(ResourceDeleteReport { path: rel.to_string(), - dangled: dangled.into_iter().collect(), + dangled, }) } @@ -129,52 +94,75 @@ pub fn delete_resource(ctx: ProjectionCtx, rel: &str) -> Result Result { - let (conn, root) = (ctx.conn, ctx.root); // The UI only sends tree-derived paths, so an invalid input (empty, absolute, // escaping, a dotfolder) is refused as "no such folder" rather than growing a // delete-specific destination error. - let dir = crate::pathspec::normalize_rel_dir(dir_input) + let dir = pathspec::normalize_rel_dir(dir_input) .map_err(|_| Error::DirNotFound(dir_input.trim().to_string()))?; - let abs = root.join(&dir); + let abs = ctx.root.join(&dir); if !abs.is_dir() { return Err(Error::DirNotFound(dir)); } - let notes = db::notes_under_dir(conn, &dir)?; - let resources = db::resources_under_dir(conn, &dir)?; - - // Inbound linkers that survive the delete — sources inside the folder die - // with it and must not be re-projected (their files are gone). - let prefix = format!("{dir}/"); - let mut dangled: BTreeSet = BTreeSet::new(); - for note_path in ¬es { - for e in db::inbound_edge_targets(conn, note_path)? { - if !e.src_path.starts_with(&prefix) { - dangled.insert(e.src_path); - } - } - } - for path in &resources { - for e in db::inbound_resource_edge_targets(conn, path)? { - if !e.src_path.starts_with(&prefix) { - dangled.insert(e.src_path); - } - } - } - - fs::remove_dir_all(&abs)?; - for note_path in ¬es { - conn.execute("DELETE FROM notes WHERE path = ?1", [note_path])?; - } - for path in &resources { - conn.execute("DELETE FROM resources WHERE path = ?1", [path])?; - } - reproject_dangled(ctx, &dangled)?; + let notes = db::notes_under_dir(ctx.conn, &dir)?; + let resources = db::resources_under_dir(ctx.conn, &dir)?; + let dangled = delete_set( + ctx, + ¬es.iter().map(String::as_str).collect::>(), + &resources.iter().map(String::as_str).collect::>(), + || Ok(fs::remove_dir_all(&abs)?), + )?; Ok(DirDeleteReport { dir, deleted_notes: notes.len(), deleted_resources: resources.len(), - dangled: dangled.into_iter().collect(), + dangled, }) } + +/// The one delete pipeline: the indexed `notes` and `resources` leave the disk (by +/// `remove`, which takes the whole set — one file or a folder) and the index, and the +/// surviving inbound linkers re-project so their links re-dangle. A linker that is +/// itself one of the deleted notes (a self-link, or a linker inside a deleted folder) +/// died with it and is not re-projected. Returns the survivors, sorted. +fn delete_set( + ctx: ProjectionCtx, + notes: &[&str], + resources: &[&str], + remove: impl FnOnce() -> Result<()>, +) -> Result> { + let conn = ctx.conn; + // The graph names the bounded inbound set before the rows go. + let dying: HashSet<&str> = notes.iter().copied().collect(); + let dangled: Vec = db::inbound_sources(conn, notes, resources)? + .into_iter() + .filter(|src| !dying.contains(src.as_str())) + .collect(); + + remove()?; + for path in notes { + db::delete_note_row(conn, path)?; + } + for path in resources { + db::delete_resource_row(conn, path)?; + } + // Bodies unchanged, so nothing re-chunks and no vector is touched: the edges + // re-derive against the pruned tables, re-dangling the links that pointed at the + // deleted members with the raw-path-keyed edge ids a full rebuild would derive. + for src in &dangled { + ingest::project_file(ctx, src)?; + } + Ok(dangled) +} + +/// Remove one file, tolerating a file already gone: an external delete that raced +/// us leaves exactly the state we are reconciling toward, so the projection +/// cleanup must still run rather than abort. +fn remove_file_if_present(abs: &Path) -> Result<()> { + match fs::remove_file(abs) { + Ok(()) => Ok(()), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } +} diff --git a/crates/b2-core/tests/mv.rs b/crates/b2-core/tests/mv.rs index 5f6b7de..c304138 100644 --- a/crates/b2-core/tests/mv.rs +++ b/crates/b2-core/tests/mv.rs @@ -798,3 +798,206 @@ fn re_running_a_move_interrupted_before_its_rename_finishes_it() { "the link resolves at the destination" ); } + +// --- one grammar, one pipeline ------------------------------------------------- +// +// The move reads link text with ingest's own scanner and runs every kind of move through +// one pipeline, so what a move repairs is what ingest projected, and each kind leaves +// the index a rebuild would produce. + +/// Every projected edge and inventory row, sorted — the index state a move must leave +/// equal to a from-scratch rebuild (S3). Edge ids are derived from the resolved +/// target, so a wrongly (un)resolved edge shows up here too. +fn projection(root: &Path) -> Vec { + let conn = common::index_conn(root); + let mut rows: Vec = Vec::new(); + let mut edges = conn + .prepare( + "SELECT id, src_path, dst_path, dst_resource_path, dst_path_raw, type + FROM edges", + ) + .unwrap(); + rows.extend( + edges + .query_map([], |r| { + Ok(format!( + "edge {} {} {:?} {:?} {} {}", + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, Option>(2)?, + r.get::<_, Option>(3)?, + r.get::<_, String>(4)?, + r.get::<_, String>(5)?, + )) + }) + .unwrap() + .map(Result::unwrap), + ); + let mut resources = conn + .prepare("SELECT path, class, size, content_hash FROM resources") + .unwrap(); + rows.extend( + resources + .query_map([], |r| { + Ok(format!( + "resource {} {} {} {}", + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, i64>(2)?, + r.get::<_, String>(3)?, + )) + }) + .unwrap() + .map(Result::unwrap), + ); + rows.sort(); + rows +} + +/// Assert the index at `root` equals what dropping it and rebuilding from the Markdown +/// alone produces. +fn assert_matches_a_rebuild(vault: Vault, root: &Path) { + let after_move = projection(root); + drop(vault); + fs::remove_dir_all(root.join(".b2")).unwrap(); + Vault::open(root).unwrap().reindex().unwrap(); + assert_eq!(projection(root), after_move); +} + +/// A vault linking one note and one resource from both syntaxes and both +/// conventions, with a self-link and a linker in a subfolder. +fn linked_vault(dir: &Path) -> (Vault, std::path::PathBuf) { + small_vault( + dir, + &[ + ("a.md", "Me: [[a]]. Pic: ![p](img.png)\n"), + ("img.png", "png bytes"), + ( + "b.md", + "See [[a|A]] and [a](a.md).\n![[img.png|cap]] and ![alt](img.png)\n", + ), + ( + "sub/c.md", + "Up: ![x](../img.png) and [[img.png]] and [[a.md#Part]].\n", + ), + ], + ) +} + +#[test] +fn a_move_rewrites_a_link_after_a_stray_open_bracket() { + // Ingest scans per line, so `see [[a]]` is an edge even though an earlier line + // left a `[[` open. The move used to scan the whole file, pair that stray `[[` + // with the `]]` below, miss the link, and leave the backlink dangling. + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = small_vault( + tmp.path(), + &[("a.md", "Target.\n"), ("b.md", "broken [[x\nsee [[a]]\n")], + ); + + let report = vault.move_note("a.md", "archive/a.md").unwrap(); + assert_eq!(report.links_rewritten, 1); + assert_eq!( + fs::read_to_string(root.join("b.md")).unwrap(), + "broken [[x\nsee [[archive/a]]\n" + ); + assert_eq!( + inbound(&vault, "archive/a.md"), + vec![("referenced-by".to_string(), "b.md".to_string())], + "the backlink resolves at the destination" + ); +} + +#[test] +fn a_self_linking_note_is_reported_under_its_new_path() { + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = linked_vault(tmp.path()); + + let report = vault.move_note("a.md", "z/a.md").unwrap(); + assert_eq!( + report.rewrote, + vec![ + "b.md".to_string(), + "sub/c.md".to_string(), + "z/a.md".to_string() + ], + "every rewritten file is named where it is after the move" + ); + assert_eq!( + fs::read_to_string(root.join("z/a.md")).unwrap(), + "Me: [[z/a]]. Pic: ![p](img.png)\n", + "the self-link follows the note" + ); +} + +#[test] +fn a_heading_link_keeps_its_fragment_across_a_move() { + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = linked_vault(tmp.path()); + + vault.move_note("a.md", "z/a.md").unwrap(); + let c = fs::read_to_string(root.join("sub/c.md")).unwrap(); + assert!(c.contains("[[z/a.md#Part]]"), "{c}"); +} + +#[test] +fn a_note_move_leaves_the_index_a_rebuild_would_project() { + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = linked_vault(tmp.path()); + vault.move_note("a.md", "z/a.md").unwrap(); + assert_matches_a_rebuild(vault, &root); +} + +#[test] +fn a_resource_move_leaves_the_index_a_rebuild_would_project() { + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = linked_vault(tmp.path()); + + let report = vault.move_resource("img.png", "media/img.png").unwrap(); + assert_eq!( + report.rewrote, + vec![ + "a.md".to_string(), + "b.md".to_string(), + "sub/c.md".to_string() + ] + ); + assert_eq!( + fs::read_to_string(root.join("sub/c.md")).unwrap(), + "Up: ![x](../media/img.png) and [[media/img.png]] and [[a.md#Part]].\n", + "each syntax keeps its own convention" + ); + assert_matches_a_rebuild(vault, &root); +} + +#[test] +fn a_folder_move_leaves_the_index_a_rebuild_would_project() { + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = linked_vault(tmp.path()); + vault.move_dir("sub", "deep/er").unwrap(); + assert_matches_a_rebuild(vault, &root); +} + +#[test] +fn a_resource_move_to_a_note_path_is_refused_and_changes_nothing() { + // A `.md` path names a note: the moved bytes would be indexed as one by the next + // rebuild, so the resource would silently stop being a resource. + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = linked_vault(tmp.path()); + let before = vault_bytes(&root); + let backlinks = vault.explain_resource("img.png").unwrap().backlinks; + + for dest in ["img.md", "media/IMG.MD"] { + let err = vault.move_resource("img.png", dest).unwrap_err(); + assert!( + matches!(&err, Error::MoveDestination(m) if m.contains("note path")), + "{dest}: {err:?}" + ); + } + assert_eq!(vault_bytes(&root), before); + assert_eq!( + vault.explain_resource("img.png").unwrap().backlinks, + backlinks, + "the index is untouched" + ); +} From 12d531cf7b6684d0186d5bc0896c5c7641addcc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:44:53 +0000 Subject: [PATCH 16/30] refactor(core): drop fields nothing reads NoteFields.title (read only by tests: a note's title is its filename and the frontmatter title: key is inert, round-tripped with the raw bytes) and CoreVerb::symmetric / relation::is_symmetric (unused; contradicts being its own inverse stays pinned by a test on the inverse labels). display_title reads the file name and extension through pathspec. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/add.rs | 7 +++- crates/b2-core/src/note.rs | 23 ++++------- crates/b2-core/src/relation.rs | 63 +++++++++++-------------------- crates/b2-core/tests/roundtrip.rs | 9 ++++- 4 files changed, 42 insertions(+), 60 deletions(-) diff --git a/crates/b2-core/src/add.rs b/crates/b2-core/src/add.rs index 3d620f5..9cb8322 100644 --- a/crates/b2-core/src/add.rs +++ b/crates/b2-core/src/add.rs @@ -156,7 +156,12 @@ mod tests { assert_eq!(parsed.as_str(), out, "renders round-trip losslessly"); let f = parsed.fields(); assert!(f.r#type.is_none(), "type is not seeded (GH #80)"); - assert_eq!(f.title.as_deref(), Some("Spaced repetition")); assert_eq!(f.created.as_deref(), Some("2026-07-03")); + // `title:` is inert (the display title is the filename), so it is checked as + // the frontmatter bytes it was written as. + assert_eq!( + parsed.frontmatter(), + Some("title: \"Spaced repetition\"\ncreated: 2026-07-03\n") + ); } } diff --git a/crates/b2-core/src/note.rs b/crates/b2-core/src/note.rs index a798c8f..98ecdca 100644 --- a/crates/b2-core/src/note.rs +++ b/crates/b2-core/src/note.rs @@ -18,11 +18,9 @@ use yaml_rust2::{Yaml, YamlLoader}; #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct NoteFields { pub r#type: Option, - /// The frontmatter `title:` value, parsed only so it round-trips and can be - /// inspected. It has **no special meaning**: a note's display title is its - /// filename ([`display_title`]), and B2 never privileges this field - /// (data-model.md §1). Kept recognized so the key is understood, not stripped. - pub title: Option, + // No `title`: a note's display title is its filename ([`display_title`]), and a + // frontmatter `title:` key is inert (data-model.md §1) — round-tripped verbatim + // with the raw bytes like any key B2 does not project. pub description: Option, pub created: Option, pub updated: Option, @@ -249,7 +247,6 @@ fn extract_fields(yaml: &str) -> (NoteFields, bool) { Some(doc) => { readable = doc.as_hash().is_some() || matches!(doc, Yaml::Null); f.r#type = doc["type"].as_str().map(str::to_string); - f.title = doc["title"].as_str().map(str::to_string); f.description = doc["description"].as_str().map(str::to_string); f.created = scalar_to_string(&doc["created"]); f.updated = scalar_to_string(&doc["updated"]); @@ -335,15 +332,11 @@ fn relations_insertion(raw: &str, fm: &Frontmatter) -> Result String { - let base = path.rsplit('/').next().unwrap_or(path); - let stem = base.rsplit_once('.').map_or(base, |(stem, ext)| { - if ext.eq_ignore_ascii_case("md") && !stem.is_empty() { - stem - } else { - base - } - }); - stem.to_string() + let name = crate::pathspec::file_name(path); + match name.get(..name.len().saturating_sub(".md".len())) { + Some(stem) if crate::pathspec::is_md(name) => stem.to_string(), + _ => name.to_string(), + } } /// YAML double-quote a string (escaping `\` and `"`), so a value with `[[`, `|`, diff --git a/crates/b2-core/src/relation.rs b/crates/b2-core/src/relation.rs index 3c1194b..44afab8 100644 --- a/crates/b2-core/src/relation.rs +++ b/crates/b2-core/src/relation.rs @@ -1,17 +1,15 @@ //! The relation vocabulary (ADR-0010): a closed three-verb stance core — `references` //! (neutral), `supports` (for), `contradicts` (against) — with display-only inverse -//! labels and symmetry, plus a tolerated tail kept verbatim. The core encodes the one -//! thing embedding similarity cannot infer: stance. Adding a verb here is the whole -//! change. +//! labels, plus a tolerated tail kept verbatim. The core encodes the one thing embedding +//! similarity cannot infer: stance. Adding a verb here is the whole change. /// A core relation verb and its display metadata. pub struct CoreVerb { pub verb: &'static str, /// The label shown for an *inbound* edge of this type (display only — the - /// edge is stored once, directed). + /// edge is stored once, directed). The symmetric verb, `contradicts`, is its own + /// inverse (data-model.md §2). pub inverse: &'static str, - /// Symmetric verbs are their own inverse and traverse both ways. - pub symmetric: bool, } /// The closed core (data-model.md §2). Order mirrors the doc's table. @@ -19,17 +17,14 @@ pub const CORE: &[CoreVerb] = &[ CoreVerb { verb: "references", inverse: "referenced-by", - symmetric: false, }, CoreVerb { verb: "supports", inverse: "supported-by", - symmetric: false, }, CoreVerb { verb: "contradicts", inverse: "contradicts", - symmetric: true, }, ]; @@ -43,12 +38,6 @@ pub fn is_core(verb: &str) -> bool { core(verb).is_some() } -/// Whether `verb` is symmetric (its own inverse). Tail verbs are treated as -/// directed. -pub fn is_symmetric(verb: &str) -> bool { - core(verb).is_some_and(|c| c.symmetric) -} - /// The display label for an inbound edge of type `verb`. Core verbs map to their /// inverse; a tail verb is opaque, so the verb itself is returned (data-model §2). pub fn inverse_label(verb: &str) -> &str { @@ -74,37 +63,27 @@ mod tests { } } - /// Stance is the one thing embedding similarity cannot infer, and `contradicts` - /// is the only verb that reads the same from both ends. Its symmetry is the - /// single non-trivial fact in this module. - #[test] - fn only_contradicts_is_symmetric_and_tail_verbs_are_directed() { - assert!(is_symmetric("contradicts")); - assert!(!is_symmetric("references")); - assert!(!is_symmetric("supports")); - // An unknown verb is opaque, so it is treated as directed — never guessed - // symmetric off its spelling. - assert!(!is_symmetric("inspired-by")); - assert!(!is_symmetric("")); - } - /// Inverse labels are display-only (the edge is stored once, directed): each - /// core verb has its own, a symmetric verb is its own inverse, and a tail verb - /// falls back to itself rather than gaining an invented "-by" form. + /// directed core verb has its own, and a tail verb falls back to itself rather + /// than gaining an invented "-by" form. #[test] - fn inverse_labels_cover_core_symmetric_and_tail() { + fn inverse_labels_cover_the_directed_core_and_the_tail() { assert_eq!(inverse_label("references"), "referenced-by"); assert_eq!(inverse_label("supports"), "supported-by"); - assert_eq!(inverse_label("contradicts"), "contradicts"); assert_eq!(inverse_label("inspired-by"), "inspired-by"); - // Every core verb's inverse round-trips through the symmetry flag. - for c in CORE { - assert_eq!( - c.symmetric, - c.verb == c.inverse, - "{}: `symmetric` must agree with the inverse label", - c.verb - ); - } + } + + /// Stance is the one thing embedding similarity cannot infer, and `contradicts` + /// is the only core verb that reads the same from both ends: its inverse is + /// itself, and no other core verb's is. + #[test] + fn only_contradicts_is_its_own_inverse() { + assert_eq!(inverse_label("contradicts"), "contradicts"); + let own_inverse: Vec<&str> = CORE + .iter() + .filter(|c| c.verb == c.inverse) + .map(|c| c.verb) + .collect(); + assert_eq!(own_inverse, vec!["contradicts"]); } } diff --git a/crates/b2-core/tests/roundtrip.rs b/crates/b2-core/tests/roundtrip.rs index d68f3e4..159a7ce 100644 --- a/crates/b2-core/tests/roundtrip.rs +++ b/crates/b2-core/tests/roundtrip.rs @@ -54,7 +54,12 @@ fn extracts_queryable_fields_without_disturbing_raw() { let n = parse(&raw); let f = n.fields(); assert_eq!(f.r#type.as_deref(), Some("concept")); - // title is the logical value (quotes are a serialization detail kept in raw). - assert_eq!(f.title.as_deref(), Some("Spaced repetition")); assert_eq!(f.created.as_deref(), Some("2026-06-20")); + // A relation is the logical value (the YAML quotes are a serialization detail + // kept in raw). + assert_eq!( + f.relations, + vec!["supports [[concepts/memory|Human memory]] — applies the forgetting curve"] + ); + assert!(n.as_str().contains(" - \"supports [[concepts/memory|")); } From f6365756a4b0451856b6b50e3b50d5c929d7664e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:47:58 +0000 Subject: [PATCH 17/30] fix(core): add refuses an occupied path in the create itself add checked abs.exists() and then wrote, so a file appearing in between was clobbered. It now goes through import's create-new placement (import::place_new), where 'does it exist' and 'claim it' are one syscall and AlreadyExists is the op's own target-exists error. The refusal now also covers a folder at the path. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/add.rs | 19 +++++++++---------- crates/b2-core/src/import.rs | 36 ++++++++++++++++++++++++------------ crates/b2-core/tests/add.rs | 18 ++++++++++++++++++ 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/crates/b2-core/src/add.rs b/crates/b2-core/src/add.rs index 9cb8322..3637679 100644 --- a/crates/b2-core/src/add.rs +++ b/crates/b2-core/src/add.rs @@ -13,7 +13,7 @@ use crate::error::{Error, Result}; use crate::ingest::{self, EmbedCtx, ProjectionCtx}; use crate::note::yaml_quote; use serde::Serialize; -use std::fs; +use std::io::Write; use std::path::Path; /// What [`add_note`] did: the created note's vault-relative path @@ -26,7 +26,7 @@ pub struct AddReport { /// Create a new note at `path_input` (a `.md` suffix is optional and added if missing) /// with a minimal, valid frontmatter and `content` as its body, then project it. /// -/// Refuses to clobber: [`Error::AddTargetExists`] if a file already sits there, +/// Refuses to clobber: [`Error::AddTargetExists`] if anything already sits there, /// [`Error::AddDestination`] for an empty/absolute/vault-escaping path. Missing parent /// directories are created, mirroring `mv`. Projection **embeds** the new note's chunks, /// so the caller must open with the embedder the index was built with. @@ -75,15 +75,14 @@ fn write_new_note( created: &str, ) -> Result { let rel = crate::pathspec::normalize_rel_md(path_input).map_err(Error::AddDestination)?; - let abs = vault_root.join(&rel); - if abs.exists() { - return Err(Error::AddTargetExists(rel)); - } let doc = render_note(title, content, created); - if let Some(parent) = abs.parent() { - fs::create_dir_all(parent)?; - } - fs::write(&abs, doc)?; + // The refusal is the create itself, never a check before it: a file appearing at + // `rel` in between is refused rather than clobbered. + crate::import::place_new( + &vault_root.join(&rel), + || Error::AddTargetExists(rel.clone()), + |file| file.write_all(doc.as_bytes()), + )?; Ok(rel) } diff --git a/crates/b2-core/src/import.rs b/crates/b2-core/src/import.rs index e3f3b6f..4e64af9 100644 --- a/crates/b2-core/src/import.rs +++ b/crates/b2-core/src/import.rs @@ -108,27 +108,39 @@ fn destination_paths(vault_root: &Path, dir: &str, file_name: &str) -> Result<(S Ok((rel, abs)) } -/// Reserve the destination and fill it — or leave nothing behind. +/// Reserve the import's destination and fill it — [`place_new`], refusing an occupied +/// destination as [`Error::ImportTargetExists`]. That is also why `import_path` streams +/// instead of calling `fs::copy`, which would truncate an occupied destination. +/// +/// The destination is a file B2 created, so it carries ordinary new-file permissions +/// rather than the source's mode — byte-honesty is about content. +fn place(rel: &str, abs: &Path, fill: impl FnOnce(&mut fs::File) -> io::Result<()>) -> Result<()> { + place_new(abs, || Error::ImportTargetExists(rel.to_string()), fill) +} + +/// Create a new file at `abs` (missing parent folders included) and fill it — or leave +/// nothing behind. Shared by every op that writes a file B2 did not have before: an +/// import here, and `add`'s new note. /// /// **`create_new` is the refusal**, not a check before one: "does it exist" and "claim /// it" are a single syscall, so a file appearing in between — another window, a sync -/// client, the CLI — cannot be overwritten. That is also why `import_path` streams -/// instead of calling `fs::copy`, which would truncate an occupied destination. An -/// [`io::ErrorKind::AlreadyExists`] *is* [`Error::ImportTargetExists`], so the race and -/// the ordinary "that name is taken" reach the user as one message. +/// client, the CLI — cannot be overwritten. An [`io::ErrorKind::AlreadyExists`] *is* the +/// op's own target-exists error (`taken`), so the race and the ordinary "that name is +/// taken" reach the user as one message. /// -/// A `fill` that fails partway takes the reserved file with it: half a file is not an -/// import. The destination is a file B2 created, so it carries ordinary new-file -/// permissions rather than the source's mode — byte-honesty is about content. -fn place(rel: &str, abs: &Path, fill: impl FnOnce(&mut fs::File) -> io::Result<()>) -> Result<()> { +/// A `fill` that fails partway takes the reserved file with it: half a file is not a +/// new file. +pub(crate) fn place_new( + abs: &Path, + taken: impl FnOnce() -> Error, + fill: impl FnOnce(&mut fs::File) -> io::Result<()>, +) -> Result<()> { if let Some(parent) = abs.parent() { fs::create_dir_all(parent)?; } let mut file = match fs::File::create_new(abs) { Ok(f) => f, - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { - return Err(Error::ImportTargetExists(rel.to_string())) - } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => return Err(taken()), Err(e) => return Err(e.into()), }; if let Err(e) = fill(&mut file) { diff --git a/crates/b2-core/tests/add.rs b/crates/b2-core/tests/add.rs index 37cdb1f..cf8f3c6 100644 --- a/crates/b2-core/tests/add.rs +++ b/crates/b2-core/tests/add.rs @@ -141,6 +141,24 @@ fn add_refuses_to_clobber_an_existing_file() { ); } +/// The refusal is the create itself (one create-new open, no check before it), so it +/// covers whatever occupies the path — a folder named like the note included. +#[test] +fn add_refuses_a_path_a_folder_occupies() { + let tmp = tempfile::TempDir::new().unwrap(); + let (vault, root) = reindexed_vault(tmp.path()); + fs::create_dir_all(root.join("notes/taken.md")).unwrap(); + + let err = vault + .add_note("notes/taken", None, Some("body")) + .unwrap_err(); + assert!(matches!(err, Error::AddTargetExists(p) if p == "notes/taken.md")); + assert!( + root.join("notes/taken.md").is_dir(), + "the folder is untouched" + ); +} + #[test] fn create_note_writes_a_minimal_note_model_free() { let tmp = tempfile::TempDir::new().unwrap(); From d6f6d5eeb237c43d0658d56ff40e0b79b98e3d27 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:47:58 +0000 Subject: [PATCH 18/30] refactor(core): one passage layout, a non-panicking fake embedder, current docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ContextPassage::block is the one passage layout; ChatRequest::system_message and chat::passage_block (kept for the façade) both render through it, byte-identical. FakeEmbedder::new clamps a zero dimension to 1 instead of asserting on a production path; unpack_f32 goes through unpack_f32_into. chunk.rs cites docs/index-engine.md §1 instead of a deleted chunking spec's D1-D5 (which collided with invariants.md's D ids) and names its levers in plain words; scan and protected_regions share one fence-line rule. Stale docs fixed in embed.rs and graph.rs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/chat.rs | 11 +++---- crates/b2-core/src/chunk.rs | 55 ++++++++++++++++++++--------------- crates/b2-core/src/embed.rs | 22 +++++++------- crates/b2-core/src/graph.rs | 12 ++++---- crates/b2-core/src/llm.rs | 20 +++++++++---- crates/b2-core/tests/embed.rs | 9 ++++++ 6 files changed, 78 insertions(+), 51 deletions(-) diff --git a/crates/b2-core/src/chat.rs b/crates/b2-core/src/chat.rs index 404b8bb..13c5dcb 100644 --- a/crates/b2-core/src/chat.rs +++ b/crates/b2-core/src/chat.rs @@ -343,14 +343,11 @@ pub fn pair_line( ) } -/// One numbered passage as tool-result text — [`ChatRequest::system_message`]'s own -/// layout, so a passage looks the same whichever way it reached the model. +/// One numbered passage as tool-result text — [`ContextPassage::block`], the layout +/// [`ChatRequest::system_message`] uses too, so a passage looks the same whichever way +/// it reached the model. pub fn passage_block(marker: usize, p: &ContextPassage) -> String { - let heading = match &p.heading_path { - Some(h) => format!(" — {h}"), - None => String::new(), - }; - format!("[{marker}] {}{heading}\n{}\n", p.path, p.text) + p.block(marker) } /// The facts block, one line per finding, each naming the B2 read it came from so the diff --git a/crates/b2-core/src/chunk.rs b/crates/b2-core/src/chunk.rs index edb968f..ba9d95a 100644 --- a/crates/b2-core/src/chunk.rs +++ b/crates/b2-core/src/chunk.rs @@ -1,4 +1,4 @@ -//! Body chunker — the qmd heuristic (#19). +//! Body chunker — the qmd heuristic as adapted (docs/index-engine.md §1, GH #19). //! //! Splits a note body into **size-targeted, overlapping, Markdown-aware** chunks that //! each carry a `heading_path` breadcrumb. The shape, borrowed wholesale from @@ -26,19 +26,20 @@ //! under `cfg.prepend_heading_path` — an eval knob (default off) that prepends the //! breadcrumb into the embedded text, which the range then does not cover. -/// The tuning surface for [`chunk_body`] (spec §3, D5). Every lever that shapes a +/// The tuning surface for [`chunk_body`] (index-engine.md §1). Every lever that shapes a /// cut lives here; `Default` reproduces the shipped values, so adapters pass -/// `&ChunkConfig::default()` and the Step-3 eval sweeps parameters in one process -/// (a loop over configs) rather than one recompile per cell. Kept a plain params -/// struct — no async/generics/traits — so it stays pure, deterministic, model-free. +/// `&ChunkConfig::default()` and the eval harness (`make eval-sweep`) sweeps parameters +/// in one process (a loop over configs) rather than one recompile per cell. Kept a plain +/// params struct — no async/generics/traits — so it stays pure, deterministic, +/// model-free. #[derive(Debug, Clone, PartialEq)] pub struct ChunkConfig { - /// Target chunk size in *estimated* tokens (D1). Default 450 — headroom under - /// bge's 512-token truncation for the D2 proxy's error and any D3 breadcrumb. + /// Target chunk size in *estimated* tokens. Default 450 — headroom under bge's + /// 512-token truncation for the token proxy's error and any prepended breadcrumb. pub target_tokens: usize, - /// Fraction of a chunk re-shared with the next one (D4). Default 0.15. + /// Fraction of a chunk re-shared with the next one. Default 0.15. pub overlap_frac: f32, - /// The D2 model-free token proxy: `tokens ≈ chars / chars_per_token`. Default + /// The model-free token proxy: `tokens ≈ chars / chars_per_token`. Default /// 4.0 (English ≈ 4 chars/token; code and tables run denser — a lever, not a law). pub chars_per_token: f32, /// How far back (in estimated tokens) the boundary search looks from the target. @@ -46,8 +47,8 @@ pub struct ChunkConfig { pub backscan_tokens: usize, /// The Markdown break-point scorer (qmd's H1=100 … list=5). pub weights: BreakWeights, - /// Prepend `heading_path` into the *embedded* `text` (contextual chunk headers, - /// D3). An eval-gated retrieval knob; **default off**. Storing `heading_path` is + /// Prepend `heading_path` into the *embedded* `text` (contextual chunk headers). + /// An eval-gated retrieval knob, measured rank-neutral; **default off**. Storing `heading_path` is /// unconditional — this only controls whether it also seeds the vector. pub prepend_heading_path: bool, } @@ -65,7 +66,7 @@ impl Default for ChunkConfig { } } -/// Markdown break-point weights (qmd's boundary scorer, spec §2). Higher = a cleaner +/// Markdown break-point weights (qmd's boundary scorer, index-engine.md §1). Higher = a cleaner /// place to end a chunk. `heading[i]` is the weight of an H{i+1}; `word` is the /// lowest-value fallback that lets a giant single-line paragraph still split. #[derive(Debug, Clone, PartialEq, Eq)] @@ -103,7 +104,7 @@ pub struct Chunk { pub seq: usize, pub char_start: usize, pub char_end: usize, - /// The D2 token *estimate* (`chars / chars_per_token`) that sized this chunk — + /// The token *estimate* (`chars / chars_per_token`) that sized this chunk — /// not an exact token count (it was a whitespace word count under the old splitter). pub token_count: usize, /// The H1 › H2 › H3 breadcrumb the chunk falls under (`"A > B"`), or `None` @@ -132,8 +133,8 @@ pub fn chunk_body(body: &str, cfg: &ChunkConfig) -> Vec { // #41: byte ranges (balanced code fences, GFM tables) a boundary must not cut // *inside* — a forced cut with no clean break in the window is exactly where a - // bisected block would otherwise happen (this dense content is what the D2 proxy - // mis-sizes, spec §5). + // bisected block would otherwise happen (this dense content is what the chars/4 + // token proxy mis-sizes). let regions = protected_regions(body); let mut chunks: Vec = Vec::new(); @@ -157,7 +158,7 @@ pub fn chunk_body(body: &str, cfg: &ChunkConfig) -> Vec { ); // If that cut lands inside a fence/table, push it past the block's end so the // whole block stays in one chunk. An oversized chunk (the embedder truncates - // at 512, D2/§5) is fine; a half-fence / header-less table is not. + // at 512 tokens) is fine; a half-fence / header-less table is not. let end = snap_past_region(end, ®ions); push_chunk(&mut chunks, body, &line_paths, start, end, cfg); if end >= body_len { @@ -198,9 +199,9 @@ fn scan(body: &str, w: &BreakWeights) -> (Vec, Vec<(usize, Option for line in body.split_inclusive('\n') { let line_start = offset; offset += line.len(); - let content = line.trim_end_matches('\n').trim_end_matches('\r'); + let content = line_content(line); let trimmed = content.trim_start(); - let is_fence = trimmed.starts_with("```") || trimmed.starts_with("~~~"); + let is_fence = is_fence_line(trimmed); // Classify the line for its break score, and detect a heading (never inside // a fence — a `# comment` in code must not corrupt the breadcrumb). @@ -244,6 +245,17 @@ fn scan(body: &str, w: &BreakWeights) -> (Vec, Vec<(usize, Option (breaks, line_paths) } +/// One `split_inclusive('\n')` line without its `\n` / `\r\n` ending. +fn line_content(line: &str) -> &str { + line.trim_end_matches('\n').trim_end_matches('\r') +} + +/// Whether a left-trimmed line opens or closes a fenced code block (```` ``` ```` or +/// `~~~`) — the one fence rule [`scan`] and [`protected_regions`] both track. +fn is_fence_line(trimmed: &str) -> bool { + trimmed.starts_with("```") || trimmed.starts_with("~~~") +} + /// The ATX heading level (1..=6) if `trimmed` is a heading, else `None`. The `#` run /// must be followed by a space or end-of-line, so `#tag` is not a heading. fn heading_level(trimmed: &str) -> Option { @@ -415,11 +427,8 @@ fn protected_regions(body: &str) -> Vec<(usize, usize)> { for line in body.split_inclusive('\n') { let line_start = offset; offset += line.len(); - let trimmed = line - .trim_end_matches('\n') - .trim_end_matches('\r') - .trim_start(); - let is_fence = trimmed.starts_with("```") || trimmed.starts_with("~~~"); + let trimmed = line_content(line).trim_start(); + let is_fence = is_fence_line(trimmed); if is_fence { // A fence edge ends any table in progress, then toggles the fence. diff --git a/crates/b2-core/src/embed.rs b/crates/b2-core/src/embed.rs index 48469cf..6f1cfe1 100644 --- a/crates/b2-core/src/embed.rs +++ b/crates/b2-core/src/embed.rs @@ -41,17 +41,19 @@ pub trait Embedder { /// Deterministic, content-addressed embedder for tests/dev: identical text → /// identical vector, so KNN is reproducible and drop-&-rebuild yields the same -/// vectors. It is **not** semantic — it stands in for a real local model behind -/// the seam (testability stack, point 4) until that model lands. +/// vectors. It is **not** semantic — it stands in for `b2-embed`'s real local model +/// behind the same seam, so the engine suite runs offline (`B2_EMBEDDER=fake`, +/// docs/architecture.md). #[derive(Debug, Clone, Copy)] pub struct FakeEmbedder { dim: usize, } impl FakeEmbedder { + /// A fake of dimension `dim`. A vector has at least one component, so a `dim` of 0 + /// is clamped to 1 rather than refused. pub fn new(dim: usize) -> Self { - assert!(dim > 0, "embedding dimension must be positive"); - Self { dim } + Self { dim: dim.max(1) } } } @@ -97,8 +99,9 @@ impl Embedder for FakeEmbedder { } /// Pack a vector as a compact little-endian float32 BLOB — the stored form of every -/// vector in the index (`embeddings.vector`, `note_centroids.centroid`; build spec -/// §1.2). The query side packs the same way so an exact match has distance 0. +/// vector in the index (`embeddings.vector`, `note_centroids.centroid`; +/// index-engine.md §3). The query side packs the same way so an exact match has +/// distance 0. pub fn pack_f32(v: &[f32]) -> Vec { let mut out = Vec::with_capacity(v.len() * 4); for x in v { @@ -113,10 +116,9 @@ pub fn pack_f32(v: &[f32]) -> Vec { /// for a vector written by [`pack_f32`], so a non-multiple-of-4 length is simply /// truncated rather than treated as an error. pub fn unpack_f32(bytes: &[u8]) -> Vec { - bytes - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .collect() + let mut out = Vec::with_capacity(bytes.len() / 4); + unpack_f32_into(bytes, &mut out); + out } /// [`unpack_f32`] into a caller-owned scratch buffer, reusing its capacity. The diff --git a/crates/b2-core/src/graph.rs b/crates/b2-core/src/graph.rs index f57a010..3c7ca48 100644 --- a/crates/b2-core/src/graph.rs +++ b/crates/b2-core/src/graph.rs @@ -29,9 +29,10 @@ pub struct Neighbor { /// (data-model.md §2). Symmetric verbs read the same both ways. pub label: String, pub explanation: Option, - /// Edge provenance — `inline` (human body link), `frontmatter` (a relation B2 - /// accepted, or a human/importer authored) (data-model.md §0). `b2 explain` - /// surfaces this so a human body link reads distinctly from a B2-committed one. + /// Edge provenance — which of the two homes authored it (data-model.md §0): + /// `inline` (a body link) or `frontmatter` (a `b2_relations:` entry, whether + /// `b2 link` wrote it on the human's command or they wrote it by hand). `b2 explain` + /// surfaces this so a plain body link reads distinctly from a typed relation. pub origin: String, } @@ -128,8 +129,9 @@ pub fn unresolved_outbound(conn: &Connection, note_path: &str) -> Result Result> { let mut seen = HashSet::from([anchor.to_string()]); let mut frontier = vec![anchor.to_string()]; diff --git a/crates/b2-core/src/llm.rs b/crates/b2-core/src/llm.rs index 8e0f8ab..dbb391b 100644 --- a/crates/b2-core/src/llm.rs +++ b/crates/b2-core/src/llm.rs @@ -166,18 +166,26 @@ impl ChatRequest { } out.push_str("\n\nPassages:\n"); for (i, p) in self.passages.iter().enumerate() { - out.push_str(&format!("\n[{}] {}", i + 1, p.path)); - if let Some(h) = &p.heading_path { - out.push_str(&format!(" — {h}")); - } - out.push('\n'); - out.push_str(&p.text); out.push('\n'); + out.push_str(&p.block(i + 1)); } out } } +impl ContextPassage { + /// The passage as the model reads it, cited as `[marker]`: the marker, the path and + /// any heading breadcrumb on one line, then the text. The one layout, whichever way + /// a passage reaches the model — the system message's block or a tool result. + pub fn block(&self, marker: usize) -> String { + let heading = match &self.heading_path { + Some(h) => format!(" — {h}"), + None => String::new(), + }; + format!("[{marker}] {}{heading}\n{}\n", self.path, self.text) + } +} + /// How a completion ended and what it produced: everything delivered to the /// callback so far, plus the completed-vs-cancelled marker — so an adapter can /// render a truncated answer honestly rather than passing it off as whole. diff --git a/crates/b2-core/tests/embed.rs b/crates/b2-core/tests/embed.rs index af762ef..f255676 100644 --- a/crates/b2-core/tests/embed.rs +++ b/crates/b2-core/tests/embed.rs @@ -33,6 +33,15 @@ fn fake_embedder_is_deterministic() { assert_eq!(e.embed("x").unwrap().len(), 16); } +/// Constructing the fake is on a production path (`Vault::open`), so a zero +/// dimension degrades to the smallest real vector rather than panicking. +#[test] +fn a_zero_dimension_fake_is_clamped_to_one() { + let e = FakeEmbedder::new(0); + assert_eq!(e.dim(), 1); + assert_eq!(e.embed("x").unwrap().len(), 1); +} + #[test] fn embed_batch_matches_embed_per_element() { // The default `embed_batch` (which the fake inherits) must be a faithful map of From e948b9929d5fc386633dc56e4de90b56884578f1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:48:37 +0000 Subject: [PATCH 19/30] refactor(ui): name main.ts's small repeated steps; theme and reminder storage move out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - leaveEdits (the drawer guard + close-the-editor pair before a navigation), teardownEditor, bufferUnsaved, refusedWhileIndexing, confirmDelete (the delete confirm's button and ⏎), searchInput, and findByData behind treeRowEl/sideRowEl/gnodeEl. - executeMove calls one mover chosen by kind (all three report alike); executeDelete shares the note and resource branch. - theme.ts owns the appearance preference's values and storage, and embedreminder.ts the banner's opt-out, the way zoom.ts, panes.ts and keymap.ts own theirs. Both are tested with and without storage. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/embedreminder.test.ts | 31 ++++- ui/src/embedreminder.ts | 32 ++++- ui/src/main.ts | 232 +++++++++++++++++------------------ ui/src/settingstabs.test.ts | 7 ++ ui/src/theme.test.ts | 47 +++++++ ui/src/theme.ts | 43 +++++++ 6 files changed, 273 insertions(+), 119 deletions(-) create mode 100644 ui/src/theme.test.ts create mode 100644 ui/src/theme.ts diff --git a/ui/src/embedreminder.test.ts b/ui/src/embedreminder.test.ts index be3a1e1..d4bcf60 100644 --- a/ui/src/embedreminder.test.ts +++ b/ui/src/embedreminder.test.ts @@ -1,7 +1,12 @@ // The install-banner gating (embedreminder.ts), pinned. Pure boolean logic — no DOM — // so node runs it straight off the source via its native type-stripping: `npm test`. // Dependency-free like newentry.test.ts (hand-rolled assert; no @types/node). -import { shouldPromptEmbedInstall, type EmbedReminderInputs } from "./embedreminder.ts"; +import { + loadReminderOptOut, + saveReminderOptOut, + shouldPromptEmbedInstall, + type EmbedReminderInputs, +} from "./embedreminder.ts"; let passed = 0; @@ -63,4 +68,28 @@ check("respects a dismissal (session ✕ or persisted opt-out)", () => { ); }); +// --- the opt-out's storage ---------------------------------------------------------- + +check("no storage at all reads as not opted out, and saving into none is not an error", () => { + // node has no `localStorage` — the shape of a browser refusing it in private mode. + assert(!loadReminderOptOut(), "the reminder still shows"); + saveReminderOptOut(); +}); + +check("an opt-out, once saved, comes back", () => { + const store = new Map(); + const g = globalThis as unknown as { localStorage?: unknown }; + g.localStorage = { + getItem: (k: string) => store.get(k) ?? null, + setItem: (k: string, v: string) => void store.set(k, v), + }; + try { + assert(!loadReminderOptOut(), "nothing saved yet"); + saveReminderOptOut(); + assert(loadReminderOptOut(), "saved"); + } finally { + delete g.localStorage; + } +}); + console.log(`embedreminder: ${passed} checks passed`); diff --git a/ui/src/embedreminder.ts b/ui/src/embedreminder.ts index 9984f49..3f52ce1 100644 --- a/ui/src/embedreminder.ts +++ b/ui/src/embedreminder.ts @@ -1,5 +1,7 @@ -// Pure gating logic for the "semantic search is off" install banner — no DOM, no IPC — -// so node runs its test straight off the source (`npm test`), like newentry.ts/panes.ts. +// The "semantic search is off" install banner's gate, and where its opt-out persists — no +// DOM, no IPC, so node runs its test straight off the source (`npm test`). The two +// functions that touch `localStorage` are at the bottom, the shape keymap.ts and zoom.ts +// use for theirs. // // The problem it addresses: on a fresh install with no embedding model, opening a vault // runs the model-free projection pass (keyword index + graph) and then *silently* stops @@ -41,3 +43,29 @@ export function shouldPromptEmbedInstall(i: EmbedReminderInputs): boolean { i.notesTotal > 0 ); } + +// --- persistence ----------------------------------------------------------------------- +// +// The "Don't remind me again" opt-out — the persisted half of `dismissed` (the ✕ only +// hides the banner for the session, which is state, not storage). localStorage, like the +// theme and the zoom: a viewing choice, never vault state. + +const KEY = "b2:embed-reminder-off"; + +/** Has the user opted out for good? Unavailable storage (private mode) reads as no, so + * the reminder still shows. */ +export function loadReminderOptOut(): boolean { + try { + return localStorage.getItem(KEY) === "1"; + } catch { + return false; + } +} + +export function saveReminderOptOut(): void { + try { + localStorage.setItem(KEY, "1"); + } catch { + // Non-fatal: the opt-out still holds for this session if it can't persist. + } +} diff --git a/ui/src/main.ts b/ui/src/main.ts index 9f00170..da720e5 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -74,6 +74,8 @@ import { type SettingsTabId, } from "./settingstabs"; import { reindexMeterHtml } from "./widgets"; +import { isThemePref, loadThemePref, saveThemePref, themeAttr } from "./theme"; +import { loadReminderOptOut, saveReminderOptOut } from "./embedreminder"; import { externalUrl, isInPageAnchor } from "./links"; import { embedImagesField, livePreview, setEmbedImages, wikilink } from "./livepreview"; import { @@ -664,8 +666,7 @@ function adoptResource(resource: ResourceExplainView, picture: string | null): v // the editor — and the user's buffer — alive instead. A successful load records the // document in the history stack (#52); back/forward call `loadNote` directly. async function openNote(ref: string): Promise { - if (!fmEditGuard()) return; - if (!(await closeEditor())) return; + if (!(await leaveEdits())) return; await loadNote(ref, (path) => navPush({ kind: "note", path })); } @@ -835,8 +836,7 @@ async function loadResource(path: string, commit: (path: string) => void): Promi // openNote — same edit-mode flush, same one-document-owns-the-pane rule, same // history push. async function openResource(path: string): Promise { - if (!fmEditGuard()) return; - if (!(await closeEditor())) return; + if (!(await leaveEdits())) return; await loadResource(path, (p) => navPush({ kind: "resource", path: p })); } @@ -921,9 +921,21 @@ const TREE_ROW = ".tree-row[data-tree-row]"; /** The DOM row for a vault path. Looked up by data attribute rather than by CSS * selector because a filename may contain anything a selector would choke on. */ function treeRowEl(path: string | null): HTMLElement | null { - if (path === null) return null; - const rows = el("tree-pane").querySelectorAll(TREE_ROW); - for (const row of rows) if (row.dataset.treeRow === path) return row; + return findByData(el("tree-pane"), TREE_ROW, "treeRow", path); +} + +/** The first element under `root` matching `selector` whose `data-*` field `key` is + * exactly `value` (null finds nothing). Iterated rather than put in the selector: the + * value is a path or a key carrying one, and a path may contain anything a selector + * would choke on. */ +function findByData( + root: Element, + selector: string, + key: string, + value: string | null, +): E | null { + if (value === null) return null; + for (const node of root.querySelectorAll(selector)) if (node.dataset[key] === value) return node; return null; } @@ -973,20 +985,14 @@ function focusTreeRow(path: string): void { /** The DOM row for a `sidenav.ts` row key. */ function sideRowEl(key: string | null): HTMLElement | null { - if (key === null) return null; - const rows = el("side-pane").querySelectorAll("[data-side-row]"); - for (const row of rows) if (row.dataset.sideRow === key) return row; - return null; + return findByData(el("side-pane"), "[data-side-row]", "sideRow", key); } /** The graph node for a scene id (graph.ts `GraphNode.id`) — the note pane's `sideRowEl`, * and iterating rather than selecting for the same reason: a node id carries a vault * path, and a path may contain anything a selector would choke on. */ function gnodeEl(id: string | null): SVGElement | null { - if (id === null) return null; - const nodes = el("note-pane").querySelectorAll("[data-gnode]"); - for (const node of nodes) if (node.dataset.gnode === id) return node; - return null; + return findByData(el("note-pane"), "[data-gnode]", "gnode", id); } /** The row carrying the roving tabstop, as painted (sidenav.ts `rovingSideKey`). */ @@ -1319,8 +1325,7 @@ async function navGo(delta: -1 | 1): Promise { if (state.loading) return; // The guards first (closeEditor can await a save flush); the cursor math after, // against whatever the stack is once navigation is actually allowed to proceed. - if (!fmEditGuard()) return; - if (!(await closeEditor())) return; + if (!(await leaveEdits())) return; const target = navCursor + delta; if (target < 0 || target >= navStack.length) return; const entry = navStack[target]; @@ -1768,11 +1773,7 @@ function droppedFiles(dt: DataTransfer | null): DroppedFile[] { /** Shared refusal gate: an import writes, so it queues behind the same runs a move does. */ function canImportNow(): boolean { if (importInFlight || state.vaultRoot === null) return false; - if (state.reindexing) { - flash("Indexing is running — try the import again when it finishes."); - return false; - } - return true; + return !refusedWhileIndexing("import"); } /** The drop half: send each accepted file's bytes, then report once. */ @@ -1863,6 +1864,14 @@ async function finishImport(dir: string, imported: string[], refused: string[]): // reconcileExternalChange re-reads the open note by path, so if it still pointed at // the old path it would flash "moved or removed" for a move we made ourselves. +/** Refuse a write while an index run is live, and say so — true when refused. The run + * and the write would race the same index. */ +function refusedWhileIndexing(what: string): boolean { + if (!state.reindexing) return false; + flash(`Indexing is running — try the ${what} again when it finishes.`); + return true; +} + /** A move/rename is in flight — further gestures are ignored (no queueing in v1). */ let moveInFlight = false; @@ -1911,40 +1920,32 @@ function openMoveModal(node: TreeNodeRef): void { */ async function executeMove(node: TreeNodeRef, to: string): Promise { if (moveInFlight) return false; - if (state.reindexing) { - flash("Indexing is running — try the move again when it finishes."); - return false; - } + if (refusedWhileIndexing("move")) return false; // If the open document is affected, flush and close the editor first so the save // chain never targets the old path (a conflict keeps the editor and aborts the move). const curPath = openDocPath(state); const affected = curPath !== null && (node.nodeKind === "folder" ? remapPath(curPath, node.path, to) !== null : curPath === node.path); + // Not `leaveEdits`: with no editor open this must not yield before the in-flight flag + // is set below, or a second gesture could slip past it. if (affected && !fmEditGuard()) return false; if (affected && state.editing && !(await closeEditor())) return false; moveInFlight = true; if (node.nodeKind === "folder") flash(`Moving ${node.path}/…`); try { - let from: string; - let rewritten: number; - if (node.nodeKind === "note") { - const r = await api.moveNote(node.path, to); - from = r.from; - to = r.to; // the host normalizes (e.g. appends .md) - rewritten = r.links_rewritten; - } else if (node.nodeKind === "resource") { - const r = await api.moveResource(node.path, to); - from = r.from; - to = r.to; - rewritten = r.links_rewritten; - } else { - const r = await api.moveDir(node.path, to); - from = r.from; - to = r.to; - rewritten = r.links_rewritten; - } + // One report shape for all three kinds. + const move = + node.nodeKind === "note" + ? api.moveNote + : node.nodeKind === "resource" + ? api.moveResource + : api.moveDir; + const r = await move(node.path, to); + const from = r.from; + to = r.to; // the host normalizes (e.g. appends .md) + const rewritten = r.links_rewritten; // Re-point open/tree state through the move before the watcher pulse re-reads it. state.expandedDirs = new Set( @@ -2007,6 +2008,15 @@ function requestDelete(node: TreeNodeRef): void { render(); } +/** Commit the folder-delete confirm — its button and its ⏎ alike: the dialog closes and + * the delete runs. */ +function confirmDelete(): void { + const node = state.deleteTarget; + if (!node) return; + state.deleteTarget = null; + void executeDelete(node); +} + /** Forget tree state pointing into a deleted folder subtree. */ function dropDirState(dir: string): void { const gone = (d: string) => isWithin(d, dir); @@ -2021,10 +2031,7 @@ function dropDirState(dir: string): void { */ async function executeDelete(node: TreeNodeRef): Promise { if (deleteInFlight) return; - if (state.reindexing) { - flash("Indexing is running — try the delete again when it finishes."); - return; - } + if (refusedWhileIndexing("delete")) return; // If the open document dies with the delete, close the editor first so no save // chain targets a file that's about to be removed (a conflict aborts the delete, // keeping the buffer alive — the executeMove posture). @@ -2032,6 +2039,8 @@ async function executeDelete(node: TreeNodeRef): Promise { const affected = curPath !== null && (node.nodeKind === "folder" ? isWithin(curPath, node.path) : curPath === node.path); + // Not `leaveEdits`: with no editor open this must not yield before the in-flight flag + // is set below, or a second gesture could slip past it. if (affected && !fmEditGuard()) return; if (affected && state.editing && !(await closeEditor())) return; @@ -2046,18 +2055,15 @@ async function executeDelete(node: TreeNodeRef): Promise { try { let what: string; let dangled: number; - if (node.nodeKind === "note") { - const r = await api.deleteNote(node.path); - what = r.path; - dangled = r.dangled.length; - } else if (node.nodeKind === "resource") { - const r = await api.deleteResource(node.path); - what = r.path; - dangled = r.dangled.length; - } else { + if (node.nodeKind === "folder") { const r = await api.deleteDir(node.path); what = `${r.dir}/`; dangled = r.dangled.length; + } else { + // A note and a resource report alike. + const r = await (node.nodeKind === "note" ? api.deleteNote : api.deleteResource)(node.path); + what = r.path; + dangled = r.dangled.length; } // Clear state that pointed into the deleted subtree — before the watcher's @@ -2234,9 +2240,14 @@ function resetSearch(): void { state.searchVouched = null; } +/** The top bar's vault-search box. */ +function searchInput(): HTMLInputElement | null { + return document.getElementById("search-input") as HTMLInputElement | null; +} + function clearSearch(): void { resetSearch(); - const input = document.getElementById("search-input") as HTMLInputElement | null; + const input = searchInput(); if (input) input.value = ""; render(); } @@ -2798,28 +2809,17 @@ async function copyPath(path: string): Promise { // `data-theme` attribute on that those rules' overrides key on. Persisted in // localStorage — a viewing choice, never vault state, so it doesn't touch the host. -const THEME_KEY = "b2:theme"; - -function isThemePref(v: string | null): v is ThemePref { - return v === "system" || v === "light" || v === "dark"; -} - /** Reflect `state.theme` onto : absent attribute ⇒ follow the OS. */ function applyTheme(): void { const root = document.documentElement; - if (state.theme === "system") root.removeAttribute("data-theme"); - else root.setAttribute("data-theme", state.theme); + const attr = themeAttr(state.theme); + if (attr === null) root.removeAttribute("data-theme"); + else root.setAttribute("data-theme", attr); } /** Read the saved preference into state and apply it (once, first thing on boot). */ function loadTheme(): void { - let saved: string | null = null; - try { - saved = localStorage.getItem(THEME_KEY); - } catch { - // localStorage can be unavailable (e.g. private mode) — fall back to System. - } - state.theme = isThemePref(saved) ? saved : "system"; + state.theme = loadThemePref(); applyTheme(); } @@ -2827,11 +2827,7 @@ function loadTheme(): void { function setTheme(theme: ThemePref): void { if (state.theme === theme) return; state.theme = theme; - try { - localStorage.setItem(THEME_KEY, theme); - } catch { - // Non-fatal: the choice still applies for this session if it can't persist. - } + saveThemePref(theme); applyTheme(); render(); } @@ -3112,28 +3108,16 @@ function resetAllChords(): void { // persists the opt-out so a keyword-only user isn't pestered. Same localStorage idiom as // the appearance preference above (a viewing choice, never vault state). -const EMBED_REMINDER_KEY = "b2:embed-reminder-off"; - /** Read the persisted "don't remind me" opt-out into state (once, on boot). */ function loadEmbedReminderPref(): void { - try { - state.embedReminderDismissed = localStorage.getItem(EMBED_REMINDER_KEY) === "1"; - } catch { - // localStorage unavailable (e.g. private mode): default to showing the reminder. - } + state.embedReminderDismissed = loadReminderOptOut(); } /** Turn the banner off. `persist` writes the opt-out so it survives relaunch (the * checkbox); the bare ✕ passes false and only hides it for this session. */ function dismissEmbedReminder(persist: boolean): void { state.embedReminderDismissed = true; - if (persist) { - try { - localStorage.setItem(EMBED_REMINDER_KEY, "1"); - } catch { - // Non-fatal: the opt-out still holds for this session if it can't persist. - } - } + if (persist) saveReminderOptOut(); render(); } @@ -3197,8 +3181,7 @@ async function switchVault(): Promise { // Flush + leave edit mode before the picker (same hook as openNote); then drop any // pending trailing embed — it belongs to the vault we may be about to leave, and // its DB-derived pending set heals on that vault's next embed/reindex anyway. - if (!fmEditGuard()) return; - if (!(await closeEditor())) return; + if (!(await leaveEdits())) return; if (embedTimer !== undefined) { clearTimeout(embedTimer); embedTimer = undefined; @@ -3231,7 +3214,7 @@ async function switchVault(): Promise { state.dirs = []; // loadNotes below re-lists the new vault's structure state.treeCreate = null; navClear(); // history is per-vault: the old stack's paths mean nothing here - const input = document.getElementById("search-input") as HTMLInputElement | null; + const input = searchInput(); if (input) input.value = ""; state.loading = true; render(); @@ -3841,10 +3824,7 @@ async function conflictReload(): Promise { try { const fresh = await api.readNote(cur.path); state.current = fresh; - editorView?.destroy(); - editorView = null; - trailingDirty = false; - autosavePaused = false; + teardownEditor(); state.editConflict = false; mountEditor(fresh.body); void refreshConnections(); // the external edit may have changed edges too @@ -3873,6 +3853,35 @@ async function conflictKeepMine(): Promise { } } +/** Destroy the live editor and forget its save-chain flags — what a remount (Reload) and + * a close share. */ +function teardownEditor(): void { + editorView?.destroy(); + editorView = null; + trailingDirty = false; + autosavePaused = false; +} + +/** Does the live editor hold text the open note on disk doesn't — a flush that failed? */ +function bufferUnsaved(): boolean { + return ( + editorView !== null && + state.current !== null && + editorView.state.doc.toString() !== state.current.body + ); +} + +/** + * Resolve both editors before the pane changes what it shows: the frontmatter drawer must + * be settled (`fmEditGuard`), and the body editor flushed and closed. False when either + * holds the user back — a conflict, a failed save, a drawer edit to resolve first — and + * the caller abandons whatever triggered it rather than drop an edit. + */ +async function leaveEdits(): Promise { + if (!fmEditGuard()) return false; + return closeEditor(); +} + /** * Flush and leave edit mode. Returns false when the buffer could not be saved — a * conflict (the bar is up) or a failed save — so the caller must abandon whatever @@ -3882,16 +3891,13 @@ async function closeEditor(): Promise { if (!state.editing) return true; await saveNow(); if (state.editConflict) return false; - if (editorView && state.current && editorView.state.doc.toString() !== state.current.body) - return false; // the flush failed (its error already toasted) — keep the buffer alive + // The flush failed (its error already toasted) — keep the buffer alive. + if (bufferUnsaved()) return false; if (autosaveTimer !== undefined) { clearTimeout(autosaveTimer); autosaveTimer = undefined; } - editorView?.destroy(); - editorView = null; - trailingDirty = false; - autosavePaused = false; + teardownEditor(); state.editing = false; state.editConflict = false; return true; @@ -3972,8 +3978,7 @@ async function commitDroppedLink(card: DraggedCard): Promise { // (already toasted). The link stays in the buffer either way — it is the user's edit — // but nothing may claim it landed, and the card stays where it is. if (state.editConflict) return; - if (editorView && state.current && editorView.state.doc.toString() !== state.current.body) - return; + if (bufferUnsaved()) return; if (state.current?.path !== src.path) return; // navigated away while the save ran // By the card, not by either of its strings: `withoutCard` is keyed on the path, and // taking the pair is what stops the target being handed to a path comparison (the review @@ -4222,7 +4227,7 @@ function syncFind(noteSwapped: boolean): void { /** ⇧⌘F: hand the keyboard to the global vault-search box in the top bar. */ function focusGlobalSearch(): void { - const input = document.getElementById("search-input") as HTMLInputElement | null; + const input = searchInput(); input?.focus(); input?.select(); } @@ -4711,9 +4716,7 @@ function wireClicks(): void { } // The folder-delete confirm: the Delete button commits and closes it. if (target.closest("#delete-confirm") && state.deleteTarget) { - const node = state.deleteTarget; - state.deleteTarget = null; - void executeDelete(node); + confirmDelete(); return; } // The Move… modal: clicking a destination row commits the move and closes it. @@ -5065,8 +5068,7 @@ function wireForms(): void { document.addEventListener("submit", (e) => { if ((e.target as HTMLElement).id === "search-form") { e.preventDefault(); - const input = document.getElementById("search-input") as HTMLInputElement | null; - void doSearch(input?.value ?? ""); + void doSearch(searchInput()?.value ?? ""); } // The chat composer is a form so its Ask button is a submit button — the platform's // own "this field's default action", which is what makes ⏎ work in it without B2 @@ -5365,9 +5367,7 @@ function wireChords(): void { // Enter commits the folder-delete confirm (its keyboard sibling of the button). if (state.deleteTarget && isBound(e, "delete.confirm")) { e.preventDefault(); - const node = state.deleteTarget; - state.deleteTarget = null; - void executeDelete(node); + confirmDelete(); return; } // ⏎ / Space on a focused graph node. SVG has no native button activation, so the diff --git a/ui/src/settingstabs.test.ts b/ui/src/settingstabs.test.ts index 61fcaca..b4d305c 100644 --- a/ui/src/settingstabs.test.ts +++ b/ui/src/settingstabs.test.ts @@ -12,6 +12,7 @@ import { DEFAULT_SETTINGS_TAB, SETTINGS_TABS, isSettingsTab, + tabDomId, tabMove, tabNavFor, tabStep, @@ -110,4 +111,10 @@ check("a key the rail has no move for is left alone", () => { ); }); +check("every tab has its own element id, the one the rail paints and focus returns to", () => { + const ids = SETTINGS_TABS.map((t) => tabDomId(t.id)); + assert(new Set(ids).size === ids.length, "unique"); + assert(tabDomId("keyboard") === "settings-tab-keyboard", "and spelled as main.ts looks for it"); +}); + console.log(`settingstabs: ${passed} checks passed`); diff --git a/ui/src/theme.test.ts b/ui/src/theme.test.ts new file mode 100644 index 0000000..5299c8e --- /dev/null +++ b/ui/src/theme.test.ts @@ -0,0 +1,47 @@ +// The appearance preference (theme.ts): what it may be, how carries it, and its +// storage — including storage that isn't there (node has none, which is the shape of a +// browser refusing it in private mode) and a stored value that was hand-edited. + +import { strict as assert } from "node:assert"; +import test from "node:test"; +import { isThemePref, loadThemePref, saveThemePref, themeAttr } from "./theme.ts"; + +/** Run `fn` with a working in-memory localStorage installed, then take it away again. */ +function withStorage(seed: Record, fn: (store: Map) => void): void { + const store = new Map(Object.entries(seed)); + const g = globalThis as unknown as { localStorage?: unknown }; + g.localStorage = { + getItem: (k: string) => store.get(k) ?? null, + setItem: (k: string, v: string) => void store.set(k, v), + }; + try { + fn(store); + } finally { + delete g.localStorage; + } +} + +test("only the three preferences are preferences", () => { + for (const t of ["system", "light", "dark"]) assert.ok(isThemePref(t), t); + for (const t of [null, "", "Dark", "sepia"]) assert.ok(!isThemePref(t), String(t)); +}); + +test("system follows the OS: no attribute; the others pin one", () => { + assert.equal(themeAttr("system"), null); + assert.equal(themeAttr("light"), "light"); + assert.equal(themeAttr("dark"), "dark"); +}); + +test("no storage at all is System, and saving into none is not an error", () => { + assert.equal(loadThemePref(), "system"); + saveThemePref("dark"); +}); + +test("a saved choice comes back; a hand-edited one is System", () => { + withStorage({}, (store) => { + saveThemePref("dark"); + assert.equal(store.get("b2:theme"), "dark"); + assert.equal(loadThemePref(), "dark"); + }); + withStorage({ "b2:theme": "sepia" }, () => assert.equal(loadThemePref(), "system")); +}); diff --git a/ui/src/theme.ts b/ui/src/theme.ts new file mode 100644 index 0000000..4dd7670 --- /dev/null +++ b/ui/src/theme.ts @@ -0,0 +1,43 @@ +// The appearance preference (Settings → General → Theme): what it may be, how +// carries it, and where it persists — main.ts applies it. Its own module like zoom.ts, +// panes.ts and keymap.ts own theirs, so the storage rules sit next to the value they keep. +// +// "system" (the default) defers to the OS via the stylesheet's `prefers-color-scheme` +// rules; "light"/"dark" pin a theme by stamping a `data-theme` attribute on that +// those rules' overrides key on. Persisted in localStorage — a viewing choice, never vault +// state, so it doesn't touch the host. + +import type { ThemePref } from "./state.ts"; + +const KEY = "b2:theme"; + +/** Does an untrusted string (a stored value, a `data-theme-choice`) name a preference? */ +export function isThemePref(v: string | null): v is ThemePref { + return v === "system" || v === "light" || v === "dark"; +} + +/** The `data-theme` value carries for a preference, or null to follow the OS. */ +export function themeAttr(theme: ThemePref): string | null { + return theme === "system" ? null : theme; +} + +// --- persistence ----------------------------------------------------------------------- + +/** The saved preference. Unreadable, unrecognised or unavailable storage is "system". */ +export function loadThemePref(): ThemePref { + try { + const saved = localStorage.getItem(KEY); + return isThemePref(saved) ? saved : "system"; + } catch { + // localStorage can be unavailable (e.g. private mode) — fall back to System. + return "system"; + } +} + +export function saveThemePref(theme: ThemePref): void { + try { + localStorage.setItem(KEY, theme); + } catch { + // Non-fatal: the choice still applies for this session if it can't persist. + } +} From 848e9628ea626a3ce44fc5269bd4ff73a9c1b5aa Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:49:39 +0000 Subject: [PATCH 20/30] docs(ui): fix stale comments in state.ts and api.ts state.ts: contextMenu covers the file tree as well as a discovery card, current is the centre pane's note (not a "left pane"), and Settings is a full-window surface rather than a modal box. api.ts: menuChords is read once to check menukeys.ts's mirror; the keyboard reference never listed from it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/api.ts | 5 +++-- ui/src/state.ts | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ui/src/api.ts b/ui/src/api.ts index 14590c0..599d661 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -399,8 +399,9 @@ export const api = { * Every chord the app's **menu bar** takes (#119) — the host declares the menu * (b2-desktop `menu.rs`), so this is the authority on chords the webview never sees: * AppKit dispatches a menu key equivalent before the key window's responder chain. - * The keyboard reference lists them from here, and `menukeys.ts` holds the mirror this - * is checked against at boot. + * Read once, at boot, to check `menukeys.ts`'s mirror against (main.ts + * `checkMenuDrift`); everything else — the recorder's refusal of a menu chord included + * — reads that mirror, never this. */ menuChords: (): Promise => invoke("menu_chords"), diff --git a/ui/src/state.ts b/ui/src/state.ts index 90288d5..e6e54e4 100644 --- a/ui/src/state.ts +++ b/ui/src/state.ts @@ -164,7 +164,7 @@ export interface AppState { * gesture itself is the intent. */ deleteTarget: TreeNodeRef | null; - /** The open note (left pane), or null before one is opened. */ + /** The note the centre pane shows, or null (nothing open yet, or a resource is). */ current: NoteView | null; /** * The selected resource's fallback card (mutually exclusive with `current`: @@ -254,7 +254,8 @@ export interface AppState { * the exceptions. Reset on note-open — the keys belong to the note just closed. */ collapsedCards: Set; - /** An open right-click menu on a discovery card, or null. */ + /** An open right-click menu — on a discovery card (or graph ghost), or on the file + * tree — or null. `ContextMenuState` says which. */ contextMenu: ContextMenuState | null; /** * The open note's unresolved (dangling) outbound links — a `[[folder]]` or a typo @@ -336,7 +337,8 @@ export interface AppState { linkTarget: LinkTarget | null; /** The verb selected in the link modal. */ linkRelation: string; - /** The settings modal (⌘,) is open. */ + /** Settings is open — a full-window surface over the app, not a floating box + * (settingsview.ts's `settingsScreenHtml` says why). */ settingsOpen: boolean; /** * Which section of the settings dialog is showing (settingstabs.ts). Outlives a From d43544bcbf649c02a93c241da00ce11ca4f8d153 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:51:06 +0000 Subject: [PATCH 21/30] docs(desktop): renderMarkdown now lives in ui/src/markdown.ts Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-desktop/CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/b2-desktop/CLAUDE.md b/crates/b2-desktop/CLAUDE.md index 71f37e2..57ae838 100644 --- a/crates/b2-desktop/CLAUDE.md +++ b/crates/b2-desktop/CLAUDE.md @@ -98,7 +98,7 @@ mirrored in `ui/src/menukeys.ts` (checked at boot): change the two together. repo), and a chat answer is generated from notes by a model. Both are hostile input rendered into B2's own window. -- **One Markdown-to-HTML path, and it sanitizes.** `renderMarkdown` (`ui/src/render.ts`) runs +- **One Markdown-to-HTML path, and it sanitizes.** `renderMarkdown` (`ui/src/markdown.ts`) runs DOMPurify (`ui/src/sanitize.ts`) as its last step, so every surface is covered by construction. - **Anything that reaches `innerHTML`** comes from `renderMarkdown` (note bodies) or `escapeHtml` (every value B2 interpolates into chrome). There is no third option. From 3d7f2ddcd326bc1125e3145ba65b54a677d34302 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:57:22 +0000 Subject: [PATCH 22/30] refactor(desktop): one Slot for both long ops, one-line commands, shared wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `Slot` (slot.rs) replaces the two hand-rolled state machines (reindex_running/reindex_cancel and ask_running/ask_cancel, ReindexGuard and AskGuard, claim-then-arm at each call site): `try_claim` returns a guard that releases on drop and clears a stale cancel; `cancel`, `cancelled`, `flow`, `cancel_and_wait`. Tests re-pointed; the two pure state-machine tests move to slot.rs. - `read_op` / `semantic_op` make every command body one line; the `*_impl` fns the tests call stay, as one-liners over them. - Wiring goes through b2_embed::embedder_for, b2_embed::fake_requested, b2_llm::provider and b2_llm::fake_requested. open_vault returns the id of the model it actually loaded, so `embed` no longer re-reads the config (which could name a different model) to attribute its time. - Dead IPC removed: the `ping` and `neighbors` commands (only ui/src/api.ts wrappers referenced them) and the ping test. - `open_resource` reads the root once, so the inventory check and the path handed to the OS belong to the same vault. - stats returns the IPC row itself (`EmbedStat`, ModelStat flattened — same model/total_ms/chunks/runs keys, pinned by a test) instead of a mirrored struct plus a hand-written From. - state_file.rs: one path + best-effort-write helper behind last-vault, chat.json and embed-stats.json. - The app failing to start logs and exits 1 instead of `.expect()`. - error.rs: the relation verbs come from b2_core::relation::CORE, and a broken embedder config names config.toml instead of "check the vault". - logging.rs: the implied-filter comment no longer claims the CLI differs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-desktop/src/chat.rs | 48 ++--- crates/b2-desktop/src/commands.rs | 294 ++++++++++------------------ crates/b2-desktop/src/error.rs | 43 +++- crates/b2-desktop/src/logging.rs | 6 +- crates/b2-desktop/src/main.rs | 263 +++++++------------------ crates/b2-desktop/src/slot.rs | 122 ++++++++++++ crates/b2-desktop/src/state_file.rs | 34 ++++ crates/b2-desktop/src/stats.rs | 85 ++++---- 8 files changed, 448 insertions(+), 447 deletions(-) create mode 100644 crates/b2-desktop/src/slot.rs create mode 100644 crates/b2-desktop/src/state_file.rs diff --git a/crates/b2-desktop/src/chat.rs b/crates/b2-desktop/src/chat.rs index 0e41a22..0ff89b3 100644 --- a/crates/b2-desktop/src/chat.rs +++ b/crates/b2-desktop/src/chat.rs @@ -19,11 +19,15 @@ //! resolver's; this adapter states its sources and doesn't rank them. use crate::keychain::KeyStore; -use b2_core::llm::{FakeLlm, LlmProvider}; -use b2_llm::{ApiKeySource, LlmConfig, OpenAiCompatProvider}; +use crate::state_file; +use b2_core::llm::LlmProvider; +use b2_llm::{ApiKeySource, LlmConfig}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; +/// The state file holding the persisted half of [`ChatPrefs`] (see [`state_file`]). +const PREFS_FILE: &str = "chat.json"; + /// The desktop's chat preferences: the Settings section's two persisted fields, /// plus the key in force. `None` means "whatever the environment and the /// defaults say" — so a user who has never opened the section is exactly the CLI @@ -102,33 +106,22 @@ impl ChatPrefs { } } -/// Whether the deterministic fake chat provider is forced (`B2_LLM=fake`) — the -/// `B2_EMBEDDER=fake` sibling, honored identically to the CLI so the two adapters -/// behave the same offline. -pub fn use_fake_llm() -> bool { - matches!(std::env::var("B2_LLM").ok().as_deref(), Some("fake")) -} - -/// Pick + wire the chat provider — `main.rs`'s `open_vault` for the second seam. +/// Pick + wire the chat provider — `main.rs`'s `open_vault` for the second seam, over the +/// same `b2_llm::provider` rule the CLI uses (`B2_LLM=fake` or the configured endpoint). /// /// Unlike the CLI's `open_llm` this does **not** probe: the desktop probes once when the /// chat surface opens, and a round trip per turn would be a per-question tax on a cloud /// endpoint. A server that dies between the probe and the question surfaces as the same /// actionable message from `Error::Llm`. pub fn provider(prefs: &ChatPrefs) -> Box { - if use_fake_llm() { - return Box::new(FakeLlm); - } - Box::new(OpenAiCompatProvider::new(prefs.config())) + b2_llm::provider(prefs.config()) } -/// Where the persisted half lives: `/b2/chat.json`, beside the -/// remembered vault (`main.rs`'s `last_vault_file`) and under the same `b2/` -/// vendor dir as the model cache. `None` only if the platform has no data dir, -/// in which case remembering is silently skipped — chat still works, it just -/// forgets the endpoint at quit. +/// Where the persisted half lives: `/b2/chat.json`, beside the remembered vault +/// ([`state_file`]). `None` only if the platform has no data dir — chat still works, it +/// just forgets the endpoint at quit. pub fn prefs_file() -> Option { - dirs::data_dir().map(|d| d.join("b2").join("chat.json")) + state_file::path(PREFS_FILE) } /// The preferences this launch starts from: the endpoint and model out of the @@ -225,25 +218,18 @@ pub fn apply_tool_cap(prev: &ChatPrefs, typed: Option<&str>) -> Option { /// opened vault: a write failure is logged and swallowed, never failing the /// setting the user just made — the change is already live in memory either way. pub fn persist_prefs(prefs: &ChatPrefs) { - let Some(file) = prefs_file() else { - eprintln!("[b2] could not remember chat settings: no platform data directory"); - return; - }; - if let Err(e) = write_prefs_to(&file, prefs) { - eprintln!("[b2] could not remember chat settings: {e}"); - } + state_file::update(PREFS_FILE, "remember chat settings", |file| { + write_prefs_to(file, prefs) + }); } /// [`persist_prefs`] against an explicit path — the testable core. Creates the /// parent dir if needed. The API key is `#[serde(skip)]`, so what lands on disk /// is the endpoint and the model and nothing else. pub fn write_prefs_to(file: &Path, prefs: &ChatPrefs) -> std::io::Result<()> { - if let Some(parent) = file.parent() { - std::fs::create_dir_all(parent)?; - } let json = serde_json::to_string_pretty(prefs) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - std::fs::write(file, json) + state_file::write(file, json.as_bytes()) } #[cfg(test)] diff --git a/crates/b2-desktop/src/commands.rs b/crates/b2-desktop/src/commands.rs index 7331120..1b0e221 100644 --- a/crates/b2-desktop/src/commands.rs +++ b/crates/b2-desktop/src/commands.rs @@ -10,19 +10,20 @@ //! synchronous (ADR-0011); `(async)` is only the "don't block the UI" knob. //! //! The thin `*_impl` split lets the command layer be unit-tested against a real vault -//! without a Tauri runtime. +//! without a Tauri runtime, and [`read_op`] / [`semantic_op`] are the whole of "open a +//! fresh vault, make one façade call, map its error" — so a command body is one line. use crate::chat::ChatPrefs; use crate::error::CmdError; use crate::watch::VaultWatcher; -use crate::{open_read, open_semantic, open_vault, AppState, AskGuard, ReindexGuard}; +use crate::{open_read, open_semantic, open_vault, open_vault_at, AppState}; use b2_core::add::AddReport; use b2_core::ingest::ReindexProgress; use b2_core::llm::{ChatTurn, LlmProvider}; use b2_core::vault::{ AnswerView, DeleteReport, DirCreateReport, DirDeleteReport, DirMoveReport, EmbedReport, - ExplainView, ImportReport, LinkReport, MoveReport, NeighborView, NoteSummary, NoteView, - ProjectReport, ResourceDeleteReport, ResourceExplainView, ResourceMoveReport, ResourceSummary, + ExplainView, ImportReport, LinkReport, MoveReport, NoteSummary, NoteView, ProjectReport, + ResourceDeleteReport, ResourceExplainView, ResourceMoveReport, ResourceSummary, SearchEvidenceView, SimilarExplainView, SimilarView, Vault, WriteReport, }; use b2_embed::{EmbedConfig, ModelChoice}; @@ -49,11 +50,22 @@ pub struct VaultInfo { pub notes_total: usize, } -/// Step 0's seam-proving command: the frontend `invoke('ping')` round-trips this to -/// confirm the Rust↔JS bridge before any real surface exists. -#[tauri::command] -pub fn ping() -> &'static str { - "pong" +/// One façade call over a fresh **read-path** vault (the fake embedder, no model load) — +/// the whole body of a model-free command. The call's own error is mapped once, here. +fn read_op( + state: &AppState, + op: impl FnOnce(&Vault) -> b2_core::Result, +) -> Result { + Ok(op(&open_read(state)?)?) +} + +/// [`read_op`] over the **real** model, for a command that embeds a query or re-projects +/// with the model the index was built with (fail-fast "run `b2 init`" when absent). +fn semantic_op( + state: &AppState, + op: impl FnOnce(&Vault) -> b2_core::Result, +) -> Result { + Ok(op(&open_semantic(state)?)?) } #[tauri::command(async)] @@ -110,8 +122,7 @@ pub fn list_notes(state: State<'_, AppState>) -> Result, CmdErr /// per-kind composition the locked design prefers over a union type (research §9b #10). #[tauri::command(async)] pub fn list_resources(state: State<'_, AppState>) -> Result, CmdError> { - let vault = open_read(state.inner())?; - Ok(vault.list_resources()?) + read_op(state.inner(), |v| v.list_resources()) } /// The file tree's structure half: every folder in the vault, **empty ones @@ -139,8 +150,7 @@ pub fn explain_resource( state: State<'_, AppState>, path: String, ) -> Result { - let vault = open_read(state.inner())?; - Ok(vault.explain_resource(&path)?) + read_op(state.inner(), |v| v.explain_resource(&path)) } /// A resource's **bytes**, base64, for the viewer the card shows in place of the @@ -157,8 +167,7 @@ pub fn explain_resource( /// the host validates, exactly as [`open_resource`] does. #[tauri::command(async)] pub fn read_resource(state: State<'_, AppState>, path: String) -> Result { - let vault = open_read(state.inner())?; - Ok(BASE64.encode(vault.read_resource_bytes(&path)?)) + read_op(state.inner(), |v| v.read_resource_bytes(&path)).map(|bytes| BASE64.encode(bytes)) } /// *Open in system default* on the fallback card — an **OS handoff**, never @@ -166,11 +175,16 @@ pub fn read_resource(state: State<'_, AppState>, path: String) -> Result, path: String) -> Result<(), CmdError> { - let vault = open_read(state.inner())?; - vault.explain_resource(&path)?; // inventory check: unknown paths refuse, never open let root = state.current_root().ok_or(CmdError::VaultRequired)?; + let (vault, _) = open_vault_at(&root, false)?; + vault.explain_resource(&path)?; // inventory check: unknown paths refuse, never open tauri_plugin_opener::open_path(root.join(&path), None::<&str>) .map_err(|e| CmdError::OpenFailed(e.to_string())) } @@ -387,8 +401,7 @@ pub fn similar( ) -> Result, CmdError> { // The ranked list is what the façade serves (GH #197) — no raw/floored mode // to route, so the command is the one call it always should have been. - let vault = open_read(state.inner())?; - Ok(vault.similar(¬e, limit)?) + read_op(state.inner(), |v| v.similar(¬e, limit)) } /// **Explain** one Similar card (GH #236): the model-free Compare view's data, the GUI @@ -401,8 +414,9 @@ pub fn explain_similar( candidate: String, limit: usize, ) -> Result { - let vault = open_read(state.inner())?; - Ok(vault.explain_similar(&anchor, &candidate, limit)?) + read_op(state.inner(), |v| { + v.explain_similar(&anchor, &candidate, limit) + }) } /// Hybrid search **with its evidence reading** (invariants.md D2, GH #202) — the @@ -418,20 +432,12 @@ pub fn search( limit: usize, ) -> Result { // Semantic: the query is embedded, so this opens the real model (fail-fast if absent). - let vault = open_semantic(state.inner())?; - Ok(vault.search_evidence(&query, limit)?) -} - -#[tauri::command(async)] -pub fn neighbors(state: State<'_, AppState>, note: String) -> Result, CmdError> { - let vault = open_read(state.inner())?; - Ok(vault.neighbors(¬e)?) + semantic_op(state.inner(), |v| v.search_evidence(&query, limit)) } #[tauri::command(async)] pub fn explain(state: State<'_, AppState>, note: String) -> Result { - let vault = open_read(state.inner())?; - Ok(vault.explain(¬e)?) + read_op(state.inner(), |v| v.explain(¬e)) } #[tauri::command(async)] @@ -443,8 +449,9 @@ pub fn link( explanation: Option, ) -> Result { // Re-projects the source note → opens the same real model the index was built with. - let vault = open_semantic(state.inner())?; - Ok(vault.link(&src, &dst, &relation, explanation.as_deref())?) + semantic_op(state.inner(), |v| { + v.link(&src, &dst, &relation, explanation.as_deref()) + }) } /// The **projection pass** — the fast, model-free half of a reindex. One façade call over @@ -483,7 +490,7 @@ pub fn embed( /// writes. A no-op if nothing is running. #[tauri::command(async)] pub fn cancel_reindex(state: State<'_, AppState>) { - state.request_reindex_cancel(); + state.reindex.cancel(); } /// The settings picker's model list: every model B2 offers ([`b2_embed::AVAILABLE_MODELS`]), @@ -537,36 +544,12 @@ pub fn embed_device() -> &'static str { b2_embed::active_device_label() } -/// One model's cumulative embedding cost, for the Settings pane (`stats.rs`). Flat view -/// over [`crate::stats::ModelStat`] so it crosses IPC as a plain payload. -#[derive(Debug, Clone, Serialize)] -pub struct EmbedStat { - pub model: String, - pub total_ms: u64, - pub chunks: u64, - pub runs: u64, -} - -impl From<(String, crate::stats::ModelStat)> for EmbedStat { - fn from((model, s): (String, crate::stats::ModelStat)) -> Self { - Self { - model, - total_ms: s.total_ms, - chunks: s.chunks, - runs: s.runs, - } - } -} - /// The per-model embedding-time ledger (`stats.rs`) — what the Settings pane renders so a /// model swap can be judged on real speed. Infallible: no data / an unreadable ledger is /// an empty list, never an error (the totals are diagnostic, never load-bearing). #[tauri::command(async)] -pub fn embed_stats() -> Vec { +pub fn embed_stats() -> Vec { crate::stats::read_all() - .into_iter() - .map(EmbedStat::from) - .collect() } /// Every chord the app's **menu bar** takes, in menu order (`menu.rs`, #119). The UI folds @@ -669,7 +652,7 @@ pub fn why_similar( /// streaming. #[tauri::command(async)] pub fn cancel_ask(state: State<'_, AppState>) { - state.request_ask_cancel(); + state.ask.cancel(); } /// What the chat surface needs to draw itself before a question is asked: the endpoint and @@ -759,7 +742,7 @@ fn set_chat_config_impl( /// same note), else one probe of the configured endpoint. fn chat_setup_impl(state: &AppState) -> ChatSetup { let config = state.chat_prefs().config(); - if crate::chat::use_fake_llm() { + if b2_llm::fake_requested() { ChatSetup::fake(&config) } else { b2_llm::probe_setup(&config) @@ -799,66 +782,45 @@ fn why_similar_impl( }) } -/// The delivery every streamed answer shares: claim the single answer slot, arm the -/// cancel flag, and run `call` — the one façade op — with a token callback that feeds -/// `sink` and reads the cancel flag at every token. +/// The delivery every streamed answer shares: claim the answer slot and run `call` — the +/// one façade op — with a token callback that feeds `sink` and reads the slot's cancel at +/// every token. fn stream_answer( state: &AppState, sink: &dyn Fn(&str), call: impl FnOnce(&mut dyn FnMut(&str) -> ControlFlow<()>) -> b2_core::Result, ) -> Result { // Single-in-flight: two answers at once would share one cancel flag, so the second - // one's `arm` would quietly un-cancel the first. The pane already refuses a second + // one's claim would quietly un-cancel the first. The pane already refuses a second // turn while one is streaming, so this is the belt-and-suspenders half. - if !state.try_start_ask() { - return Err(CmdError::AskInFlight); - } - let _guard = AskGuard(state); - // Clear any stale cancel now that *this* answer owns the slot (the Esc that stopped - // the previous turn must not stop this one before its first token). - state.arm_ask(); - + let _held = state.ask.try_claim().ok_or(CmdError::AskInFlight)?; Ok(call(&mut |token| { sink(token); - if state.ask_cancelled() { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } + state.ask.flow() })?) } /// The testable core of `project`: one façade call over the fake vault (projection /// is model-free by construction — it never touches the embedding space). fn project_impl(state: &AppState) -> Result { - let vault = open_read(state)?; - Ok(vault.project(false)?) + read_op(state, |v| v.project(false)) } -/// The testable core of `embed`, split from the Tauri `State` wrapper. Guards -/// single-in-flight, arms the cancel flag, opens the real-model vault, and streams -/// progress while consulting the cancel flag at each batch. +/// The testable core of `embed`, split from the Tauri `State` wrapper. Claims the +/// reindex slot, opens the real-model vault, and streams progress while consulting the +/// slot's cancel at each batch. fn embed_impl( state: &AppState, on_event: &Channel, ) -> Result { // Single-in-flight: refuse a second embed rather than race two writers on one DB. // The UI also disables the button, so this is rarely hit. - if !state.try_start_reindex() { - return Err(CmdError::ReindexInFlight); - } - let _guard = ReindexGuard(state); - // Clear any stale cancel now that *this* run owns the slot (a prior switch/cancel - // must not kill a fresh embed). - state.arm_reindex(); - - // Fills missing vectors → needs the real model. `semantic` is false only under - // `B2_EMBEDDER=fake` (dev/offline): don't attribute fake-embed time to the real model. - let (vault, semantic) = open_vault(state, true)?; - // Attribute the time to whatever model this vault embeds with (config.toml / default). - let model = EmbedConfig::load() - .map(|c| c.model) - .unwrap_or_else(|_| b2_embed::DEFAULT_MODEL.to_string()); + let _held = state.reindex.try_claim().ok_or(CmdError::ReindexInFlight)?; + + // Fills missing vectors → needs the real model. `model` is the id of the model this + // open actually loaded, and `None` only under `B2_EMBEDDER=fake` (dev/offline) — + // fake-embed time is never attributed to a real model. + let (vault, model) = open_vault(state, true)?; // Time the embed pass itself — the clock starts *after* the model load above, so the // recorded total is embedding throughput, not one-time setup. `chunks_done` is // cumulative, so its last value is this run's chunk count. @@ -869,15 +831,11 @@ fn embed_impl( // Forward progress to the webview; a send error (the window navigated/closed) // is not fatal to the index — keep embedding. let _ = on_event.send(p); - if state.reindex_cancelled() { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } + state.reindex.flow() })?; // Record the run's cost (best-effort). Skip when nothing embedded (an up-to-date // vault) or under the fake embedder, so the ledger stays clean and correctly attributed. - if semantic && chunks_this_run > 0 { + if let Some(model) = model.filter(|_| chunks_this_run > 0) { crate::stats::record(&model, start.elapsed().as_millis() as u64, chunks_this_run); } Ok(report) @@ -892,8 +850,7 @@ fn vault_info_impl(state: &AppState) -> Result { // `semantic` is a file *probe* (`semantic_available`), not a model load: it stays // "is a model installed", while `notes_embedded/total` is the precise fraction the // UI flags keyword-only from. - let vault = open_read(state)?; - let status = vault.embed_status()?; + let status = read_op(state, |v| v.embed_status())?; Ok(VaultInfo { root: root.display().to_string(), semantic: crate::semantic_available(), @@ -907,29 +864,25 @@ fn vault_info_impl(state: &AppState) -> Result { /// first**, waiting for it to wind down before repointing the root, so a reindex can never /// keep writing the vault the app has left. fn set_vault_root_impl(state: &AppState, root: &Path) -> Result { - state.cancel_and_wait_for_reindex(); + state.reindex.cancel_and_wait(); state.set_root(root); vault_info_impl(state) } fn read_note_impl(state: &AppState, note: &str) -> Result { - let vault = open_read(state)?; - Ok(vault.read(note)?) + read_op(state, |v| v.read(note)) } fn list_notes_impl(state: &AppState) -> Result, CmdError> { - let vault = open_read(state)?; - Ok(vault.list_notes()?) + read_op(state, |v| v.list_notes()) } fn list_dirs_impl(state: &AppState) -> Result, CmdError> { - let vault = open_read(state)?; - Ok(vault.list_dirs()?) + read_op(state, |v| v.list_dirs()) } fn create_dir_impl(state: &AppState, dir: &str) -> Result { - let vault = open_read(state)?; - Ok(vault.create_dir(dir)?) + read_op(state, |v| v.create_dir(dir)) } fn write_note_impl( @@ -938,8 +891,7 @@ fn write_note_impl( body: &str, base_revision: &str, ) -> Result { - let vault = open_read(state)?; - Ok(vault.write(note, body, base_revision)?) + read_op(state, |v| v.write(note, body, base_revision)) } fn write_frontmatter_impl( @@ -948,13 +900,13 @@ fn write_frontmatter_impl( frontmatter: &str, base_revision: &str, ) -> Result { - let vault = open_read(state)?; - Ok(vault.write_frontmatter(note, frontmatter, base_revision)?) + read_op(state, |v| { + v.write_frontmatter(note, frontmatter, base_revision) + }) } fn create_note_impl(state: &AppState, path: &str) -> Result { - let vault = open_read(state)?; - Ok(vault.create_note(path)?) + read_op(state, |v| v.create_note(path)) } fn import_file_impl( @@ -966,18 +918,15 @@ fn import_file_impl( let bytes = BASE64 .decode(data) .map_err(|e| CmdError::ImportPayload(e.to_string()))?; - let vault = open_read(state)?; - Ok(vault.import_file(dir, name, &bytes)?) + read_op(state, |v| v.import_file(dir, name, &bytes)) } fn import_path_impl(state: &AppState, dir: &str, source: &str) -> Result { - let vault = open_read(state)?; - Ok(vault.import_path(dir, Path::new(source))?) + read_op(state, |v| v.import_path(dir, Path::new(source))) } fn move_note_impl(state: &AppState, note: &str, to: &str) -> Result { - let vault = open_semantic(state)?; - Ok(vault.move_note(note, to)?) + semantic_op(state, |v| v.move_note(note, to)) } fn move_resource_impl( @@ -985,28 +934,23 @@ fn move_resource_impl( path: &str, to: &str, ) -> Result { - let vault = open_semantic(state)?; - Ok(vault.move_resource(path, to)?) + semantic_op(state, |v| v.move_resource(path, to)) } fn move_dir_impl(state: &AppState, from: &str, to: &str) -> Result { - let vault = open_semantic(state)?; - Ok(vault.move_dir(from, to)?) + semantic_op(state, |v| v.move_dir(from, to)) } fn delete_note_impl(state: &AppState, note: &str) -> Result { - let vault = open_read(state)?; - Ok(vault.delete_note(note)?) + read_op(state, |v| v.delete_note(note)) } fn delete_resource_impl(state: &AppState, path: &str) -> Result { - let vault = open_read(state)?; - Ok(vault.delete_resource(path)?) + read_op(state, |v| v.delete_resource(path)) } fn delete_dir_impl(state: &AppState, dir: &str) -> Result { - let vault = open_read(state)?; - Ok(vault.delete_dir(dir)?) + read_op(state, |v| v.delete_dir(dir)) } /// The testable core of `set_model`. `EmbedConfig::set_model` validates the id against the @@ -1227,11 +1171,6 @@ mod tests { ); } - #[test] - fn ping_round_trips() { - assert_eq!(ping(), "pong"); - } - #[test] fn list_models_returns_the_registry() { // Global config, no vault needed. Deterministic w.r.t. ambient config only in the @@ -1281,26 +1220,13 @@ mod tests { // switch-cancels-first, all model-free (no reindex actually runs — the core's own // suite covers the cancel *behavior*; here we prove the host's control bits). - #[test] - fn reindex_slot_is_single_in_flight() { - let state = AppState::new(None); - assert!(state.try_start_reindex(), "first claim wins the slot"); - assert!(state.reindex_in_flight()); - assert!( - !state.try_start_reindex(), - "second claim is refused while running" - ); - state.finish_reindex(); - assert!(!state.reindex_in_flight()); - assert!(state.try_start_reindex(), "slot is reusable once released"); - } - #[test] fn a_second_embed_is_refused_before_touching_the_model() { // With the slot already held, `embed_impl` must refuse *before* opening the // real-model vault — so this needs no model and can't hang on one. let state = AppState::new(None); - assert!(state.try_start_reindex()); // stand in for a running embed + let running = state.reindex.try_claim(); // stand in for a running embed + assert!(running.is_some()); let channel = Channel::::new(|_| Ok(())); let err = embed_impl(&state, &channel).unwrap_err(); assert!(matches!(err, CmdError::ReindexInFlight)); @@ -1370,7 +1296,7 @@ mod tests { assert_eq!(report.path, "resources/dropped.png"); assert!(!report.note); // a non-`.md` file is routed as a resource assert_eq!(fs::read(root.join("resources/dropped.png")).unwrap(), bytes); - let listed = open_read(&state).unwrap().list_resources().unwrap(); + let listed = read_op(&state, |v| v.list_resources()).unwrap(); assert!(listed.iter().any(|r| r.path == "resources/dropped.png")); } @@ -1644,7 +1570,8 @@ mod tests { golden_indexed(&root); let state = AppState::new(Some(root)); - assert!(state.try_start_reindex()); // stand in for an in-flight embed + let running = state.reindex.try_claim(); // stand in for an in-flight embed + assert!(running.is_some()); let note = read_note_impl(&state, "concepts/memory").unwrap(); write_note_impl( &state, @@ -1671,7 +1598,8 @@ mod tests { // Hold the slot (a stand-in for an in-flight embed): `project` is deliberately // unguarded (index-engine.md) and must still run. - assert!(state.try_start_reindex()); + let running = state.reindex.try_claim(); + assert!(running.is_some()); let report = project_impl(&state).unwrap(); assert_eq!(report.indexed, 2); @@ -1744,19 +1672,6 @@ mod tests { ); } - #[test] - fn arm_clears_a_stale_cancel_but_request_sets_it() { - let state = AppState::new(None); - state.request_reindex_cancel(); - assert!(state.reindex_cancelled()); - // A fresh run arming the slot clears a cancel left by a prior switch/cancel… - state.arm_reindex(); - assert!(!state.reindex_cancelled()); - // …and a new cancel request is then observable again. - state.request_reindex_cancel(); - assert!(state.reindex_cancelled()); - } - #[test] fn vault_switch_cancels_and_waits_for_the_inflight_reindex() { let tmp = tempfile::TempDir::new().unwrap(); @@ -1765,17 +1680,18 @@ mod tests { let state = AppState::new(Some(root.clone())); // Simulate a reindex holding the slot. - assert!(state.try_start_reindex()); - assert!(state.reindex_in_flight()); + let held = state.reindex.try_claim().expect("the slot is free"); + assert!(state.reindex.in_flight()); std::thread::scope(|s| { // A stand-in reindex worker: spin until asked to cancel, then wind down — - // exactly what the real embed-loop closure does at a batch boundary. + // exactly what the real embed-loop closure does at a batch boundary, where + // returning drops the guard. s.spawn(|| { - while !state.reindex_cancelled() { + while !state.reindex.cancelled() { std::thread::sleep(std::time::Duration::from_millis(1)); } - state.finish_reindex(); + drop(held); }); // Switching vaults must request cancel AND block until the worker released @@ -1783,7 +1699,7 @@ mod tests { let info = set_vault_root_impl(&state, &root).unwrap(); assert_eq!(info.root, root.display().to_string()); // If the switch returned, the in-flight run has already wound down. - assert!(!state.reindex_in_flight()); + assert!(!state.reindex.in_flight()); }); } @@ -1824,7 +1740,7 @@ mod tests { "the fake cites every passage it was handed: {answer:?}" ); // The slot is released on the way out, so the next turn can claim it. - assert!(state.try_start_ask()); + assert!(state.ask.try_claim().is_some()); } #[test] @@ -1837,7 +1753,7 @@ mod tests { // from another thread; setting it inside the sink is that race, made deterministic. let answer = ask_impl(&state, &vault, &FakeLlm, "memory", &[], &|t| { streamed.borrow_mut().push(t.to_string()); - state.request_ask_cancel(); + state.ask.cancel(); }) .unwrap(); @@ -1856,7 +1772,8 @@ mod tests { let (state, vault) = ask_state(&tmp); // Stand in for an answer already streaming (the real one holds the slot for the // duration of its call). - assert!(state.try_start_ask()); + let streaming = state.ask.try_claim(); + assert!(streaming.is_some()); let err = ask_impl(&state, &vault, &FakeLlm, "memory", &[], &|_| {}).unwrap_err(); assert!(matches!(err, CmdError::AskInFlight)); @@ -1890,27 +1807,28 @@ mod tests { streamed.borrow_mut().clear(); let stopped = why_similar_impl(&state, &vault, &FakeLlm, anchor, candidate, 10, &|t| { streamed.borrow_mut().push(t.to_string()); - state.request_ask_cancel(); + state.ask.cancel(); }) .unwrap(); assert!(stopped.cancelled); assert_eq!(streamed.borrow().len(), 1); // One answer slot for both kinds of turn. - assert!(state.try_start_ask()); + let streaming = state.ask.try_claim(); + assert!(streaming.is_some()); let err = why_similar_impl(&state, &vault, &FakeLlm, anchor, candidate, 10, &|_| {}).unwrap_err(); assert!(matches!(err, CmdError::AskInFlight)); } - /// A stale cancel must not kill the *next* answer: `arm_ask` clears it once the fresh - /// turn owns the slot, which is the whole reason the flag is armed rather than reset - /// by whoever set it. + /// A stale cancel must not kill the *next* answer: claiming the slot clears it once the + /// fresh turn owns it, which is the whole reason the flag is cleared on claim rather + /// than reset by whoever set it. #[test] fn a_stale_cancel_does_not_stop_the_next_answer() { let tmp = tempfile::TempDir::new().unwrap(); let (state, vault) = ask_state(&tmp); - state.request_ask_cancel(); // …from a turn that already ended + state.ask.cancel(); // …from a turn that already ended let answer = ask_impl(&state, &vault, &FakeLlm, "memory", &[], &|_| {}).unwrap(); assert!(!answer.cancelled); } diff --git a/crates/b2-desktop/src/error.rs b/crates/b2-desktop/src/error.rs index 9397ffa..0dbcf48 100644 --- a/crates/b2-desktop/src/error.rs +++ b/crates/b2-desktop/src/error.rs @@ -5,7 +5,7 @@ //! `Result` hands the webview a safe message and never a sqlite/io/serde //! internal. `B2_DEBUG` opts into the raw detail, exactly as the CLI does. -use b2_embed::EmbedError; +use b2_embed::{EmbedConfig, EmbedError}; use serde::{Serialize, Serializer}; /// The host's error, composing the crates it drives. Kept internal; it is only ever @@ -93,8 +93,20 @@ pub fn user_message(err: &CmdError) -> String { "The embedding model failed to load. Try downloading it again, or pick a different model in Settings." .to_string() } + // `config.toml` didn't parse, or its `source` names a folder missing a model file: + // the fix is in that one file, so name it. + CmdError::Embed(EmbedError::Config(_)) => format!( + "B2 couldn't use its embedder settings. Check the [embedder] table in {}, then try again.", + EmbedConfig::config_path() + .map_or_else(|| "config.toml".to_string(), |p| p.display().to_string()) + ), CmdError::Core(b2_core::Error::InvalidRelation(v)) => format!( - "'{v}' isn't a known relation type. Use one of: references, supports, contradicts." + "'{v}' isn't a known relation type. Use one of: {}.", + b2_core::relation::CORE + .iter() + .map(|c| c.verb) + .collect::>() + .join(", ") ), CmdError::Core(b2_core::Error::MoveTargetExists(p)) => format!( "Can't move: something already exists at '{p}'. Choose a different name or destination." @@ -198,7 +210,7 @@ pub fn user_message(err: &CmdError) -> String { // Everything else in the two composed crates is an internal (sqlite/io/serde/…) // the webview must never see. Spelled out rather than `_` so adding a CmdError // variant fails to compile here instead of silently degrading to the catch-all. - CmdError::Core(_) | CmdError::Embed(_) => { + CmdError::Core(_) | CmdError::Embed(EmbedError::Io(_)) => { "Something went wrong. Please check the vault and try again.".to_string() } }; @@ -270,6 +282,31 @@ mod tests { assert!(!json.to_lowercase().contains("utf-8"), "no internal leaks"); } + /// The verb list is `b2_core::relation::CORE`'s, so a verb added there is offered here. + #[test] + fn an_invalid_relation_lists_every_core_verb() { + let msg = user_message(&CmdError::Core(b2_core::Error::InvalidRelation( + "refutes".into(), + ))); + for verb in b2_core::relation::CORE { + assert!(msg.contains(verb.verb), "{msg}"); + } + } + + /// A config that won't parse used to read as "check the vault" — a vault it never + /// involved. It names the settings instead, and keeps the parser's detail internal. + #[test] + fn a_broken_embedder_config_names_the_settings_not_the_vault() { + let msg = user_message(&CmdError::Embed(EmbedError::Config( + "config.toml: expected `=`".into(), + ))); + assert!(msg.contains("[embedder]"), "{msg}"); + assert!( + !msg.contains("check the vault") && !msg.contains("expected"), + "{msg}" + ); + } + /// The move family maps to specific, actionable messages (not the catch-all), /// mirroring the CLI's `user_message` arms. #[test] diff --git a/crates/b2-desktop/src/logging.rs b/crates/b2-desktop/src/logging.rs index a301cf5..4ee4a09 100644 --- a/crates/b2-desktop/src/logging.rs +++ b/crates/b2-desktop/src/logging.rs @@ -36,9 +36,9 @@ pub fn init_logging() -> Option { let log_file = std::env::var_os("B2_LOG_FILE"); let directive = match std::env::var("B2_LOG") { Ok(v) if !v.trim().is_empty() => v, - // Implied default scoped to the kernel's targets (`b2::sqlite`/`vault`/`ingest`/ - // `search`) — not the CLI's bare `debug` — so Tauri/wry/hyper tracing doesn't - // pollute the file. See the module doc for why this one knob diverges. + // Implied default scoped to B2's own targets (`b2::sqlite`/`vault`/`ingest`/ + // `search`/`llm`/`embed`) — the same scoping the CLI uses — so Tauri/wry/hyper + // tracing doesn't pollute the file (see the module doc). _ if std::env::var_os("B2_DEBUG").is_some() || log_file.is_some() => "b2=debug".to_string(), _ => return None, }; diff --git a/crates/b2-desktop/src/main.rs b/crates/b2-desktop/src/main.rs index f306bdf..9fcb9e2 100644 --- a/crates/b2-desktop/src/main.rs +++ b/crates/b2-desktop/src/main.rs @@ -9,8 +9,8 @@ //! remembers the pick. Every command opens a *fresh* vault from the current root, //! exactly as the one-process-per-command CLI does. //! * **Embedder wiring** — pure reads open with the deterministic fake; anything that -//! embeds a query or writes vectors opens the real [`LocalEmbedder`] and fails fast -//! with "run `b2 init`" if absent. `project` opens the fake, so the first tree paint +//! embeds a query or writes vectors opens the real model (`b2_embed::embedder_for`, +//! the rule the CLI uses too) and fails fast with "run `b2 init`" if absent. `project` opens the fake, so the first tree paint //! never waits on a model load. `B2_EMBEDDER=fake` forces the fake everywhere. //! //! And one it hands off: the **menu bar** is declared in [`menu`] rather than inherited, so @@ -25,25 +25,23 @@ mod error; mod keychain; mod logging; mod menu; +mod slot; +mod state_file; mod stats; mod watch; -use b2_core::embed::Embedder; use b2_core::vault::Vault; -use b2_embed::{EmbedConfig, LocalEmbedder}; +use b2_embed::EmbedConfig; use chat::ChatPrefs; use error::CmdError; +use slot::Slot; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Mutex; -use std::time::Duration; use tauri::{Emitter, Manager}; use watch::VaultWatcher; -/// How often [`AppState::cancel_and_wait_for_reindex`] re-asserts the cancel flag and -/// re-checks whether the in-flight reindex has wound down. Short enough to feel -/// instant on a vault switch, long enough not to spin hot. -const CANCEL_POLL: Duration = Duration::from_millis(25); +/// The state file remembering the last opened vault (see [`state_file`]). +const LAST_VAULT_FILE: &str = "last-vault"; /// The host's shared state: the active vault root plus the background-reindex and chat /// control bits. The root is resolved once at startup and swappable at runtime by the @@ -51,27 +49,20 @@ const CANCEL_POLL: Duration = Duration::from_millis(25); /// [`Vault`] over the *current* root, the faithful mirror of the CLI opening a fresh vault /// per invocation. `None` means no vault is configured. /// -/// The reindex and chat bits are host **infrastructure**, not engine logic: *how the window -/// drives and interrupts* a long façade op stays here, *what* it computes stays in the -/// core. `reindex_running` is a single-in-flight guard for the vector-writing embed pass -/// (the model-free `project` runs outside it by design), and a running embed checks -/// `reindex_cancel` at each batch boundary. `ask_cancel` is read by the token callback at -/// every token — the seam's own cancellation checkpoint — and `ask_running` keeps one -/// turn's stale cancel from killing the next. +/// The reindex and chat slots are host **infrastructure**, not engine logic: *how the +/// window drives and interrupts* a long façade op stays here, *what* it computes stays in +/// the core. Each is a [`Slot`]: single-in-flight, released on every exit path, cancelled +/// cooperatively from another thread. pub struct AppState { root: Mutex>, - /// Set by `cancel_reindex` (and a vault switch); the running reindex closure - /// observes it at each batch boundary and returns `ControlFlow::Break`. - reindex_cancel: AtomicBool, - /// `true` while a reindex is in flight — the single-in-flight guard (a second - /// `reindex` is refused; see [`AppState::try_start_reindex`]). - reindex_running: AtomicBool, - /// Set by `cancel_ask` (the chat pane's Esc); the running answer's token - /// callback observes it and returns `ControlFlow::Break`, which the seam - /// reports honestly as a cancelled — not failed — completion. - ask_cancel: AtomicBool, - /// `true` while an answer is streaming — the single-in-flight guard. - ask_running: AtomicBool, + /// The vector-writing embed pass (the model-free `project` runs outside it by + /// design). Cancelled by `cancel_reindex` and by a vault switch; a running embed + /// reads it at each batch boundary. + pub reindex: Slot, + /// The streaming answer (`ask` and `why_similar` share it). Cancelled by + /// `cancel_ask` (the chat pane's Esc); the token callback reads it at every token, + /// and the seam reports a stop honestly as a cancelled — not failed — completion. + pub ask: Slot, /// The chat endpoint, model, and the key in force. Behind a `Mutex` for the /// vault root's reason: Settings changes it at runtime and every later ask /// resolves a fresh provider from it. **Never vault or index state** @@ -98,10 +89,8 @@ impl AppState { pub fn with_chat(root: Option, chat: ChatPrefs) -> Self { Self { root: Mutex::new(root), - reindex_cancel: AtomicBool::new(false), - reindex_running: AtomicBool::new(false), - ask_cancel: AtomicBool::new(false), - ask_running: AtomicBool::new(false), + reindex: Slot::default(), + ask: Slot::default(), chat: Mutex::new(chat), chat_saving: Mutex::new(()), } @@ -110,7 +99,7 @@ impl AppState { /// Claim the Settings save for its whole read-modify-write — see /// [`chat_saving`](Self::chat_saving). Blocks rather than refusing: a save is /// short and a user who pressed Save twice meant both, so the second must - /// *follow* the first rather than be dropped (contrast `try_start_ask`, + /// *follow* the first rather than be dropped (contrast the `ask` slot, /// where two answers at once is a genuine conflict). pub fn begin_chat_save(&self) -> std::sync::MutexGuard<'_, ()> { lock_recover(&self.chat_saving) @@ -128,37 +117,6 @@ impl AppState { *lock_recover(&self.chat) = prefs; } - /// Claim the single answer slot — [`try_start_reindex`](Self::try_start_reindex) - /// for chat. `true` if this call won it (release via [`AskGuard`]). - pub fn try_start_ask(&self) -> bool { - self.ask_running - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - } - - /// Release the answer slot (always, even on error — see [`AskGuard`]). - pub fn finish_ask(&self) { - self.ask_running.store(false, Ordering::SeqCst); - } - - /// Clear the cancel flag now that a fresh answer owns the slot, so the Esc - /// that stopped the *last* answer can't stop this one before its first token. - pub fn arm_ask(&self) { - self.ask_cancel.store(false, Ordering::SeqCst); - } - - /// Whether the streaming answer has been asked to stop (read at every token). - pub fn ask_cancelled(&self) -> bool { - self.ask_cancel.load(Ordering::SeqCst) - } - - /// Signal the streaming answer to stop at its next token (the `cancel_ask` - /// command — the pane's Esc). Cooperative, like the reindex's: the partial - /// text stands and is rendered honestly. A no-op if nothing is streaming. - pub fn request_ask_cancel(&self) { - self.ask_cancel.store(true, Ordering::SeqCst); - } - /// The current vault root, cloned out so the lock is **not** held while a command /// opens a vault (which may load the model — slow). `None` when unconfigured. pub fn current_root(&self) -> Option { @@ -171,60 +129,6 @@ impl AppState { *self.lock_root() = Some(root.to_path_buf()); } - /// Claim the single reindex slot. Returns `true` if this call won the slot (the - /// caller must release it via [`finish_reindex`](Self::finish_reindex)), `false` - /// if a reindex is already in flight — the belt-and-suspenders half of the - /// single-in-flight guard (the UI also disables the button). - pub fn try_start_reindex(&self) -> bool { - self.reindex_running - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - } - - /// Release the reindex slot (always, even on error — see [`ReindexGuard`]). - /// Idempotent. - pub fn finish_reindex(&self) { - self.reindex_running.store(false, Ordering::SeqCst); - } - - /// Whether a reindex is currently in flight. - pub fn reindex_in_flight(&self) -> bool { - self.reindex_running.load(Ordering::SeqCst) - } - - /// Clear the cancel flag — called once a fresh reindex has claimed the slot, so a - /// stale cancel from a previous run (or a prior vault switch) can't kill it. - pub fn arm_reindex(&self) { - self.reindex_cancel.store(false, Ordering::SeqCst); - } - - /// Whether the running reindex has been asked to stop (checked at each batch). - pub fn reindex_cancelled(&self) -> bool { - self.reindex_cancel.load(Ordering::SeqCst) - } - - /// Signal the running reindex to stop at its next batch boundary (the - /// `cancel_reindex` command). Cooperative — never a thread kill, so no torn writes. - /// A no-op if nothing is running. - pub fn request_reindex_cancel(&self) { - self.reindex_cancel.store(true, Ordering::SeqCst); - } - - /// Cancel any in-flight reindex and **block until it winds down** — used before a - /// vault switch so a reindex can never keep writing the vault the app has left. - /// Re-asserts the cancel flag on every poll so it wins - /// even against a reindex that armed (cleared) it a moment after starting; returns - /// immediately when nothing is running. - pub fn cancel_and_wait_for_reindex(&self) { - loop { - self.request_reindex_cancel(); - if !self.reindex_in_flight() { - return; - } - std::thread::sleep(CANCEL_POLL); - } - } - /// The critical sections here are a single clone or store — neither can panic — /// so the lock can never be poisoned; see [`lock_recover`]. fn lock_root(&self) -> std::sync::MutexGuard<'_, Option> { @@ -232,24 +136,6 @@ impl AppState { } } -/// Releases the single-in-flight reindex slot on drop, so it is freed on **every** -/// exit path — normal return, an early `?` (e.g. model-not-provisioned), or a panic. -pub(crate) struct ReindexGuard<'a>(pub(crate) &'a AppState); -impl Drop for ReindexGuard<'_> { - fn drop(&mut self) { - self.0.finish_reindex(); - } -} - -/// [`ReindexGuard`] for the chat seam: releases the single answer slot on every -/// exit path — a finished stream, an early `?` (no vault, no model), a panic. -pub(crate) struct AskGuard<'a>(pub(crate) &'a AppState); -impl Drop for AskGuard<'_> { - fn drop(&mut self) { - self.0.finish_ask(); - } -} - /// Lock a mutex, recovering the inner value rather than unwrapping if the lock is ever /// poisoned (the no-panic rule). Every mutex in this host guards a critical section of /// a single clone/store/drop — none can panic, so poisoning is effectively impossible, @@ -258,38 +144,49 @@ pub(crate) fn lock_recover(m: &Mutex) -> std::sync::MutexGuard<'_, T> { m.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// Whether the deterministic fake embedder is forced (`B2_EMBEDDER=fake`) — the CLI's -/// offline/dev switch, honored identically so the two adapters behave the same. -fn use_fake_embedder() -> bool { - matches!(std::env::var("B2_EMBEDDER").ok().as_deref(), Some("fake")) -} - /// Open a fresh vault over the configured root with the right embedder — the desktop -/// mirror of the CLI's `open_vault`. `needs_semantic` commands (`search`/`link`/ -/// `embed`) load the real [`LocalEmbedder`] (fail-fast "run `b2 init`" if absent); -/// pure reads — and `project`, which never touches vectors — use the fake. Returns -/// the vault and whether its embedder is semantic (used only for honest UI). Errors -/// with [`CmdError::VaultRequired`] if no vault is set. -pub fn open_vault(state: &AppState, needs_semantic: bool) -> Result<(Vault, bool), CmdError> { +/// mirror of the CLI's `open_vault`, over the same `b2_embed::embedder_for` rule. +/// `needs_semantic` commands (`search`/`link`/`embed`) load the real model (fail-fast +/// "run `b2 init`" if absent); pure reads — and `project`, which never touches vectors — +/// use the fake. Errors with [`CmdError::VaultRequired`] if no vault is set. +/// +/// Returns the vault and, when the real model was loaded, its configured id — which is +/// what `embed` attributes its time to, read off the model actually loaded rather than a +/// second look at a config that may have changed since. +pub fn open_vault( + state: &AppState, + needs_semantic: bool, +) -> Result<(Vault, Option), CmdError> { let root = state.current_root().ok_or(CmdError::VaultRequired)?; - if needs_semantic && !use_fake_embedder() { - let config = EmbedConfig::load()?; - let embedder = LocalEmbedder::load(&config)?; - let vault = Vault::open_with_embedder(&root, Box::new(embedder) as Box)?; - Ok((vault, true)) - } else { - Ok((Vault::open(&root)?, false)) - } + open_vault_at(&root, needs_semantic) +} + +/// [`open_vault`] over an explicit root — for a command that needs the root itself as +/// well, so the two can't come from either side of a vault switch. +pub fn open_vault_at( + root: &Path, + needs_semantic: bool, +) -> Result<(Vault, Option), CmdError> { + Ok(match b2_embed::embedder_for(needs_semantic)? { + Some(embedder) => { + let model = embedder.configured_model().to_string(); + ( + Vault::open_with_embedder(root, Box::new(embedder))?, + Some(model), + ) + } + None => (Vault::open(root)?, None), + }) } /// Read-path open: a fresh vault over the fake embedder (no model load), for commands -/// that never embed. [`open_vault`] with the semantic flag discarded. +/// that never embed. [`open_vault`] with the model id discarded. pub fn open_read(state: &AppState) -> Result { Ok(open_vault(state, false)?.0) } /// Wants-the-real-model open: a fresh vault over the real embedder (fail-fast "run -/// `b2 init`" if absent), for commands that embed. [`open_vault`] with the flag discarded. +/// `b2 init`" if absent), for commands that embed. [`open_vault`] with the id discarded. pub fn open_semantic(state: &AppState) -> Result { Ok(open_vault(state, true)?.0) } @@ -305,7 +202,7 @@ pub fn open_semantic(state: &AppState) -> Result { /// model reads as `semantic: true` and fails at the first `search`/`reindex` instead, which /// is already fail-fast and actionable. pub fn semantic_available() -> bool { - if use_fake_embedder() { + if b2_embed::fake_requested() { return false; } EmbedConfig::load().is_ok_and(|c| c.is_model_provisioned(&c.model)) @@ -339,20 +236,12 @@ fn pick_root( arg.or(remembered).or(env) } -/// Path of the "last opened vault" state file: `/b2/last-vault` (macOS: -/// `~/Library/Application Support/b2/…`, Linux: `~/.local/share/b2/…`) — the same -/// `dirs`-resolved `b2/` vendor dir b2-embed uses for its model cache. `None` only if the -/// platform has no data dir, in which case remembering is silently skipped. -fn last_vault_file() -> Option { - dirs::data_dir().map(|d| d.join("b2").join("last-vault")) -} - /// The remembered vault root from the last picker choice, or `None` if there is none, /// the file is unreadable/empty, **or the remembered directory no longer exists** (moved /// or deleted). Falling through on a stale entry lets startup drop back to the env /// default rather than opening a vault whose every command would then error. fn read_last_vault() -> Option { - read_last_vault_from(&last_vault_file()?) + read_last_vault_from(&state_file::path(LAST_VAULT_FILE)?) } /// [`read_last_vault`] against an explicit file path — the testable core (a tempfile @@ -365,27 +254,20 @@ fn read_last_vault_from(file: &Path) -> Option { } /// Remember `root` as the last opened vault so the next launch reopens it. **Best-effort -/// host state**: a write failure (no data dir, unwritable disk) is logged to stderr and -/// swallowed — remembering must never fail the vault switch the user just made. Called -/// only from the `choose_vault` command wrapper (an explicit user pick), never from the -/// unit-tested state transition, so tests don't touch the real data dir. +/// host state** ([`state_file::update`]): remembering must never fail the vault switch the +/// user just made. Called only from the `choose_vault` command wrapper (an explicit user +/// pick), never from the unit-tested state transition, so tests don't touch the real data +/// dir. fn persist_last_vault(root: &Path) { - let Some(file) = last_vault_file() else { - eprintln!("[b2] could not remember last vault: no platform data directory"); - return; - }; - if let Err(e) = persist_last_vault_to(&file, root) { - eprintln!("[b2] could not remember last vault: {e}"); - } + state_file::update(LAST_VAULT_FILE, "remember the last vault", |file| { + persist_last_vault_to(file, root) + }); } /// [`persist_last_vault`] against an explicit file path — the testable core. Creates the /// parent dir if needed and writes the path as the file's sole line. fn persist_last_vault_to(file: &Path, root: &Path) -> std::io::Result<()> { - if let Some(parent) = file.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(file, root.to_string_lossy().as_bytes()) + state_file::write(file, root.to_string_lossy().as_bytes()) } fn main() { @@ -400,7 +282,7 @@ fn main() { // with no cloud model configured has no Keychain item, so the common launch asks for // nothing and prompts for nothing. let state = AppState::with_chat(resolve_root(), chat::read_prefs(&keychain::Keychain)); - tauri::Builder::default() + let app = tauri::Builder::default() // The menu bar, declared (#119). Without this call Tauri installs // `Menu::default()`, whose dozen accelerators nothing in the app can enumerate // — and AppKit dispatches them before the webview sees a key, so the keyboard @@ -441,7 +323,6 @@ fn main() { Ok(()) }) .invoke_handler(tauri::generate_handler![ - commands::ping, commands::vault_info, commands::choose_vault, commands::read_note, @@ -469,7 +350,6 @@ fn main() { commands::similar, commands::explain_similar, commands::search, - commands::neighbors, commands::explain, commands::link, commands::project, @@ -489,8 +369,13 @@ fn main() { commands::chat_setup, commands::set_chat_config, ]) - .run(tauri::generate_context!()) - .expect("error while running the B2 desktop app"); + .run(tauri::generate_context!()); + // The one failure left to report is the app itself not starting (no window, no + // webview). Say so and exit non-zero — the no-panic rule, even here. + if let Err(e) = app { + eprintln!("[b2] the desktop app could not start: {e}"); + std::process::exit(1); + } } #[cfg(test)] diff --git a/crates/b2-desktop/src/slot.rs b/crates/b2-desktop/src/slot.rs new file mode 100644 index 0000000..9e17774 --- /dev/null +++ b/crates/b2-desktop/src/slot.rs @@ -0,0 +1,122 @@ +//! A single-in-flight slot for a long, cancellable façade op — host **infrastructure**, +//! not engine logic. The window drives two such ops, the background embed and a streaming +//! answer, and both need the same four things: refuse a second run while one is going, +//! release on every exit path, let another thread ask the run to stop, and keep a stale +//! stop request from killing the *next* run. + +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +/// How often [`Slot::cancel_and_wait`] re-asserts the cancel flag and re-checks whether +/// the run has wound down. Short enough to feel instant on a vault switch, long enough +/// not to spin hot. +const CANCEL_POLL: Duration = Duration::from_millis(25); + +/// One op's slot: whether a run holds it, and whether that run has been asked to stop. +#[derive(Debug, Default)] +pub struct Slot { + running: AtomicBool, + cancel: AtomicBool, +} + +/// Proof of holding a [`Slot`]; dropping it releases the slot, so it is freed on +/// **every** exit path — a normal return, an early `?` (no vault, no model), a panic. +#[derive(Debug)] +pub struct SlotGuard<'a>(&'a Slot); + +impl Drop for SlotGuard<'_> { + fn drop(&mut self) { + self.0.running.store(false, Ordering::SeqCst); + } +} + +impl Slot { + /// Claim the slot for a fresh run, or `None` when one is already in flight. Winning + /// also clears any stale cancel — the request that stopped the *previous* run (or a + /// vault switch) must not stop this one before it starts. + pub fn try_claim(&self) -> Option> { + self.running + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .ok()?; + self.cancel.store(false, Ordering::SeqCst); + Some(SlotGuard(self)) + } + + /// Whether a run holds the slot right now. + pub fn in_flight(&self) -> bool { + self.running.load(Ordering::SeqCst) + } + + /// Ask the running op to stop at its next checkpoint. Cooperative — never a thread + /// kill, so no torn writes. A no-op if nothing is running. + pub fn cancel(&self) { + self.cancel.store(true, Ordering::SeqCst); + } + + /// Whether the running op has been asked to stop. + pub fn cancelled(&self) -> bool { + self.cancel.load(Ordering::SeqCst) + } + + /// The cancel flag as the façade's loops read it: [`ControlFlow::Break`] once a stop + /// has been asked for, else [`ControlFlow::Continue`]. + pub fn flow(&self) -> ControlFlow<()> { + if self.cancelled() { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } + + /// Cancel any run in flight and **block until it winds down**. Re-asserts the cancel + /// on every poll, so it wins even against a run that claimed the slot (clearing the + /// flag) a moment after the first request; returns at once when nothing is running. + pub fn cancel_and_wait(&self) { + loop { + self.cancel(); + if !self.in_flight() { + return; + } + std::thread::sleep(CANCEL_POLL); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn one_claim_at_a_time_and_the_guard_releases_it() { + let slot = Slot::default(); + let held = slot.try_claim(); + assert!(held.is_some(), "the first claim wins the slot"); + assert!(slot.in_flight()); + assert!( + slot.try_claim().is_none(), + "a second claim is refused while running" + ); + drop(held); + assert!(!slot.in_flight()); + assert!( + slot.try_claim().is_some(), + "the slot is reusable once released" + ); + } + + #[test] + fn claiming_clears_a_stale_cancel_and_a_new_request_sets_it() { + let slot = Slot::default(); + slot.cancel(); + assert!(slot.cancelled()); + assert!(slot.flow().is_break()); + // A fresh run clears the cancel a prior run (or vault switch) left behind… + let _held = slot.try_claim(); + assert!(!slot.cancelled()); + assert!(slot.flow().is_continue()); + // …and a new request is observable again. + slot.cancel(); + assert!(slot.flow().is_break()); + } +} diff --git a/crates/b2-desktop/src/state_file.rs b/crates/b2-desktop/src/state_file.rs new file mode 100644 index 0000000..16a2e0e --- /dev/null +++ b/crates/b2-desktop/src/state_file.rs @@ -0,0 +1,34 @@ +//! The host's per-machine state files — the last opened vault, the chat settings, the +//! embed-time ledger. All live under `/b2/` (macOS: +//! `~/Library/Application Support/b2/`, Linux: `~/.local/share/b2/`), the same vendor dir +//! `b2-embed` keeps its model cache in, and all are **best-effort**: remembering something +//! must never fail the action the user just took. Never vault or index state. + +use std::path::{Path, PathBuf}; + +/// `/b2/`, or `None` when the platform has no data dir — in which case +/// the caller simply doesn't remember. +pub fn path(name: &str) -> Option { + dirs::data_dir().map(|d| d.join("b2").join(name)) +} + +/// Write `contents` to `file`, creating its parent dir first — the testable core every +/// state file's writer shares. +pub fn write(file: &Path, contents: &[u8]) -> std::io::Result<()> { + if let Some(parent) = file.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(file, contents) +} + +/// Run `update` against the state file `name`, best-effort: no data dir, or a failed +/// write, is logged to stderr (`could not {what}: …`) and swallowed. +pub fn update(name: &str, what: &str, update: impl FnOnce(&Path) -> std::io::Result<()>) { + let Some(file) = path(name) else { + eprintln!("[b2] could not {what}: no platform data directory"); + return; + }; + if let Err(e) = update(&file) { + eprintln!("[b2] could not {what}: {e}"); + } +} diff --git a/crates/b2-desktop/src/stats.rs b/crates/b2-desktop/src/stats.rs index 3ae1703..1af780a 100644 --- a/crates/b2-desktop/src/stats.rs +++ b/crates/b2-desktop/src/stats.rs @@ -9,9 +9,13 @@ //! corpus (ADR-0007) and the ledger must restart with the vectors. Purely diagnostic and //! best-effort — a read/write failure never fails an embed. +use crate::state_file; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; +use std::path::Path; + +/// The ledger's state file, `/b2/embed-stats.json` (see [`state_file`]). +const STATS_FILE: &str = "embed-stats.json"; /// One model's accumulated embedding cost. `total_ms / chunks` is the throughput the /// Settings pane shows; `runs` counts the embed passes that contributed. @@ -27,27 +31,34 @@ pub struct ModelStat { pub runs: u64, } +/// One model's row as the Settings pane reads it (`embed_stats`, `ui/src/types.ts`'s +/// `EmbedStat`): the id beside its [`ModelStat`] fields, flattened, so the IPC payload is +/// the ledger's own bucket rather than a hand-copied mirror of it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct EmbedStat { + pub model: String, + #[serde(flatten)] + pub stat: ModelStat, +} + /// The on-disk ledger: model id → its cumulative stat. #[derive(Debug, Clone, Default, Serialize, Deserialize)] struct StatsFile { models: BTreeMap, } -/// The ledger's path: `/b2/embed-stats.json` (the same vendor dir as -/// `last-vault` and the model cache). `None` when the platform has no data dir, in which -/// case recording is silently skipped. -fn stats_file() -> Option { - dirs::data_dir().map(|d| d.join("b2").join("embed-stats.json")) -} - -/// The whole ledger, one `(model_id, stat)` per model, or empty when there's no data dir -/// / no file / an unreadable-or-corrupt file — stats are never load-bearing, so a bad -/// file degrades cleanly to "no history" rather than surfacing an error. -pub fn read_all() -> Vec<(String, ModelStat)> { - let Some(path) = stats_file() else { +/// The whole ledger, one row per model, or empty when there's no data dir / no file / an +/// unreadable-or-corrupt file — stats are never load-bearing, so a bad file degrades +/// cleanly to "no history" rather than surfacing an error. +pub fn read_all() -> Vec { + let Some(path) = state_file::path(STATS_FILE) else { return Vec::new(); }; - read_from(&path).models.into_iter().collect() + read_from(&path) + .models + .into_iter() + .map(|(model, stat)| EmbedStat { model, stat }) + .collect() } /// [`read_all`] against an explicit path — the testable core. A missing or malformed file @@ -60,24 +71,20 @@ fn read_from(path: &Path) -> StatsFile { } /// Serialize the ledger and rewrite it at `path` — the shared write tail of -/// [`record_to`] and [`reset_in`]. Creating the parent dir stays the caller's job: -/// only [`record_to`] may create the file (a no-op reset must not). +/// [`record_to`] and [`reset_in`]. Only a write that has something to say reaches here +/// (a no-op reset returns before it), so creating the parent dir on the way is safe. fn write_ledger(path: &Path, file: &StatsFile) -> std::io::Result<()> { let text = serde_json::to_string_pretty(file).map_err(std::io::Error::other)?; - std::fs::write(path, text) + state_file::write(path, text.as_bytes()) } /// Add one embed run's `(elapsed_ms, chunks)` to `model`'s running total. Best-effort: /// a missing data dir or a write failure is logged to stderr and swallowed — recording a /// measurement must never fail the embed the user actually asked for. pub fn record(model: &str, elapsed_ms: u64, chunks: u64) { - let Some(path) = stats_file() else { - eprintln!("[b2] embed stats: no platform data directory; not recording"); - return; - }; - if let Err(e) = record_to(&path, model, elapsed_ms, chunks) { - eprintln!("[b2] embed stats: could not record ({e})"); - } + state_file::update(STATS_FILE, "record embed stats", |path| { + record_to(path, model, elapsed_ms, chunks) + }); } /// [`record`] against an explicit path — the testable core. Read-modify-write: load the @@ -89,9 +96,6 @@ fn record_to(path: &Path, model: &str, elapsed_ms: u64, chunks: u64) -> std::io: entry.total_ms = entry.total_ms.saturating_add(elapsed_ms); entry.chunks = entry.chunks.saturating_add(chunks); entry.runs = entry.runs.saturating_add(1); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } write_ledger(path, &file) } @@ -102,12 +106,9 @@ fn record_to(path: &Path, model: &str, elapsed_ms: u64, chunks: u64) -> std::io: /// the switched-to model is touched, so the others' history survives for comparison. /// Best-effort like [`record`]. pub fn reset(model: &str) { - let Some(path) = stats_file() else { - return; // no data dir ⇒ nothing was ever recorded ⇒ nothing to reset - }; - if let Err(e) = reset_in(&path, model) { - eprintln!("[b2] embed stats: could not reset ({e})"); - } + state_file::update(STATS_FILE, "reset embed stats", |path| { + reset_in(path, model) + }); } /// [`reset`] against an explicit path — the testable core. Drops `model`'s bucket and @@ -203,6 +204,24 @@ mod tests { ); } + /// The Settings pane reads these four keys (`ui/src/types.ts`'s `EmbedStat`), so the + /// flattened row must serialize exactly as the hand-written mirror it replaced. + #[test] + fn a_ledger_row_crosses_ipc_with_the_ui_field_names() { + let row = EmbedStat { + model: "m/base".into(), + stat: ModelStat { + total_ms: 3500, + chunks: 100, + runs: 2, + }, + }; + assert_eq!( + serde_json::to_value(&row).unwrap(), + serde_json::json!({"model": "m/base", "total_ms": 3500, "chunks": 100, "runs": 2}) + ); + } + #[test] fn missing_or_corrupt_file_reads_as_empty() { let tmp = tempfile::TempDir::new().unwrap(); From e9e1265552efd2c9bdc8b0b416a4de112a267e68 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:57:44 +0000 Subject: [PATCH 23/30] refactor(eval): split eval.rs into eval/ modules along its report blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eval.rs had grown to ~4,500 lines. It is now `examples/eval/main.rs` (still the `eval` example, so `cargo run -p b2-embed --example eval` and every Makefile target are unchanged) plus one module per block of the report: labels (sets + lint), metrics, retrieval, discovery (per-mate, strangers, z dump), fold (GH #200 bake-off), evidence (D2 calibration + bake-off), tail (GH #206 bake-off + cross-bench join), dense (the single-domain fixture), ablation (--stemmer / --sweep), report (default table + JSONL row), instrument (batch ≡ single, timed embed, pool-blind warning) and gate (the exit-gate constants and assertions). main.rs is the run's order of operations and nothing else. Moved verbatim apart from visibility, imports and doc links, with three shape changes: - `result_row`'s 18 positional parameters become `RunId` (who scored the run; also taken by `dense_row`) plus a `RowInputs` view whose four default-row-only readings travel together as `Calibration`. Every row key and value is unchanged; the ablation rows' calibration keys still read null. - The stemmer and sweep blocks become `ablation::stemmer`/`ablation::sweep` over a `Baseline` view of the default passes; the stemmer's `.expect` on the unstemmed BM25 pass is gone (the pass is handed over by `if let`). - The exit gate becomes `gate::passes`, returning `false` where `run` returned `Ok(false)`. Also reattaches `QueryEvidence`'s doc comment, which had drifted onto `ServedRow`, and points docs/evals.md's exit-gate link at eval/gate.rs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-embed/examples/eval.rs | 4425 ------------------- crates/b2-embed/examples/eval/ablation.rs | 239 + crates/b2-embed/examples/eval/dense.rs | 396 ++ crates/b2-embed/examples/eval/discovery.rs | 450 ++ crates/b2-embed/examples/eval/evidence.rs | 667 +++ crates/b2-embed/examples/eval/fold.rs | 715 +++ crates/b2-embed/examples/eval/gate.rs | 207 + crates/b2-embed/examples/eval/instrument.rs | 98 + crates/b2-embed/examples/eval/labels.rs | 181 + crates/b2-embed/examples/eval/main.rs | 318 ++ crates/b2-embed/examples/eval/metrics.rs | 88 + crates/b2-embed/examples/eval/report.rs | 575 +++ crates/b2-embed/examples/eval/retrieval.rs | 89 + crates/b2-embed/examples/eval/tail.rs | 668 +++ docs/evals.md | 4 +- 15 files changed, 4693 insertions(+), 4427 deletions(-) delete mode 100644 crates/b2-embed/examples/eval.rs create mode 100644 crates/b2-embed/examples/eval/ablation.rs create mode 100644 crates/b2-embed/examples/eval/dense.rs create mode 100644 crates/b2-embed/examples/eval/discovery.rs create mode 100644 crates/b2-embed/examples/eval/evidence.rs create mode 100644 crates/b2-embed/examples/eval/fold.rs create mode 100644 crates/b2-embed/examples/eval/gate.rs create mode 100644 crates/b2-embed/examples/eval/instrument.rs create mode 100644 crates/b2-embed/examples/eval/labels.rs create mode 100644 crates/b2-embed/examples/eval/main.rs create mode 100644 crates/b2-embed/examples/eval/metrics.rs create mode 100644 crates/b2-embed/examples/eval/report.rs create mode 100644 crates/b2-embed/examples/eval/retrieval.rs create mode 100644 crates/b2-embed/examples/eval/tail.rs diff --git a/crates/b2-embed/examples/eval.rs b/crates/b2-embed/examples/eval.rs deleted file mode 100644 index f5bbcb1..0000000 --- a/crates/b2-embed/examples/eval.rs +++ /dev/null @@ -1,4425 +0,0 @@ -//! Semantic-retrieval + discovery eval — the pass that scores model quality **out of CI** -//! (ADR-0013). It lives as an *example*, not a test, so it never runs in the deterministic -//! `cargo test` suite and quality can never flake CI. -//! -//! ```console -//! cargo run -p b2-embed --example eval # score the configured model -//! cargo run -p b2-embed --example eval -- --sweep # + chunker A/B (the #44 gate) -//! cargo run -p b2-embed --example eval -- --stemmer # + FTS tokenizer A/B (the #157 gate) -//! ``` -//! -//! Any other argument is refused (exit 1) rather than ignored: a typo'd `--swep` would -//! otherwise score only the default config while the reader believes the A/B ran. -//! -//! **`docs/evals.md` is the notebook of record** — the corpus, what the exit code -//! enforces, every verdict this harness has ruled, and the process rules. Read it before -//! touching the corpus, the labels, or a constant here. What this comment carries is only -//! what a reader of *this file* needs: -//! -//! One run builds throwaway vaults from the labelled corpora in `evals/` and scores, through -//! the real pipeline: a **BM25 baseline** (after `project` only — the floor the model must -//! clear, since the labelled queries avoid their target's keywords); **hybrid retrieval** -//! plus a **vector-only ablation** (GH #158), whose delta is the measured value of the one -//! AI seam; **passage rank** at chunk level, which is where chunking levers show; and -//! **discovery**, scored per labelled mate rather than per anchor (GH #183 — the per-anchor -//! metric saturates), with the strangers a positive anchor serves counted, named, and -//! deliberately ungated, since the cheapest way to shrink that count is to label one. -//! -//! Two calibration blocks re-derive their windows **every run** rather than quoting a -//! reading: the discovery **z dump** (GH #187) and the **search evidence bake-off** -//! (ADR-0015, GH #201/#202). That is the house rule — constants in code, measurements in the -//! harness — and it exists because the GH #150 floors were frozen into a docstring and went -//! stale the first time the corpus grew a shape they were never read against. -//! -//! The **dense single-domain fixture** (`evals/corpus-dense/`) is scored in its own vault -//! and its own row, never averaged in: fifteen genuinely inter-related notes with no loner, -//! the geometry that broke every anchor-local existence gate (ADR-0014) and killed the first -//! lexical evidence rule (ADR-0015). The orthogonal corpus is structurally incapable of -//! expressing topical concentration, so a run that judges a bar only there judges it on the -//! geometry it survives. -//! -//! What this corpus mostly **cannot** score is *candidate width*: while its chunk count sits -//! at or under a signal's candidate pool, that list is never truncated, widening it cannot -//! add a candidate, and every number is invariant under that view's headroom and -//! `search::pool_size` (GH #141). The corpus has since grown a few chunks past the narrower -//! (passage-view) pool (GH #183), so the run reads the live counts and warns (`pool_blind`) -//! only when blindness actually holds; the property itself is measured by -//! `--example stability`, on a vault big enough for the pools to bind. `RRF_K` re-weights -//! the *same* lists, so it does move scores here and needs no separate instrument. -//! -//! `--sweep` re-chunks + re-embeds the same vault under variant [`ChunkConfig`]s. `--stemmer` -//! swaps `chunks_fts` between the shipped `porter unicode61` and the unstemmed ablation over -//! **identical** chunk rows and vectors, so every rank move is the tokenizer's alone. -//! -//! Every scored run appends one JSON line to `evals/results.jsonl` (gitignored), so runs -//! accumulate into a comparable dataset. - -// `result_row`'s JSON literal is one `json!` expansion per key, and the row has -// grown a key per instrument (GH #158, #141, #183, #187, #188). Raising the -// limit keeps the row's shape legible in one place — the alternative is -// scattering its subtrees across helper functions to satisfy a macro, which -// costs the thing the row is for: a reader seeing every recorded field at once. -#![recursion_limit = "256"] - -mod common; - -use b2_core::chunk::ChunkConfig; -use b2_core::db::FtsTokenizer; -use b2_core::embed::Embedder; -use b2_core::search::EvidenceBar; -use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, Vault}; -use b2_embed::{provision, EmbedConfig, LocalEmbedder}; -use common::{ - append_result, cosine_of, git_short_sha, has_flag, passage_z, pile_stats, reject_unknown_flags, - term_coverage, title_query, truncate, Band, ScratchVault, -}; -use serde::Deserialize; -use std::collections::HashMap; -use std::ops::ControlFlow; -use std::path::Path; -use std::time::Instant; - -/// How deep we look for a relevant note/chunk when scoring. -const K: usize = 10; -/// How many `similar` candidates we look at per anchor. -const SIM_K: usize = 5; -/// How deep the z dump reads (GH #187) — every candidate note in a corpus this -/// size, since a threshold is calibrated against the whole population it has to -/// cut, not against the prefix a human-facing `limit` would show. -const Z_SCAN_LIMIT: usize = 500; -/// The reciprocity depths GH #200's candidate 1 is shown at in the headline -/// table and the per-anchor lists — [`SIM_K`] (the pane itself) with one depth -/// either side of it. They are a *display* choice only: the window below is -/// derived from the whole sweep, so no verdict turns on these three. -const FOLD_MUTUAL_K: [usize; 3] = [3, 5, 10]; -/// How far the reciprocity depth is swept when re-deriving candidate 1's -/// admissible window each run (GH #187's idiom on the disclosure axis — a -/// measurement in the harness, never a constant frozen into a docstring). Half -/// the orthogonal corpus's note count: a `k` past that admits most of the vault -/// as "mutually near", which is the rule going vacuous rather than a depth -/// anyone would ship. -const FOLD_K_SWEEP: usize = 15; -/// The soft reference floor on the default config's hybrid note hit@1. -/// Untouched by the GH #197 re-derivation: retrieval never had a gate to -/// retire. -const FLOOR_HIT1: f64 = 0.75; -/// The floor on **per-mate** discovery MRR@[`SIM_K`] (GH #188) — the non-saturating rank -/// metric, gated once a baseline existed to price it. Re-derived for always-serve -/// (ADR-0014): the reading moved 0.633 -> 0.650 when the existence gate retired. -/// -/// Placed **below** the reading, never at it: a gate pinned to today's number fails on the -/// first legitimate corpus edit, which trains the one habit this harness must never train -/// (process rule 2 — editing a *label* to get green). Sizing is measured, not guessed: five -/// consecutive runs on an unchanged corpus/model/build reproduce every rank exactly, so the -/// noise floor is 0 and the headroom exists for *corpus* drift. At n = 15 mates one mate -/// lost from rank 1 costs 1/15, and this floor sits ~2 such losses under the reading. -const FLOOR_MATE_MRR: f64 = 0.52; -/// The floor on the **dense fixture's** per-mate MRR@[`SIM_K`] (ADR-0014, Phase 0b), gated -/// once its baseline existed — the same measure-then-calibrate order as [`FLOOR_MATE_MRR`]. -/// -/// The reading is 0.467 (n = 14): the model recovers the within-cluster labels and ranks the -/// three cross-cluster claims lower, which is headroom in both directions. (Its first -/// reading was 0.502; a one-word grammar fix moved one mate a rank — the worked example of -/// why these floors carry corpus-drift margin.) At n = 14 one mate lost from rank 1 costs -/// 1/14, and this sits ~2 such losses under. Process rule 2 binds hard here: in a corpus -/// where everything relates, relabelling toward the model's order would always look -/// plausible — a red reading argues about the notes. -const FLOOR_DENSE_MATE_MRR: f64 = 0.32; -/// How many **labelled negative queries** the shipped evidence bar may still serve -/// (ADR-0015, GH #202). Zero: this is the defect the bar exists to fix. -/// -/// A floor at its measured value rather than below it, which is the deliberate exception to -/// the house sizing method — the "headroom" a floor normally carries would be *permission to -/// serve a nonsense query*. A new negative the bar serves is either a real regression or a -/// mislabelled query; both want a red reading. -const MAX_NEGATIVES_SERVED: usize = 0; -/// How many **labelled relevant queries** the bar may cut (GH #202) — the search-side -/// tripwire ADR-0015 asserts at zero with no headroom, and the direction that costs a user -/// something real: a served nonsense row costs a little trust, a cut positive costs the -/// answer. Its precondition was met by GH #208, which labelled the date-shaped query pile. -/// The reading is 0 of 44. A nonzero value is never a calibration nudge: it means the *rule* -/// is wrong for a shape the corpus now carries. -const MAX_POSITIVES_CUT: usize = 0; -/// How many of the **dense fixture's title-as-query probes** the bar may cut (GH #202). -/// Zero, and this is the assertion that would have caught the losing rule: a note's own -/// title names a note the vault demonstrably holds, so cutting one is indefensible whatever -/// a labelled corpus says. A third row rather than headroom on [`MAX_POSITIVES_CUT`] because -/// it gates a different *geometry*: the labelled corpus minimizes shared vocabulary by -/// construction, so topical concentration is only expressible here. Titles need no labels, -/// so nothing in this reading can be relabelled to clear it. -const MAX_DENSE_TITLES_CUT: usize = 0; - -#[derive(Deserialize)] -struct QuerySet { - queries: Vec, -} - -#[derive(Deserialize)] -struct Labelled { - query: String, - /// The vault-relative path(s) that should rank first. **Empty = a negative - /// query** (invariants.md D2, GH #201): the labelled answer is "no - /// matches" — the vault holds no evidence for the query, so everything - /// served is junk by label, the query-side sibling of similar.json's - /// negative anchors. Negative queries are excluded from every rank - /// aggregate (adding one moves no pre-existing number) and are scored only - /// by the search evidence calibration. - relevant: Vec, - /// A short verbatim phrase from the target passage; when present the query is - /// also scored at chunk level (does a top-K chunk of a relevant note contain - /// it?). See queries.json's description for the labelling rules. - #[serde(default)] - passage: Option, - /// Notes beyond `relevant` that are honest evidence for the query (GH #206) — - /// the per-hit tail depth. The judgement is **exhaustive** for every positive - /// query: a served note in neither `relevant` nor here is irrelevant *by - /// label*, which is the statement the tail bake-off is judged on. Never enters - /// a rank aggregate — `relevant` alone says what should rank first. - #[serde(default)] - tail_relevant: Vec, -} - -#[derive(Deserialize)] -struct SimilarSet { - anchors: Vec, -} - -#[derive(Deserialize)] -struct SimilarLabel { - anchor: String, - /// Corpus notes a human says belong next to `anchor`. **Empty = a negative - /// anchor**: the labelled answer is "nothing relates", so the right result is - /// zero candidates and everything surfaced is junk by label (similar.json). - expected: Vec, -} - -/// One query's ranks in one retrieval mode: 1-based note rank, 1-based chunk rank -/// (only for passage-labelled queries), and the top note hit for display. -struct QueryScore { - note: Option, - chunk: Option, - top: String, -} - -/// Running hit@1 / hit@3 / MRR@K over a set of 1-based ranks. ("hit@k" — each -/// query has essentially one relevant target, so precision@k and recall@k -/// coincide with it.) -#[derive(Default)] -struct Agg { - n: usize, - hit1: usize, - hit3: usize, - rr: f64, -} - -impl Agg { - fn add(&mut self, rank: Option) { - self.n += 1; - if let Some(r) = rank { - self.rr += 1.0 / r as f64; - if r <= 1 { - self.hit1 += 1; - } - if r <= 3 { - self.hit3 += 1; - } - } - } - fn hit1(&self) -> f64 { - self.hit1 as f64 / self.n.max(1) as f64 - } - fn hit3(&self) -> f64 { - self.hit3 as f64 / self.n.max(1) as f64 - } - fn mrr(&self) -> f64 { - self.rr / self.n.max(1) as f64 - } -} - -/// A full pass over the query set in the vault's current state (keyword-only -/// before `embed`, hybrid after): per-query scores plus note- and chunk-level -/// aggregates. -struct Pass { - scores: Vec, - note: Agg, - chunk: Agg, -} - -/// A full pass over the discovery labels: rank aggregates over the positive -/// anchors, the suppression tally over the negative ones, and every surfaced -/// candidate's cosine sorted into the two calibration piles the quality floor -/// is read from (index-engine.md §3's ruling, PR #145). -#[derive(Default)] -struct SimilarPass { - /// Rank of the first `expected` hit per positive anchor — the pre-existing - /// hit@1 / hit@3 / MRR discovery metrics, unchanged. - rank: Agg, - /// **Per-mate** ranks: one entry per `(anchor, expected mate)` pair rather than one per - /// anchor (GH #183). [`Self::rank`] takes the *first* labelled mate it finds and stops, - /// so an anchor scores a hit on its easiest one and every harder mate is invisible — - /// which is how that metric sat pinned at 1.000 across the ordering change it was - /// supposed to judge. Worse, *adding* a hard mate makes `rank` strictly easier. - /// - /// Scoring each mate on its own is the non-saturating readout GH #183 asked for: a mate - /// sliding from rank 2 to rank 4 moves this number even while `rank` stays at ceiling. A - /// `None` is honest — that mate did not surface at all. - /// - /// **In the exit gate** since GH #188, at [`FLOOR_MATE_MRR`]. It shipped reporting-only - /// for one issue's worth of time on the measure-then-calibrate precedent, and that is - /// where the baseline came from. - /// - /// (A pass-vs-pass suppression diff — `mate_raw` / `mate_suppressed`, asserted at zero — - /// stood beside this until GH #217: once ADR-0014 retired the existence gate, both - /// passes read the *identical* `similar` call, so the diff compared a call against - /// itself and could never fire. An existence gate returning to the path is what the - /// dense fixture's zero-empty-panes assertion and both per-mate MRR floors catch — - /// against ground truth, not against a second read of the same surface.) - mate: Agg, - /// **Strangers**: unlabelled notes the shipped surface serves on a *positive* anchor, - /// within the same top-`SIM_K` the ranks are read at — `(anchor, path)` per card, so the - /// smoke alarm comes with the list you argue against it with (process rule 1). - /// - /// This is discovery's **precision** side, and the harness had none: the gated numbers - /// watch mate ranks and negative anchors, and the negatives gate is structurally blind to - /// a member bar — while `member_z <= leader_z`, a negative anchor is clean iff its leader - /// is cut, so a relaxed member bar spends its entire cost here (GH #187/#188). - /// - /// Deliberately **reported, not gated**, and the reason is the gaming direction: the - /// cheapest way to shrink this count is to *label the stranger*, which silently moves the - /// per-mate metric too. The labels are not exhaustive, so an unlabelled note served is - /// not proof of junk — it reads as a smoke alarm with names attached. - strangers: Vec<(String, String)>, - /// Positive anchors serving at least one stranger — the spread behind - /// [`Self::strangers`], since one anchor with a long tail and five anchors - /// with one card each are different failures at the same count. - stranger_anchors: usize, - /// Negative anchors asked. - neg_n: usize, - /// Negative anchors that surfaced zero candidates. Under always-serve - /// (GH #197) an anchor with scorable candidates always serves, so this - /// reads 0 on this corpus — recorded for row comparability (the key's - /// meaning, "anchors serving nothing", is unchanged), no longer asserted: - /// the labels still say "nothing relates", and what the served cards - /// *claim* is measured by their bands (A2's readout, calibration block). - neg_clean: usize, - /// Candidates surfaced across all negative anchors — cards whose labelled - /// answer was "nothing". - neg_cards: usize, - /// Cosines of surfaced candidates a human labelled genuinely related. - related: Vec, - /// Cosines of everything else surfaced — non-expected candidates of positive - /// anchors, and every candidate of a negative anchor. - junk: Vec, - /// Every anchor's surfaced list in rank order. The flat piles above judge the - /// absolute floor; the *relative* drop-off cutoff is judged within one - /// anchor's list (how far did #2 fall from #1?), and naming the pair behind a - /// pile value needs the anchor too — so the order is recorded, not just the - /// distribution. - detail: Vec, -} - -/// One anchor's surfaced candidates, in rank order, for the results log. -struct AnchorDetail { - anchor: String, - /// True for a negative anchor (empty `expected`). - negative: bool, - /// (candidate path, cosine, human-labelled related) per surfaced candidate. - candidates: Vec<(String, f64, bool)>, -} - -/// One row of the z dump: a candidate in the band's unit, with the human label -/// attached. -struct ZCand { - path: String, - /// The stage-2 best-passage z (GH #192's unit; the strength band's input, - /// gating nothing since GH #197) — straight from the engine's own - /// statistics. - z: f64, - /// The same z recomputed harness-side from the served scores (z over squared - /// best-pair distance, nearer = higher). An instrument check, not a reading: - /// if this drifts from `z`, the engine's statistic moved and the harness's - /// model of it is stale. `None` only when the population had zero variance - /// and no z exists. - z_recheck: Option, - /// The stage-2 score as served (negated best chunk-pair L2) — kept so the - /// row also records the model-comparable cosine, not only the - /// anchor-relative z. - score: f64, - /// Labelled a mate of this anchor. - mate: bool, -} - -/// One anchor's **complete** reading in the band's unit — every candidate note -/// in the corpus, in z order, with the human label attached (GH #187; stage-2 -/// best-passage z since GH #192; the z travels ungated on the one shipped -/// surface since GH #197, so the dump *is* the served list read deep). -/// -/// The cosine piles above are the same numbers in a model-comparable unit; this -/// is the anchor-relative unit a z existence bar would judge in, which is why -/// the piles could never re-derive one's constants. -struct AnchorZ { - anchor: String, - /// True for a negative anchor (empty `expected`) — its whole list is - /// strangers by label, and its leader is what a leader gate would answer to. - negative: bool, - /// Every candidate in served order, which *is* descending z. - candidates: Vec, -} - -impl AnchorZ { - /// The top candidate's z — what a leader gate would read. `None` only if - /// the anchor produced no scorable candidate at all. - fn leader(&self) -> Option { - self.candidates.first().map(|c| c.z) - } - /// This anchor's labelled mates' z's (empty for a negative anchor). - fn mates(&self) -> impl Iterator + '_ { - self.candidates.iter().filter(|c| c.mate).map(|c| c.z) - } - /// Everything on this anchor's list a human did *not* label — on a positive - /// anchor, exactly the population a member bar would have to cut. - fn strangers(&self) -> impl Iterator + '_ { - self.candidates.iter().filter(|c| !c.mate).map(|c| c.z) - } - /// The worst disagreement between the engine's z and the harness's own - /// recomputation from the served scores, across this anchor's candidates — - /// the statistic-level drift check ([`ZCand::z_recheck`]). - fn recheck_delta(&self) -> f64 { - self.candidates - .iter() - .filter_map(|c| c.z_recheck.map(|r| (c.z - r).abs())) - .fold(0.0, f64::max) - } -} - -/// The z dump across every discovery anchor, split into the populations an existence bar -/// would answer to (GH #187; the unit is the stage-2 best-passage z, and it gates nothing — -/// ADR-0014). -/// -/// Three populations, because a two-bar rule's constants answer to different ones — the -/// conflation is what made "the negatives gate would catch a bad `member_z`" look true when -/// it was not. A leader gate is calibrated by negative-anchor leaders against positive-anchor -/// leaders; a member bar by strangers against labelled mates. Both windows are re-derived -/// every run — the standing record of why no such rule ships, and the first reading any -/// Phase-2 candidate answers to. -#[derive(Default)] -struct FloorZ { - /// Every anchor with computed statistics, in label order. - anchors: Vec, - /// Anchors whose candidate pool was under `STATS_MIN_POPULATION` or had zero - /// variance, so no z exists to calibrate from. Named rather than silently - /// dropped: on a corpus small enough for this to happen, every window below - /// is measured on fewer anchors than the labels suggest. - ungraded: Vec, -} - -impl FloorZ { - /// (a) Labelled mates' z's — what a member bar would have to keep. - fn mate_z(&self) -> Vec { - self.anchors.iter().flat_map(|a| a.mates()).collect() - } - /// (b) Strangers on **positive** anchors — what a member bar would have to - /// cut. The negative anchors' own candidates are deliberately excluded: - /// they are the leader pair's business, and folding them in here is what - /// would let a member window look wider than it is. - fn stranger_z(&self) -> Vec { - self.anchors - .iter() - .filter(|a| !a.negative) - .flat_map(|a| a.strangers()) - .collect() - } - /// (c) Negative anchors' leaders — what a leader gate would have to cut. - fn neg_leader_z(&self) -> Vec { - self.anchors - .iter() - .filter(|a| a.negative) - .filter_map(|a| a.leader()) - .collect() - } - /// Positive anchors' leaders — what a leader gate would have to keep, or it - /// empties a list that has a real mate on it. - fn pos_leader_z(&self) -> Vec { - self.anchors - .iter() - .filter(|a| !a.negative) - .filter_map(|a| a.leader()) - .collect() - } - - /// The worst engine-vs-recomputed z disagreement across every anchor — the - /// statistic-level instrument check (see [`ZCand::z_recheck`]). Small fp - /// noise is expected (the engine z-scores f32 squared distances; the - /// harness recomputes them from the f32-sqrt'd scores), so the printed - /// check tolerates 1e-3 z and reports the observed maximum. - fn recheck_delta(&self) -> f64 { - self.anchors - .iter() - .map(|a| a.recheck_delta()) - .fold(0.0, f64::max) - } -} - -/// The admissible interval for one threshold, read straight off two labelled -/// populations: a bar `t` keeps everything in `keep` iff `t ≤ min(keep)`, and -/// cuts everything in `cut` iff `t > max(cut)`, so every workable constant lies -/// in `(cut_max, keep_min]` — and **no constant works at all** when the two -/// populations overlap, which is a measured result rather than a failure to -/// search harder. -struct Window { - cut_max: f64, - keep_min: f64, -} - -impl Window { - /// Both edges, or `None` while either population is empty (nothing to - /// separate — an honest no-reading, not a wide-open window). - fn read(cut: &[f64], keep: &[f64]) -> Option { - let cut_max = cut.iter().copied().fold(f64::NEG_INFINITY, f64::max); - let keep_min = keep.iter().copied().fold(f64::INFINITY, f64::min); - (cut_max.is_finite() && keep_min.is_finite()).then_some(Self { cut_max, keep_min }) - } - /// Whether any constant separates the two populations. - fn open(&self) -> bool { - self.cut_max < self.keep_min - } -} - -fn main() { - match run() { - Err(e) => { - eprintln!("eval failed: {e}"); - std::process::exit(1); - } - Ok(passed) => { - if !passed { - std::process::exit(2); - } - } - } -} - -/// Returns whether the default-config hybrid pass cleared the reference floor. -fn run() -> Result> { - // A bare `--` is dropped rather than refused (`make stability`'s posture: a pasted - // `cargo run … -- --sweep` must not fail on its own separator); anything else - // unrecognised is refused, since a typo'd flag would otherwise run the default - // measurement while the reader believes an A/B ran. - let args: Vec = std::env::args().skip(1).filter(|a| a != "--").collect(); - reject_unknown_flags(&args, &["--sweep", "--stemmer"], &[])?; - let sweep = has_flag(&args, "--sweep"); - let stemmer = has_flag(&args, "--stemmer"); - let evals_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("evals"); - let corpus_dir = evals_dir.join("corpus"); - let results_path = evals_dir.join("results.jsonl"); - - // Load the labelled sets. Queries split on their labels: positives carry - // rank labels; an empty `relevant` is a negative query (invariants.md D2, - // GH #201), scored only by the search evidence calibration so its presence - // moves no rank aggregate. - let set: QuerySet = - serde_json::from_str(&std::fs::read_to_string(evals_dir.join("queries.json"))?)?; - let (negatives, positives): (Vec, Vec) = - set.queries.into_iter().partition(|q| q.relevant.is_empty()); - let sim_set: SimilarSet = - serde_json::from_str(&std::fs::read_to_string(evals_dir.join("similar.json"))?)?; - let dense_set: SimilarSet = serde_json::from_str(&std::fs::read_to_string( - evals_dir.join("similar-dense.json"), - )?)?; - - // Lint the labels against the corpora they claim to describe, before - // anything expensive runs. A typo'd label path fails nothing on its own — - // it just reads as a permanent miss (a rank of `None`, a served row - // downgraded to filler by label) and gets chased as an engine regression — - // so the run refuses to score against labels the corpus cannot honour - // (exit 1: the run broke, not the gate). - lint_labels( - &corpus_dir, - &evals_dir.join("corpus-dense"), - &positives, - &negatives, - &sim_set, - &dense_set, - )?; - - // Ensure the model is available, then load it. (Provision is idempotent, so an - // already-installed model is a no-op; a missing one is fetched here.) - let config = EmbedConfig::load()?; - provision(&config, |line| eprintln!("[init] {line}"))?; - let embedder = LocalEmbedder::load(&config)?; - let model_id = embedder.model_id().to_string(); - let dim = embedder.dim(); - eprintln!("[eval] model = {model_id} (dim {dim})\n"); - - // A correctness gate, not a score: every number below is computed from batched - // embeddings, so they only mean anything if batching is faithful. - check_batch_matches_single(&embedder)?; - - // Build a throwaway vault from the corpus. - let scratch = ScratchVault::copy_flat(&corpus_dir)?; - let vault_root = scratch.root(); - let mut vault = Vault::open_with_embedder(vault_root, Box::new(embedder))?; - - // ---- Phase 1: projection only → the BM25-only baseline. ------------------ - // The vector space does not exist yet, so `search`/`search_chunks` run - // keyword-only (index-engine.md) — the ablation costs nothing - // extra: it is the same vault, paused between the two passes. - let report = vault.project(false)?; - let bm25 = score_pass(&vault, &positives, Retrieval::Fused)?; - eprintln!( - "[eval] projected {} notes; BM25-only baseline scored\n", - report.indexed - ); - - // The stemmer instrument's lexical arm is scored HERE, while the vault is - // still projected-but-unembedded, so `search` is honestly BM25-only under - // both tokenizers; the vault is handed back to the shipped default before - // anything embeds. - let bm25_unstemmed = if stemmer { - vault.rebuild_fts(FtsTokenizer::Unicode61)?; - let pass = score_pass(&vault, &positives, Retrieval::Fused)?; - vault.rebuild_fts(FtsTokenizer::PorterUnicode61)?; - Some(pass) - } else { - None - }; - - // ---- Phase 2: embed → vector-only ablation + hybrid + discovery. --------- - let (chunks, embed_secs) = timed_embed(&vault)?; - let vector = score_pass(&vault, &positives, Retrieval::VectorOnly)?; - let hybrid = score_pass(&vault, &positives, Retrieval::Fused)?; - let similar = score_similar(&vault, &sim_set)?; - // The fold bake-off (GH #200, Phase B) — the candidate default-disclosure - // rules judged on the same served lists `similar` above was scored from, so - // no rule is compared against a surface the others did not see. - let fold = score_fold(&vault, &sim_set, "orthogonal", false)?; - // The z calibration dump (GH #187) — the same shipped surface read deep, - // since the z travels ungated on it (GH #197). - let floor_z = score_floor_z(&vault, &sim_set)?; - // The search evidence dump (invariants.md D2, GH #201) — the query-side - // sibling of the z calibration, read over the same built vault. - let evidence = score_search_evidence(vault_root, &vault, &positives, &negatives)?; - eprintln!( - "[eval] embedded {chunks} chunks in {embed_secs:.1}s ({} candidates per signal at K={K}, \ - {} for the passage view)\n", - note_candidate_pool(K), - chunk_candidate_pool(K) - ); - warn_if_pool_blind(chunks); - - print_default_report(&positives, &bm25, &vector, &hybrid, &similar, &floor_z); - print_fold_bench(&fold); - print_search_evidence(&evidence); - print_search_bakeoff(&evidence, &bake_off(&evidence), &model_id); - // The per-hit tail bake-off (GH #206) — judged from the same served lists - // the evidence dump above recorded, against the tail_relevant keep-set. - let tail = score_search_tail(&evidence); - print_search_tail(&tail); - - let git = git_short_sha(); - append_result( - &results_path, - &result_row( - &git, - &model_id, - dim, - "default", - &ChunkConfig::default(), - FtsTokenizer::PorterUnicode61.sql(), - report.indexed, - chunks, - embed_secs, - &positives, - Some(&bm25), - Some(&vector), - &hybrid, - Some(&similar), - Some(&floor_z), - Some(&evidence), - Some(&fold), - Some(&tail), - ), - )?; - - // ---- Phase 3: the dense single-domain fixture (GH #196/#197, Phase 0b). -- - // Its own throwaway vault, its own model load, its own results row (corpus - // id `dense`) — the fixture measures a *vault-level* geometry, so nothing - // about it may share state with the orthogonal corpus's run above. - let dense = score_dense(&evals_dir, &dense_set)?; - print_dense_report(&dense); - print_fold_bench(&dense.fold); - append_result(&results_path, &dense_row(&git, &model_id, dim, &dense))?; - // The tail bake-off's cross-bench join (GH #206) — printable only here, - // where both corpora's readings exist in one run. - print_tail_join(&evidence, &tail, &dense.search.titles); - - // ---- Optional: the FTS tokenizer ablation (the #157 instrument). --------- - // One lever, isolated: `rebuild_fts` swaps the tokenizer over the identical - // chunk rows and vectors — the shipped `porter unicode61` against the - // unstemmed `unicode61` the A/B retired, kept measurable so the verdict can - // be re-tried as the corpus grows. Discovery is deliberately not re-scored — - // `similar` never touches FTS (centroid shortlist + chunk vectors), so its - // numbers cannot move and the ablation row records no `similar` keys. The - // dense ablation IS re-scored, as an instrument check: FTS cannot reach it - // either, so a moved dense rank means the harness is broken, not the engine. - if stemmer { - let bm25_unstemmed = bm25_unstemmed - .as_ref() - .expect("scored while unembedded above"); - vault.rebuild_fts(FtsTokenizer::Unicode61)?; - let vec_unstemmed = score_pass(&vault, &positives, Retrieval::VectorOnly)?; - let hybrid_unstemmed = score_pass(&vault, &positives, Retrieval::Fused)?; - - println!("\n{}", "=".repeat(78)); - println!( - "stemmer ablation — chunks_fts rebuilt `unicode61` (unstemmed) over identical chunks + vectors (GH #157)" - ); - let dense_moved = (0..positives.len()) - .filter(|&i| vec_unstemmed.scores[i].note != vector.scores[i].note) - .count(); - if dense_moved == 0 { - println!(" [check] dense ablation identical across the flip, as it must be"); - } else { - println!( - " [FAULT] {dense_moved} dense rank(s) moved across an FTS-only flip — \ - the harness is broken; distrust this run" - ); - } - println!(" bm25-only, unstemmed vs shipped porter:"); - print_rank_moves(&positives, &bm25, bm25_unstemmed); - println!(" hybrid, unstemmed vs shipped porter:"); - print_rank_moves(&positives, &hybrid, &hybrid_unstemmed); - println!( - " unstemmed aggregates: bm25 note {:.2} / {:.3} hybrid note {:.2} / {:.3} chunk {:.2} / {:.3}", - bm25_unstemmed.note.hit1(), - bm25_unstemmed.note.mrr(), - hybrid_unstemmed.note.hit1(), - hybrid_unstemmed.note.mrr(), - hybrid_unstemmed.chunk.hit1(), - hybrid_unstemmed.chunk.mrr(), - ); - append_result( - &results_path, - &result_row( - &git, - &model_id, - dim, - "unicode61", - &ChunkConfig::default(), - FtsTokenizer::Unicode61.sql(), - report.indexed, - chunks, - embed_secs, - &positives, - Some(bm25_unstemmed), - Some(&vec_unstemmed), - &hybrid_unstemmed, - None, - None, - None, - None, - None, - ), - )?; - // Hand the vault back untainted, so a `--sweep` in the same run (and the - // reference numbers above) stay under the shipped default tokenizer. - vault.rebuild_fts(FtsTokenizer::PorterUnicode61)?; - } - - // ---- Optional: the in-process chunker sweep (the #44 A/B). --------------- - if sweep { - // The #44 grid: both directions on each swept knob, plus the one - // interaction worth a row. `target_tokens` brackets the 450 default - // (250 / 350 / 600); `overlap_frac` brackets 0.15 (0.0 / 0.30); - // `target-250+heading-path` exists because a smaller chunk carries less - // of its own context, which is exactly when the breadcrumb prefix (D3) - // is most plausibly worth its tokens. `chars_per_token` and - // `backscan_tokens` stay unswept: they are calibration constants of the - // token proxy and the boundary search, not retrieval-quality levers. - let variants: Vec<(&str, ChunkConfig)> = vec![ - ( - "target-250", - ChunkConfig { - target_tokens: 250, - ..ChunkConfig::default() - }, - ), - ( - "target-350", - ChunkConfig { - target_tokens: 350, - ..ChunkConfig::default() - }, - ), - ( - "target-600", - ChunkConfig { - target_tokens: 600, - ..ChunkConfig::default() - }, - ), - ( - "overlap-0", - ChunkConfig { - overlap_frac: 0.0, - ..ChunkConfig::default() - }, - ), - ( - "overlap-30", - ChunkConfig { - overlap_frac: 0.30, - ..ChunkConfig::default() - }, - ), - ( - "prepend-heading-path", - ChunkConfig { - prepend_heading_path: true, - ..ChunkConfig::default() - }, - ), - ( - "target-250+heading-path", - ChunkConfig { - target_tokens: 250, - prepend_heading_path: true, - ..ChunkConfig::default() - }, - ), - ]; - println!("\n{}", "=".repeat(78)); - println!("chunker sweep (same model, same corpus; default row above for reference)"); - // `mate MRR` rather than the per-anchor `similar h@3` this column used - // to carry: that number saturates (GH #183), so as a *comparison* - // column across variants it could only ever print 1.00 (GH #188). - // `strangers` replaced `neg clean` when GH #197 retired the existence - // gate: under always-serve a negative anchor always serves, so that - // column could only ever print 0/5 — the same cannot-move failure. - println!( - "{:<24} {:>7} {:>8} note h@1/MRR vec h@1/MRR chunk h@1/MRR mate MRR strangers", - "config", "chunks", "embed_s" - ); - for (label, cfg) in variants { - vault.set_chunk_config(cfg.clone()); - vault.project(true)?; // force: re-chunk everything, clearing vectors - let (chunks, embed_secs) = timed_embed(&vault)?; - let vec_pass = score_pass(&vault, &positives, Retrieval::VectorOnly)?; - let pass = score_pass(&vault, &positives, Retrieval::Fused)?; - let sim = score_similar(&vault, &sim_set)?; - println!( - "{:<24} {:>7} {:>8.1} {:.2} / {:.3} {:.2} / {:.3} {:.2} / {:.3} {:.3} {}", - label, - chunks, - embed_secs, - pass.note.hit1(), - pass.note.mrr(), - vec_pass.note.hit1(), - vec_pass.note.mrr(), - pass.chunk.hit1(), - pass.chunk.mrr(), - sim.mate.mrr(), - sim.strangers.len(), - ); - // The readout the A/B is actually judged on: at this n every aggregate - // delta above is 1–2 queries, so the aggregate is a smoke alarm and the - // per-query win/loss list is the data (docs/evals.md, the - // process rules). - print_rank_moves(&positives, &hybrid, &pass); - append_result( - &results_path, - &result_row( - &git, - &model_id, - dim, - label, - &cfg, - FtsTokenizer::PorterUnicode61.sql(), - report.indexed, - chunks, - embed_secs, - &positives, - None, - Some(&vec_pass), - &pass, - Some(&sim), - None, - None, - None, - None, - ), - )?; - } - } - - eprintln!("\n[eval] appended run to {}", results_path.display()); - - // The soft floors, on the DEFAULT config's passes — so this can double as - // a manual quality gate. Not a CI test. - if hybrid.note.hit1() < FLOOR_HIT1 { - eprintln!( - "\n[warn] hybrid hit@1 {:.2} is below the {FLOOR_HIT1} reference floor — inspect the misses above.", - hybrid.note.hit1() - ); - return Ok(false); - } - // The negatives' suppression assertion RETIRED with the gate it watched (ADR-0014): - // under always-serve a loner anchor serves its ranked nearest — that is the ruling, not - // a regression. The anchors stay labelled, the strangers instrument keeps counting, and - // what the served cards *claim* is the calibration block's band readout. (The - // pass-vs-pass suppression assertion retired later, for a different reason: it compared - // a call against itself and could never fire — GH #217. A returning existence gate is - // the dense zero-empty-panes assertion's and the per-mate floors' to catch.) - // - // Discovery **rank** (GH #188). The rank floor sits below its measured reading (corpus - // drift headroom, run noise being zero). - if similar.mate.mrr() < FLOOR_MATE_MRR { - eprintln!( - "\n[warn] per-mate MRR@{SIM_K} {:.3} is below the {FLOOR_MATE_MRR:.2} floor — discovery ranking regressed \ - (read the per-mate line's mates, not the aggregate; and do NOT relabel to clear this).", - similar.mate.mrr() - ); - return Ok(false); - } - // The dense fixture's existence assertion (GH #196/#197): every note in a - // corpus where everything relates must serve candidates. An empty pane here - // can only come from an anchor-local statistic claiming "nothing relates" - // on a vault where that is false — the exact failure GH #196 measured. - if !dense.empty_panes.is_empty() { - eprintln!( - "\n[warn] {} of {} dense-fixture notes serve an EMPTY pane ({}) — an existence \ - gate is refusing a vault whose every note genuinely relates (GH #196).", - dense.empty_panes.len(), - dense.notes, - dense.empty_panes.join(", ") - ); - return Ok(false); - } - // …and its rank floor, the dense sibling of FLOOR_MATE_MRR. - if dense.mate.mrr() < FLOOR_DENSE_MATE_MRR { - eprintln!( - "\n[warn] dense per-mate MRR@{SIM_K} {:.3} is below the {FLOOR_DENSE_MATE_MRR:.2} floor — \ - single-domain discovery ranking regressed (argue with the notes, not the labels).", - dense.mate.mrr() - ); - return Ok(false); - } - // The search-evidence rows (ADR-0015, GH #202), landed with the surfaces that consume - // the verdict. Everything above is discovery's and is deliberately UNCHANGED: search's - // bar moves no discovery rank and no reachability, so movement up there is a bug. - // - // All three sit at their structural zeros with no headroom — the exception to the house - // sizing method rather than an oversight, since headroom here would read as permission - // to serve a nonsense query or cut a real one. Skipped entirely when the model has no - // calibrated bar (ADR-0007): asserting the absence of a verdict would fail every run on - // a model the harness has simply not measured yet. - match read_shipped_bar(&evidence, &model_id) { - None => eprintln!( - "\n[note] no calibrated evidence bar for {model_id} — D2's exit-gate rows are not \ - asserted this run (M2)." - ), - Some(reading) => { - if reading.neg_served > MAX_NEGATIVES_SERVED { - eprintln!( - "\n[warn] the shipped evidence bar serves {} of {} labelled NEGATIVE queries where \ - D2 permits {MAX_NEGATIVES_SERVED} — a query the vault holds nothing for is being \ - answered with rows (read the per-query lines above, and do NOT relabel to clear \ - this).", - reading.neg_served, - evidence.negatives.len() - ); - return Ok(false); - } - // The tripwire, and the direction that costs a user the answer rather - // than a little trust. Its precondition is GH #208's date-shaped pile: - // the assertion is only worth what the query shapes behind it are. - if reading.pos_cut > MAX_POSITIVES_CUT { - eprintln!( - "\n[warn] the shipped evidence bar CUTS {} of {} labelled relevant queries where D2 \ - permits {MAX_POSITIVES_CUT} — a note the vault holds is unreachable for a query \ - naming it. Change the RULE, not the constant (the df ceiling died exactly here).", - reading.pos_cut, - evidence.positives.len() - ); - return Ok(false); - } - } - } - // The same two directions on the dense fixture — a different geometry, not a - // different threshold. Topical concentration is what killed the losing rule - // and is structurally inexpressible on the orthogonal corpus (process rule 2's - // token audit minimizes shared vocabulary), so these are their own rows. - // Titles carry no labels, so nothing here can be relabelled to clear it. - let titles_cut = dense - .search - .titles - .iter() - .filter(|p| p.vouched == Some(false)) - .count(); - if titles_cut > MAX_DENSE_TITLES_CUT { - eprintln!( - "\n[warn] the evidence bar cuts {titles_cut} of {} dense-fixture titles where \ - {MAX_DENSE_TITLES_CUT} is permitted — a note's own title is a query naming a note the \ - vault demonstrably holds, and the lexical half has gone inert on a single-subject vault \ - (GH #201's transfer check, as an assertion).", - dense.search.titles.len() - ); - return Ok(false); - } - let nonsense_served = dense - .search - .nonsense - .iter() - .filter(|p| p.vouched == Some(true)) - .count(); - if nonsense_served > MAX_NEGATIVES_SERVED { - eprintln!( - "\n[warn] the evidence bar serves {nonsense_served} of {} nonsense queries on the dense \ - fixture where {MAX_NEGATIVES_SERVED} is permitted — nonsense needs no token audit in any \ - vault, which is exactly why this reading transfers.", - dense.search.nonsense.len() - ); - return Ok(false); - } - Ok(true) -} - -/// Every note in one corpus dir, as `file name → lowercased content` — the -/// ground the label lint checks against. Lowercased once, so the passage check -/// matches the way [`score_pass`] will (case-insensitive containment). -fn corpus_texts(dir: &Path) -> Result, Box> { - let mut out = HashMap::new(); - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - if entry.file_type()?.is_file() { - out.insert( - entry.file_name().to_string_lossy().into_owned(), - std::fs::read_to_string(entry.path())?.to_lowercase(), - ); - } - } - Ok(out) -} - -/// Refuse to score against labels the corpus cannot honour (see the call site). -/// -/// Checks: every labelled path (`relevant`, `tail_relevant`, `anchor`, -/// `expected`) names a note in its corpus; every `passage` occurs verbatim -/// (case-insensitively) in a note the query labels relevant — the containment -/// chunk scoring will test; and a negative query carries neither a `passage` -/// nor a `tail_relevant` (its whole list is junk by label already, per the -/// queries.json rules). Faults are all printed before the run refuses, so one -/// run names every problem rather than the first. -fn lint_labels( - corpus_dir: &Path, - dense_dir: &Path, - positives: &[Labelled], - negatives: &[Labelled], - sim_set: &SimilarSet, - dense_set: &SimilarSet, -) -> Result<(), Box> { - let corpus = corpus_texts(corpus_dir)?; - let dense = corpus_texts(dense_dir)?; - let mut faults: Vec = Vec::new(); - - for q in positives { - for (key, paths) in [ - ("relevant", &q.relevant), - ("tail_relevant", &q.tail_relevant), - ] { - for p in paths { - if !corpus.contains_key(p) { - faults.push(format!( - "queries.json: `{key}` names no corpus note: {p:?} (query {:?})", - q.query - )); - } - } - } - if let Some(passage) = &q.passage { - // A blank passage is the opposite defect from a typo'd one: every - // string contains "", so it would lint clean here and then "match" - // every top-K chunk of a relevant note in the chunk scoring — - // silently inflating chunk rank instead of reading a miss - // (PR #221 review). - if passage.trim().is_empty() { - faults.push(format!( - "queries.json: `passage` is blank (query {:?}) — it would match every chunk \ - and inflate chunk rank", - q.query - )); - } else { - let needle = passage.to_lowercase(); - let found = q - .relevant - .iter() - .filter_map(|p| corpus.get(p)) - .any(|text| text.contains(&needle)); - if !found { - faults.push(format!( - "queries.json: `passage` {passage:?} is not verbatim in any relevant note \ - (query {:?}) — chunk rank would read a permanent miss", - q.query - )); - } - } - } - } - for q in negatives { - if q.passage.is_some() { - faults.push(format!( - "queries.json: negative query {:?} carries a `passage` — negatives score no rank", - q.query - )); - } - if !q.tail_relevant.is_empty() { - faults.push(format!( - "queries.json: negative query {:?} carries `tail_relevant` — its whole served \ - list is junk by label", - q.query - )); - } - } - for (file, set, texts) in [ - ("similar.json", sim_set, &corpus), - ("similar-dense.json", dense_set, &dense), - ] { - for label in &set.anchors { - if !texts.contains_key(&label.anchor) { - faults.push(format!( - "{file}: `anchor` names no corpus note: {:?}", - label.anchor - )); - } - for e in &label.expected { - if !texts.contains_key(e) { - faults.push(format!( - "{file}: `expected` names no corpus note: {e:?} (anchor {:?})", - label.anchor - )); - } - } - } - } - if faults.is_empty() { - return Ok(()); - } - for f in &faults { - eprintln!("[lint] {f}"); - } - Err(format!( - "{} label lint fault(s) — a label the corpus cannot honour scores as a permanent miss \ - and reads as an engine regression; fix the label file (or the note) before scoring", - faults.len() - ) - .into()) -} - -/// Score the dense single-domain fixture (GH #196/#197, Phase 0b) in a throwaway -/// vault of its own: per-mate discovery ranks against `similar-dense.json`, and -/// the empty-pane sweep across **every** note in the fixture — not only the -/// labelled anchors, because the assertion is about the surface ("no pane in a -/// dense vault is dark"), not about the labels. -fn score_dense( - evals_dir: &Path, - set: &SimilarSet, -) -> Result> { - let corpus_dir = evals_dir.join("corpus-dense"); - // A second model load rather than sharing the first vault's: the embedder was - // moved into that vault, and the fixture's whole point is an isolated run. - let config = EmbedConfig::load()?; - let embedder = LocalEmbedder::load(&config)?; - let model_id = embedder.model_id().to_string(); - let scratch = ScratchVault::copy_flat(&corpus_dir)?; - let vault = Vault::open_with_embedder(scratch.root(), Box::new(embedder))?; - vault.project(false)?; - let (chunks, embed_secs) = timed_embed(&vault)?; - - let mut pass = DensePass { - notes: 0, - chunks, - embed_secs, - mate: Agg::default(), - mate_ranks: Vec::new(), - empty_panes: Vec::new(), - detail: Vec::new(), - // The bake-off's absolute bench (GH #200): every note is an anchor here, - // because the ruling being tested is about the *surface* — a vault where - // everything relates may never default to "nothing relates" — and the - // labels cover only a few of these notes. - fold: score_fold(&vault, set, "dense", true)?, - // The bar's hardest bench, for the same reason the fold's is: this is the - // geometry that disqualified the rule that lost (GH #201). - search: score_dense_search(&vault, &model_id)?, - }; - // The pane sweep: every note is an anchor, labelled or not. - for note in vault.list_notes()? { - pass.notes += 1; - if vault.similar(¬e.path, SIM_K)?.is_empty() { - pass.empty_panes.push(note.path); - } - } - // Per-mate ranks on the labelled anchors, the orthogonal corpus's metric - // re-used verbatim (GH #183's non-saturating readout). - for label in &set.anchors { - let candidates = vault.similar(&label.anchor, SIM_K)?; - for expected in &label.expected { - let rank = candidates - .iter() - .position(|c| paths_match(&c.path, expected)) - .map(|p| p + 1); - pass.mate.add(rank); - pass.mate_ranks - .push((label.anchor.clone(), expected.clone(), rank)); - } - pass.detail.push(AnchorDetail { - anchor: label.anchor.clone(), - negative: false, - candidates: candidates - .iter() - .map(|c| { - let related = label.expected.iter().any(|e| paths_match(&c.path, e)); - (c.path.clone(), cosine_of(c.score), related) - }) - .collect(), - }); - } - Ok(pass) -} - -/// The dense fixture's reading (see [`score_dense`]). -struct DensePass { - notes: usize, - chunks: usize, - embed_secs: f64, - /// Per-mate ranks at [`SIM_K`] — the fixture's rank metric. - mate: Agg, - /// (anchor, mate, rank) per labelled mate, for the printed lines and the row. - mate_ranks: Vec<(String, String, Option)>, - /// Notes whose discovery pane served nothing — asserted empty (GH #196/#197). - empty_panes: Vec, - detail: Vec, - /// The fold bake-off on this fixture (GH #200) — swept over **every** note, - /// which is where the candidates' hardest bench is: a rule whose default - /// view goes dark on a single-domain vault is disqualified, not re-tuned. - fold: FoldBench, - /// D2's shipped bar replayed on this fixture (GH #201) — see - /// [`score_dense_search`]. - search: DenseSearch, -} - -/// The shipped search evidence bar's reading **on the single-domain fixture** (ADR-0015). -/// -/// This exists because the bar's first form died here and nowhere else. A hard `df <= 10%` -/// content ceiling read 0 cut / 0 served on the labelled orthogonal corpus — clean by every -/// number that bench can produce — and then classed `drone` (df 3) and `comb` (df 7) as -/// stopwords in a vault about beekeeping, cutting 3 of 15 answerable queries. The lexical -/// rule's hazard is **topical concentration**, which the orthogonal corpus cannot express, so -/// a run that judges the bar only there judges it on the geometry it survives. -/// -/// The reading was first taken once, by hand. Taking it *once* is the thing GH #187 named, -/// so it is re-derived every run, here, beside the fold bench that already sweeps this -/// fixture. **In the exit gate** since GH #202, as a row of its own rather than headroom on -/// the labelled corpus's, because it watches a different *geometry*. -struct DenseSearch { - /// Every note's own title replayed as a query — the **tripwire direction** - /// (D2: a labelled-relevant query cut is zero with no headroom). Titles need - /// no labels and so nothing here can be relabelled to clear a reading. - titles: Vec, - /// Nonsense, the defect direction. See [`DENSE_NONSENSE`]. - nonsense: Vec, - /// `None` when the active model has no calibrated bar (M2) — the coverage - /// readings still print, the verdicts do not exist to print. - bar: Option, -} - -/// One query's reading on the dense fixture: the two absolute signals D2 judges, -/// and the engine's own verdict rather than a restatement of it. -struct SearchProbe { - query: String, - /// IDF-weighted term coverage; `None` when no term carries any weight (the - /// lexical half abstaining, not scoring zero). - coverage: Option, - best_cos: Option, - /// `Vault::search_evidence`'s verdict — what would actually ship. `None` - /// mirrors [`DenseSearch::bar`]. - vouched: Option, - /// The served list with its per-hit provenance — the tail bake-off's dense - /// bench (GH #206). On a title query `keep` is true for every row **by - /// geometry**, not by label: a single-domain vault's lists are all real - /// matches, so a tail rule that truncates one is disqualified — the same - /// absolute GH #200 enforced for discovery, and the reason this fixture - /// needs no `tail_relevant` labels. - rows: Vec, -} - -/// The negatives replayed on the dense fixture: **nonsense only**. The labelled negatives in -/// `queries.json` are the *orthogonal* corpus's, and process rule 2's token audit is what -/// makes them negatives — an audit that says nothing about a different corpus. Against -/// `corpus-dense` the phrase-shaped ones disqualify themselves on their merits ("why parrots -/// mimic speech" shares `mimic` with `robbing-behavior.md`, which is a thing the rule -/// deliberately serves). Nonsense needs no audit in any vault, which is why it transfers. -const DENSE_NONSENSE: [&str; 2] = ["shjfasd", "vrelqip zonktar wembleforth"]; - -/// Replay the shipped bar over the dense fixture (see [`DenseSearch`]). -/// -/// Coverage is read off [`b2_core::vault::QueryTermView::idf`] — the view's own -/// weights, not a second copy of the formula ([`term_coverage`]). The orthogonal -/// corpus's bake-off re-derives its arithmetic deliberately, as a drift check -/// against the engine; one such check is the check, and a second would only be two -/// places to fix. -fn score_dense_search( - vault: &Vault, - model_id: &str, -) -> Result> { - let read = |query: &str, keep: bool| -> Result> { - let view = vault.search_evidence(query, K)?; - Ok(SearchProbe { - query: query.to_string(), - coverage: term_coverage(&view), - best_cos: view.best_cos, - vouched: view.vouched, - rows: view - .results - .iter() - .map(|r| ServedRow { - path: r.result.path.clone(), - bm25_rank: r.bm25_rank, - cos: r.cos, - keep, - }) - .collect(), - }) - }; - let mut titles = Vec::new(); - for note in vault.list_notes()? { - // The fixture's notes carry no frontmatter title, so the slug is the - // query — `drone-comb` → "drone comb", which is the pair of words the - // retired ceiling called stopwords. - if let Some(title) = title_query(¬e) { - titles.push(read(&title, true)?); - } - } - Ok(DenseSearch { - titles, - nonsense: DENSE_NONSENSE - .iter() - .map(|q| read(q, false)) - .collect::>()?, - bar: b2_core::search::EvidenceBar::for_model(model_id), - }) -} - -/// Print the dense fixture's search-evidence reading (see [`DenseSearch`]). -fn print_dense_search(search: &DenseSearch) { - println!( - " search bar D2's shipped bar replayed on this geometry (GH #201; GATED since GH #202)" - ); - // The coverage reading comes FIRST, and above the bar, because it is - // **model-free**: a fact about this vault's vocabulary, and the lexical - // half's whole premise. Gating it behind a calibrated bar would print - // nothing at all on a vault the harness can still say something true about — - // the defect PR #205's review already fixed in `calibrate.rs` (c03f8cd), met - // again here (PR #207 review). - let covs: Vec = search.titles.iter().filter_map(|p| p.coverage).collect(); - let cov_line = match pile_stats(&covs) { - Some((min, med, max)) => format!("{min:.2}/{med:.2}/{max:.2}"), - None => "— (no query carried weight)".to_string(), - }; - println!(" title-as-query coverage min/med/max {cov_line}"); - - // Only the *verdicts* below need a bar, so only they stop here. - let Some(bar) = search.bar else { - println!(" no calibrated bar for this model — no verdict is offered (M2)"); - return; - }; - let cut: Vec<&SearchProbe> = search - .titles - .iter() - .filter(|p| p.vouched == Some(false)) - .collect(); - let served: Vec<&SearchProbe> = search - .nonsense - .iter() - .filter(|p| p.vouched == Some(true)) - .collect(); - println!( - " bar under test: coverage ≥ {:.2} or cos ≥ {:.3}", - bar.min_term_coverage, bar.min_cos, - ); - println!( - " cuts {}/{} title queries ← the TRIPWIRE direction; the retired df ceiling \ - cut 3 here", - cut.len(), - search.titles.len() - ); - for p in &cut { - println!( - " [CUT] {:<32} cov {:>5} cos {:>6}", - truncate(&p.query, 32), - p.coverage - .map(|c| format!("{c:.2}")) - .unwrap_or_else(|| "—".to_string()), - p.best_cos - .map(|c| format!("{c:.3}")) - .unwrap_or_else(|| "—".to_string()), - ); - } - println!( - " serves {}/{} nonsense queries ← the reported defect, on this corpus", - served.len(), - search.nonsense.len() - ); -} - -/// The dense fixture's search reading as JSON (`search_transfer` in the dense -/// row) — every probe, so any bar is re-derivable from a row without re-running -/// the model, the `discovery_fold` convention. -fn dense_search_json(search: &DenseSearch) -> serde_json::Value { - let probes = |pile: &[SearchProbe]| { - pile.iter() - .map(|p| { - serde_json::json!({ - "query": p.query, - "coverage": p.coverage.map(|c| (c * 1e4).round() / 1e4), - "best_cos": p.best_cos.map(|c| (c * 1e4).round() / 1e4), - "vouched": p.vouched, - // NEW subkey (absent before GH #206): the served list's - // per-hit provenance, so the tail constraints below are - // re-derivable from a row without re-running the model. - "rows": p.rows.iter().map(|row| serde_json::json!({ - "path": row.path, - "bm25_rank": row.bm25_rank, - "cos": row.cos.map(|c| (c * 1e4).round() / 1e4), - })).collect::>(), - }) - }) - .collect::>() - }; - serde_json::json!({ - "bar": search.bar.map(|b| serde_json::json!({ - "min_term_coverage": b.min_term_coverage, - "min_cos": b.min_cos, - })), - "titles_cut": search.titles.iter().filter(|p| p.vouched == Some(false)).count(), - "nonsense_served": search.nonsense.iter().filter(|p| p.vouched == Some(true)).count(), - "titles": probes(&search.titles), - "nonsense": probes(&search.nonsense), - // NEW subkey (absent before GH #206): the per-hit tail families' - // dense-bench constraints — the absolute this fixture supplies. - "tail": dense_tail_json(&search.titles), - }) -} - -fn print_dense_report(dense: &DensePass) { - println!("\n{}", "=".repeat(78)); - println!( - "dense fixture — corpus-dense/ ({} notes / {} chunks, single-domain, no loner; GH #196/#197)", - dense.notes, dense.chunks - ); - println!( - " per-mate hit@1={:.2} hit@3={:.2} MRR@{SIM_K}={:.3} (n={} mates, GATED at MRR@{SIM_K} ≥ {FLOOR_DENSE_MATE_MRR:.2})", - dense.mate.hit1(), - dense.mate.hit3(), - dense.mate.mrr(), - dense.mate.n - ); - for (anchor, mate, rank) in &dense.mate_ranks { - println!( - " {:>5} {anchor} → {mate}", - rank_str_at(*rank, SIM_K) - ); - } - if dense.empty_panes.is_empty() { - println!( - " panes {}/{} notes serve candidates — zero empty panes (ASSERTED: a dense vault \ - may never read as \"nothing relates\")", - dense.notes, dense.notes - ); - } else { - println!( - " panes {} of {} notes serve an EMPTY pane: {}", - dense.empty_panes.len(), - dense.notes, - dense.empty_panes.join(", ") - ); - } - print_dense_search(&dense.search); - // Model-geometry reading, not a verdict, so it prints with or without a - // calibrated bar — the same posture as the coverage line above it. - print_dense_tail(&dense.search.titles); -} - -/// The dense fixture's own JSONL row. Tagged `"corpus": "dense"` — the key that -/// keeps rows from ever averaging across corpora (the orthogonal rows carry -/// `"corpus": "orthogonal"`); a smaller shape than the main row on purpose, -/// since the fixture scores discovery alone. -fn dense_row( - git: &Option, - model: &str, - dim: usize, - dense: &DensePass, -) -> serde_json::Value { - let ts = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - serde_json::json!({ - "ts": ts, - "git": git, - "model": model, - "dim": dim, - "corpus": "dense", - "notes": dense.notes, - "chunks": dense.chunks, - "embed_secs": dense.embed_secs, - "similar_per_mate": { "n": dense.mate.n, "hit1": dense.mate.hit1(), "hit3": dense.mate.hit3(), "mrr": dense.mate.mrr() }, - "mates": dense.mate_ranks.iter().map(|(anchor, mate, rank)| serde_json::json!({ - "anchor": anchor, "mate": mate, "rank": rank, - })).collect::>(), - "empty_panes": { "n": dense.notes, "empty": dense.empty_panes.len(), "detail": dense.empty_panes }, - "discovery_fold": fold_json(&dense.fold), - // NEW key (absent from rows before 2026-08-22): the shipped search bar - // replayed on this fixture (GH #201). Deliberately NOT the orthogonal - // row's `search_evidence`: that key holds the *labelled* bake-off, and - // this is the label-free transfer reading — a different measurement, so - // it takes a different name. Same convention as every key above: new, - // never a redefinition, so no reader has to branch on `corpus` to learn - // which shape it is holding (PR #207 review). - "search_transfer": dense_search_json(&dense.search), - "similar_detail": dense.detail.iter().map(|d| serde_json::json!({ - "anchor": d.anchor, - "negative": d.negative, - "candidates": d.candidates.iter().map(|(path, cos, related)| serde_json::json!({ - "path": path, - "cos": (cos * 1e4).round() / 1e4, - "related": related, - })).collect::>(), - })).collect::>(), - }) -} - -/// [`rank_str`] at a caller-named depth (the discovery metrics read at -/// [`SIM_K`], not [`K`]). -fn rank_str_at(rank: Option, depth: usize) -> String { - match rank { - Some(1) => "✓1".to_string(), - Some(r) => format!("·{r}"), - None => format!("✗>{depth}"), - } -} - -/// Which retrieval a pass scores. `Fused` is the shipped path (`Vault::search` — -/// BM25-only before embed, hybrid after) plus chunk-level scoring for -/// passage-labelled queries; `VectorOnly` is the dense ablation -/// (`Vault::search_vector_only`, GH #158), note-level only — its chunk aggregate -/// stays empty. The ablation column is what lets a run say whether fusion paid -/// rent: the finding that RRF demotes dense rank-1 hits was established -/// by hand-decomposing fused scores once; this makes it a standing measurement. -#[derive(Clone, Copy, PartialEq)] -enum Retrieval { - Fused, - VectorOnly, -} - -/// Score every labelled query against the vault's current state: note rank via -/// the selected retrieval, and — for passage-labelled queries on the fused path — -/// chunk rank via `search_chunks` (the first top-K chunk that belongs to a -/// relevant note AND contains the labelled phrase, case-insensitively). -fn score_pass( - vault: &Vault, - queries: &[Labelled], - retrieval: Retrieval, -) -> Result> { - let mut scores = Vec::with_capacity(queries.len()); - let mut note_agg = Agg::default(); - let mut chunk_agg = Agg::default(); - for q in queries { - let results = match retrieval { - Retrieval::Fused => vault.search(&q.query, K)?, - Retrieval::VectorOnly => vault.search_vector_only(&q.query, K)?, - }; - let note = results - .iter() - .position(|r| q.relevant.iter().any(|rel| paths_match(&r.path, rel))) - .map(|p| p + 1); - let top = results - .first() - .map(|r| r.path.clone()) - .unwrap_or_else(|| "—".to_string()); - note_agg.add(note); - - let chunk = match (&q.passage, retrieval) { - (None, _) | (_, Retrieval::VectorOnly) => None, - (Some(passage), Retrieval::Fused) => { - let needle = passage.to_lowercase(); - let hits = vault.search_chunks(&q.query, K)?; - let rank = hits - .iter() - .position(|h| { - q.relevant.iter().any(|rel| paths_match(&h.path, rel)) - && h.text.to_lowercase().contains(&needle) - }) - .map(|p| p + 1); - chunk_agg.add(rank); - rank - } - }; - scores.push(QueryScore { note, chunk, top }); - } - Ok(Pass { - scores, - note: note_agg, - chunk: chunk_agg, - }) -} - -/// Score the discovery labels — **one pass, one surface** (GH #197). -/// -/// Everything reads `Vault::similar`, the always-served ranked list: the ranks, -/// the strangers, the negatives' card tally, and the cosine piles + per-anchor -/// detail the calibration blocks consume. (A second pass with a pass-vs-pass -/// suppression diff existed while the GH #150 existence gate shipped; under -/// always-serve it re-read the identical call and could never fire, so it -/// retired — GH #217, and see [`SimilarPass::mate`] for what catches a -/// returning gate instead.) -fn score_similar( - vault: &Vault, - set: &SimilarSet, -) -> Result> { - let mut pass = SimilarPass::default(); - for label in &set.anchors { - let candidates = vault.similar(&label.anchor, SIM_K)?; - let negative = label.expected.is_empty(); - if negative { - pass.neg_n += 1; - if candidates.is_empty() { - pass.neg_clean += 1; - } - pass.neg_cards += candidates.len(); - } else { - let rank = candidates - .iter() - .position(|c| label.expected.iter().any(|e| paths_match(&c.path, e))) - .map(|p| p + 1); - pass.rank.add(rank); - // …and again per labelled mate, so a hard one can't hide behind an - // easy one (GH #183 — see `SimilarPass::mate`). - for expected in &label.expected { - let mate_rank = candidates - .iter() - .position(|c| paths_match(&c.path, expected)) - .map(|p| p + 1); - pass.mate.add(mate_rank); - } - // The precision side of the same list (GH #188 — see - // `SimilarPass::strangers`): everything served here that no label - // claims. Scored on the shipped surface at the ranks' own depth, - // because that is the list a human is handed. - let before = pass.strangers.len(); - for c in &candidates { - if !label.expected.iter().any(|e| paths_match(&c.path, e)) { - pass.strangers.push((label.anchor.clone(), c.path.clone())); - } - } - if pass.strangers.len() > before { - pass.stranger_anchors += 1; - } - } - // The calibration piles + per-anchor detail, off the same served list. - let mut ordered = Vec::with_capacity(candidates.len()); - for c in &candidates { - let related = !negative && label.expected.iter().any(|e| paths_match(&c.path, e)); - let cos = cosine_of(c.score); - if related { - pass.related.push(cos); - } else { - pass.junk.push(cos); - } - ordered.push((c.path.clone(), cos, related)); - } - pass.detail.push(AnchorDetail { - anchor: label.anchor.clone(), - negative, - candidates: ordered, - }); - } - Ok(pass) -} - -// --------------------------------------------------------------------------- -// The discovery fold bake-off (GH #200, Phase B) — the *default disclosure -// boundary*, priced on every bench this harness carries. -// --------------------------------------------------------------------------- - -/// A **default disclosure rule**: a candidate answer to "how much of the ranked -/// list does the default view vouch for?" (invariants.md D1 as redrafted; -/// GH #200). Every variant is a **prefix** rule *by construction* — it returns a -/// depth, never a set — so a rule that admits rank 5 while folding rank 2 is not -/// expressible here. D1 states that admissibility requirement; this type makes -/// violating it unrepresentable, which is the difference between a check and a -/// guarantee. -#[derive(Clone, Copy, PartialEq)] -enum FoldRule { - /// **Candidate 3, the incumbent**: no fold. The default view is the whole - /// served prefix — GH #197's always-serve ruling, admissible per D1 and the - /// thing the other candidates have to beat. - NoFold, - /// **Candidate 1**: mutual-kNN reciprocity in prefix form. Candidate B is - /// *reciprocal* for anchor A iff A sits in B's own top `k` candidates, and - /// the default view is the ranked list's longest reciprocal prefix. Rank- - /// based (the hubness-correction family: mutual proximity, CSLS), so it - /// carries no cosine or z constant — though `k` itself is measured below to - /// be scale-dependent, which is a different objection and the decisive one. - Mutual(usize), -} - -impl FoldRule { - /// The rule's name in every table, row and window line — one spelling, so a - /// printed verdict and a recorded row can never name the same rule - /// differently. - fn label(self) -> String { - match self { - FoldRule::NoFold => "no fold".to_string(), - FoldRule::Mutual(k) => format!("mutual-{k}"), - } - } - - /// How many of `rows` this rule's default view vouches for. - fn fold(self, rows: &[FoldRow]) -> usize { - match self { - FoldRule::NoFold => rows.len(), - FoldRule::Mutual(k) => rows.iter().take_while(|r| r.reciprocal_at(k)).count(), - } - } -} - -/// One served candidate on one anchor's list, carrying everything a candidate -/// rule judges: the label, the score, and — the reciprocity signal — **where the -/// anchor sits in this candidate's own ranked list**. -struct FoldRow { - path: String, - cos: f64, - /// A labelled mate of this anchor (always `false` on a negative or - /// unlabelled anchor). - mate: bool, - /// The **anchor's** rank in *this candidate's* full-depth candidate list, or - /// `None` if the anchor is not in it at all. Storing the rank rather than a - /// per-`k` boolean is what makes the whole sweep below free: reciprocity at - /// depth `k` is `rank ≤ k`, so every `k` is priced off one discovery pass - /// per note instead of one per note per depth. - recip_rank: Option, -} - -impl FoldRow { - /// Whether this candidate is *reciprocal* at depth `k` — i.e. the anchor - /// sits in its own top `k`. Reciprocity is necessary but not sufficient for - /// being above the fold: the fold is the longest reciprocal **prefix**, so a - /// reciprocal candidate ranked after a non-reciprocal one is below it too. - fn reciprocal_at(&self, k: usize) -> bool { - self.recip_rank.is_some_and(|r| r <= k) - } -} - -/// One anchor's served list, read once and judged by every candidate rule — so -/// the rules are compared on identical rows rather than on separate passes that -/// could drift. -struct FoldAnchor { - anchor: String, - /// A labelled **loner** (empty `expected`): its correct default view is - /// *empty above the fold* — the assertion GH #197 retired, returning on the - /// disclosure axis. - negative: bool, - /// Labelled mates in label order. - expected: Vec, - /// The served prefix (`limit` = [`SIM_K`]) in rank order. - rows: Vec, -} - -impl FoldAnchor { - /// This anchor's labelled mates that the *served prefix* reaches, as - /// `(mate, rank)`. A mate ranked past `limit` is not among them — it is - /// always-serve's own miss at this depth, and counting it against a fold - /// would charge every rule for a cost none of them caused. - fn served_mates(&self) -> Vec<(String, usize)> { - self.expected - .iter() - .filter_map(|m| { - self.rows - .iter() - .position(|r| paths_match(&r.path, m)) - .map(|p| (m.clone(), p + 1)) - }) - .collect() - } -} - -/// What one rule reads on one bench — GH #200's judged quantities, each carrying -/// the per-anchor list it is argued from (process rule 1: the aggregate is a -/// smoke alarm, the named list is the data). -struct FoldReading { - rule: FoldRule, - /// Labelled mates the served prefix reaches but the default view does - /// **not** vouch for: `(anchor, mate, rank)`. **The fold's own cost**, and - /// the quantity GH #200 judges a candidate at zero on — a human-labelled - /// relation hidden by default. Reported rather than gated while nothing - /// folds (it reads a structural 0 under always-serve) — and GH #202 shipped - /// no fold to charge, so it stays reporting-only; it becomes an exit-gate - /// row with the first fold that does ship. - mates_folded: Vec<(String, String, usize)>, - /// Labelled mates above the fold — the other half of the same count. - mates_above: usize, - /// Unlabelled notes the default view vouches for on a *positive* anchor — - /// the strangers count, read above the fold instead of over the whole served - /// prefix. Expected to shrink; deliberately ungated for the standing reason - /// (the cheapest way to shrink it is to label the stranger). - strangers_above: Vec<(String, String)>, - /// Negative (loner) anchors whose default view is empty — the honest answer - /// their label always claimed, and what always-serve cannot assert. - neg_empty: usize, - /// Non-negative anchors whose default view is empty. On the dense fixture - /// **any** entry here is disqualifying (a vault where everything relates may - /// never *default* to "nothing relates"); on the orthogonal corpus it is the - /// same event read through the labels. - dark_panes: Vec, - /// Cards above the fold, summed — the default view's size against - /// always-serve's. - cards_above: usize, -} - -/// The **admissible-`k` window** for candidate 1, re-derived from this run -/// (GH #187's idiom, moved onto the disclosure axis: print the window a rule -/// *would* have rather than freezing a constant into a docstring). Two bounds -/// that must overlap for any `k` to be shippable on this bench: -/// -/// - `keep_min` — the smallest `k` that folds **no** labelled mate and darkens -/// **no** pane. Below it the rule hides relations a human labelled. -/// - `loner_max` — the largest `k` at which **every** labelled loner's default -/// view is empty. Above it the fold stops making the claim it exists to make, -/// and a loner's pane fills up again. -struct FoldWindow { - keep_min: Option, - loner_max: Option, - /// The smallest swept `k` whose fold equals always-serve on every anchor — - /// where the rule becomes vacuous (it vouches for everything). - vacuous_at: Option, - /// The best the loner claim ever gets on this bench, as `(empty, largest k - /// still achieving it)` — printed because "never all of them" and "four of - /// five up to k = 11" are different findings, and only the second says how - /// close the rule came. - loner_best: Option<(usize, usize)>, - /// The largest swept `k` that still darkens a pane. Every `k` at or below it - /// is disqualified outright on a corpus where everything relates (D1's - /// absolute), so this is the hard floor under any window. - dark_below: Option, -} - -impl FoldWindow { - /// Whether some swept `k` satisfies both bounds at once. A bench with no - /// labelled loner has no upper bound to satisfy and so can never report an - /// open window on its own — it contributes the lower bound to the - /// bake-off's joint window, which is what the printed verdict says. - fn open(&self) -> bool { - matches!((self.keep_min, self.loner_max), (Some(a), Some(b)) if a <= b) - } -} - -/// One bench's complete bake-off reading. -struct FoldBench { - corpus: &'static str, - anchors: Vec, - /// The headline rules: always-serve plus [`FOLD_MUTUAL_K`]'s detail depths. - readings: Vec, - /// `(k, reading)` across the whole swept range — what the window is derived - /// from, and printed as a compact table so the shape of the trade is visible - /// rather than asserted. - sweep: Vec<(usize, FoldReading)>, - window: FoldWindow, - /// Negative anchors on this bench (0 on the dense fixture, which has no - /// loner by construction). - neg_n: usize, - /// Labelled mates the served prefix never reaches at `limit` — always-serve's - /// own miss, identical under every rule. Printed beside the folded count so - /// no rule is charged for it. - mates_unserved: usize, - /// The median *full-depth* candidate pool an anchor here has — what a - /// reciprocity depth is a fraction OF. A `k` is only meaningful against it: - /// "top 7" of a 14-candidate pool and "top 7" of a 30-candidate one are - /// different claims, and the two corpora's windows below are only - /// comparable in this unit. - pool_median: usize, - /// Authored edges in the built vault — **candidate 2's** entire calibration - /// population ("what related looks like in this vault", read off the human's - /// own committed links). Both eval corpora are link-free by construction, so - /// this reads 0 and candidate 2 is *unpriceable here*: it is measured where - /// its population exists, by `make calibrate` on a real vault. - authored_edges: usize, -} - -/// Every note's own ranked candidate list as `path → rank`, read at full depth — -/// the reciprocity lookup candidate 1 is judged on. Read over **every note in -/// the vault**, not only the labelled anchors: reciprocity is decided by the -/// *candidate's* list, and most candidates are not anchors. -fn reciprocity_ranks( - vault: &Vault, -) -> Result>, Box> { - let mut out = HashMap::new(); - for note in vault.list_notes()? { - let ranks = vault - .similar(¬e.path, Z_SCAN_LIMIT)? - .into_iter() - .enumerate() - .map(|(i, c)| (c.path, i + 1)) - .collect::>(); - out.insert(note.path, ranks); - } - Ok(out) -} - -/// Score the fold bake-off on one built vault. `sweep_all` reads **every note** -/// as an anchor (the dense fixture's pane sweep, where the assertion is about -/// the surface rather than the labels); otherwise only the labelled anchors are -/// read, matching [`score_similar`]'s depth and population exactly. -fn score_fold( - vault: &Vault, - set: &SimilarSet, - corpus: &'static str, - sweep_all: bool, -) -> Result> { - let recip = reciprocity_ranks(vault)?; - - let mut anchors: Vec<(String, Option<&SimilarLabel>)> = Vec::new(); - if sweep_all { - for note in vault.list_notes()? { - let label = set - .anchors - .iter() - .find(|l| paths_match(¬e.path, &l.anchor)); - anchors.push((note.path, label)); - } - } else { - for label in &set.anchors { - anchors.push((label.anchor.clone(), Some(label))); - } - } - - // Outbound only. `neighbors` answers "what is 1 hop from here" and so returns - // an edge from *both* of its endpoints; summing the raw counts over every - // note would census edge **endpoints** and report candidate 2's population at - // twice its size (PR #204 review). Counting each edge once from its source is - // the number the rule would calibrate from. - let mut authored_edges = 0; - for note in vault.list_notes()? { - authored_edges += vault - .neighbors(¬e.path)? - .iter() - .filter(|n| n.direction == "outbound") - .count(); - } - - let mut rows_by_anchor: Vec = Vec::new(); - let mut neg_n = 0; - for (anchor, label) in anchors { - let expected: Vec = label.map(|l| l.expected.clone()).unwrap_or_default(); - // A *labelled* loner. An unlabelled anchor on the dense sweep is not a - // negative — that fixture carries no loner at all, and reading "no - // label" as "nothing relates" would invent the claim the bench exists - // to test. - let negative = label.is_some() && expected.is_empty(); - if negative { - neg_n += 1; - } - let rows = vault - .similar(&anchor, SIM_K)? - .into_iter() - .map(|c| FoldRow { - mate: expected.iter().any(|e| paths_match(&c.path, e)), - recip_rank: recip.get(&c.path).and_then(|ranks| { - ranks - .iter() - .find(|(p, _)| paths_match(p, &anchor)) - .map(|(_, r)| *r) - }), - cos: cosine_of(c.score), - path: c.path, - }) - .collect(); - rows_by_anchor.push(FoldAnchor { - anchor, - negative, - expected, - rows, - }); - } - - let mates_unserved = rows_by_anchor - .iter() - .map(|a| a.expected.len() - a.served_mates().len()) - .sum(); - let readings = std::iter::once(FoldRule::NoFold) - .chain(FOLD_MUTUAL_K.iter().map(|&k| FoldRule::Mutual(k))) - .map(|rule| read_fold(&rows_by_anchor, rule)) - .collect(); - let sweep: Vec<(usize, FoldReading)> = (1..=FOLD_K_SWEEP) - .map(|k| (k, read_fold(&rows_by_anchor, FoldRule::Mutual(k)))) - .collect(); - let window = FoldWindow { - keep_min: sweep - .iter() - .find(|(_, r)| r.mates_folded.is_empty() && r.dark_panes.is_empty()) - .map(|(k, _)| *k), - loner_max: (neg_n > 0) - .then(|| { - sweep - .iter() - .filter(|(_, r)| r.neg_empty == neg_n) - .map(|(k, _)| *k) - .next_back() - }) - .flatten(), - vacuous_at: sweep - .iter() - .find(|(_, r)| { - rows_by_anchor - .iter() - .all(|a| r.rule.fold(&a.rows) == a.rows.len()) - }) - .map(|(k, _)| *k), - loner_best: (neg_n > 0) - .then(|| { - let best = sweep.iter().map(|(_, r)| r.neg_empty).max()?; - let last = sweep - .iter() - .filter(|(_, r)| r.neg_empty == best) - .map(|(k, _)| *k) - .next_back()?; - Some((best, last)) - }) - .flatten(), - dark_below: sweep - .iter() - .filter(|(_, r)| !r.dark_panes.is_empty()) - .map(|(k, _)| *k) - .next_back(), - }; - - let pool_median = { - let mut sizes: Vec = recip.values().map(|r| r.len()).collect(); - sizes.sort_unstable(); - sizes.get(sizes.len() / 2).copied().unwrap_or(0) - }; - - Ok(FoldBench { - corpus, - pool_median, - anchors: rows_by_anchor, - readings, - sweep, - window, - neg_n, - mates_unserved, - authored_edges, - }) -} - -/// Judge one rule against one bench's rows. -fn read_fold(anchors: &[FoldAnchor], rule: FoldRule) -> FoldReading { - let mut reading = FoldReading { - rule, - mates_folded: Vec::new(), - mates_above: 0, - strangers_above: Vec::new(), - neg_empty: 0, - dark_panes: Vec::new(), - cards_above: 0, - }; - for a in anchors { - let fold = rule.fold(&a.rows); - reading.cards_above += fold; - if fold == 0 { - if a.negative { - reading.neg_empty += 1; - } else if !a.rows.is_empty() { - reading.dark_panes.push(a.anchor.clone()); - } - } - for (mate, rank) in a.served_mates() { - if rank <= fold { - reading.mates_above += 1; - } else { - reading.mates_folded.push((a.anchor.clone(), mate, rank)); - } - } - // The precision side, read above the fold: unlabelled cards the default - // view vouches for on a positive anchor. - if !a.negative && !a.expected.is_empty() { - for row in a.rows.iter().take(fold) { - if !row.mate { - reading - .strangers_above - .push((a.anchor.clone(), row.path.clone())); - } - } - } - } - reading -} - -/// The bake-off's printed readout: the headline rules side by side, the swept -/// `k` the window is derived from, and the per-anchor folds — which is the list -/// the verdict is argued from (process rule 1). -fn print_fold_bench(bench: &FoldBench) { - println!("\n{}", "=".repeat(78)); - println!( - "discovery fold bake-off — the default disclosure boundary on the {} corpus \ - (GH #200, Phase B; invariants.md D1)", - bench.corpus - ); - println!( - " {} anchors read at limit={SIM_K} over a median {}-candidate pool; a fold is a PREFIX of \ - the served order and everything below it stays served (D1)", - bench.anchors.len(), - bench.pool_median - ); - println!( - "\n {:<10} {:>11} {:>12} {:>15} {:>12} {:>10}", - "rule", "cards above", "mates folded", "strangers above", "loners empty", "dark panes" - ); - for r in &bench.readings { - println!( - " {:<10} {:>11} {:>12} {:>15} {:>12} {:>10}", - r.rule.label(), - r.cards_above, - r.mates_folded.len(), - r.strangers_above.len(), - format!("{}/{}", r.neg_empty, bench.neg_n), - r.dark_panes.len(), - ); - } - println!( - " (mates folded: served within limit={SIM_K} but below the fold — the fold's OWN cost, and \ - the quantity GH #200 judges a candidate at 0 on. Reported, not gated: nothing folds today, \ - and GH #202 shipped no fold either, so the exit gate has nothing here to watch until one \ - does. {} further labelled mate(s) \ - rank past limit={SIM_K} under every rule, always-serve included, so no fold is charged \ - for them.)", - bench.mates_unserved - ); - - // The named cost of every rule that has one — the list, not the count. - for r in &bench.readings { - if r.mates_folded.is_empty() { - continue; - } - println!(" {} folds labelled mates:", r.rule.label()); - for (anchor, mate, rank) in &r.mates_folded { - println!(" rank {rank} {anchor} → {mate}"); - } - } - - // The swept window — the #187 idiom on the disclosure axis: derive the - // rule's admissible range from THIS run rather than quote a constant. - println!( - "\n mutual-k sweep (k = 1..{FOLD_K_SWEEP}; the window a shippable k would have to sit in)" - ); - println!( - " {:>3} {:>11} {:>12} {:>15} {:>12} {:>10}", - "k", "cards above", "mates folded", "strangers above", "loners empty", "dark panes" - ); - for (k, r) in &bench.sweep { - println!( - " {k:>3} {:>11} {:>12} {:>15} {:>12} {:>10}", - r.cards_above, - r.mates_folded.len(), - r.strangers_above.len(), - format!("{}/{}", r.neg_empty, bench.neg_n), - r.dark_panes.len(), - ); - } - let frac = |k: usize| { - if bench.pool_median == 0 { - String::new() - } else { - format!( - " (= {:.2} of the {}-candidate pool)", - k as f64 / bench.pool_median as f64, - bench.pool_median - ) - } - }; - match bench.window.keep_min { - Some(k) => println!( - " k ≥ {k}{} folds no labelled mate and darkens no pane on this corpus", - frac(k) - ), - None => println!(" (no swept k folds zero mates with zero dark panes on this corpus)"), - } - match bench.window.loner_max { - Some(k) => println!( - " k ≤ {k}{} folds every labelled loner's default view to empty", - frac(k) - ), - None if bench.neg_n == 0 => { - println!( - " (no labelled loner on this corpus — the upper bound is the other bench's)" - ) - } - None => println!(" (no swept k empties every loner's default view on this corpus)"), - } - if let Some(k) = bench.window.vacuous_at { - println!( - " k ≥ {k} the fold equals always-serve on every anchor here — the rule stops \ - claiming anything" - ); - } - if let Some(k) = bench.window.dark_below { - println!( - " k ≤ {k} darkens at least one pane here — disqualified outright where the labels \ - say everything relates (D1's absolute)" - ); - } - if let Some((best, last)) = bench.window.loner_best { - println!( - " best loner claim: {best}/{} empty, holding to k = {last}", - bench.neg_n - ); - } - // The verdict names *which* bound failed. "No k works" and "this corpus - // cannot decide alone" are different sentences, and a bench with no loner - // can only ever supply the lower bound. - println!( - " → window {}", - match (bench.neg_n, bench.window.keep_min, bench.window.loner_max) { - (0, Some(k), _) => format!( - "UNDECIDABLE on this corpus alone — no loner here, so it supplies only the lower \ - bound (k ≥ {k}) and the absolute above; the upper bound is the other bench's" - ), - (0, None, _) => "UNDECIDABLE on this corpus alone — no loner here, and no swept k is \ - even clean on the labels" - .to_string(), - (_, _, None) => format!( - "EMPTY — no swept k empties every loner's default view, so the fold never fully \ - makes the claim it exists to make ({})", - bench - .window - .keep_min - .map(|k| format!("its lower bound here is k ≥ {k}")) - .unwrap_or_else(|| "and no swept k is clean on the labels either".into()) - ), - (_, Some(a), Some(b)) if a <= b => format!( - "OPEN on this corpus at k ∈ [{a}, {b}] — the other benches are the rest \ - of the claim" - ), - (_, Some(a), Some(b)) => format!( - "EMPTY — the k that stops folding labelled mates (≥ {a}) is past the k that still \ - empties every loner's view (≤ {b}), so no constant separates them" - ), - (_, None, Some(b)) => format!( - "EMPTY — no swept k is clean on the labels at all, while the loner claim holds \ - only to k ≤ {b}" - ), - } - ); - - // The per-anchor lists at the headline depths. - println!("\n per anchor — served / above the fold, by rule:"); - print!(" {:<40} {:>7}", "anchor", "served"); - for r in &bench.readings { - if r.rule != FoldRule::NoFold { - print!(" {:>9}", r.rule.label()); - } - } - println!(); - for a in &bench.anchors { - print!( - " {:<40} {:>7}", - format!( - "{}{}", - truncate(&a.anchor, 36), - if a.negative { " [loner]" } else { "" } - ), - a.rows.len() - ); - for r in &bench.readings { - if r.rule != FoldRule::NoFold { - let fold = r.rule.fold(&a.rows); - let lost = a - .served_mates() - .iter() - .filter(|(_, rank)| *rank > fold) - .count(); - print!( - " {:>9}", - if lost > 0 { - format!("{fold}(-{lost})") - } else { - fold.to_string() - } - ); - } - } - println!(); - } - - // Candidate 2's census — printed on every bench, because "we did not measure - // it" and "there was nothing to measure it on" are different sentences and - // only the second is an argument. - println!( - "\n candidate 2 (authored-edge reference bar): {} authored edges in this corpus — {}", - bench.authored_edges, - if bench.authored_edges == 0 { - "UNPRICEABLE here (a link-free corpus offers the rule no population to calibrate from; \ - it is measured where one exists, by `make calibrate` on a real vault)" - } else { - "priceable — see the calibrate replay" - } - ); -} - -/// The bake-off as one JSON subtree (`discovery_fold`): every rule's reading, -/// the swept window it was judged against, and the per-anchor rows both were -/// computed from — so a verdict cited from a row can be re-derived without -/// re-running the model, which is what `results.jsonl` is for. -fn fold_json(bench: &FoldBench) -> serde_json::Value { - let reading = |r: &FoldReading| { - serde_json::json!({ - "rule": r.rule.label(), - "cards_above": r.cards_above, - "mates_above": r.mates_above, - "mates_folded": r.mates_folded.iter().map(|(a, m, rank)| serde_json::json!({ - "anchor": a, "mate": m, "rank": rank, - })).collect::>(), - "strangers_above": r.strangers_above.iter().map(|(a, p)| serde_json::json!({ - "anchor": a, "path": p, - })).collect::>(), - "neg_empty": r.neg_empty, - "dark_panes": r.dark_panes, - }) - }; - serde_json::json!({ - "corpus": bench.corpus, - "limit": SIM_K, - "pool_median": bench.pool_median, - "neg_n": bench.neg_n, - // Always-serve's own miss at this limit, recorded apart from every - // rule's cost so the two can never be added together by a later reader. - "mates_unserved": bench.mates_unserved, - // Candidate 2's population, recorded even at zero: "unpriceable here" - // is a measurement, and a row that omitted it would read as untried. - "authored_edges": bench.authored_edges, - "rules": bench.readings.iter().map(reading).collect::>(), - "sweep": bench.sweep.iter().map(|(k, r)| serde_json::json!({ - "k": k, - "reading": reading(r), - })).collect::>(), - "window": { - "swept_to": FOLD_K_SWEEP, - "keep_min": bench.window.keep_min, - "loner_max": bench.window.loner_max, - "vacuous_at": bench.window.vacuous_at, - "dark_below": bench.window.dark_below, - "loner_best": bench.window.loner_best.map(|(empty, k)| serde_json::json!({ - "empty": empty, "of": bench.neg_n, "holds_to_k": k, - })), - "open": bench.window.open(), - }, - "anchors": bench.anchors.iter().map(|a| serde_json::json!({ - "anchor": a.anchor, - "negative": a.negative, - "served": a.rows.len(), - "folds": bench.readings.iter().map(|r| serde_json::json!({ - "rule": r.rule.label(), - "fold": r.rule.fold(&a.rows), - })).collect::>(), - "rows": a.rows.iter().map(|row| serde_json::json!({ - "path": row.path, - "cos": (row.cos * 1e4).round() / 1e4, - "mate": row.mate, - // The anchor's rank in THIS candidate's own list — the whole - // reciprocity signal, from which any k is re-derivable. - "recip_rank": row.recip_rank, - })).collect::>(), - })).collect::>(), - }) -} - -/// Dump every candidate's **band-unit z** (stage-2 best-passage z, GH #192) on -/// every discovery anchor — the deep discovery pass (GH #187). -/// -/// Since GH #197 the z travels ungated on the one shipped surface, so the dump -/// is simply `similar` read at [`Z_SCAN_LIMIT`] rather than `SIM_K`: a bar is -/// calibrated against the population it has to cut, and the strangers just -/// under a served prefix are precisely the ones a lower bar would admit. (This -/// used to need the engine's floor moved out of the way with `-∞` bars; the -/// machinery went with the gate.) -fn score_floor_z(vault: &Vault, set: &SimilarSet) -> Result> { - let mut dump = FloorZ::default(); - for label in &set.anchors { - let candidates = vault.similar(&label.anchor, Z_SCAN_LIMIT)?; - // z is uniform within one query by construction (`discover::candidates` - // gives every surviving candidate a z or none of them), so the leader's - // presence decides the whole list — an anchor with no statistics is - // named, never half-counted. - let Some(true) = candidates.first().map(|c| c.z.is_some()) else { - dump.ungraded.push(label.anchor.clone()); - continue; - }; - // The band z, recomputed harness-side over the same population the - // engine just served: z over squared best-pair distance (score is - // negated L2, so d² = score²), oriented nearer = higher. Should equal - // the engine's own z to fp noise — the statistic-level drift check. - let d2: Vec = candidates.iter().map(|c| c.score * c.score).collect(); - let recheck = passage_z(&d2); - dump.anchors.push(AnchorZ { - anchor: label.anchor.clone(), - negative: label.expected.is_empty(), - candidates: candidates - .iter() - .enumerate() - .filter_map(|(i, c)| { - let mate = label.expected.iter().any(|e| paths_match(&c.path, e)); - c.z.map(|z| ZCand { - path: c.path.clone(), - z, - z_recheck: recheck.as_ref().map(|r| r[i]), - score: c.score, - mate, - }) - }) - .collect(), - }); - } - Ok(dump) -} - -/// One labelled query's evidence reading (invariants.md D2; GH #201, Phase A of -/// the disclosure work): the signals a query-level evidence rule would judge, -/// dumped before any rule exists so GH #201 derives its rule from measurement -/// rather than assumption. Nothing here gates anything. -/// One served row of a labelled query's list (GH #206): the per-hit provenance -/// RRF discards, and the row's relevance **by label** — the two things a per-hit -/// tail rule is judged between. Rank is the row's position in `rows`, never a -/// stored field: the fused order is the identity D1's prefix requirement binds. -struct ServedRow { - path: String, - /// 0-based rank in the BM25 list; `None` = the lexical half never ranked - /// this chunk — the "dense-only" row. - bm25_rank: Option, - /// This row's own cosine to the query; `None` when the dense half never - /// ranked it (a row is in at least one list, so `bm25_rank` and `cos` are - /// never both absent). - cos: Option, - /// In the keep-set — `relevant` ∪ `tail_relevant`, exhaustive by label - /// since GH #206: a false here is a judgement ("filler"), not an omission. - keep: bool, -} - -struct QueryEvidence { - query: String, - /// Chunks the OR-sanitized FTS5 expression matches at all. **Not** a - /// lexical-anchor test on phrase queries — `fts5_query` ORs every - /// alphanumeric term, so stopwords saturate this count; recorded precisely - /// so that saturation stays measured instead of assumed away. - bm25_hits: usize, - /// Best BM25 score over those matches, sign-flipped so higher = better - /// (FTS5's `rank` is more-negative-is-better). `None` when nothing matches - /// — the honest zero the fused surface currently cannot say. - bm25_best: Option, - /// Best cosine between the embedded query and any stored chunk vector (the - /// dense top-1) — the strongest semantic evidence the vault holds for this - /// query, which RRF discards before the surface sees it. - best_cos: Option, - /// The note that dense top-1 belongs to, naming what the number points at. - top: String, - /// The served list at [`K`], in fused order, one entry per row — path, - /// per-hit provenance, and the row's relevance **by label** (GH #206). For a - /// negative query every row's `keep` is false: the whole list is junk by - /// label. `served`/`dense_only` below read this, so the counts and the rows - /// they summarize cannot drift apart. - rows: Vec, - /// Chunks in the index — the denominator every `df` below is judged against. - chunk_total: usize, - /// Every query term with its document frequency, in query order. The - /// population the lexical-anchor rule is swept over: raw hit count and raw - /// best-BM25 both failed to separate the piles (Phase A), so the anchor is - /// derived from *these* rather than from either of those. - terms: Vec<(String, usize)>, - /// **The engine's own verdict** for this query — `Vault::search_evidence`'s `vouched`, - /// i.e. exactly what the surfaces act on (GH #202). The exit gate counts on *this*, never - /// on the harness's restatement below: an assertion about what ships must read what - /// ships. `None` when the model has no calibrated bar. - /// - /// The restatement is kept beside it because the sweep needs it — [`bake_off`] evaluates - /// the rule at coverages the engine cannot be asked about — and because two independent - /// readings of one rule are a drift check when compared. So they are: [`read_shipped_bar`] - /// prints a `[FAULT]` on any query where they disagree. - vouched: Option, -} - -impl QueryEvidence { - /// What the shipped surface serves at [`K`] today. For a negative query - /// this is the measured D2 defect: `limit` confident-looking results for a - /// query the vault holds nothing for. - fn served(&self) -> usize { - self.rows.len() - } - - /// Of the served rows, how many the **lexical half never ranked at all** - /// (`bm25_rank: None`) — the per-hit shape of the same defect, and the - /// signal the `lexical` tail rule folds on (GH #206). - fn dense_only(&self) -> usize { - self.rows.iter().filter(|r| r.bm25_rank.is_none()).count() - } - - /// The share of this query's term IDF the vault carries — the harness's own - /// restatement of [`b2_core::search::LexicalEvidence::term_coverage`], kept - /// separate so a drift in the engine's rule shows up as the two disagreeing - /// rather than as silence. `None` when nothing in the query carries weight. - fn coverage(&self) -> Option { - let idf = |df: usize| ((self.chunk_total as f64 + 1.0) / (df as f64 + 1.0)).ln(); - let total: f64 = self.terms.iter().map(|(_, df)| idf(*df)).sum(); - if total <= f64::EPSILON { - return None; - } - let present: f64 = self - .terms - .iter() - .filter(|(_, df)| *df >= 1) - .map(|(_, df)| idf(*df)) - .fold(0.0, |a, b| a + b); - Some(present / total) - } - - /// Whether this query has a lexical anchor at `min_coverage`. - fn anchored(&self, min_coverage: f64) -> bool { - self.coverage().is_some_and(|c| c >= min_coverage) - } -} - -/// The `min_term_coverage` grid the bake-off sweeps: how much of a query's own -/// weight the vault must carry for the lexical half to vouch for it. Spans "a -/// twentieth" to "all of it", so the printed rows show the rule's whole -/// behaviour rather than a neighbourhood of the shipped constant. -const COVERAGES: [f64; 10] = [0.05, 0.10, 0.15, 0.20, 0.25, 0.34, 0.50, 0.67, 0.85, 1.00]; - -/// One coverage-bar cell of the bake-off: how the lexical half alone splits the -/// labelled piles there, and what the cosine half would then have to do for the -/// queries it leaves undecided. -struct EvidenceCell { - coverage: f64, - /// Positives the lexical half already vouches for. - pos_anchored: usize, - /// Negatives the lexical half wrongly vouches for. **Any nonzero value - /// disqualifies the cell**: an anchored negative is served whatever the - /// cosine bar says, so no `min_cos` can rescue it. - neg_anchored: usize, - /// Cosines of the positives the lexical half left undecided — the pile a - /// `min_cos` must KEEP. - undecided_pos: Vec, - /// Cosines of the negatives left undecided — the pile it must CUT. - undecided_neg: Vec, -} - -impl EvidenceCell { - /// The cosine window this cell leaves for `min_cos`, over **only** the - /// queries the lexical half did not already decide — which is the whole - /// point of reading it here rather than over every query: D2's rule is - /// lexical OR semantic, so a positive with an anchor never needs its cosine - /// kept, and a pure-cosine window (Phase A's) overstates the keep set. - fn window(&self) -> Option { - Window::read(&self.undecided_neg, &self.undecided_pos) - } - - /// Whether some `min_cos` completes this cell into a rule that keeps every - /// positive and cuts every negative. - /// - /// Three ways to be admissible, and the two degenerate ones are real - /// readings rather than edge-case bookkeeping: with no undecided negatives - /// the cosine half is inert (the lexical rule did the whole job), and with - /// no undecided positives any bar above the negatives' best cosine works. - fn admissible(&self) -> bool { - if self.neg_anchored > 0 { - return false; - } - match self.window() { - Some(w) => w.open(), - None => true, - } - } - - /// The lowest `min_cos` that cuts every undecided negative — the bar's - /// measured floor. `None` when nothing is left to cut. - fn cut_floor(&self) -> Option { - let max = self - .undecided_neg - .iter() - .copied() - .fold(f64::NEG_INFINITY, f64::max); - max.is_finite().then_some(max) - } - - /// The highest `min_cos` that still keeps every undecided positive — the - /// bar's measured ceiling. `None` when nothing is left to keep. - fn keep_ceiling(&self) -> Option { - let min = self - .undecided_pos - .iter() - .copied() - .fold(f64::INFINITY, f64::min); - min.is_finite().then_some(min) - } -} - -/// Sweep the lexical rule over [`COVERAGES`], reading each cell's piles from the -/// labelled queries (GH #201, Phase C). -fn bake_off(ev: &SearchEvidence) -> Vec { - let mut cells = Vec::new(); - for coverage in COVERAGES { - let split = |rows: &[QueryEvidence]| -> (usize, Vec) { - let anchored = rows.iter().filter(|r| r.anchored(coverage)).count(); - let undecided = rows - .iter() - .filter(|r| !r.anchored(coverage)) - .filter_map(|r| r.best_cos) - .collect(); - (anchored, undecided) - }; - let (pos_anchored, undecided_pos) = split(&ev.positives); - let (neg_anchored, undecided_neg) = split(&ev.negatives); - cells.push(EvidenceCell { - coverage, - pos_anchored, - neg_anchored, - undecided_pos, - undecided_neg, - }); - } - cells -} - -/// The search evidence dump (GH #201): every labelled query's reading, split -/// into the piles a query-level evidence bar answers to — positives it must -/// keep reachable, negatives it must answer "no matches". -struct SearchEvidence { - positives: Vec, - negatives: Vec, -} - -/// One side's best-cos pile — the population the query-window derivation and -/// the printed pile lines both read (queries with no dense reading drop out). -fn best_cos_pile(rows: &[QueryEvidence]) -> Vec { - rows.iter().filter_map(|r| r.best_cos).collect() -} - -/// Score the search evidence calibration (invariants.md D2; GH #201) — a pure -/// read over the already-built vault. Per labelled query: the lexical half's -/// reading via a direct `chunks_fts` probe (the engine's own sanitized -/// expression — the absolute signals RRF discards), the dense top-1 via the -/// vector-only ablation path, and the shipped surface's served count. -fn score_search_evidence( - vault_root: &Path, - vault: &Vault, - positives: &[Labelled], - negatives: &[Labelled], -) -> Result> { - // A second read connection beside the Vault's own — C1: readers are - // unrestricted, and the probe wants FTS5's `rank`, which no façade read - // exposes (deliberately: bm25 units are engine internals everywhere but - // this instrument). - let conn = b2_core::open(&vault_root.join(".b2").join("b2.sqlite"))?; - let rows = |queries: &[Labelled]| -> Result, Box> { - queries - .iter() - .map(|q| { - let (bm25_hits, bm25_best) = bm25_probe(&conn, &q.query)?; - let dense = vault.search_vector_only(&q.query, 1)?; - let best_cos = dense.first().map(|h| cosine_of(h.score)); - let top = dense - .first() - .map(|h| h.path.clone()) - .unwrap_or_else(|| "—".to_string()); - // The façade's own evidence read (GH #201) — the term dfs, the - // per-hit provenance, and the served list in one call, so the - // sweep judges exactly what the engine would. - let view = vault.search_evidence(&q.query, K)?; - Ok(QueryEvidence { - query: q.query.clone(), - bm25_hits, - bm25_best, - best_cos, - top, - rows: view - .results - .iter() - .map(|r| ServedRow { - path: r.result.path.clone(), - bm25_rank: r.bm25_rank, - cos: r.cos, - keep: q - .relevant - .iter() - .chain(q.tail_relevant.iter()) - .any(|rel| paths_match(&r.result.path, rel)), - }) - .collect(), - chunk_total: view.chunk_total, - terms: view.terms.iter().map(|t| (t.term.clone(), t.df)).collect(), - vouched: view.vouched, - }) - }) - .collect() - }; - Ok(SearchEvidence { - positives: rows(positives)?, - negatives: rows(negatives)?, - }) -} - -/// The lexical half's reading for one raw query, probed directly over -/// `chunks_fts` with the engine's own sanitized expression -/// ([`b2_core::search::fts5_query`]): how many chunks match at all, and the -/// best BM25 score among them (FTS5 `rank`, sign-flipped so higher = better). -/// These are exactly the absolute signals the fused path computes and then -/// discards at RRF — which is why the instrument reads them raw. -fn bm25_probe( - conn: &rusqlite::Connection, - query: &str, -) -> Result<(usize, Option), Box> { - let expr = b2_core::search::fts5_query(query); - if expr.is_empty() { - return Ok((0, None)); - } - let hits: i64 = conn.query_row( - "SELECT count(*) FROM chunks_fts WHERE chunks_fts MATCH ?1", - [&expr], - |r| r.get(0), - )?; - let hits = usize::try_from(hits).unwrap_or(0); - if hits == 0 { - return Ok((0, None)); - } - let best: f64 = conn.query_row( - "SELECT rank FROM chunks_fts WHERE chunks_fts MATCH ?1 ORDER BY rank LIMIT 1", - [&expr], - |r| r.get(0), - )?; - Ok((hits, Some(-best))) -} - -/// Print the search evidence calibration (invariants.md D2; GH #201) — the -/// query-side sibling of [`print_floor_windows`]. Reported, nothing gates: the -/// piles are what GH #201's query-level bar is argued from, and any constant -/// read off them owes process rule 5's real-vault transfer check before it -/// ships. The window's caveat is structural: D2's rule is lexical OR semantic -/// evidence, so a *pure-cosine* window overstates what a real bar must keep — -/// a positive query with a lexical anchor never needs its cosine kept. -fn print_search_evidence(ev: &SearchEvidence) { - println!( - " search evidence calibration (D2 — reported, nothing gates; the query bar is GH #201's \ - to earn)" - ); - let pos_cos = best_cos_pile(&ev.positives); - let neg_cos = best_cos_pile(&ev.negatives); - for (label, pile, role) in [ - ( - "pos best-cos", - &pos_cos, - "a pure-cosine bar would have to KEEP", - ), - ("neg best-cos", &neg_cos, "any query bar would have to CUT"), - ] { - match pile_stats(pile) { - Some((min, med, max)) => println!( - " {label:<12} n={:<4} min/med/max {min:.3}/{med:.3}/{max:.3} ← {role}", - pile.len() - ), - None => println!(" {label:<12} n=0 (nothing labelled — no reading)"), - } - } - let hits: Vec = ev.positives.iter().map(|r| r.bm25_hits as f64).collect(); - if let Some((min, med, max)) = pile_stats(&hits) { - println!( - " pos bm25 hits min/med/max {min:.0}/{med:.0}/{max:.0} (OR-saturated: stopwords \ - match too, so a raw count is not a lexical anchor — GH #201 derives one from this dump)" - ); - } - match Window::read(&neg_cos, &pos_cos) { - None => println!( - " cos window no reading (a pile is empty — label negative queries to give GH #201 \ - its CUT side)" - ), - Some(w) if w.open() => println!( - " cos window ({:.3}, {:.3}] — open on THIS corpus for a pure-cosine query bar; \ - the real keep-set is smaller (lexical evidence keeps its own), and a real vault is the \ - other half of any such claim (process rule 5, `make calibrate`)", - w.cut_max, w.keep_min - ), - Some(w) => println!( - " cos window EMPTY for a pure-cosine bar — negatives reach {:.3} while positives \ - start at {:.3}; D2's two-signal rule (lexical OR semantic) is argued from the per-query \ - lines, not this window", - w.cut_max, w.keep_min - ), - } - if ev.negatives.is_empty() { - println!( - " negatives n=0 (none labelled — queries.json takes empty `relevant` as \ - \"no matches\")" - ); - } else { - println!( - " negatives (labelled answer: NO MATCHES — what the shipped surface serves instead):" - ); - for r in &ev.negatives { - println!( - " {:<40} bm25 {:>3} hit{} best {:>6} cos {:>6} served {:>2}/{K} top {}", - truncate(&r.query, 40), - r.bm25_hits, - if r.bm25_hits == 1 { " " } else { "s" }, - r.bm25_best - .map(|b| format!("{b:.2}")) - .unwrap_or_else(|| "—".to_string()), - r.best_cos - .map(|c| format!("{c:.3}")) - .unwrap_or_else(|| "—".to_string()), - r.served(), - r.top, - ); - } - } -} - -/// Print the **search evidence bake-off** (ADR-0015, GH #201) — the query-level rule's -/// window, re-derived from the labelled piles on every run rather than quoted from the day it -/// was read. The GH #187 idiom on search's side: the constant lives in -/// [`b2_core::search::BGE_BASE_EVIDENCE_BAR`] and its *justification* is recomputed here, -/// including whether the shipped bar still sits inside the window it was read from. -fn print_search_bakeoff(ev: &SearchEvidence, cells: &[EvidenceCell], model_id: &str) { - println!( - " search evidence bake-off (D2 — the query-level rule, re-derived every run; GH #201)" - ); - for line in [ - "the rule: serve iff the query has a LEXICAL ANCHOR (the vault carries ≥ coverage of the", - " query's term IDF — a word in most chunks weighs ~0, a word in none weighs the", - " most) OR its dense top-1 clears a cosine bar. Two signals on purpose: a", - " one-signal test cannot tell \"nothing matches\" from \"everything matches\" (#196).", - "`cos need` reads over ONLY the queries the lexical half left undecided — a positive with", - " an anchor never needs its cosine kept, so a pure-cosine window overstates.", - ] { - println!(" {line}"); - } - println!( - " n = {} positives / {} negatives labelled", - ev.positives.len(), - ev.negatives.len() - ); - println!( - " {:>5} {:>9} {:>9} {:>15} verdict", - "cov", "pos anch", "neg anch", "cos need" - ); - for cell in cells { - let need = match (cell.cut_floor(), cell.keep_ceiling()) { - (Some(cut), Some(keep)) => format!("({cut:.3},{keep:.3}]"), - (Some(cut), None) => format!("> {cut:.3}"), - (None, Some(keep)) => format!("≤ {keep:.3} (inert)"), - (None, None) => "inert".to_string(), - }; - let verdict = if cell.neg_anchored > 0 { - format!( - "✗ {} negative{} anchored — no cosine bar can rescue them", - cell.neg_anchored, - if cell.neg_anchored == 1 { "" } else { "s" } - ) - } else if cell.admissible() { - "✓ admissible".to_string() - } else { - "✗ cosine window empty".to_string() - }; - println!( - " {:>5.2} {:>4}/{:<4} {:>4}/{:<4} {:>15} {}", - cell.coverage, - cell.pos_anchored, - ev.positives.len(), - cell.neg_anchored, - ev.negatives.len(), - need, - verdict, - ); - } - let admissible: Vec<&EvidenceCell> = cells.iter().filter(|c| c.admissible()).collect(); - if admissible.is_empty() { - for line in [ - "→ NO admissible cell on this corpus: every lexical rule either vouches for a", - " labelled negative or leaves an empty cosine window. D2's bar is not earned,", - " and no rule ships (the GH #200 outcome, on search's side).", - ] { - println!(" {line}"); - } - } else { - println!( - " → {} of {} cells admissible; the widest cosine window belongs to {}", - admissible.len(), - cells.len(), - widest(&admissible), - ); - } - print_shipped_bar(ev, model_id); - let dense_only = |rows: &[QueryEvidence]| { - ( - rows.iter().map(|r| r.dense_only()).sum::(), - rows.iter().map(|r| r.served()).sum::(), - ) - }; - let (pos_only, pos_served) = dense_only(&ev.positives); - let (neg_only, neg_served) = dense_only(&ev.negatives); - println!(" tail reading: served rows the lexical half never ranked —"); - println!( - " positives {pos_only}/{pos_served}, negatives {neg_only}/{neg_served}. The per-hit \ - rules over these are" - ); - println!(" the tail bake-off's, below (GH #206)."); -} - -/// Name the admissible cells with the most cosine headroom, and within that band -/// the **most conservative corner** — the tightest `df` fraction and the -/// strictest coverage that still buy the widest window. -/// -/// The tie matters more than the maximum does, and printing only a winner would -/// hide it: the grid's cells are not distinct rules but a plateau, and the -/// reading to carry into process rule 5's transfer check is "anywhere in this -/// band", not "at this point". A constant placed at a lone maximum would be -/// fitted to the grid's resolution. -fn widest(cells: &[&EvidenceCell]) -> String { - let best = cells - .iter() - .map(|c| headroom(c)) - .fold(f64::NEG_INFINITY, f64::max); - if !best.is_finite() { - let inert = cells.iter().filter(|c| !headroom(c).is_finite()).count(); - return format!( - "{inert} cell(s) where the cosine half is INERT — the lexical rule alone decides \ - every labelled query, so no `min_cos` is constrained there" - ); - } - let band: Vec<&&EvidenceCell> = cells - .iter() - .filter(|c| (headroom(c) - best).abs() < 1e-9) - .collect(); - match band.iter().max_by(|a, b| { - a.coverage - .partial_cmp(&b.coverage) - .unwrap_or(std::cmp::Ordering::Equal) - }) { - None => "—".to_string(), - Some(c) => format!( - "a plateau of {} cells at headroom {best:.3}; its strictest is coverage ≥ {:.2}", - band.len(), - c.coverage, - ), - } -} - -/// A cell's cosine headroom: how much room a `min_cos` has between the -/// negatives it must cut and the positives it must keep. An inert cosine half -/// (nothing left undecided on one side) has unbounded room and is reported as -/// such rather than scored against bounded cells. -fn headroom(cell: &EvidenceCell) -> f64 { - match (cell.cut_floor(), cell.keep_ceiling()) { - (Some(cut), Some(keep)) => keep - cut, - _ => f64::INFINITY, - } -} - -/// Where the **shipped** constant stands against this run's piles: the tripwire (a labelled -/// positive the bar would cut — ADR-0015 asserts zero, no headroom) and the defect it exists -/// to fix (a labelled negative it still serves). -/// -/// **In the exit gate** since GH #202, at [`MAX_POSITIVES_CUT`] and [`MAX_NEGATIVES_SERVED`]. -/// Read here rather than in the printer, so the number asserted and the number explained are -/// one number. `None` when the active model has no calibrated bar. -struct ShippedBar { - bar: EvidenceBar, - /// Labelled positives the bar cuts — the tripwire's direction. - pos_cut: usize, - /// Labelled negatives it still serves — the defect's direction. - neg_served: usize, - /// Queries where the engine's verdict and the harness's independent - /// restatement of the same rule **disagree** — a drift between the two, which - /// is silence unless something looks. Expected empty; printed, not gated, - /// because it accuses the instrument as readily as the engine and the reader - /// has to say which. - faults: Vec, -} - -fn read_shipped_bar(ev: &SearchEvidence, model_id: &str) -> Option { - let bar = EvidenceBar::for_model(model_id)?; - // The ENGINE's verdict, not a restatement of it: this is what the gate - // asserts and what the surfaces act on. `None` cannot occur here — the model - // has a bar or this function returned already — so it is counted as served, - // the same direction the surfaces take it (M2: no verdict is not "no match"). - Some(ShippedBar { - bar, - pos_cut: ev - .positives - .iter() - .filter(|r| r.vouched == Some(false)) - .count(), - neg_served: ev - .negatives - .iter() - .filter(|r| r.vouched != Some(false)) - .count(), - faults: ev - .positives - .iter() - .chain(ev.negatives.iter()) - .filter(|r| { - let restated = r.anchored(bar.min_term_coverage) - || r.best_cos.is_some_and(|c| c >= bar.min_cos); - r.vouched != Some(restated) - }) - .map(|r| r.query.clone()) - .collect(), - }) -} - -fn print_shipped_bar(ev: &SearchEvidence, model_id: &str) { - let Some(ShippedBar { - bar, - pos_cut, - neg_served, - faults, - }) = read_shipped_bar(ev, model_id) - else { - println!(" shipped bar: none for this model — no verdict is offered (M2)"); - return; - }; - // Every line below reads the ENGINE's verdict, so the numbers printed are the - // numbers gated — one reading, not two that can drift apart. - let vouches = |r: &QueryEvidence| r.vouched != Some(false); - println!( - " shipped bar: coverage ≥ {:.2}, cos ≥ {:.3}", - bar.min_term_coverage, bar.min_cos - ); - println!( - " positives it would cut {pos_cut}/{} ← the search-side TRIPWIRE (D2: zero, no \ - headroom; GATED at ≤ {MAX_POSITIVES_CUT})", - ev.positives.len() - ); - println!( - " negatives it still serves {neg_served}/{} ← the defect the bar exists to fix \ - (GATED at ≤ {MAX_NEGATIVES_SERVED})", - ev.negatives.len() - ); - for r in ev.positives.iter().filter(|r| !vouches(r)) { - println!( - " [CUT] {:<40} cov {:>5} cos {:>6}", - truncate(&r.query, 40), - r.coverage() - .map(|c| format!("{c:.2}")) - .unwrap_or_else(|| "—".to_string()), - r.best_cos - .map(|c| format!("{c:.3}")) - .unwrap_or_else(|| "—".to_string()), - ); - } - if !faults.is_empty() { - println!( - " [FAULT] the engine's verdict and this harness's restatement of the same rule \ - disagree on {} query/queries: {}. One of the two has drifted — read the engine's \ - `LexicalEvidence`, not this line's wording.", - faults.len(), - faults.join(", ") - ); - } - for r in &ev.negatives { - println!( - " {:<40} cov {:>5} cos {:>6} → {}", - truncate(&r.query, 40), - r.coverage() - .map(|c| format!("{c:.2}")) - .unwrap_or_else(|| "—".to_string()), - r.best_cos - .map(|c| format!("{c:.3}")) - .unwrap_or_else(|| "—".to_string()), - if vouches(r) { "SERVED" } else { "no matches" }, - ); - } -} - -/// A candidate **per-hit tail rule** family (GH #206) — the fold where a real -/// query's evidence runs out, D2's per-hit half. Every family folds as a -/// **prefix cut** (invariants.md D1): the default view ends at the first served -/// row failing the family's test, and every row below folds with it — a passing -/// row under a failing one folds too, because a fold that punched holes in the -/// fused order would let row order and fold visibly disagree. -/// -/// The consequence that does all the work below: a filler row served *above* a -/// keep-labelled row must pass the test too, or the keep row under it folds. So -/// each family's constraint is read over the **keep-prefix** — every row at or -/// above a list's deepest keep row — not over the keep rows alone. -#[derive(Clone, Copy, PartialEq)] -enum TailRule { - /// Fold at the first row the lexical half never ranked (`bm25_rank: None`) - /// — the dense-only fold, the signal GH #201 measured (0 of 410 positive - /// rows vs 20 of 50 negative ones). Parameterless, so nothing - /// distributional to place — but **not** scale-free: `bm25_rank` is a rank - /// in a pool-truncated list (`search::pool_size`), so "never ranked" is - /// partly a fact about pool depth against vault size. `make calibrate - /// VAULT= ARGS=--search` measures that rather than assuming it - /// (process rule 5's posture, owed even without a constant). - Lexical, - /// Fold at the first row that is dense-only **and** under a per-hit cosine - /// bar — the shipped query rule's shape (lexical OR semantic) read per hit. - LexOrCos, - /// Fold at the first row under a per-hit cosine bar, lexical rank ignored - /// (a row the dense half never ranked fails at any bar). The single-signal - /// baseline the two-signal family must beat — the pure-cosine window at hit - /// granularity, included to be retired the way GH #201 retired its - /// query-level twin. - Cos, - /// Fold at the first row whose cosine sits more than δ under the list's own - /// best — the "drop-off" shape, scale-adaptive in the query and still - /// distributional in δ. - CosDrop, -} - -impl TailRule { - const ALL: [TailRule; 4] = [ - TailRule::Lexical, - TailRule::LexOrCos, - TailRule::Cos, - TailRule::CosDrop, - ]; - - fn label(self) -> &'static str { - match self { - TailRule::Lexical => "lexical (dense-only fold)", - TailRule::LexOrCos => "lex-or-cos ≥ c", - TailRule::Cos => "cos ≥ c", - TailRule::CosDrop => "cos ≥ best − δ", - } - } - - /// Whether `row` passes this family's per-hit test at constant `p` - /// (ignored by `Lexical`; `best` is the row's list's best served cosine, - /// read only by `CosDrop`). - fn passes(self, row: &ServedRow, p: f64, best: f64) -> bool { - match self { - TailRule::Lexical => row.bm25_rank.is_some(), - TailRule::LexOrCos => row.bm25_rank.is_some() || row.cos.is_some_and(|c| c >= p), - TailRule::Cos => row.cos.is_some_and(|c| c >= p), - TailRule::CosDrop => row.cos.is_some_and(|c| c >= best - p), - } - } - - /// Where this family folds `rows` at constant `p`: the index of the first - /// failing row, `rows.len()` when nothing folds. The prefix-cut definition - /// lives here and nowhere else. - fn fold(self, rows: &[ServedRow], p: f64) -> usize { - let best = best_served_cos(rows); - rows.iter() - .position(|r| !self.passes(r, p, best)) - .unwrap_or(rows.len()) - } -} - -/// The best served cosine of one list — [`TailRule::CosDrop`]'s reference -/// point. Read off the *served* rows, not the vault-wide dense top-1: the drop -/// rule is a claim about a list's own shape. -fn best_served_cos(rows: &[ServedRow]) -> f64 { - rows.iter() - .filter_map(|r| r.cos) - .fold(f64::NEG_INFINITY, f64::max) -} - -/// The keep-prefix of one served list: every row at or above its deepest -/// keep-labelled row. Empty when nothing served is keep-labelled — such a list -/// constrains no rule (there is nothing a fold could wrongly hide). -fn keep_prefix(rows: &[ServedRow]) -> &[ServedRow] { - match rows.iter().rposition(|r| r.keep) { - Some(last) => &rows[..=last], - None => &[], - } -} - -/// One family's constraint, re-derived from a set of served lists (GH #187's -/// idiom on the per-hit axis): the range of its constant, if any, at which no -/// keep-prefix row anywhere fails. Both edges carry the row that set them, -/// because a window edge is only arguable once the pair is named. -struct TailConstraint { - /// A keep-prefix row that cannot pass at **any** constant — the family is - /// dead on this bench (e.g. a row with no cosine under a cosine family). - dead: Option<(String, String)>, - /// `Cos`/`LexOrCos`: the highest admissible bar (the binding keep-prefix - /// row's own cosine). `CosDrop`: the lowest admissible δ (the binding - /// row's drop from its list's best). `None` = unconstrained — no - /// keep-prefix row engages the family's test at all. - edge: Option, - /// The (query-or-title, path) that set `edge`. - edge_row: Option<(String, String)>, - /// `Lexical` only: keep-prefix rows failing its fixed test. Nonzero = - /// inadmissible, and each failure is a keep row the fold would hide. - lexical_violations: usize, -} - -/// Read one family's [`TailConstraint`] over `(list name, full rows, keep-prefix -/// length)` triples, where every keep-prefix row must pass. The dense fixture -/// reuses this with whole lists as keep-prefixes: on a single-domain vault every -/// served row is a real match, so the absolute "truncate nothing" is the same -/// constraint with the keep-set saturated. The full list rides along because -/// [`TailRule::CosDrop`]'s reference is the **list's** best served cosine — -/// reading it off the prefix alone could understate a required δ when the best -/// row sits below the prefix, and the constraint must read exactly what the -/// fold would. -fn tail_constraint<'a>( - rule: TailRule, - lists: impl Iterator, -) -> TailConstraint { - let mut out = TailConstraint { - dead: None, - edge: None, - edge_row: None, - lexical_violations: 0, - }; - for (name, rows, prefix_len) in lists { - let best = best_served_cos(rows); - for row in &rows[..prefix_len] { - match rule { - TailRule::Lexical => { - if row.bm25_rank.is_none() { - out.lexical_violations += 1; - if out.dead.is_none() { - out.dead = Some((name.to_string(), row.path.clone())); - } - } - } - // The cosine arms read the row's cosine through a finiteness - // filter (PR #212 review): a NaN would sail through `c < e` - // comparisons as silently-true-nowhere — worse, `is_none_or` - // admits the FIRST row unconditionally, so a NaN first row - // would seed the edge. A non-finite reading is *no reading*, - // and the honest arm for that is `dead`, with the row named — - // never a silent skip that reports the family unconstrained. - TailRule::LexOrCos => { - if row.bm25_rank.is_none() { - match row.cos.filter(|c| c.is_finite()) { - // A served row is in at least one list, so a - // dense-only row carries a cosine unless it is - // non-finite — either way, no finite reading means - // this row can pass at no bar. - None => { - if out.dead.is_none() { - out.dead = Some((name.to_string(), row.path.clone())); - } - } - Some(c) => { - if out.edge.is_none_or(|e| c < e) { - out.edge = Some(c); - out.edge_row = Some((name.to_string(), row.path.clone())); - } - } - } - } - } - TailRule::Cos => match row.cos.filter(|c| c.is_finite()) { - None => { - if out.dead.is_none() { - out.dead = Some((name.to_string(), row.path.clone())); - } - } - Some(c) => { - if out.edge.is_none_or(|e| c < e) { - out.edge = Some(c); - out.edge_row = Some((name.to_string(), row.path.clone())); - } - } - }, - TailRule::CosDrop => match row.cos.filter(|c| c.is_finite()) { - None => { - if out.dead.is_none() { - out.dead = Some((name.to_string(), row.path.clone())); - } - } - Some(c) => { - let drop = best - c; - if out.edge.is_none_or(|e| drop > e) { - out.edge = Some(drop); - out.edge_row = Some((name.to_string(), row.path.clone())); - } - } - }, - } - } - } - out -} - -/// What a family buys at constant `p` on the labelled lists: rows cut, split by -/// label. At an admissible constant `kept_cut` is zero **by construction** — -/// the constraint above is exactly "every keep-prefix row passes" — so a -/// nonzero here is an arithmetic fault, printed as such rather than assumed -/// away. -struct TailPayoff { - filler_cut: usize, - kept_cut: usize, -} - -fn tail_payoff(rule: TailRule, lists: &[&QueryEvidence], p: f64) -> TailPayoff { - let mut out = TailPayoff { - filler_cut: 0, - kept_cut: 0, - }; - for q in lists { - let fold = rule.fold(&q.rows, p); - for row in &q.rows[fold..] { - if row.keep { - out.kept_cut += 1; - } else { - out.filler_cut += 1; - } - } - } - out -} - -/// The per-hit tail bake-off's labelled-corpus reading (GH #206): every -/// family's re-derived constraint plus its payoff at the constraint's own edge -/// — the most aggressive admissible point, since payoff is monotone in the -/// constant. Judged over the **positives**: the query-level bar already -/// answers the negatives whole (GH #201), so a tail rule ships riding on a -/// vouched verdict; the negatives' would-be reading is kept as context for -/// what the rule would buy where a query bar had missed. -struct TailBench { - keep_rows: usize, - filler_rows: usize, - /// Positives whose keep-prefix is the whole served list — a fold has - /// nothing to cut there under any admissible rule. - saturated: usize, - /// The **oracle** ceiling: rows below each list's last keep row — what a - /// perfect prefix fold (one placed by the labels themselves) would cut. - /// Every family's payoff is read against this, because "cuts N rows" means - /// nothing until the reachable maximum is beside it. - oracle: usize, - families: Vec, - /// Junk rows the parameterless lexical fold would cut on the negatives' - /// served lists, of their total — the GH #201 `dense_only` reading, priced - /// as a fold. - neg_lexical_cut: usize, - neg_rows: usize, -} - -struct TailFamilyReading { - rule: TailRule, - constraint: TailConstraint, - /// Payoff at the constraint's edge; `None` when the family is dead here or - /// (for the constant families) unconstrained-and-therefore-identical to a - /// simpler family. - payoff: Option, -} - -fn score_search_tail(ev: &SearchEvidence) -> TailBench { - let positives: Vec<&QueryEvidence> = ev.positives.iter().collect(); - let keep_rows = positives - .iter() - .flat_map(|q| q.rows.iter()) - .filter(|r| r.keep) - .count(); - let filler_rows = positives - .iter() - .flat_map(|q| q.rows.iter()) - .filter(|r| !r.keep) - .count(); - let saturated = positives - .iter() - .filter(|q| !q.rows.is_empty() && keep_prefix(&q.rows).len() == q.rows.len()) - .count(); - let oracle = positives - .iter() - .map(|q| q.rows.len() - keep_prefix(&q.rows).len()) - .sum(); - let families = TailRule::ALL - .iter() - .map(|&rule| { - let constraint = tail_constraint( - rule, - positives.iter().map(|q| { - ( - q.query.as_str(), - q.rows.as_slice(), - keep_prefix(&q.rows).len(), - ) - }), - ); - let payoff = match rule { - TailRule::Lexical => (constraint.lexical_violations == 0) - .then(|| tail_payoff(rule, &positives, f64::NAN)), - _ => match (&constraint.dead, constraint.edge) { - (None, Some(edge)) => Some(tail_payoff(rule, &positives, edge)), - _ => None, - }, - }; - TailFamilyReading { - rule, - constraint, - payoff, - } - }) - .collect(); - let neg_lexical_cut = ev - .negatives - .iter() - .map(|q| q.rows.len() - TailRule::Lexical.fold(&q.rows, f64::NAN)) - .sum(); - TailBench { - keep_rows, - filler_rows, - saturated, - oracle, - families, - neg_lexical_cut, - neg_rows: ev.negatives.iter().map(|q| q.rows.len()).sum(), - } -} - -fn print_search_tail(bench: &TailBench) { - println!( - " search tail bake-off (D2 per-hit — GH #206; the labels are GH #206's tail_relevant)" - ); - for line in [ - "the rule under audition: end the DEFAULT VIEW at the first served row failing a", - " per-hit evidence test — a PREFIX CUT (D1), so a filler row above a keep row", - " must pass too or the keep row folds with it. Constraint re-derived per run", - " over the keep-prefixes; payoff read at the constraint's own edge. \"No tail", - " fold\" is an admissible winner (the GH #200 outcome, on search's side).", - ] { - println!(" {line}"); - } - println!( - " served rows over the positives: {} keep / {} filler by label; {} list(s) saturated \ - (keep to the last served row)", - bench.keep_rows, bench.filler_rows, bench.saturated - ); - println!( - " oracle ceiling: a fold placed by the labels themselves (each list's last keep row) \ - would cut {} of {} — every payoff below is read against this", - bench.oracle, bench.filler_rows - ); - for f in &bench.families { - let constraint = match f.rule { - TailRule::Lexical => { - if f.constraint.lexical_violations == 0 { - "admissible (no dense-only keep-prefix row)".to_string() - } else { - let (q, p) = f.constraint.dead.as_ref().expect("violation names a row"); - format!( - "✗ {} keep-prefix row(s) are dense-only — first: {} → {}", - f.constraint.lexical_violations, - truncate(q, 28), - p - ) - } - } - _ => match (&f.constraint.dead, f.constraint.edge) { - (Some((q, p)), _) => { - format!( - "DEAD — {} → {} has no cosine to clear any bar", - truncate(q, 28), - p - ) - } - (None, None) => "unconstrained (no keep-prefix row engages the test)".to_string(), - (None, Some(edge)) => { - let (q, p) = f - .constraint - .edge_row - .as_ref() - .map(|(q, p)| (truncate(q, 28), p.clone())) - .unwrap_or_default(); - match f.rule { - TailRule::CosDrop => format!("δ ≥ {edge:.3} (set by {q} → {p})"), - _ => format!("c ≤ {edge:.3} (set by {q} → {p})"), - } - } - }, - }; - let payoff = match &f.payoff { - None => "—".to_string(), - Some(p) if p.kept_cut > 0 => format!( - "[FAULT] cuts {} keep row(s) at its own edge — the constraint arithmetic is wrong", - p.kept_cut - ), - Some(p) => format!("cuts {} of {} filler rows", p.filler_cut, bench.filler_rows), - }; - println!(" {:<24} {:<58} {}", f.rule.label(), constraint, payoff); - } - println!( - " negatives context: the lexical fold alone would cut {}/{} of their junk rows — the \ - query-level bar already cuts all of them (GH #201), so a tail rule is judged on what it \ - buys ABOVE that bar", - bench.neg_lexical_cut, bench.neg_rows - ); -} - -/// The tail bake-off's JSON (`search_tail` in the orthogonal row): each -/// family's re-derived constraint and edge payoff. The served rows it was read -/// from are already in `search_evidence` per query, so any other constant is -/// re-derivable from the row — the `discovery_fold` convention. -fn tail_json(bench: &TailBench) -> serde_json::Value { - let round = |v: f64| (v * 1e4).round() / 1e4; - serde_json::json!({ - "keep_rows": bench.keep_rows, - "filler_rows": bench.filler_rows, - "saturated_lists": bench.saturated, - "oracle": bench.oracle, - "neg_lexical_cut": bench.neg_lexical_cut, - "neg_rows": bench.neg_rows, - "families": bench.families.iter().map(|f| serde_json::json!({ - "rule": f.rule.label(), - "dead": f.constraint.dead.as_ref().map(|(q, p)| serde_json::json!([q, p])), - "edge": f.constraint.edge.map(round), - "edge_row": f.constraint.edge_row.as_ref().map(|(q, p)| serde_json::json!([q, p])), - "lexical_violations": f.constraint.lexical_violations, - "filler_cut_at_edge": f.payoff.as_ref().map(|p| p.filler_cut), - "kept_cut_at_edge": f.payoff.as_ref().map(|p| p.kept_cut), - })).collect::>(), - }) -} - -/// One family's dense-fixture reading (GH #206): the same [`TailConstraint`] -/// read over **whole** title lists — every served row is a real match by -/// geometry, so the keep-prefix is the entire list and the constraint *is* the -/// absolute ("truncate nothing"). `lexical_cut` is the parameterless family's -/// visible cost here: rows its fold would remove from title queries' views. -struct TailFamilyTransfer { - rule: TailRule, - constraint: TailConstraint, - lexical_cut: usize, -} - -fn dense_tail_families(titles: &[SearchProbe]) -> Vec { - TailRule::ALL - .iter() - .map(|&rule| TailFamilyTransfer { - rule, - constraint: tail_constraint( - rule, - titles - .iter() - .map(|t| (t.query.as_str(), t.rows.as_slice(), t.rows.len())), - ), - lexical_cut: match rule { - TailRule::Lexical => titles - .iter() - .map(|t| t.rows.len() - rule.fold(&t.rows, f64::NAN)) - .sum(), - _ => 0, - }, - }) - .collect() -} - -/// Print the dense fixture's tail-transfer reading (GH #206) — the per-hit -/// sibling of the query bar's `search_transfer` block, and the bench that -/// carries D1's absolute for this bake-off. -fn print_dense_tail(titles: &[SearchProbe]) { - println!( - " tail the per-hit tail bench on this geometry (GH #206): every served row of a" - ); - println!( - " title query is a real match, so a rule that folds one is disqualified —" - ); - println!(" the GH #200 absolute, on search's side"); - for f in dense_tail_families(titles) { - let reading = match f.rule { - TailRule::Lexical => { - if f.lexical_cut == 0 { - "cuts 0 title rows".to_string() - } else { - let (q, p) = f.constraint.dead.as_ref().expect("cut names a row"); - format!( - "✗ cuts {} title row(s) — first: {} → {}", - f.lexical_cut, - truncate(q, 24), - p - ) - } - } - _ => match (&f.constraint.dead, f.constraint.edge) { - (Some((q, p)), _) => { - format!("DEAD — {} → {} has no cosine", truncate(q, 24), p) - } - (None, None) => "unconstrained (no row engages the test)".to_string(), - (None, Some(edge)) => { - let (q, p) = f - .constraint - .edge_row - .as_ref() - .map(|(q, p)| (truncate(q, 24), p.clone())) - .unwrap_or_default(); - match f.rule { - TailRule::CosDrop => { - format!("needs δ ≥ {edge:.3} to fold nothing (set by {q} → {p})") - } - _ => format!("needs c ≤ {edge:.3} to fold nothing (set by {q} → {p})"), - } - } - }, - }; - println!(" {:<24} {}", f.rule.label(), reading); - } -} - -/// The dense tail-transfer reading as JSON, nested under the dense row's -/// `search_transfer` key (additive subkey, per the row conventions). -fn dense_tail_json(titles: &[SearchProbe]) -> serde_json::Value { - let round = |v: f64| (v * 1e4).round() / 1e4; - serde_json::json!(dense_tail_families(titles) - .iter() - .map(|f| serde_json::json!({ - "rule": f.rule.label(), - "dead": f.constraint.dead.as_ref().map(|(q, p)| serde_json::json!([q, p])), - "edge": f.constraint.edge.map(round), - "edge_row": f.constraint.edge_row.as_ref().map(|(q, p)| serde_json::json!([q, p])), - "lexical_cut": f.lexical_cut, - })) - .collect::>()) -} - -/// The tail bake-off's **cross-bench join** (GH #206), printed once both -/// corpora have been read in the same run: a family ships only if some -/// constant is admissible on the labelled corpus *and* folds nothing on the -/// dense fixture *and* still cuts labelled filler there — read at the joint -/// edge, the most aggressive point both benches allow. -fn print_tail_join(ev: &SearchEvidence, orth: &TailBench, titles: &[SearchProbe]) { - println!("\n{}", "=".repeat(78)); - println!("search tail — the cross-bench join (GH #206; both corpora, one run)"); - let positives: Vec<&QueryEvidence> = ev.positives.iter().collect(); - let dense = dense_tail_families(titles); - let mut winner = false; - for (orth_f, dense_f) in orth.families.iter().zip(&dense) { - let rule = orth_f.rule; - let verdict = - match rule { - TailRule::Lexical => { - if orth_f.constraint.lexical_violations > 0 { - format!( - "✗ inadmissible on the labelled corpus ({} keep-prefix row(s) dense-only)", - orth_f.constraint.lexical_violations - ) - } else if dense_f.lexical_cut > 0 { - format!( - "✗ disqualified on the dense fixture (cuts {} title rows)", - dense_f.lexical_cut - ) - } else { - let cut = orth_f.payoff.as_ref().map(|p| p.filler_cut).unwrap_or(0); - if cut == 0 { - "✓ admissible and VACUOUS — cuts 0 of the labelled filler, so it buys \ - nothing the query bar has not already bought" - .to_string() - } else { - winner = true; - format!( - "✓ admissible on both benches — cuts {cut} of the {} an oracle \ - fold reaches", - orth.oracle - ) - } - } - } - _ => { - let orth_dead = orth_f.constraint.dead.is_some(); - let dense_dead = dense_f.constraint.dead.is_some(); - if orth_dead || dense_dead { - format!( - "✗ DEAD on the {} bench (a required row has no cosine)", - if orth_dead { "labelled" } else { "dense" } - ) - } else { - // Joint edge: the tighter bench binds. `None` = that bench - // leaves the constant free. - let joint = match rule { - TailRule::CosDrop => { - match (orth_f.constraint.edge, dense_f.constraint.edge) { - (Some(a), Some(b)) => Some(a.max(b)), - (a, b) => a.or(b), - } - } - _ => match (orth_f.constraint.edge, dense_f.constraint.edge) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }, - }; - match joint { - None => { - "degenerates to the lexical fold (no row on either bench engages \ - the test) — see that family's verdict" - .to_string() - } - Some(edge) => { - let payoff = tail_payoff(rule, &positives, edge); - if payoff.kept_cut > 0 { - format!( - "[FAULT] cuts {} keep row(s) at the joint edge {edge:.3} — \ - the join arithmetic is wrong", - payoff.kept_cut - ) - } else if payoff.filler_cut == 0 { - format!( - "✓ admissible to {} {edge:.3} and VACUOUS — cuts 0 labelled \ - filler there", - if rule == TailRule::CosDrop { "δ ≥" } else { "c ≤" } - ) - } else { - winner = true; - format!( - "✓ admissible at {} {edge:.3} on both benches — cuts {} of \ - the {} an oracle fold reaches", - if rule == TailRule::CosDrop { "δ ≥" } else { "c ≤" }, - payoff.filler_cut, - orth.oracle - ) - } - } - } - } - } - }; - println!(" {:<24} {}", rule.label(), verdict); - } - if winner { - println!( - " → the ✓ family/families above survive both corpora at their joint edges. That is \ - ADMISSIBILITY, not a shipping order: a joint edge sits AT a bench's own binding row \ - (zero headroom — the constant placement the house sizing method forbids), each payoff \ - reads against the oracle ceiling above, and a shipped constant owes process rule 5's \ - real-vault reading besides (`make calibrate VAULT= ARGS=--search`, the tail block). The ruling of \ - record lives in docs/evals.md." - ); - } else { - println!( - " → NO family survives both benches with a nonzero payoff: the incumbent — no \ - per-hit tail fold — stands, the GH #200 outcome on search's side. The filler the \ - complaint names is above every admissible fold, so the honesty still rides on the \ - query-level bar and the copy." - ); - } -} - -/// `LocalEmbedder::embed_batch` must be a faithful map of `embed`: right-padding short rows -/// to the batch's longest and masking them out has to leave each row's CLS vector unchanged. -/// The reindex path batches freely, so a regression here would silently corrupt every stored -/// vector — and every score this eval prints. -/// -/// It lives in the eval rather than `cargo test` because it needs the provisioned model, -/// which the fast suite deliberately never touches (ADR-0013). Running it here means it -/// actually runs, instead of sitting behind an `#[ignore]` nobody passes `--ignored` to. -fn check_batch_matches_single(model: &LocalEmbedder) -> Result<(), Box> { - // Deliberately varied lengths, so batching pads the short rows to the longest. - let texts = [ - "Spaced repetition schedules reviews at increasing intervals.", - "Sleep consolidates memory.", - "Short.", - "Focus and sustained attention shape what is later recalled from long-term memory across days.", - ]; - let refs: Vec<&str> = texts.to_vec(); - let batched = model.embed_batch(&refs)?; - if batched.len() != texts.len() { - return Err(format!( - "embed_batch returned {} rows for {} texts", - batched.len(), - texts.len() - ) - .into()); - } - let mut worst = f32::INFINITY; - for (text, batched_row) in texts.iter().zip(&batched) { - let single = model.embed(text)?; - if batched_row.len() != single.len() { - return Err(format!("batched/single dim mismatch for {text:?}").into()); - } - // Both rows are L2-normalized, so the dot product is cosine similarity; - // padding must not move it off ~1.0. Non-finite is checked first and - // explicitly: every comparison against a NaN is false, so `cos <= 0.9999` - // alone would wave a NaN row *through* the gate — the one failure mode a - // correctness check must not have. - let cos: f32 = batched_row.iter().zip(&single).map(|(a, b)| a * b).sum(); - if !cos.is_finite() { - return Err(format!("batched embedding is non-finite for {text:?}: {cos}").into()); - } - worst = worst.min(cos); - if cos <= 0.9999 { - return Err(format!( - "batched embedding differs from single for {text:?}: cosine {cos}" - ) - .into()); - } - } - eprintln!("[eval] batch ≡ single: worst-row cosine {worst:.6}\n"); - Ok(()) -} - -/// Run the embed pass, timing it and counting the chunks it filled. -fn timed_embed(vault: &Vault) -> Result<(usize, f64), Box> { - let mut chunks = 0usize; - let t0 = Instant::now(); - vault.embed(&mut |p| { - chunks = p.chunks_done; - ControlFlow::Continue(()) - })?; - Ok((chunks, t0.elapsed().as_secs_f64())) -} - -/// State the one thing this corpus **cannot** measure, on every run that can't. -/// -/// A corpus with no more chunks than a signal's candidate pool truncates *neither* list, so -/// both are already complete, widening cannot add a candidate, and every score above is -/// invariant under **candidate width**: a change to either view's headroom or to -/// `search::pool_size` prints bit-identical numbers here while genuinely reordering a real -/// vault (GH #141). Judged on the **narrower** of the two pools, since blindness is a claim -/// about every number the run prints. -/// -/// Scoped deliberately to width — `RRF_K` re-weights the *same* two lists, so it reorders -/// results on any corpus, and this eval sees that. A warning, not a gate: the point is that a -/// reader must not take an unmoved number as evidence of no change. The property itself is -/// measured by `--example stability`, on a vault big enough for the pool to bind. -fn warn_if_pool_blind(chunks: usize) { - let pool = chunk_candidate_pool(K).min(note_candidate_pool(K)); - if chunks <= pool { - eprintln!( - "[warn] {chunks} chunks ≤ {pool}-candidate pool — neither signal is truncated here, so a\n\ - \x20 candidate-width change (either hit pool, pool_size) cannot move any number in this run\n\ - \x20 (GH #141). `make stability` measures that property on a large vault. (RRF_K is\n\ - \x20 not in that set — it re-weights the same lists, and this corpus does see it.)\n" - ); - } -} - -fn print_default_report( - queries: &[Labelled], - bm25: &Pass, - vector: &Pass, - hybrid: &Pass, - similar: &SimilarPass, - floor_z: &FloorZ, -) { - println!( - "{:>5} {:>6} {:>6} {:>6} {:<40} top hybrid hit", - "bm25", "vec", "hybrid", "chunk", "query" - ); - println!("{}", "-".repeat(102)); - for (i, q) in queries.iter().enumerate() { - println!( - "{:>5} {:>6} {:>6} {:>6} {:<40} {}", - rank_str(bm25.scores[i].note), - rank_str(vector.scores[i].note), - rank_str(hybrid.scores[i].note), - match q.passage { - Some(_) => rank_str(hybrid.scores[i].chunk), - None => "".to_string(), - }, - truncate(&q.query, 40), - hybrid.scores[i].top, - ); - } - - println!("\n{}", "=".repeat(78)); - println!("note rank (n={}, K={K}):", queries.len()); - println!( - " bm25-only hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", - bm25.note.hit1(), - bm25.note.hit3(), - bm25.note.mrr() - ); - println!( - " vec-only hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", - vector.note.hit1(), - vector.note.hit3(), - vector.note.mrr() - ); - println!( - " hybrid hit@1={:.2} hit@3={:.2} MRR@{K}={:.3} semantic lift: {:+.2} hit@1", - hybrid.note.hit1(), - hybrid.note.hit3(), - hybrid.note.mrr(), - hybrid.note.hit1() - bm25.note.hit1(), - ); - // The standing form of the fusion finding (GH #158): every query - // where fusing the two signals ranked the labelled answer WORSE than the - // dense signal alone would have. RRF's consensus bias makes some of this - // inevitable; the point is that it is counted and named on every run instead - // of rediscovered by hand-decomposing scores. - let demoted: Vec = (0..queries.len()) - .filter(|&i| { - let Some(v) = vector.scores[i].note else { - return false; - }; - match hybrid.scores[i].note { - Some(h) => h > v, - None => true, - } - }) - .collect(); - if demoted.is_empty() { - println!(" fusion no query ranks worse under hybrid than under vector alone"); - } else { - println!( - " fusion {} quer{} rank worse under hybrid than under vector alone:", - demoted.len(), - if demoted.len() == 1 { "y" } else { "ies" } - ); - for &i in &demoted { - println!( - " vec {} → hybrid {} {}", - rank_str(vector.scores[i].note), - rank_str(hybrid.scores[i].note), - truncate(&queries[i].query, 48) - ); - } - } - if hybrid.chunk.n > 0 { - println!("chunk rank (passage-labelled, n={}):", hybrid.chunk.n); - println!( - " bm25-only hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", - bm25.chunk.hit1(), - bm25.chunk.hit3(), - bm25.chunk.mrr() - ); - println!( - " hybrid hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", - hybrid.chunk.hit1(), - hybrid.chunk.hit3(), - hybrid.chunk.mrr() - ); - } - println!( - "similar (n={} positive + {} negative, K={SIM_K}):", - similar.rank.n, similar.neg_n - ); - // The per-anchor metric (first mate found, then stop) is **no longer - // printed** (GH #188): it read 1.000 across every change it was meant to - // judge, and a line that cannot move is a line that trains skimming. It is - // still recorded — `results.jsonl`'s `"similar"` key is unchanged, so rows - // stay comparable back to the first run — so retiring the line costs the - // dataset nothing. - println!( - " per-mate hit@1={:.2} hit@3={:.2} MRR@{SIM_K}={:.3} (n={} mates, GATED at MRR@{SIM_K} ≥ {FLOOR_MATE_MRR:.2})", - similar.mate.hit1(), - similar.mate.hit3(), - similar.mate.mrr(), - similar.mate.n - ); - // Discovery's precision side (GH #188) — reported with names, never gated; - // see `SimilarPass::strangers` for why gating it would reward labelling. - println!( - " strangers {} card{} on {}/{} positive anchors (unlabelled notes served in the top-{SIM_K} — \ - a smoke alarm, not a gate: labels aren't exhaustive)", - similar.strangers.len(), - if similar.strangers.len() == 1 { "" } else { "s" }, - similar.stranger_anchors, - similar.rank.n - ); - for (anchor, path) in &similar.strangers { - println!(" {anchor} → {path}"); - } - if similar.neg_n > 0 { - // Under always-serve (GH #197) a negative anchor serves its ranked - // nearest like any other — that is the ruling, not a regression. What - // these anchors measure now is what the served cards *claim*: their - // bands (A2's readout, printed per leader in the calibration block). - println!( - " negatives {} loner anchors serve {} cards under always-serve (labels still say \ - \"nothing relates\"; the bands carry the honesty — leaders below)", - similar.neg_n, similar.neg_cards - ); - } - // The two calibration piles. If they separate, the gap IS the floor, read - // off measured data; if they overlap, no simple floor can hold and the - // escalation path (a discovery-side pair-scorer) is justified by data. - if let (Some((r_min, r_med, r_max)), Some((j_min, j_med, j_max))) = - (pile_stats(&similar.related), pile_stats(&similar.junk)) - { - println!( - " piles(cos) related n={:<3} min/med/max {:.3}/{:.3}/{:.3}", - similar.related.len(), - r_min, - r_med, - r_max - ); - println!( - " junk n={:<3} min/med/max {:.3}/{:.3}/{:.3}", - similar.junk.len(), - j_min, - j_med, - j_max - ); - let gap = r_min - j_max; - println!( - " related-min − junk-max = {:+.3} ({})", - gap, - if gap > 0.0 { - "piles separate — the gap is the floor" - } else { - "piles overlap — a simple floor cuts both" - } - ); - } - // The same question in the floor's own anchor-relative unit, where an - // answer is actionable: the piles are absolute cosines, and no constant in - // the code is ever compared against one. - print_floor_windows(floor_z); -} - -/// Re-derive the admissible windows a z existence rule *would* have on the -/// corpus as it stands today (GH #187) — the standing record of why none ships -/// (GH #197), and the first reading any Phase-2 bake-off candidate answers to. -/// -/// Printed every run, and recorded in the row, precisely so no number here ever -/// gets copied into a doc comment again: the #150 windows were measured once, -/// frozen into a rustdoc, and silently falsified the first time the corpus grew -/// a note shape they were never derived from. The instrument is the citable -/// thing; any constant stays in the code it governs. -fn print_floor_windows(z: &FloorZ) { - let (mates, strangers) = (z.mate_z(), z.stranger_z()); - let (neg_leaders, pos_leaders) = (z.neg_leader_z(), z.pos_leader_z()); - println!( - " discovery z calibration (stage-2 best-passage z — the band's input; gates NOTHING \ - since GH #197)" - ); - // The statistic-level instrument check: the dump's z is recomputed from the - // served scores, so a drift in the engine's statistic can't pass silently - // (tolerance covers f32 sqrt/square round-trip noise). - let recheck = z.recheck_delta(); - if recheck <= 1e-3 { - println!(" [check] harness recomputation matches the engine z (max Δ {recheck:.1e})"); - } else { - println!( - " [FAULT] harness recomputation disagrees with the engine z by up to {recheck:.3} — \ - the statistic moved; distrust every reading below" - ); - } - for (label, pile, role) in [ - ("mates", &mates, "a member bar would have to KEEP"), - ( - "strangers", - &strangers, - "a member bar would have to CUT (positive anchors)", - ), - ( - "neg leaders", - &neg_leaders, - "a leader gate would have to CUT", - ), - ( - "pos leaders", - &pos_leaders, - "a leader gate would have to KEEP", - ), - ] { - match pile_stats(pile) { - Some((min, med, max)) => println!( - " {label:<12} n={:<4} min/med/max {min:+.3}/{med:+.3}/{max:+.3} ← {role}", - pile.len() - ), - None => println!(" {label:<12} n=0 (nothing labelled — no reading)"), - } - } - for (name, win) in [ - ("leader", Window::read(&neg_leaders, &pos_leaders)), - ("member", Window::read(&strangers, &mates)), - ] { - match win { - None => println!(" {name} window no reading (a population was empty)"), - Some(w) if w.open() => println!( - " {name} window ({:+.3}, {:+.3}] — open on THIS corpus; a real vault is the \ - other half of any such claim (process rule 5, `make calibrate`)", - w.cut_max, w.keep_min - ), - Some(w) => println!( - " {name} window EMPTY — the population it must cut reaches {:+.3} while the one \n\ - \x20 it must keep starts at {:+.3}; the two INVERT, and no constant \n\ - \x20 separates an inversion", - w.cut_max, w.keep_min - ), - } - } - // Every negative anchor's leader with the band it paints — A2's readout: - // under always-serve these cards ARE served, and the band is what they - // claim to the human whose labels say "nothing relates". - println!( - " negative anchors' leaders (served under always-serve; the band carries the honesty):" - ); - for a in z.anchors.iter().filter(|a| a.negative) { - match a.candidates.first() { - Some(c) => println!( - " {} → {} {:+.3} {}", - a.anchor, - c.path, - c.z, - Band::of(c.z).glyph() - ), - None => println!(" {} (no candidates)", a.anchor), - } - } - if !z.ungraded.is_empty() { - println!( - " [warn] no z statistics for {} anchor(s) ({}) — pool under STATS_MIN_POPULATION or \ - zero variance; the windows above are measured without them", - z.ungraded.len(), - z.ungraded.join(", ") - ); - } -} - -/// The paired per-query diff between two fused passes — what an A/B is actually -/// judged on. At this corpus's n, every aggregate delta is worth 1–2 queries, so -/// "hit@1 +0.05" and "these two queries flipped, this one broke" are the same -/// fact — but only the second form can be argued with, per-query, against the -/// labels (docs/evals.md, the process rules). Prints nothing but a -/// no-moves line when the variant reproduced the reference ranking exactly — -/// which, per the same rules, is itself a claim to verify against a -/// continuous quantity (the piles), never bare proof of "no effect". -fn print_rank_moves(queries: &[Labelled], reference: &Pass, variant: &Pass) { - let improved = |a: Option, b: Option| match (a, b) { - (None, Some(_)) => true, - (Some(x), Some(y)) => y < x, - _ => false, - }; - let mut note_up = 0usize; - let mut note_down = 0usize; - let mut lines = Vec::new(); - for (i, q) in queries.iter().enumerate() { - let (a, b) = (reference.scores[i].note, variant.scores[i].note); - if a != b { - if improved(a, b) { - note_up += 1; - } else { - note_down += 1; - } - lines.push(format!( - " Δ note {:>5} → {:<5} {}", - rank_str(a), - rank_str(b), - truncate(&q.query, 48) - )); - } - if q.passage.is_some() { - let (a, b) = (reference.scores[i].chunk, variant.scores[i].chunk); - if a != b { - lines.push(format!( - " Δ chunk {:>5} → {:<5} {}", - rank_str(a), - rank_str(b), - truncate(&q.query, 48) - )); - } - } - } - if lines.is_empty() { - println!(" Δ vs default: no per-query rank moved (verify against the piles before reading this as \"no effect\")"); - return; - } - println!(" Δ vs default — note ranks: {note_up} improved, {note_down} worsened",); - for line in lines { - println!("{line}"); - } -} - -/// One appendable JSONL row for a scored configuration. -#[allow(clippy::too_many_arguments)] -fn result_row( - git: &Option, - model: &str, - dim: usize, - label: &str, - cfg: &ChunkConfig, - tokenizer: &str, - notes: usize, - chunks: usize, - embed_secs: f64, - queries: &[Labelled], - bm25: Option<&Pass>, - vector: Option<&Pass>, - hybrid: &Pass, - similar: Option<&SimilarPass>, - floor_z: Option<&FloorZ>, - evidence: Option<&SearchEvidence>, - fold: Option<&FoldBench>, - tail: Option<&TailBench>, -) -> serde_json::Value { - let ts = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - let agg = |a: &Agg| serde_json::json!({ "n": a.n, "hit1": a.hit1(), "hit3": a.hit3(), "mrr": a.mrr() }); - serde_json::json!({ - "ts": ts, - "git": git, - "model": model, - "dim": dim, - // NEW key (absent from rows before 2026-08-18): which corpus this row - // scored — the dense single-domain fixture appends its own `"dense"` - // rows (GH #196/#197), and rows must never average across corpora. - // Absent = this corpus, so every older row stays comparable unchanged. - "corpus": "orthogonal", - "config": { - "label": label, - "target_tokens": cfg.target_tokens, - "overlap_frac": cfg.overlap_frac, - "chars_per_token": cfg.chars_per_token, - "backscan_tokens": cfg.backscan_tokens, - "prepend_heading_path": cfg.prepend_heading_path, - }, - // NEW key (absent from rows before 2026-08-11, which were all scored - // under the then-default `unicode61`): the chunks_fts tokenizer this row - // was scored under (GH #157). Top-level rather than inside `config`, - // which stays the ChunkConfig alone. - "tokenizer": tokenizer, - "notes": notes, - "chunks": chunks, - // The caveat travels with the numbers: a row whose corpus fit inside the - // retrieval pool had both candidate lists complete, so no candidate-width - // change could have affected it and comparing it across one proves nothing - // (GH #141). Equality is blind too — a pool exactly the size of the corpus - // truncates nothing either. Both depths are recorded because since #142 the - // two views differ; the flat `pool` key of earlier rows is deliberately - // *not* reused, so a reader of a mixed file sees a missing field rather than - // one number silently meaning something narrower than it used to. - "pool_note": note_candidate_pool(K), - "pool_chunk": chunk_candidate_pool(K), - "pool_blind": chunks <= chunk_candidate_pool(K).min(note_candidate_pool(K)), - "embed_secs": embed_secs, - "note": { - "bm25": bm25.map(|p| agg(&p.note)), - // NEW key (absent from rows before 2026-08-10): the dense ablation — - // `Vault::search_vector_only`, the single-signal baseline fusion is - // judged against (GH #158). Same convention as pool_note/pool_chunk: - // a new key, never a redefined one. - "vector": vector.map(|p| agg(&p.note)), - "hybrid": agg(&hybrid.note), - }, - "chunk": { - "bm25": bm25.map(|p| agg(&p.chunk)), - "hybrid": agg(&hybrid.chunk), - }, - // "similar" keeps its pre-negative shape (the positive anchors' rank agg) - // so rows stay comparable across the change; the negative-anchor tally and - // the calibration piles are NEW keys, absent from older rows rather than - // redefining an existing one — the same convention as pool_note/pool_chunk. - "similar": similar.map(|s| agg(&s.rank)), - // NEW key (absent from rows before 2026-08-17): the per-mate, - // non-saturating companion to "similar" (GH #183). Same convention - // again — a new key, never a redefined one, so every older row stays - // comparable on "similar" itself. - "similar_per_mate": similar.map(|s| agg(&s.mate)), - // `similar_per_mate_raw` / `similar_mates_suppressed` retired with the - // pass-vs-pass tripwire (GH #217): under always-serve both passes read - // the one surface through the same call, so the diff was a tautology. - // Absent from newer rows, never redefined — the row conventions. - // NEW key (absent from rows before 2026-08-17): discovery's precision - // side — unlabelled notes served on positive anchors at the ranks' own - // depth (GH #188). Same convention as every key above: new, never a - // redefinition. Recorded with the pairs, because the count alone is - // unarguable and the pairs are what a reader checks against the notes. - "similar_strangers": similar.map(|s| serde_json::json!({ - "cards": s.strangers.len(), - "anchors": s.stranger_anchors, - "positives": s.rank.n, - // The depth the count is read at — the ranks' own top-K, so a - // future SIM_K change shows up in the row instead of silently - // redefining the number. - "depth": SIM_K, - // The pairs, because a bare count is unarguable: this metric is a - // smoke alarm whose correct answer is sometimes "that label is - // missing", and that argument needs the notes named. - "detail": s.strangers.iter().map(|(anchor, path)| serde_json::json!({ - "anchor": anchor, - "path": path, - })).collect::>(), - })), - "similar_negatives": similar.map(|s| serde_json::json!({ - "n": s.neg_n, "clean": s.neg_clean, "cards": s.neg_cards, - })), - // Cosine, 4 decimals: enough to place a floor, short enough to keep rows - // readable. Related = human-labelled matches; junk = everything else - // surfaced (see score_similar). - "similar_piles": similar.map(|s| serde_json::json!({ - "related": s.related.iter().map(|c| (c * 1e4).round() / 1e4).collect::>(), - "junk": s.junk.iter().map(|c| (c * 1e4).round() / 1e4).collect::>(), - })), - // The same scores with their per-anchor rank order kept: the relative - // drop-off cutoff is judged within one anchor's list, and tracing a pile - // value back to its pair needs the anchor. The piles above are this, - // flattened — kept anyway, because the flat distributions are what a - // quick jq/pandas histogram wants. - "similar_detail": similar.map(|s| s.detail.iter().map(|d| serde_json::json!({ - "anchor": d.anchor, - "negative": d.negative, - "candidates": d.candidates.iter().map(|(path, cos, related)| serde_json::json!({ - "path": path, - "cos": (cos * 1e4).round() / 1e4, - "related": related, - })).collect::>(), - })).collect::>()), - // NEW key (absent from rows before 2026-08-17): the floor's own - // calibration data in the floor's own unit — every candidate's ungated - // judge z, the populations the two constants answer to, and both - // re-derived windows (GH #187; the unit changed from stage-1 centroid z - // to stage-2 best-passage z with GH #192 — the row's "unit" field is - // what tells the two apart). Same convention as every key above: new, - // never a redefinition. This is the row the next recalibration reads, and - // the reason no window belongs in a doc comment — `null` on the ablation - // rows, which do not re-derive it. - "discovery_z": floor_z.map(|z| { - let (mates, strangers) = (z.mate_z(), z.stranger_z()); - let (neg_leaders, pos_leaders) = (z.neg_leader_z(), z.pos_leader_z()); - let win = |cut: &[f64], keep: &[f64]| Window::read(cut, keep).map(|w| serde_json::json!({ - "cut_max": (w.cut_max * 1e4).round() / 1e4, - "keep_min": (w.keep_min * 1e4).round() / 1e4, - "open": w.open(), - })); - let round = |v: &[f64]| v.iter().map(|z| (z * 1e4).round() / 1e4).collect::>(); - serde_json::json!({ - // The unit these piles/windows are measured in. Rows before - // GH #192 carried no key here and are stage-1 centroid z — a - // different unit; never compare across the flip. Since GH #197 - // the z gates nothing (the "shipped" and "replay_faults" keys - // of earlier rows retired with the gate — absent, per the - // convention, rather than redefined). - "unit": "stage2-best-passage", - "piles": { - "mates": round(&mates), - "strangers_on_positives": round(&strangers), - "negative_leaders": round(&neg_leaders), - "positive_leaders": round(&pos_leaders), - }, - "window_leader": win(&neg_leaders, &pos_leaders), - "window_member": win(&strangers, &mates), - "ungraded_anchors": z.ungraded, - // ~0 is the only trustworthy reading — see FloorZ::recheck_delta. - "z_recheck_max_delta": z.recheck_delta(), - // Per-anchor and per-candidate, because a window edge is only - // arguable once you can name the pair that set it. `z` is the - // stage-2 best-passage z, `cos` the same pair's cosine (the - // model-comparable unit). - "detail": z.anchors.iter().map(|a| serde_json::json!({ - "anchor": a.anchor, - "negative": a.negative, - "candidates": a.candidates.iter().map(|c| serde_json::json!({ - "path": c.path, - "z": (c.z * 1e4).round() / 1e4, - "cos": (cosine_of(c.score) * 1e4).round() / 1e4, - "mate": c.mate, - })).collect::>(), - })).collect::>(), - }) - }), - // NEW key (absent from rows before 2026-08-22): the search evidence - // dump (invariants.md D2, GH #201) — per labelled query, the absolute - // signals RRF discards (BM25 hit count + best score, dense top-1 - // cosine) and the shipped surface's served count, positives and - // negatives apart, with the would-be pure-cosine query window. Same - // convention as every key above: new, never a redefinition. `null` on - // ablation/sweep rows, which do not re-derive it. - "search_evidence": evidence.map(|e| { - let bar = EvidenceBar::for_model(model); - let row = |r: &QueryEvidence| serde_json::json!({ - "q": r.query, - "bm25_hits": r.bm25_hits, - "bm25_best": r.bm25_best.map(|b| (b * 1e4).round() / 1e4), - "best_cos": r.best_cos.map(|c| (c * 1e4).round() / 1e4), - "top": r.top, - "served": r.served(), - // NEW keys (absent from rows before 2026-08-22, GH #201 Phase C): - // the per-query evidence the bake-off is swept over, recorded raw - // so any (fraction, coverage, cos) cell is re-derivable from a row - // without re-running the model — the `discovery_fold` convention. - "dense_only": r.dense_only(), - // NEW key (absent from rows before GH #206): the served list - // itself, per row — path, per-hit provenance, and relevance by - // label — so any per-hit tail rule is re-derivable from a row - // without re-running the model, the `discovery_fold` convention - // at hit granularity. `keep` reads `relevant` ∪ `tail_relevant`. - "rows": r.rows.iter().map(|row| serde_json::json!({ - "path": row.path, - "bm25_rank": row.bm25_rank, - "cos": row.cos.map(|c| (c * 1e4).round() / 1e4), - "keep": row.keep, - })).collect::>(), - "chunk_total": r.chunk_total, - "terms": r.terms.iter().map(|(t, df)| serde_json::json!([t, df])) - .collect::>(), - // The ENGINE's verdict, matching the dense row's convention and - // what the exit gate asserts — recording the harness's - // restatement here instead would mask exactly the drift the - // [FAULT] check exists to surface (the restatement is - // re-derivable from `terms` + `best_cos` + `bar` regardless). - "vouched": r.vouched, - }); - let (pos_cos, neg_cos) = (best_cos_pile(&e.positives), best_cos_pile(&e.negatives)); - serde_json::json!({ - // The depth `served` is read at, so a future K change shows up - // in the row instead of silently redefining the number. - "k": K, - "positives": e.positives.iter().map(row).collect::>(), - "negatives": e.negatives.iter().map(row).collect::>(), - "window_best_cos": Window::read(&neg_cos, &pos_cos).map(|w| serde_json::json!({ - "cut_max": (w.cut_max * 1e4).round() / 1e4, - "keep_min": (w.keep_min * 1e4).round() / 1e4, - "open": w.open(), - })), - // NEW key (GH #201, Phase C): the whole bake-off grid, so the - // admissible window is re-derivable from the row — including the - // shipped bar's own three constants, which is what makes a later - // reader able to see that a bar has drifted out of the window it - // was read from. - "bar": bar.map(|b| serde_json::json!({ - "min_term_coverage": b.min_term_coverage, - "min_cos": b.min_cos, - })), - "bakeoff": bake_off(e).iter().map(|c| serde_json::json!({ - "min_term_coverage": c.coverage, - "pos_anchored": c.pos_anchored, - "neg_anchored": c.neg_anchored, - "cos_cut_floor": c.cut_floor().map(|v| (v * 1e4).round() / 1e4), - "cos_keep_ceiling": c.keep_ceiling().map(|v| (v * 1e4).round() / 1e4), - "admissible": c.admissible(), - })).collect::>(), - }) - }), - // The fold bake-off (GH #200, Phase B) — every candidate rule's reading - // on this run, with the per-anchor folds it was read from. Recorded on - // the default row only: the sweep's variants re-chunk the corpus, and a - // disclosure rule judged on a non-shipped chunker is a number about the - // chunker. - "discovery_fold": fold.map(fold_json), - // NEW key (absent from rows before GH #206): the per-hit tail bake-off - // — each family's re-derived constraint and edge payoff. Default row - // only, for the same reason as `discovery_fold`; the served rows it was - // read from are under `search_evidence`, so any other constant is - // re-derivable from the row. - "search_tail": tail.map(tail_json), - "queries": queries.iter().enumerate().map(|(i, q)| serde_json::json!({ - "q": q.query, - "bm25": bm25.map(|p| p.scores[i].note), - "vector": vector.map(|p| p.scores[i].note), - "hybrid": hybrid.scores[i].note, - "chunk": hybrid.scores[i].chunk, - })).collect::>(), - }) -} - -/// Corpus notes are copied flat into the vault, so a result path equals (or ends -/// with) the labelled relevant path. -fn paths_match(result_path: &str, relevant: &str) -> bool { - result_path == relevant || result_path.ends_with(&format!("/{relevant}")) -} - -fn rank_str(rank: Option) -> String { - match rank { - Some(1) => "✓1".to_string(), - Some(r) => format!("·{r}"), - None => format!("✗>{K}"), - } -} diff --git a/crates/b2-embed/examples/eval/ablation.rs b/crates/b2-embed/examples/eval/ablation.rs new file mode 100644 index 0000000..ec3aad0 --- /dev/null +++ b/crates/b2-embed/examples/eval/ablation.rs @@ -0,0 +1,239 @@ +//! The opt-in A/Bs, each judged against the default config's passes and logged as rows +//! of their own beside its row: `--stemmer` (the FTS tokenizer ablation, GH #157) and +//! `--sweep` (the in-process chunker sweep, GH #44). + +use crate::common::append_result; +use crate::discovery::score_similar; +use crate::instrument::timed_embed; +use crate::labels::{Labelled, SimilarSet}; +use crate::report::{print_rank_moves, result_row, RowInputs, RunId}; +use crate::retrieval::{score_pass, Pass, Retrieval}; +use b2_core::chunk::ChunkConfig; +use b2_core::db::FtsTokenizer; +use b2_core::vault::Vault; +use std::path::Path; + +/// The default config's scored run — what every ablation row is judged against +/// (the paired per-query moves) and logged beside. A short-lived, read-only view +/// over values `run` owns, built once and handed to each A/B. +#[derive(Clone, Copy)] +pub struct Baseline<'a> { + /// The results log every row is appended to. + pub log: &'a Path, + pub run: RunId<'a>, + pub notes: usize, + pub chunks: usize, + pub embed_secs: f64, + /// The labelled positives every pass below was scored over. + pub queries: &'a [Labelled], + pub bm25: &'a Pass, + pub vector: &'a Pass, + pub hybrid: &'a Pass, +} + +/// The FTS tokenizer ablation (the #157 instrument). +/// +/// One lever, isolated: `rebuild_fts` swaps the tokenizer over the identical +/// chunk rows and vectors — the shipped `porter unicode61` against the +/// unstemmed `unicode61` the A/B retired, kept measurable so the verdict can +/// be re-tried as the corpus grows. Discovery is deliberately not re-scored — +/// `similar` never touches FTS (centroid shortlist + chunk vectors), so its +/// numbers cannot move and the ablation row records no `similar` keys. The +/// dense ablation IS re-scored, as an instrument check: FTS cannot reach it +/// either, so a moved dense rank means the harness is broken, not the engine. +/// +/// `bm25_unstemmed` is the lexical arm, which `run` scores while the vault is still +/// projected-but-unembedded (so `search` is honestly BM25-only under both tokenizers). +pub fn stemmer( + vault: &Vault, + base: &Baseline, + bm25_unstemmed: &Pass, +) -> Result<(), Box> { + let positives = base.queries; + vault.rebuild_fts(FtsTokenizer::Unicode61)?; + let vec_unstemmed = score_pass(vault, positives, Retrieval::VectorOnly)?; + let hybrid_unstemmed = score_pass(vault, positives, Retrieval::Fused)?; + + println!("\n{}", "=".repeat(78)); + println!( + "stemmer ablation — chunks_fts rebuilt `unicode61` (unstemmed) over identical chunks + vectors (GH #157)" + ); + let dense_moved = (0..positives.len()) + .filter(|&i| vec_unstemmed.scores[i].note != base.vector.scores[i].note) + .count(); + if dense_moved == 0 { + println!(" [check] dense ablation identical across the flip, as it must be"); + } else { + println!( + " [FAULT] {dense_moved} dense rank(s) moved across an FTS-only flip — \ + the harness is broken; distrust this run" + ); + } + println!(" bm25-only, unstemmed vs shipped porter:"); + print_rank_moves(positives, base.bm25, bm25_unstemmed); + println!(" hybrid, unstemmed vs shipped porter:"); + print_rank_moves(positives, base.hybrid, &hybrid_unstemmed); + println!( + " unstemmed aggregates: bm25 note {:.2} / {:.3} hybrid note {:.2} / {:.3} chunk {:.2} / {:.3}", + bm25_unstemmed.note.hit1(), + bm25_unstemmed.note.mrr(), + hybrid_unstemmed.note.hit1(), + hybrid_unstemmed.note.mrr(), + hybrid_unstemmed.chunk.hit1(), + hybrid_unstemmed.chunk.mrr(), + ); + append_result( + base.log, + &result_row( + base.run, + RowInputs { + label: "unicode61", + cfg: &ChunkConfig::default(), + tokenizer: FtsTokenizer::Unicode61.sql(), + notes: base.notes, + chunks: base.chunks, + embed_secs: base.embed_secs, + queries: positives, + bm25: Some(bm25_unstemmed), + vector: Some(&vec_unstemmed), + hybrid: &hybrid_unstemmed, + similar: None, + calibration: None, + }, + ), + )?; + // Hand the vault back untainted, so a `--sweep` in the same run (and the + // reference numbers above) stay under the shipped default tokenizer. + vault.rebuild_fts(FtsTokenizer::PorterUnicode61)?; + Ok(()) +} + +/// The in-process chunker sweep (the #44 A/B): re-chunk and re-embed the same vault +/// under each variant, and judge it on the paired per-query moves against the default. +pub fn sweep( + vault: &mut Vault, + base: &Baseline, + sim_set: &SimilarSet, +) -> Result<(), Box> { + let positives = base.queries; + // The #44 grid: both directions on each swept knob, plus the one + // interaction worth a row. `target_tokens` brackets the 450 default + // (250 / 350 / 600); `overlap_frac` brackets 0.15 (0.0 / 0.30); + // `target-250+heading-path` exists because a smaller chunk carries less + // of its own context, which is exactly when the breadcrumb prefix (D3) + // is most plausibly worth its tokens. `chars_per_token` and + // `backscan_tokens` stay unswept: they are calibration constants of the + // token proxy and the boundary search, not retrieval-quality levers. + let variants: Vec<(&str, ChunkConfig)> = vec![ + ( + "target-250", + ChunkConfig { + target_tokens: 250, + ..ChunkConfig::default() + }, + ), + ( + "target-350", + ChunkConfig { + target_tokens: 350, + ..ChunkConfig::default() + }, + ), + ( + "target-600", + ChunkConfig { + target_tokens: 600, + ..ChunkConfig::default() + }, + ), + ( + "overlap-0", + ChunkConfig { + overlap_frac: 0.0, + ..ChunkConfig::default() + }, + ), + ( + "overlap-30", + ChunkConfig { + overlap_frac: 0.30, + ..ChunkConfig::default() + }, + ), + ( + "prepend-heading-path", + ChunkConfig { + prepend_heading_path: true, + ..ChunkConfig::default() + }, + ), + ( + "target-250+heading-path", + ChunkConfig { + target_tokens: 250, + prepend_heading_path: true, + ..ChunkConfig::default() + }, + ), + ]; + println!("\n{}", "=".repeat(78)); + println!("chunker sweep (same model, same corpus; default row above for reference)"); + // `mate MRR` rather than the per-anchor `similar h@3` this column used + // to carry: that number saturates (GH #183), so as a *comparison* + // column across variants it could only ever print 1.00 (GH #188). + // `strangers` replaced `neg clean` when GH #197 retired the existence + // gate: under always-serve a negative anchor always serves, so that + // column could only ever print 0/5 — the same cannot-move failure. + println!( + "{:<24} {:>7} {:>8} note h@1/MRR vec h@1/MRR chunk h@1/MRR mate MRR strangers", + "config", "chunks", "embed_s" + ); + for (label, cfg) in variants { + vault.set_chunk_config(cfg.clone()); + vault.project(true)?; // force: re-chunk everything, clearing vectors + let (chunks, embed_secs) = timed_embed(vault)?; + let vec_pass = score_pass(vault, positives, Retrieval::VectorOnly)?; + let pass = score_pass(vault, positives, Retrieval::Fused)?; + let sim = score_similar(vault, sim_set)?; + println!( + "{:<24} {:>7} {:>8.1} {:.2} / {:.3} {:.2} / {:.3} {:.2} / {:.3} {:.3} {}", + label, + chunks, + embed_secs, + pass.note.hit1(), + pass.note.mrr(), + vec_pass.note.hit1(), + vec_pass.note.mrr(), + pass.chunk.hit1(), + pass.chunk.mrr(), + sim.mate.mrr(), + sim.strangers.len(), + ); + // The readout the A/B is actually judged on: at this n every aggregate + // delta above is 1–2 queries, so the aggregate is a smoke alarm and the + // per-query win/loss list is the data (docs/evals.md, the + // process rules). + print_rank_moves(positives, base.hybrid, &pass); + append_result( + base.log, + &result_row( + base.run, + RowInputs { + label, + cfg: &cfg, + tokenizer: FtsTokenizer::PorterUnicode61.sql(), + notes: base.notes, + chunks, + embed_secs, + queries: positives, + bm25: None, + vector: Some(&vec_pass), + hybrid: &pass, + similar: Some(&sim), + calibration: None, + }, + ), + )?; + } + Ok(()) +} diff --git a/crates/b2-embed/examples/eval/dense.rs b/crates/b2-embed/examples/eval/dense.rs new file mode 100644 index 0000000..32e2227 --- /dev/null +++ b/crates/b2-embed/examples/eval/dense.rs @@ -0,0 +1,396 @@ +//! The dense single-domain fixture (`evals/corpus-dense/`, GH #196/#197 Phase 0b), scored +//! in its own vault and its own row, never averaged in: per-mate ranks, the zero-empty-panes +//! sweep, the fold bench, and the shipped search bar replayed over every note's title. + +use crate::common::{cosine_of, pile_stats, term_coverage, title_query, truncate, ScratchVault}; +use crate::discovery::AnchorDetail; +use crate::evidence::ServedRow; +use crate::fold::{fold_json, score_fold, FoldBench}; +use crate::gate::FLOOR_DENSE_MATE_MRR; +use crate::instrument::timed_embed; +use crate::labels::SimilarSet; +use crate::metrics::{paths_match, rank_str_at, Agg}; +use crate::report::{unix_secs, RunId}; +use crate::tail::{dense_tail_json, print_dense_tail}; +use crate::{K, SIM_K}; +use b2_core::embed::Embedder; +use b2_core::vault::Vault; +use b2_embed::{EmbedConfig, LocalEmbedder}; +use std::path::Path; + +/// Score the dense single-domain fixture (GH #196/#197, Phase 0b) in a throwaway +/// vault of its own: per-mate discovery ranks against `similar-dense.json`, and +/// the empty-pane sweep across **every** note in the fixture — not only the +/// labelled anchors, because the assertion is about the surface ("no pane in a +/// dense vault is dark"), not about the labels. +pub fn score_dense( + evals_dir: &Path, + set: &SimilarSet, +) -> Result> { + let corpus_dir = evals_dir.join("corpus-dense"); + // A second model load rather than sharing the first vault's: the embedder was + // moved into that vault, and the fixture's whole point is an isolated run. + let config = EmbedConfig::load()?; + let embedder = LocalEmbedder::load(&config)?; + let model_id = embedder.model_id().to_string(); + let scratch = ScratchVault::copy_flat(&corpus_dir)?; + let vault = Vault::open_with_embedder(scratch.root(), Box::new(embedder))?; + vault.project(false)?; + let (chunks, embed_secs) = timed_embed(&vault)?; + + let mut pass = DensePass { + notes: 0, + chunks, + embed_secs, + mate: Agg::default(), + mate_ranks: Vec::new(), + empty_panes: Vec::new(), + detail: Vec::new(), + // The bake-off's absolute bench (GH #200): every note is an anchor here, + // because the ruling being tested is about the *surface* — a vault where + // everything relates may never default to "nothing relates" — and the + // labels cover only a few of these notes. + fold: score_fold(&vault, set, "dense", true)?, + // The bar's hardest bench, for the same reason the fold's is: this is the + // geometry that disqualified the rule that lost (GH #201). + search: score_dense_search(&vault, &model_id)?, + }; + // The pane sweep: every note is an anchor, labelled or not. + for note in vault.list_notes()? { + pass.notes += 1; + if vault.similar(¬e.path, SIM_K)?.is_empty() { + pass.empty_panes.push(note.path); + } + } + // Per-mate ranks on the labelled anchors, the orthogonal corpus's metric + // re-used verbatim (GH #183's non-saturating readout). + for label in &set.anchors { + let candidates = vault.similar(&label.anchor, SIM_K)?; + for expected in &label.expected { + let rank = candidates + .iter() + .position(|c| paths_match(&c.path, expected)) + .map(|p| p + 1); + pass.mate.add(rank); + pass.mate_ranks + .push((label.anchor.clone(), expected.clone(), rank)); + } + pass.detail.push(AnchorDetail { + anchor: label.anchor.clone(), + negative: false, + candidates: candidates + .iter() + .map(|c| { + let related = label.expected.iter().any(|e| paths_match(&c.path, e)); + (c.path.clone(), cosine_of(c.score), related) + }) + .collect(), + }); + } + Ok(pass) +} + +/// The dense fixture's reading (see [`score_dense`]). +pub struct DensePass { + pub notes: usize, + pub chunks: usize, + pub embed_secs: f64, + /// Per-mate ranks at [`SIM_K`] — the fixture's rank metric. + pub mate: Agg, + /// (anchor, mate, rank) per labelled mate, for the printed lines and the row. + pub mate_ranks: Vec<(String, String, Option)>, + /// Notes whose discovery pane served nothing — asserted empty (GH #196/#197). + pub empty_panes: Vec, + pub detail: Vec, + /// The fold bake-off on this fixture (GH #200) — swept over **every** note, + /// which is where the candidates' hardest bench is: a rule whose default + /// view goes dark on a single-domain vault is disqualified, not re-tuned. + pub fold: FoldBench, + /// D2's shipped bar replayed on this fixture (GH #201) — see + /// [`score_dense_search`]. + pub search: DenseSearch, +} + +/// The shipped search evidence bar's reading **on the single-domain fixture** (ADR-0015). +/// +/// This exists because the bar's first form died here and nowhere else. A hard `df <= 10%` +/// content ceiling read 0 cut / 0 served on the labelled orthogonal corpus — clean by every +/// number that bench can produce — and then classed `drone` (df 3) and `comb` (df 7) as +/// stopwords in a vault about beekeeping, cutting 3 of 15 answerable queries. The lexical +/// rule's hazard is **topical concentration**, which the orthogonal corpus cannot express, so +/// a run that judges the bar only there judges it on the geometry it survives. +/// +/// The reading was first taken once, by hand. Taking it *once* is the thing GH #187 named, +/// so it is re-derived every run, here, beside the fold bench that already sweeps this +/// fixture. **In the exit gate** since GH #202, as a row of its own rather than headroom on +/// the labelled corpus's, because it watches a different *geometry*. +pub struct DenseSearch { + /// Every note's own title replayed as a query — the **tripwire direction** + /// (D2: a labelled-relevant query cut is zero with no headroom). Titles need + /// no labels and so nothing here can be relabelled to clear a reading. + pub titles: Vec, + /// Nonsense, the defect direction. See [`DENSE_NONSENSE`]. + pub nonsense: Vec, + /// `None` when the active model has no calibrated bar (M2) — the coverage + /// readings still print, the verdicts do not exist to print. + pub bar: Option, +} + +/// One query's reading on the dense fixture: the two absolute signals D2 judges, +/// and the engine's own verdict rather than a restatement of it. +pub struct SearchProbe { + pub query: String, + /// IDF-weighted term coverage; `None` when no term carries any weight (the + /// lexical half abstaining, not scoring zero). + pub coverage: Option, + pub best_cos: Option, + /// `Vault::search_evidence`'s verdict — what would actually ship. `None` + /// mirrors [`DenseSearch::bar`]. + pub vouched: Option, + /// The served list with its per-hit provenance — the tail bake-off's dense + /// bench (GH #206). On a title query `keep` is true for every row **by + /// geometry**, not by label: a single-domain vault's lists are all real + /// matches, so a tail rule that truncates one is disqualified — the same + /// absolute GH #200 enforced for discovery, and the reason this fixture + /// needs no `tail_relevant` labels. + pub rows: Vec, +} + +/// The negatives replayed on the dense fixture: **nonsense only**. The labelled negatives in +/// `queries.json` are the *orthogonal* corpus's, and process rule 2's token audit is what +/// makes them negatives — an audit that says nothing about a different corpus. Against +/// `corpus-dense` the phrase-shaped ones disqualify themselves on their merits ("why parrots +/// mimic speech" shares `mimic` with `robbing-behavior.md`, which is a thing the rule +/// deliberately serves). Nonsense needs no audit in any vault, which is why it transfers. +pub const DENSE_NONSENSE: [&str; 2] = ["shjfasd", "vrelqip zonktar wembleforth"]; + +/// Replay the shipped bar over the dense fixture (see [`DenseSearch`]). +/// +/// Coverage is read off [`b2_core::vault::QueryTermView::idf`] — the view's own +/// weights, not a second copy of the formula ([`term_coverage`]). The orthogonal +/// corpus's bake-off re-derives its arithmetic deliberately, as a drift check +/// against the engine; one such check is the check, and a second would only be two +/// places to fix. +pub fn score_dense_search( + vault: &Vault, + model_id: &str, +) -> Result> { + let read = |query: &str, keep: bool| -> Result> { + let view = vault.search_evidence(query, K)?; + Ok(SearchProbe { + query: query.to_string(), + coverage: term_coverage(&view), + best_cos: view.best_cos, + vouched: view.vouched, + rows: view + .results + .iter() + .map(|r| ServedRow { + path: r.result.path.clone(), + bm25_rank: r.bm25_rank, + cos: r.cos, + keep, + }) + .collect(), + }) + }; + let mut titles = Vec::new(); + for note in vault.list_notes()? { + // The fixture's notes carry no frontmatter title, so the slug is the + // query — `drone-comb` → "drone comb", which is the pair of words the + // retired ceiling called stopwords. + if let Some(title) = title_query(¬e) { + titles.push(read(&title, true)?); + } + } + Ok(DenseSearch { + titles, + nonsense: DENSE_NONSENSE + .iter() + .map(|q| read(q, false)) + .collect::>()?, + bar: b2_core::search::EvidenceBar::for_model(model_id), + }) +} + +/// Print the dense fixture's search-evidence reading (see [`DenseSearch`]). +pub fn print_dense_search(search: &DenseSearch) { + println!( + " search bar D2's shipped bar replayed on this geometry (GH #201; GATED since GH #202)" + ); + // The coverage reading comes FIRST, and above the bar, because it is + // **model-free**: a fact about this vault's vocabulary, and the lexical + // half's whole premise. Gating it behind a calibrated bar would print + // nothing at all on a vault the harness can still say something true about — + // the defect PR #205's review already fixed in `calibrate.rs` (c03f8cd), met + // again here (PR #207 review). + let covs: Vec = search.titles.iter().filter_map(|p| p.coverage).collect(); + let cov_line = match pile_stats(&covs) { + Some((min, med, max)) => format!("{min:.2}/{med:.2}/{max:.2}"), + None => "— (no query carried weight)".to_string(), + }; + println!(" title-as-query coverage min/med/max {cov_line}"); + + // Only the *verdicts* below need a bar, so only they stop here. + let Some(bar) = search.bar else { + println!(" no calibrated bar for this model — no verdict is offered (M2)"); + return; + }; + let cut: Vec<&SearchProbe> = search + .titles + .iter() + .filter(|p| p.vouched == Some(false)) + .collect(); + let served: Vec<&SearchProbe> = search + .nonsense + .iter() + .filter(|p| p.vouched == Some(true)) + .collect(); + println!( + " bar under test: coverage ≥ {:.2} or cos ≥ {:.3}", + bar.min_term_coverage, bar.min_cos, + ); + println!( + " cuts {}/{} title queries ← the TRIPWIRE direction; the retired df ceiling \ + cut 3 here", + cut.len(), + search.titles.len() + ); + for p in &cut { + println!( + " [CUT] {:<32} cov {:>5} cos {:>6}", + truncate(&p.query, 32), + p.coverage + .map(|c| format!("{c:.2}")) + .unwrap_or_else(|| "—".to_string()), + p.best_cos + .map(|c| format!("{c:.3}")) + .unwrap_or_else(|| "—".to_string()), + ); + } + println!( + " serves {}/{} nonsense queries ← the reported defect, on this corpus", + served.len(), + search.nonsense.len() + ); +} + +/// The dense fixture's search reading as JSON (`search_transfer` in the dense +/// row) — every probe, so any bar is re-derivable from a row without re-running +/// the model, the `discovery_fold` convention. +pub fn dense_search_json(search: &DenseSearch) -> serde_json::Value { + let probes = |pile: &[SearchProbe]| { + pile.iter() + .map(|p| { + serde_json::json!({ + "query": p.query, + "coverage": p.coverage.map(|c| (c * 1e4).round() / 1e4), + "best_cos": p.best_cos.map(|c| (c * 1e4).round() / 1e4), + "vouched": p.vouched, + // NEW subkey (absent before GH #206): the served list's + // per-hit provenance, so the tail constraints below are + // re-derivable from a row without re-running the model. + "rows": p.rows.iter().map(|row| serde_json::json!({ + "path": row.path, + "bm25_rank": row.bm25_rank, + "cos": row.cos.map(|c| (c * 1e4).round() / 1e4), + })).collect::>(), + }) + }) + .collect::>() + }; + serde_json::json!({ + "bar": search.bar.map(|b| serde_json::json!({ + "min_term_coverage": b.min_term_coverage, + "min_cos": b.min_cos, + })), + "titles_cut": search.titles.iter().filter(|p| p.vouched == Some(false)).count(), + "nonsense_served": search.nonsense.iter().filter(|p| p.vouched == Some(true)).count(), + "titles": probes(&search.titles), + "nonsense": probes(&search.nonsense), + // NEW subkey (absent before GH #206): the per-hit tail families' + // dense-bench constraints — the absolute this fixture supplies. + "tail": dense_tail_json(&search.titles), + }) +} + +pub fn print_dense_report(dense: &DensePass) { + println!("\n{}", "=".repeat(78)); + println!( + "dense fixture — corpus-dense/ ({} notes / {} chunks, single-domain, no loner; GH #196/#197)", + dense.notes, dense.chunks + ); + println!( + " per-mate hit@1={:.2} hit@3={:.2} MRR@{SIM_K}={:.3} (n={} mates, GATED at MRR@{SIM_K} ≥ {FLOOR_DENSE_MATE_MRR:.2})", + dense.mate.hit1(), + dense.mate.hit3(), + dense.mate.mrr(), + dense.mate.n + ); + for (anchor, mate, rank) in &dense.mate_ranks { + println!( + " {:>5} {anchor} → {mate}", + rank_str_at(*rank, SIM_K) + ); + } + if dense.empty_panes.is_empty() { + println!( + " panes {}/{} notes serve candidates — zero empty panes (ASSERTED: a dense vault \ + may never read as \"nothing relates\")", + dense.notes, dense.notes + ); + } else { + println!( + " panes {} of {} notes serve an EMPTY pane: {}", + dense.empty_panes.len(), + dense.notes, + dense.empty_panes.join(", ") + ); + } + print_dense_search(&dense.search); + // Model-geometry reading, not a verdict, so it prints with or without a + // calibrated bar — the same posture as the coverage line above it. + print_dense_tail(&dense.search.titles); +} + +/// The dense fixture's own JSONL row. Tagged `"corpus": "dense"` — the key that +/// keeps rows from ever averaging across corpora (the orthogonal rows carry +/// `"corpus": "orthogonal"`); a smaller shape than the main row on purpose, +/// since the fixture scores discovery alone. +pub fn dense_row(run: RunId, dense: &DensePass) -> serde_json::Value { + let RunId { git, model, dim } = run; + let ts = unix_secs(); + serde_json::json!({ + "ts": ts, + "git": git, + "model": model, + "dim": dim, + "corpus": "dense", + "notes": dense.notes, + "chunks": dense.chunks, + "embed_secs": dense.embed_secs, + "similar_per_mate": { "n": dense.mate.n, "hit1": dense.mate.hit1(), "hit3": dense.mate.hit3(), "mrr": dense.mate.mrr() }, + "mates": dense.mate_ranks.iter().map(|(anchor, mate, rank)| serde_json::json!({ + "anchor": anchor, "mate": mate, "rank": rank, + })).collect::>(), + "empty_panes": { "n": dense.notes, "empty": dense.empty_panes.len(), "detail": dense.empty_panes }, + "discovery_fold": fold_json(&dense.fold), + // NEW key (absent from rows before 2026-08-22): the shipped search bar + // replayed on this fixture (GH #201). Deliberately NOT the orthogonal + // row's `search_evidence`: that key holds the *labelled* bake-off, and + // this is the label-free transfer reading — a different measurement, so + // it takes a different name. Same convention as every key above: new, + // never a redefinition, so no reader has to branch on `corpus` to learn + // which shape it is holding (PR #207 review). + "search_transfer": dense_search_json(&dense.search), + "similar_detail": dense.detail.iter().map(|d| serde_json::json!({ + "anchor": d.anchor, + "negative": d.negative, + "candidates": d.candidates.iter().map(|(path, cos, related)| serde_json::json!({ + "path": path, + "cos": (cos * 1e4).round() / 1e4, + "related": related, + })).collect::>(), + })).collect::>(), + }) +} diff --git a/crates/b2-embed/examples/eval/discovery.rs b/crates/b2-embed/examples/eval/discovery.rs new file mode 100644 index 0000000..a46ea76 --- /dev/null +++ b/crates/b2-embed/examples/eval/discovery.rs @@ -0,0 +1,450 @@ +//! Discovery (`similar`) scoring on the orthogonal corpus: per-mate ranks, strangers and +//! the cosine piles (GH #183/#188), and the z calibration dump whose windows are re-derived +//! every run (GH #187; gating nothing since GH #197). + +use crate::common::{cosine_of, passage_z, pile_stats, Band}; +use crate::labels::SimilarSet; +use crate::metrics::{paths_match, Agg, Window}; +use crate::{SIM_K, Z_SCAN_LIMIT}; +use b2_core::vault::Vault; + +/// A full pass over the discovery labels: rank aggregates over the positive +/// anchors, the suppression tally over the negative ones, and every surfaced +/// candidate's cosine sorted into the two calibration piles the quality floor +/// is read from (index-engine.md §3's ruling, PR #145). +#[derive(Default)] +pub struct SimilarPass { + /// Rank of the first `expected` hit per positive anchor — the pre-existing + /// hit@1 / hit@3 / MRR discovery metrics, unchanged. + pub rank: Agg, + /// **Per-mate** ranks: one entry per `(anchor, expected mate)` pair rather than one per + /// anchor (GH #183). [`Self::rank`] takes the *first* labelled mate it finds and stops, + /// so an anchor scores a hit on its easiest one and every harder mate is invisible — + /// which is how that metric sat pinned at 1.000 across the ordering change it was + /// supposed to judge. Worse, *adding* a hard mate makes `rank` strictly easier. + /// + /// Scoring each mate on its own is the non-saturating readout GH #183 asked for: a mate + /// sliding from rank 2 to rank 4 moves this number even while `rank` stays at ceiling. A + /// `None` is honest — that mate did not surface at all. + /// + /// **In the exit gate** since GH #188, at [`FLOOR_MATE_MRR`](crate::gate::FLOOR_MATE_MRR). It shipped reporting-only + /// for one issue's worth of time on the measure-then-calibrate precedent, and that is + /// where the baseline came from. + /// + /// (A pass-vs-pass suppression diff — `mate_raw` / `mate_suppressed`, asserted at zero — + /// stood beside this until GH #217: once ADR-0014 retired the existence gate, both + /// passes read the *identical* `similar` call, so the diff compared a call against + /// itself and could never fire. An existence gate returning to the path is what the + /// dense fixture's zero-empty-panes assertion and both per-mate MRR floors catch — + /// against ground truth, not against a second read of the same surface.) + pub mate: Agg, + /// **Strangers**: unlabelled notes the shipped surface serves on a *positive* anchor, + /// within the same top-`SIM_K` the ranks are read at — `(anchor, path)` per card, so the + /// smoke alarm comes with the list you argue against it with (process rule 1). + /// + /// This is discovery's **precision** side, and the harness had none: the gated numbers + /// watch mate ranks and negative anchors, and the negatives gate is structurally blind to + /// a member bar — while `member_z <= leader_z`, a negative anchor is clean iff its leader + /// is cut, so a relaxed member bar spends its entire cost here (GH #187/#188). + /// + /// Deliberately **reported, not gated**, and the reason is the gaming direction: the + /// cheapest way to shrink this count is to *label the stranger*, which silently moves the + /// per-mate metric too. The labels are not exhaustive, so an unlabelled note served is + /// not proof of junk — it reads as a smoke alarm with names attached. + pub strangers: Vec<(String, String)>, + /// Positive anchors serving at least one stranger — the spread behind + /// [`Self::strangers`], since one anchor with a long tail and five anchors + /// with one card each are different failures at the same count. + pub stranger_anchors: usize, + /// Negative anchors asked. + pub neg_n: usize, + /// Negative anchors that surfaced zero candidates. Under always-serve + /// (GH #197) an anchor with scorable candidates always serves, so this + /// reads 0 on this corpus — recorded for row comparability (the key's + /// meaning, "anchors serving nothing", is unchanged), no longer asserted: + /// the labels still say "nothing relates", and what the served cards + /// *claim* is measured by their bands (A2's readout, calibration block). + pub neg_clean: usize, + /// Candidates surfaced across all negative anchors — cards whose labelled + /// answer was "nothing". + pub neg_cards: usize, + /// Cosines of surfaced candidates a human labelled genuinely related. + pub related: Vec, + /// Cosines of everything else surfaced — non-expected candidates of positive + /// anchors, and every candidate of a negative anchor. + pub junk: Vec, + /// Every anchor's surfaced list in rank order. The flat piles above judge the + /// absolute floor; the *relative* drop-off cutoff is judged within one + /// anchor's list (how far did #2 fall from #1?), and naming the pair behind a + /// pile value needs the anchor too — so the order is recorded, not just the + /// distribution. + pub detail: Vec, +} + +/// One anchor's surfaced candidates, in rank order, for the results log. +pub struct AnchorDetail { + pub anchor: String, + /// True for a negative anchor (empty `expected`). + pub negative: bool, + /// (candidate path, cosine, human-labelled related) per surfaced candidate. + pub candidates: Vec<(String, f64, bool)>, +} + +/// One row of the z dump: a candidate in the band's unit, with the human label +/// attached. +pub struct ZCand { + pub path: String, + /// The stage-2 best-passage z (GH #192's unit; the strength band's input, + /// gating nothing since GH #197) — straight from the engine's own + /// statistics. + pub z: f64, + /// The same z recomputed harness-side from the served scores (z over squared + /// best-pair distance, nearer = higher). An instrument check, not a reading: + /// if this drifts from `z`, the engine's statistic moved and the harness's + /// model of it is stale. `None` only when the population had zero variance + /// and no z exists. + pub z_recheck: Option, + /// The stage-2 score as served (negated best chunk-pair L2) — kept so the + /// row also records the model-comparable cosine, not only the + /// anchor-relative z. + pub score: f64, + /// Labelled a mate of this anchor. + pub mate: bool, +} + +/// One anchor's **complete** reading in the band's unit — every candidate note +/// in the corpus, in z order, with the human label attached (GH #187; stage-2 +/// best-passage z since GH #192; the z travels ungated on the one shipped +/// surface since GH #197, so the dump *is* the served list read deep). +/// +/// The cosine piles above are the same numbers in a model-comparable unit; this +/// is the anchor-relative unit a z existence bar would judge in, which is why +/// the piles could never re-derive one's constants. +pub struct AnchorZ { + pub anchor: String, + /// True for a negative anchor (empty `expected`) — its whole list is + /// strangers by label, and its leader is what a leader gate would answer to. + pub negative: bool, + /// Every candidate in served order, which *is* descending z. + pub candidates: Vec, +} + +impl AnchorZ { + /// The top candidate's z — what a leader gate would read. `None` only if + /// the anchor produced no scorable candidate at all. + pub fn leader(&self) -> Option { + self.candidates.first().map(|c| c.z) + } + /// This anchor's labelled mates' z's (empty for a negative anchor). + pub fn mates(&self) -> impl Iterator + '_ { + self.candidates.iter().filter(|c| c.mate).map(|c| c.z) + } + /// Everything on this anchor's list a human did *not* label — on a positive + /// anchor, exactly the population a member bar would have to cut. + pub fn strangers(&self) -> impl Iterator + '_ { + self.candidates.iter().filter(|c| !c.mate).map(|c| c.z) + } + /// The worst disagreement between the engine's z and the harness's own + /// recomputation from the served scores, across this anchor's candidates — + /// the statistic-level drift check ([`ZCand::z_recheck`]). + pub fn recheck_delta(&self) -> f64 { + self.candidates + .iter() + .filter_map(|c| c.z_recheck.map(|r| (c.z - r).abs())) + .fold(0.0, f64::max) + } +} + +/// The z dump across every discovery anchor, split into the populations an existence bar +/// would answer to (GH #187; the unit is the stage-2 best-passage z, and it gates nothing — +/// ADR-0014). +/// +/// Three populations, because a two-bar rule's constants answer to different ones — the +/// conflation is what made "the negatives gate would catch a bad `member_z`" look true when +/// it was not. A leader gate is calibrated by negative-anchor leaders against positive-anchor +/// leaders; a member bar by strangers against labelled mates. Both windows are re-derived +/// every run — the standing record of why no such rule ships, and the first reading any +/// Phase-2 candidate answers to. +#[derive(Default)] +pub struct FloorZ { + /// Every anchor with computed statistics, in label order. + pub anchors: Vec, + /// Anchors whose candidate pool was under `STATS_MIN_POPULATION` or had zero + /// variance, so no z exists to calibrate from. Named rather than silently + /// dropped: on a corpus small enough for this to happen, every window below + /// is measured on fewer anchors than the labels suggest. + pub ungraded: Vec, +} + +impl FloorZ { + /// (a) Labelled mates' z's — what a member bar would have to keep. + pub fn mate_z(&self) -> Vec { + self.anchors.iter().flat_map(|a| a.mates()).collect() + } + /// (b) Strangers on **positive** anchors — what a member bar would have to + /// cut. The negative anchors' own candidates are deliberately excluded: + /// they are the leader pair's business, and folding them in here is what + /// would let a member window look wider than it is. + pub fn stranger_z(&self) -> Vec { + self.anchors + .iter() + .filter(|a| !a.negative) + .flat_map(|a| a.strangers()) + .collect() + } + /// (c) Negative anchors' leaders — what a leader gate would have to cut. + pub fn neg_leader_z(&self) -> Vec { + self.anchors + .iter() + .filter(|a| a.negative) + .filter_map(|a| a.leader()) + .collect() + } + /// Positive anchors' leaders — what a leader gate would have to keep, or it + /// empties a list that has a real mate on it. + pub fn pos_leader_z(&self) -> Vec { + self.anchors + .iter() + .filter(|a| !a.negative) + .filter_map(|a| a.leader()) + .collect() + } + + /// The worst engine-vs-recomputed z disagreement across every anchor — the + /// statistic-level instrument check (see [`ZCand::z_recheck`]). Small fp + /// noise is expected (the engine z-scores f32 squared distances; the + /// harness recomputes them from the f32-sqrt'd scores), so the printed + /// check tolerates 1e-3 z and reports the observed maximum. + pub fn recheck_delta(&self) -> f64 { + self.anchors + .iter() + .map(|a| a.recheck_delta()) + .fold(0.0, f64::max) + } +} + +/// Score the discovery labels — **one pass, one surface** (GH #197). +/// +/// Everything reads `Vault::similar`, the always-served ranked list: the ranks, +/// the strangers, the negatives' card tally, and the cosine piles + per-anchor +/// detail the calibration blocks consume. (A second pass with a pass-vs-pass +/// suppression diff existed while the GH #150 existence gate shipped; under +/// always-serve it re-read the identical call and could never fire, so it +/// retired — GH #217, and see [`SimilarPass::mate`] for what catches a +/// returning gate instead.) +pub fn score_similar( + vault: &Vault, + set: &SimilarSet, +) -> Result> { + let mut pass = SimilarPass::default(); + for label in &set.anchors { + let candidates = vault.similar(&label.anchor, SIM_K)?; + let negative = label.expected.is_empty(); + if negative { + pass.neg_n += 1; + if candidates.is_empty() { + pass.neg_clean += 1; + } + pass.neg_cards += candidates.len(); + } else { + let rank = candidates + .iter() + .position(|c| label.expected.iter().any(|e| paths_match(&c.path, e))) + .map(|p| p + 1); + pass.rank.add(rank); + // …and again per labelled mate, so a hard one can't hide behind an + // easy one (GH #183 — see `SimilarPass::mate`). + for expected in &label.expected { + let mate_rank = candidates + .iter() + .position(|c| paths_match(&c.path, expected)) + .map(|p| p + 1); + pass.mate.add(mate_rank); + } + // The precision side of the same list (GH #188 — see + // `SimilarPass::strangers`): everything served here that no label + // claims. Scored on the shipped surface at the ranks' own depth, + // because that is the list a human is handed. + let before = pass.strangers.len(); + for c in &candidates { + if !label.expected.iter().any(|e| paths_match(&c.path, e)) { + pass.strangers.push((label.anchor.clone(), c.path.clone())); + } + } + if pass.strangers.len() > before { + pass.stranger_anchors += 1; + } + } + // The calibration piles + per-anchor detail, off the same served list. + let mut ordered = Vec::with_capacity(candidates.len()); + for c in &candidates { + let related = !negative && label.expected.iter().any(|e| paths_match(&c.path, e)); + let cos = cosine_of(c.score); + if related { + pass.related.push(cos); + } else { + pass.junk.push(cos); + } + ordered.push((c.path.clone(), cos, related)); + } + pass.detail.push(AnchorDetail { + anchor: label.anchor.clone(), + negative, + candidates: ordered, + }); + } + Ok(pass) +} + +/// Dump every candidate's **band-unit z** (stage-2 best-passage z, GH #192) on +/// every discovery anchor — the deep discovery pass (GH #187). +/// +/// Since GH #197 the z travels ungated on the one shipped surface, so the dump +/// is simply `similar` read at [`Z_SCAN_LIMIT`] rather than `SIM_K`: a bar is +/// calibrated against the population it has to cut, and the strangers just +/// under a served prefix are precisely the ones a lower bar would admit. (This +/// used to need the engine's floor moved out of the way with `-∞` bars; the +/// machinery went with the gate.) +pub fn score_floor_z( + vault: &Vault, + set: &SimilarSet, +) -> Result> { + let mut dump = FloorZ::default(); + for label in &set.anchors { + let candidates = vault.similar(&label.anchor, Z_SCAN_LIMIT)?; + // z is uniform within one query by construction (`discover::candidates` + // gives every surviving candidate a z or none of them), so the leader's + // presence decides the whole list — an anchor with no statistics is + // named, never half-counted. + let Some(true) = candidates.first().map(|c| c.z.is_some()) else { + dump.ungraded.push(label.anchor.clone()); + continue; + }; + // The band z, recomputed harness-side over the same population the + // engine just served: z over squared best-pair distance (score is + // negated L2, so d² = score²), oriented nearer = higher. Should equal + // the engine's own z to fp noise — the statistic-level drift check. + let d2: Vec = candidates.iter().map(|c| c.score * c.score).collect(); + let recheck = passage_z(&d2); + dump.anchors.push(AnchorZ { + anchor: label.anchor.clone(), + negative: label.expected.is_empty(), + candidates: candidates + .iter() + .enumerate() + .filter_map(|(i, c)| { + let mate = label.expected.iter().any(|e| paths_match(&c.path, e)); + c.z.map(|z| ZCand { + path: c.path.clone(), + z, + z_recheck: recheck.as_ref().map(|r| r[i]), + score: c.score, + mate, + }) + }) + .collect(), + }); + } + Ok(dump) +} + +/// Re-derive the admissible windows a z existence rule *would* have on the +/// corpus as it stands today (GH #187) — the standing record of why none ships +/// (GH #197), and the first reading any Phase-2 bake-off candidate answers to. +/// +/// Printed every run, and recorded in the row, precisely so no number here ever +/// gets copied into a doc comment again: the #150 windows were measured once, +/// frozen into a rustdoc, and silently falsified the first time the corpus grew +/// a note shape they were never derived from. The instrument is the citable +/// thing; any constant stays in the code it governs. +pub fn print_floor_windows(z: &FloorZ) { + let (mates, strangers) = (z.mate_z(), z.stranger_z()); + let (neg_leaders, pos_leaders) = (z.neg_leader_z(), z.pos_leader_z()); + println!( + " discovery z calibration (stage-2 best-passage z — the band's input; gates NOTHING \ + since GH #197)" + ); + // The statistic-level instrument check: the dump's z is recomputed from the + // served scores, so a drift in the engine's statistic can't pass silently + // (tolerance covers f32 sqrt/square round-trip noise). + let recheck = z.recheck_delta(); + if recheck <= 1e-3 { + println!(" [check] harness recomputation matches the engine z (max Δ {recheck:.1e})"); + } else { + println!( + " [FAULT] harness recomputation disagrees with the engine z by up to {recheck:.3} — \ + the statistic moved; distrust every reading below" + ); + } + for (label, pile, role) in [ + ("mates", &mates, "a member bar would have to KEEP"), + ( + "strangers", + &strangers, + "a member bar would have to CUT (positive anchors)", + ), + ( + "neg leaders", + &neg_leaders, + "a leader gate would have to CUT", + ), + ( + "pos leaders", + &pos_leaders, + "a leader gate would have to KEEP", + ), + ] { + match pile_stats(pile) { + Some((min, med, max)) => println!( + " {label:<12} n={:<4} min/med/max {min:+.3}/{med:+.3}/{max:+.3} ← {role}", + pile.len() + ), + None => println!(" {label:<12} n=0 (nothing labelled — no reading)"), + } + } + for (name, win) in [ + ("leader", Window::read(&neg_leaders, &pos_leaders)), + ("member", Window::read(&strangers, &mates)), + ] { + match win { + None => println!(" {name} window no reading (a population was empty)"), + Some(w) if w.open() => println!( + " {name} window ({:+.3}, {:+.3}] — open on THIS corpus; a real vault is the \ + other half of any such claim (process rule 5, `make calibrate`)", + w.cut_max, w.keep_min + ), + Some(w) => println!( + " {name} window EMPTY — the population it must cut reaches {:+.3} while the one \n\ + \x20 it must keep starts at {:+.3}; the two INVERT, and no constant \n\ + \x20 separates an inversion", + w.cut_max, w.keep_min + ), + } + } + // Every negative anchor's leader with the band it paints — A2's readout: + // under always-serve these cards ARE served, and the band is what they + // claim to the human whose labels say "nothing relates". + println!( + " negative anchors' leaders (served under always-serve; the band carries the honesty):" + ); + for a in z.anchors.iter().filter(|a| a.negative) { + match a.candidates.first() { + Some(c) => println!( + " {} → {} {:+.3} {}", + a.anchor, + c.path, + c.z, + Band::of(c.z).glyph() + ), + None => println!(" {} (no candidates)", a.anchor), + } + } + if !z.ungraded.is_empty() { + println!( + " [warn] no z statistics for {} anchor(s) ({}) — pool under STATS_MIN_POPULATION or \ + zero variance; the windows above are measured without them", + z.ungraded.len(), + z.ungraded.join(", ") + ); + } +} diff --git a/crates/b2-embed/examples/eval/evidence.rs b/crates/b2-embed/examples/eval/evidence.rs new file mode 100644 index 0000000..867da1f --- /dev/null +++ b/crates/b2-embed/examples/eval/evidence.rs @@ -0,0 +1,667 @@ +//! The search evidence calibration and bake-off (invariants.md D2; ADR-0015, GH #201/#202): +//! per labelled query, the absolute signals RRF discards, the coverage × cosine grid swept +//! over them, and the shipped bar's reading the exit gate asserts. + +use crate::common::{cosine_of, pile_stats, truncate}; +use crate::gate::{MAX_NEGATIVES_SERVED, MAX_POSITIVES_CUT}; +use crate::labels::Labelled; +use crate::metrics::{paths_match, Window}; +use crate::K; +use b2_core::search::EvidenceBar; +use b2_core::vault::Vault; +use std::path::Path; + +/// One served row of a labelled query's list (GH #206): the per-hit provenance +/// RRF discards, and the row's relevance **by label** — the two things a per-hit +/// tail rule is judged between. Rank is the row's position in `rows`, never a +/// stored field: the fused order is the identity D1's prefix requirement binds. +pub struct ServedRow { + pub path: String, + /// 0-based rank in the BM25 list; `None` = the lexical half never ranked + /// this chunk — the "dense-only" row. + pub bm25_rank: Option, + /// This row's own cosine to the query; `None` when the dense half never + /// ranked it (a row is in at least one list, so `bm25_rank` and `cos` are + /// never both absent). + pub cos: Option, + /// In the keep-set — `relevant` ∪ `tail_relevant`, exhaustive by label + /// since GH #206: a false here is a judgement ("filler"), not an omission. + pub keep: bool, +} + +/// One labelled query's evidence reading (invariants.md D2; GH #201, Phase A of +/// the disclosure work): the signals a query-level evidence rule would judge, +/// dumped before any rule exists so GH #201 derives its rule from measurement +/// rather than assumption. Nothing here gates anything. +pub struct QueryEvidence { + pub query: String, + /// Chunks the OR-sanitized FTS5 expression matches at all. **Not** a + /// lexical-anchor test on phrase queries — `fts5_query` ORs every + /// alphanumeric term, so stopwords saturate this count; recorded precisely + /// so that saturation stays measured instead of assumed away. + pub bm25_hits: usize, + /// Best BM25 score over those matches, sign-flipped so higher = better + /// (FTS5's `rank` is more-negative-is-better). `None` when nothing matches + /// — the honest zero the fused surface currently cannot say. + pub bm25_best: Option, + /// Best cosine between the embedded query and any stored chunk vector (the + /// dense top-1) — the strongest semantic evidence the vault holds for this + /// query, which RRF discards before the surface sees it. + pub best_cos: Option, + /// The note that dense top-1 belongs to, naming what the number points at. + pub top: String, + /// The served list at [`K`], in fused order, one entry per row — path, + /// per-hit provenance, and the row's relevance **by label** (GH #206). For a + /// negative query every row's `keep` is false: the whole list is junk by + /// label. `served`/`dense_only` below read this, so the counts and the rows + /// they summarize cannot drift apart. + pub rows: Vec, + /// Chunks in the index — the denominator every `df` below is judged against. + pub chunk_total: usize, + /// Every query term with its document frequency, in query order. The + /// population the lexical-anchor rule is swept over: raw hit count and raw + /// best-BM25 both failed to separate the piles (Phase A), so the anchor is + /// derived from *these* rather than from either of those. + pub terms: Vec<(String, usize)>, + /// **The engine's own verdict** for this query — `Vault::search_evidence`'s `vouched`, + /// i.e. exactly what the surfaces act on (GH #202). The exit gate counts on *this*, never + /// on the harness's restatement below: an assertion about what ships must read what + /// ships. `None` when the model has no calibrated bar. + /// + /// The restatement is kept beside it because the sweep needs it — [`bake_off`] evaluates + /// the rule at coverages the engine cannot be asked about — and because two independent + /// readings of one rule are a drift check when compared. So they are: [`read_shipped_bar`] + /// prints a `[FAULT]` on any query where they disagree. + pub vouched: Option, +} + +impl QueryEvidence { + /// What the shipped surface serves at [`K`] today. For a negative query + /// this is the measured D2 defect: `limit` confident-looking results for a + /// query the vault holds nothing for. + pub fn served(&self) -> usize { + self.rows.len() + } + + /// Of the served rows, how many the **lexical half never ranked at all** + /// (`bm25_rank: None`) — the per-hit shape of the same defect, and the + /// signal the `lexical` tail rule folds on (GH #206). + pub fn dense_only(&self) -> usize { + self.rows.iter().filter(|r| r.bm25_rank.is_none()).count() + } + + /// The share of this query's term IDF the vault carries — the harness's own + /// restatement of [`b2_core::search::LexicalEvidence::term_coverage`], kept + /// separate so a drift in the engine's rule shows up as the two disagreeing + /// rather than as silence. `None` when nothing in the query carries weight. + pub fn coverage(&self) -> Option { + let idf = |df: usize| ((self.chunk_total as f64 + 1.0) / (df as f64 + 1.0)).ln(); + let total: f64 = self.terms.iter().map(|(_, df)| idf(*df)).sum(); + if total <= f64::EPSILON { + return None; + } + let present: f64 = self + .terms + .iter() + .filter(|(_, df)| *df >= 1) + .map(|(_, df)| idf(*df)) + .fold(0.0, |a, b| a + b); + Some(present / total) + } + + /// Whether this query has a lexical anchor at `min_coverage`. + pub fn anchored(&self, min_coverage: f64) -> bool { + self.coverage().is_some_and(|c| c >= min_coverage) + } +} + +/// The `min_term_coverage` grid the bake-off sweeps: how much of a query's own +/// weight the vault must carry for the lexical half to vouch for it. Spans "a +/// twentieth" to "all of it", so the printed rows show the rule's whole +/// behaviour rather than a neighbourhood of the shipped constant. +pub const COVERAGES: [f64; 10] = [0.05, 0.10, 0.15, 0.20, 0.25, 0.34, 0.50, 0.67, 0.85, 1.00]; + +/// One coverage-bar cell of the bake-off: how the lexical half alone splits the +/// labelled piles there, and what the cosine half would then have to do for the +/// queries it leaves undecided. +pub struct EvidenceCell { + pub coverage: f64, + /// Positives the lexical half already vouches for. + pub pos_anchored: usize, + /// Negatives the lexical half wrongly vouches for. **Any nonzero value + /// disqualifies the cell**: an anchored negative is served whatever the + /// cosine bar says, so no `min_cos` can rescue it. + pub neg_anchored: usize, + /// Cosines of the positives the lexical half left undecided — the pile a + /// `min_cos` must KEEP. + pub undecided_pos: Vec, + /// Cosines of the negatives left undecided — the pile it must CUT. + pub undecided_neg: Vec, +} + +impl EvidenceCell { + /// The cosine window this cell leaves for `min_cos`, over **only** the + /// queries the lexical half did not already decide — which is the whole + /// point of reading it here rather than over every query: D2's rule is + /// lexical OR semantic, so a positive with an anchor never needs its cosine + /// kept, and a pure-cosine window (Phase A's) overstates the keep set. + pub fn window(&self) -> Option { + Window::read(&self.undecided_neg, &self.undecided_pos) + } + + /// Whether some `min_cos` completes this cell into a rule that keeps every + /// positive and cuts every negative. + /// + /// Three ways to be admissible, and the two degenerate ones are real + /// readings rather than edge-case bookkeeping: with no undecided negatives + /// the cosine half is inert (the lexical rule did the whole job), and with + /// no undecided positives any bar above the negatives' best cosine works. + pub fn admissible(&self) -> bool { + if self.neg_anchored > 0 { + return false; + } + match self.window() { + Some(w) => w.open(), + None => true, + } + } + + /// The lowest `min_cos` that cuts every undecided negative — the bar's + /// measured floor. `None` when nothing is left to cut. + pub fn cut_floor(&self) -> Option { + let max = self + .undecided_neg + .iter() + .copied() + .fold(f64::NEG_INFINITY, f64::max); + max.is_finite().then_some(max) + } + + /// The highest `min_cos` that still keeps every undecided positive — the + /// bar's measured ceiling. `None` when nothing is left to keep. + pub fn keep_ceiling(&self) -> Option { + let min = self + .undecided_pos + .iter() + .copied() + .fold(f64::INFINITY, f64::min); + min.is_finite().then_some(min) + } +} + +/// Sweep the lexical rule over [`COVERAGES`], reading each cell's piles from the +/// labelled queries (GH #201, Phase C). +pub fn bake_off(ev: &SearchEvidence) -> Vec { + let mut cells = Vec::new(); + for coverage in COVERAGES { + let split = |rows: &[QueryEvidence]| -> (usize, Vec) { + let anchored = rows.iter().filter(|r| r.anchored(coverage)).count(); + let undecided = rows + .iter() + .filter(|r| !r.anchored(coverage)) + .filter_map(|r| r.best_cos) + .collect(); + (anchored, undecided) + }; + let (pos_anchored, undecided_pos) = split(&ev.positives); + let (neg_anchored, undecided_neg) = split(&ev.negatives); + cells.push(EvidenceCell { + coverage, + pos_anchored, + neg_anchored, + undecided_pos, + undecided_neg, + }); + } + cells +} + +/// The search evidence dump (GH #201): every labelled query's reading, split +/// into the piles a query-level evidence bar answers to — positives it must +/// keep reachable, negatives it must answer "no matches". +pub struct SearchEvidence { + pub positives: Vec, + pub negatives: Vec, +} + +/// One side's best-cos pile — the population the query-window derivation and +/// the printed pile lines both read (queries with no dense reading drop out). +pub fn best_cos_pile(rows: &[QueryEvidence]) -> Vec { + rows.iter().filter_map(|r| r.best_cos).collect() +} + +/// Score the search evidence calibration (invariants.md D2; GH #201) — a pure +/// read over the already-built vault. Per labelled query: the lexical half's +/// reading via a direct `chunks_fts` probe (the engine's own sanitized +/// expression — the absolute signals RRF discards), the dense top-1 via the +/// vector-only ablation path, and the shipped surface's served count. +pub fn score_search_evidence( + vault_root: &Path, + vault: &Vault, + positives: &[Labelled], + negatives: &[Labelled], +) -> Result> { + // A second read connection beside the Vault's own — C1: readers are + // unrestricted, and the probe wants FTS5's `rank`, which no façade read + // exposes (deliberately: bm25 units are engine internals everywhere but + // this instrument). + let conn = b2_core::open(&vault_root.join(".b2").join("b2.sqlite"))?; + let rows = |queries: &[Labelled]| -> Result, Box> { + queries + .iter() + .map(|q| { + let (bm25_hits, bm25_best) = bm25_probe(&conn, &q.query)?; + let dense = vault.search_vector_only(&q.query, 1)?; + let best_cos = dense.first().map(|h| cosine_of(h.score)); + let top = dense + .first() + .map(|h| h.path.clone()) + .unwrap_or_else(|| "—".to_string()); + // The façade's own evidence read (GH #201) — the term dfs, the + // per-hit provenance, and the served list in one call, so the + // sweep judges exactly what the engine would. + let view = vault.search_evidence(&q.query, K)?; + Ok(QueryEvidence { + query: q.query.clone(), + bm25_hits, + bm25_best, + best_cos, + top, + rows: view + .results + .iter() + .map(|r| ServedRow { + path: r.result.path.clone(), + bm25_rank: r.bm25_rank, + cos: r.cos, + keep: q + .relevant + .iter() + .chain(q.tail_relevant.iter()) + .any(|rel| paths_match(&r.result.path, rel)), + }) + .collect(), + chunk_total: view.chunk_total, + terms: view.terms.iter().map(|t| (t.term.clone(), t.df)).collect(), + vouched: view.vouched, + }) + }) + .collect() + }; + Ok(SearchEvidence { + positives: rows(positives)?, + negatives: rows(negatives)?, + }) +} + +/// The lexical half's reading for one raw query, probed directly over +/// `chunks_fts` with the engine's own sanitized expression +/// ([`b2_core::search::fts5_query`]): how many chunks match at all, and the +/// best BM25 score among them (FTS5 `rank`, sign-flipped so higher = better). +/// These are exactly the absolute signals the fused path computes and then +/// discards at RRF — which is why the instrument reads them raw. +pub fn bm25_probe( + conn: &rusqlite::Connection, + query: &str, +) -> Result<(usize, Option), Box> { + let expr = b2_core::search::fts5_query(query); + if expr.is_empty() { + return Ok((0, None)); + } + let hits: i64 = conn.query_row( + "SELECT count(*) FROM chunks_fts WHERE chunks_fts MATCH ?1", + [&expr], + |r| r.get(0), + )?; + let hits = usize::try_from(hits).unwrap_or(0); + if hits == 0 { + return Ok((0, None)); + } + let best: f64 = conn.query_row( + "SELECT rank FROM chunks_fts WHERE chunks_fts MATCH ?1 ORDER BY rank LIMIT 1", + [&expr], + |r| r.get(0), + )?; + Ok((hits, Some(-best))) +} + +/// Print the search evidence calibration (invariants.md D2; GH #201) — the +/// query-side sibling of [`print_floor_windows`](crate::discovery::print_floor_windows). Reported, nothing gates: the +/// piles are what GH #201's query-level bar is argued from, and any constant +/// read off them owes process rule 5's real-vault transfer check before it +/// ships. The window's caveat is structural: D2's rule is lexical OR semantic +/// evidence, so a *pure-cosine* window overstates what a real bar must keep — +/// a positive query with a lexical anchor never needs its cosine kept. +pub fn print_search_evidence(ev: &SearchEvidence) { + println!( + " search evidence calibration (D2 — reported, nothing gates; the query bar is GH #201's \ + to earn)" + ); + let pos_cos = best_cos_pile(&ev.positives); + let neg_cos = best_cos_pile(&ev.negatives); + for (label, pile, role) in [ + ( + "pos best-cos", + &pos_cos, + "a pure-cosine bar would have to KEEP", + ), + ("neg best-cos", &neg_cos, "any query bar would have to CUT"), + ] { + match pile_stats(pile) { + Some((min, med, max)) => println!( + " {label:<12} n={:<4} min/med/max {min:.3}/{med:.3}/{max:.3} ← {role}", + pile.len() + ), + None => println!(" {label:<12} n=0 (nothing labelled — no reading)"), + } + } + let hits: Vec = ev.positives.iter().map(|r| r.bm25_hits as f64).collect(); + if let Some((min, med, max)) = pile_stats(&hits) { + println!( + " pos bm25 hits min/med/max {min:.0}/{med:.0}/{max:.0} (OR-saturated: stopwords \ + match too, so a raw count is not a lexical anchor — GH #201 derives one from this dump)" + ); + } + match Window::read(&neg_cos, &pos_cos) { + None => println!( + " cos window no reading (a pile is empty — label negative queries to give GH #201 \ + its CUT side)" + ), + Some(w) if w.open() => println!( + " cos window ({:.3}, {:.3}] — open on THIS corpus for a pure-cosine query bar; \ + the real keep-set is smaller (lexical evidence keeps its own), and a real vault is the \ + other half of any such claim (process rule 5, `make calibrate`)", + w.cut_max, w.keep_min + ), + Some(w) => println!( + " cos window EMPTY for a pure-cosine bar — negatives reach {:.3} while positives \ + start at {:.3}; D2's two-signal rule (lexical OR semantic) is argued from the per-query \ + lines, not this window", + w.cut_max, w.keep_min + ), + } + if ev.negatives.is_empty() { + println!( + " negatives n=0 (none labelled — queries.json takes empty `relevant` as \ + \"no matches\")" + ); + } else { + println!( + " negatives (labelled answer: NO MATCHES — what the shipped surface serves instead):" + ); + for r in &ev.negatives { + println!( + " {:<40} bm25 {:>3} hit{} best {:>6} cos {:>6} served {:>2}/{K} top {}", + truncate(&r.query, 40), + r.bm25_hits, + if r.bm25_hits == 1 { " " } else { "s" }, + r.bm25_best + .map(|b| format!("{b:.2}")) + .unwrap_or_else(|| "—".to_string()), + r.best_cos + .map(|c| format!("{c:.3}")) + .unwrap_or_else(|| "—".to_string()), + r.served(), + r.top, + ); + } + } +} + +/// Print the **search evidence bake-off** (ADR-0015, GH #201) — the query-level rule's +/// window, re-derived from the labelled piles on every run rather than quoted from the day it +/// was read. The GH #187 idiom on search's side: the constant lives in +/// [`b2_core::search::BGE_BASE_EVIDENCE_BAR`] and its *justification* is recomputed here, +/// including whether the shipped bar still sits inside the window it was read from. +pub fn print_search_bakeoff(ev: &SearchEvidence, cells: &[EvidenceCell], model_id: &str) { + println!( + " search evidence bake-off (D2 — the query-level rule, re-derived every run; GH #201)" + ); + for line in [ + "the rule: serve iff the query has a LEXICAL ANCHOR (the vault carries ≥ coverage of the", + " query's term IDF — a word in most chunks weighs ~0, a word in none weighs the", + " most) OR its dense top-1 clears a cosine bar. Two signals on purpose: a", + " one-signal test cannot tell \"nothing matches\" from \"everything matches\" (#196).", + "`cos need` reads over ONLY the queries the lexical half left undecided — a positive with", + " an anchor never needs its cosine kept, so a pure-cosine window overstates.", + ] { + println!(" {line}"); + } + println!( + " n = {} positives / {} negatives labelled", + ev.positives.len(), + ev.negatives.len() + ); + println!( + " {:>5} {:>9} {:>9} {:>15} verdict", + "cov", "pos anch", "neg anch", "cos need" + ); + for cell in cells { + let need = match (cell.cut_floor(), cell.keep_ceiling()) { + (Some(cut), Some(keep)) => format!("({cut:.3},{keep:.3}]"), + (Some(cut), None) => format!("> {cut:.3}"), + (None, Some(keep)) => format!("≤ {keep:.3} (inert)"), + (None, None) => "inert".to_string(), + }; + let verdict = if cell.neg_anchored > 0 { + format!( + "✗ {} negative{} anchored — no cosine bar can rescue them", + cell.neg_anchored, + if cell.neg_anchored == 1 { "" } else { "s" } + ) + } else if cell.admissible() { + "✓ admissible".to_string() + } else { + "✗ cosine window empty".to_string() + }; + println!( + " {:>5.2} {:>4}/{:<4} {:>4}/{:<4} {:>15} {}", + cell.coverage, + cell.pos_anchored, + ev.positives.len(), + cell.neg_anchored, + ev.negatives.len(), + need, + verdict, + ); + } + let admissible: Vec<&EvidenceCell> = cells.iter().filter(|c| c.admissible()).collect(); + if admissible.is_empty() { + for line in [ + "→ NO admissible cell on this corpus: every lexical rule either vouches for a", + " labelled negative or leaves an empty cosine window. D2's bar is not earned,", + " and no rule ships (the GH #200 outcome, on search's side).", + ] { + println!(" {line}"); + } + } else { + println!( + " → {} of {} cells admissible; the widest cosine window belongs to {}", + admissible.len(), + cells.len(), + widest(&admissible), + ); + } + print_shipped_bar(ev, model_id); + let dense_only = |rows: &[QueryEvidence]| { + ( + rows.iter().map(|r| r.dense_only()).sum::(), + rows.iter().map(|r| r.served()).sum::(), + ) + }; + let (pos_only, pos_served) = dense_only(&ev.positives); + let (neg_only, neg_served) = dense_only(&ev.negatives); + println!(" tail reading: served rows the lexical half never ranked —"); + println!( + " positives {pos_only}/{pos_served}, negatives {neg_only}/{neg_served}. The per-hit \ + rules over these are" + ); + println!(" the tail bake-off's, below (GH #206)."); +} + +/// Name the admissible cells with the most cosine headroom, and within that band +/// the **most conservative corner** — the tightest `df` fraction and the +/// strictest coverage that still buy the widest window. +/// +/// The tie matters more than the maximum does, and printing only a winner would +/// hide it: the grid's cells are not distinct rules but a plateau, and the +/// reading to carry into process rule 5's transfer check is "anywhere in this +/// band", not "at this point". A constant placed at a lone maximum would be +/// fitted to the grid's resolution. +pub fn widest(cells: &[&EvidenceCell]) -> String { + let best = cells + .iter() + .map(|c| headroom(c)) + .fold(f64::NEG_INFINITY, f64::max); + if !best.is_finite() { + let inert = cells.iter().filter(|c| !headroom(c).is_finite()).count(); + return format!( + "{inert} cell(s) where the cosine half is INERT — the lexical rule alone decides \ + every labelled query, so no `min_cos` is constrained there" + ); + } + let band: Vec<&&EvidenceCell> = cells + .iter() + .filter(|c| (headroom(c) - best).abs() < 1e-9) + .collect(); + match band.iter().max_by(|a, b| { + a.coverage + .partial_cmp(&b.coverage) + .unwrap_or(std::cmp::Ordering::Equal) + }) { + None => "—".to_string(), + Some(c) => format!( + "a plateau of {} cells at headroom {best:.3}; its strictest is coverage ≥ {:.2}", + band.len(), + c.coverage, + ), + } +} + +/// A cell's cosine headroom: how much room a `min_cos` has between the +/// negatives it must cut and the positives it must keep. An inert cosine half +/// (nothing left undecided on one side) has unbounded room and is reported as +/// such rather than scored against bounded cells. +pub fn headroom(cell: &EvidenceCell) -> f64 { + match (cell.cut_floor(), cell.keep_ceiling()) { + (Some(cut), Some(keep)) => keep - cut, + _ => f64::INFINITY, + } +} + +/// Where the **shipped** constant stands against this run's piles: the tripwire (a labelled +/// positive the bar would cut — ADR-0015 asserts zero, no headroom) and the defect it exists +/// to fix (a labelled negative it still serves). +/// +/// **In the exit gate** since GH #202, at [`MAX_POSITIVES_CUT`] and [`MAX_NEGATIVES_SERVED`]. +/// Read here rather than in the printer, so the number asserted and the number explained are +/// one number. `None` when the active model has no calibrated bar. +pub struct ShippedBar { + pub bar: EvidenceBar, + /// Labelled positives the bar cuts — the tripwire's direction. + pub pos_cut: usize, + /// Labelled negatives it still serves — the defect's direction. + pub neg_served: usize, + /// Queries where the engine's verdict and the harness's independent + /// restatement of the same rule **disagree** — a drift between the two, which + /// is silence unless something looks. Expected empty; printed, not gated, + /// because it accuses the instrument as readily as the engine and the reader + /// has to say which. + pub faults: Vec, +} + +pub fn read_shipped_bar(ev: &SearchEvidence, model_id: &str) -> Option { + let bar = EvidenceBar::for_model(model_id)?; + // The ENGINE's verdict, not a restatement of it: this is what the gate + // asserts and what the surfaces act on. `None` cannot occur here — the model + // has a bar or this function returned already — so it is counted as served, + // the same direction the surfaces take it (M2: no verdict is not "no match"). + Some(ShippedBar { + bar, + pos_cut: ev + .positives + .iter() + .filter(|r| r.vouched == Some(false)) + .count(), + neg_served: ev + .negatives + .iter() + .filter(|r| r.vouched != Some(false)) + .count(), + faults: ev + .positives + .iter() + .chain(ev.negatives.iter()) + .filter(|r| { + let restated = r.anchored(bar.min_term_coverage) + || r.best_cos.is_some_and(|c| c >= bar.min_cos); + r.vouched != Some(restated) + }) + .map(|r| r.query.clone()) + .collect(), + }) +} + +pub fn print_shipped_bar(ev: &SearchEvidence, model_id: &str) { + let Some(ShippedBar { + bar, + pos_cut, + neg_served, + faults, + }) = read_shipped_bar(ev, model_id) + else { + println!(" shipped bar: none for this model — no verdict is offered (M2)"); + return; + }; + // Every line below reads the ENGINE's verdict, so the numbers printed are the + // numbers gated — one reading, not two that can drift apart. + let vouches = |r: &QueryEvidence| r.vouched != Some(false); + println!( + " shipped bar: coverage ≥ {:.2}, cos ≥ {:.3}", + bar.min_term_coverage, bar.min_cos + ); + println!( + " positives it would cut {pos_cut}/{} ← the search-side TRIPWIRE (D2: zero, no \ + headroom; GATED at ≤ {MAX_POSITIVES_CUT})", + ev.positives.len() + ); + println!( + " negatives it still serves {neg_served}/{} ← the defect the bar exists to fix \ + (GATED at ≤ {MAX_NEGATIVES_SERVED})", + ev.negatives.len() + ); + for r in ev.positives.iter().filter(|r| !vouches(r)) { + println!( + " [CUT] {:<40} cov {:>5} cos {:>6}", + truncate(&r.query, 40), + r.coverage() + .map(|c| format!("{c:.2}")) + .unwrap_or_else(|| "—".to_string()), + r.best_cos + .map(|c| format!("{c:.3}")) + .unwrap_or_else(|| "—".to_string()), + ); + } + if !faults.is_empty() { + println!( + " [FAULT] the engine's verdict and this harness's restatement of the same rule \ + disagree on {} query/queries: {}. One of the two has drifted — read the engine's \ + `LexicalEvidence`, not this line's wording.", + faults.len(), + faults.join(", ") + ); + } + for r in &ev.negatives { + println!( + " {:<40} cov {:>5} cos {:>6} → {}", + truncate(&r.query, 40), + r.coverage() + .map(|c| format!("{c:.2}")) + .unwrap_or_else(|| "—".to_string()), + r.best_cos + .map(|c| format!("{c:.3}")) + .unwrap_or_else(|| "—".to_string()), + if vouches(r) { "SERVED" } else { "no matches" }, + ); + } +} diff --git a/crates/b2-embed/examples/eval/fold.rs b/crates/b2-embed/examples/eval/fold.rs new file mode 100644 index 0000000..0c35dad --- /dev/null +++ b/crates/b2-embed/examples/eval/fold.rs @@ -0,0 +1,715 @@ +//! The discovery fold bake-off (GH #200, Phase B) — the *default disclosure boundary*, +//! priced on every bench this harness carries. The ruling of record is **no fold ships** +//! (docs/evals.md); the bake-off keeps re-deriving the window every run. + +use crate::common::{cosine_of, truncate}; +use crate::labels::{SimilarLabel, SimilarSet}; +use crate::metrics::paths_match; +use crate::{SIM_K, Z_SCAN_LIMIT}; +use b2_core::vault::Vault; +use std::collections::HashMap; + +/// The reciprocity depths GH #200's candidate 1 is shown at in the headline +/// table and the per-anchor lists — [`SIM_K`] (the pane itself) with one depth +/// either side of it. They are a *display* choice only: the window below is +/// derived from the whole sweep, so no verdict turns on these three. +pub const FOLD_MUTUAL_K: [usize; 3] = [3, 5, 10]; + +/// How far the reciprocity depth is swept when re-deriving candidate 1's +/// admissible window each run (GH #187's idiom on the disclosure axis — a +/// measurement in the harness, never a constant frozen into a docstring). Half +/// the orthogonal corpus's note count: a `k` past that admits most of the vault +/// as "mutually near", which is the rule going vacuous rather than a depth +/// anyone would ship. +pub const FOLD_K_SWEEP: usize = 15; + +/// A **default disclosure rule**: a candidate answer to "how much of the ranked +/// list does the default view vouch for?" (invariants.md D1 as redrafted; +/// GH #200). Every variant is a **prefix** rule *by construction* — it returns a +/// depth, never a set — so a rule that admits rank 5 while folding rank 2 is not +/// expressible here. D1 states that admissibility requirement; this type makes +/// violating it unrepresentable, which is the difference between a check and a +/// guarantee. +#[derive(Clone, Copy, PartialEq)] +pub enum FoldRule { + /// **Candidate 3, the incumbent**: no fold. The default view is the whole + /// served prefix — GH #197's always-serve ruling, admissible per D1 and the + /// thing the other candidates have to beat. + NoFold, + /// **Candidate 1**: mutual-kNN reciprocity in prefix form. Candidate B is + /// *reciprocal* for anchor A iff A sits in B's own top `k` candidates, and + /// the default view is the ranked list's longest reciprocal prefix. Rank- + /// based (the hubness-correction family: mutual proximity, CSLS), so it + /// carries no cosine or z constant — though `k` itself is measured below to + /// be scale-dependent, which is a different objection and the decisive one. + Mutual(usize), +} + +impl FoldRule { + /// The rule's name in every table, row and window line — one spelling, so a + /// printed verdict and a recorded row can never name the same rule + /// differently. + pub fn label(self) -> String { + match self { + FoldRule::NoFold => "no fold".to_string(), + FoldRule::Mutual(k) => format!("mutual-{k}"), + } + } + + /// How many of `rows` this rule's default view vouches for. + pub fn fold(self, rows: &[FoldRow]) -> usize { + match self { + FoldRule::NoFold => rows.len(), + FoldRule::Mutual(k) => rows.iter().take_while(|r| r.reciprocal_at(k)).count(), + } + } +} + +/// One served candidate on one anchor's list, carrying everything a candidate +/// rule judges: the label, the score, and — the reciprocity signal — **where the +/// anchor sits in this candidate's own ranked list**. +pub struct FoldRow { + pub path: String, + pub cos: f64, + /// A labelled mate of this anchor (always `false` on a negative or + /// unlabelled anchor). + pub mate: bool, + /// The **anchor's** rank in *this candidate's* full-depth candidate list, or + /// `None` if the anchor is not in it at all. Storing the rank rather than a + /// per-`k` boolean is what makes the whole sweep below free: reciprocity at + /// depth `k` is `rank ≤ k`, so every `k` is priced off one discovery pass + /// per note instead of one per note per depth. + pub recip_rank: Option, +} + +impl FoldRow { + /// Whether this candidate is *reciprocal* at depth `k` — i.e. the anchor + /// sits in its own top `k`. Reciprocity is necessary but not sufficient for + /// being above the fold: the fold is the longest reciprocal **prefix**, so a + /// reciprocal candidate ranked after a non-reciprocal one is below it too. + pub fn reciprocal_at(&self, k: usize) -> bool { + self.recip_rank.is_some_and(|r| r <= k) + } +} + +/// One anchor's served list, read once and judged by every candidate rule — so +/// the rules are compared on identical rows rather than on separate passes that +/// could drift. +pub struct FoldAnchor { + pub anchor: String, + /// A labelled **loner** (empty `expected`): its correct default view is + /// *empty above the fold* — the assertion GH #197 retired, returning on the + /// disclosure axis. + pub negative: bool, + /// Labelled mates in label order. + pub expected: Vec, + /// The served prefix (`limit` = [`SIM_K`]) in rank order. + pub rows: Vec, +} + +impl FoldAnchor { + /// This anchor's labelled mates that the *served prefix* reaches, as + /// `(mate, rank)`. A mate ranked past `limit` is not among them — it is + /// always-serve's own miss at this depth, and counting it against a fold + /// would charge every rule for a cost none of them caused. + pub fn served_mates(&self) -> Vec<(String, usize)> { + self.expected + .iter() + .filter_map(|m| { + self.rows + .iter() + .position(|r| paths_match(&r.path, m)) + .map(|p| (m.clone(), p + 1)) + }) + .collect() + } +} + +/// What one rule reads on one bench — GH #200's judged quantities, each carrying +/// the per-anchor list it is argued from (process rule 1: the aggregate is a +/// smoke alarm, the named list is the data). +pub struct FoldReading { + pub rule: FoldRule, + /// Labelled mates the served prefix reaches but the default view does + /// **not** vouch for: `(anchor, mate, rank)`. **The fold's own cost**, and + /// the quantity GH #200 judges a candidate at zero on — a human-labelled + /// relation hidden by default. Reported rather than gated while nothing + /// folds (it reads a structural 0 under always-serve) — and GH #202 shipped + /// no fold to charge, so it stays reporting-only; it becomes an exit-gate + /// row with the first fold that does ship. + pub mates_folded: Vec<(String, String, usize)>, + /// Labelled mates above the fold — the other half of the same count. + pub mates_above: usize, + /// Unlabelled notes the default view vouches for on a *positive* anchor — + /// the strangers count, read above the fold instead of over the whole served + /// prefix. Expected to shrink; deliberately ungated for the standing reason + /// (the cheapest way to shrink it is to label the stranger). + pub strangers_above: Vec<(String, String)>, + /// Negative (loner) anchors whose default view is empty — the honest answer + /// their label always claimed, and what always-serve cannot assert. + pub neg_empty: usize, + /// Non-negative anchors whose default view is empty. On the dense fixture + /// **any** entry here is disqualifying (a vault where everything relates may + /// never *default* to "nothing relates"); on the orthogonal corpus it is the + /// same event read through the labels. + pub dark_panes: Vec, + /// Cards above the fold, summed — the default view's size against + /// always-serve's. + pub cards_above: usize, +} + +/// The **admissible-`k` window** for candidate 1, re-derived from this run +/// (GH #187's idiom, moved onto the disclosure axis: print the window a rule +/// *would* have rather than freezing a constant into a docstring). Two bounds +/// that must overlap for any `k` to be shippable on this bench: +/// +/// - `keep_min` — the smallest `k` that folds **no** labelled mate and darkens +/// **no** pane. Below it the rule hides relations a human labelled. +/// - `loner_max` — the largest `k` at which **every** labelled loner's default +/// view is empty. Above it the fold stops making the claim it exists to make, +/// and a loner's pane fills up again. +pub struct FoldWindow { + pub keep_min: Option, + pub loner_max: Option, + /// The smallest swept `k` whose fold equals always-serve on every anchor — + /// where the rule becomes vacuous (it vouches for everything). + pub vacuous_at: Option, + /// The best the loner claim ever gets on this bench, as `(empty, largest k + /// still achieving it)` — printed because "never all of them" and "four of + /// five up to k = 11" are different findings, and only the second says how + /// close the rule came. + pub loner_best: Option<(usize, usize)>, + /// The largest swept `k` that still darkens a pane. Every `k` at or below it + /// is disqualified outright on a corpus where everything relates (D1's + /// absolute), so this is the hard floor under any window. + pub dark_below: Option, +} + +impl FoldWindow { + /// Whether some swept `k` satisfies both bounds at once. A bench with no + /// labelled loner has no upper bound to satisfy and so can never report an + /// open window on its own — it contributes the lower bound to the + /// bake-off's joint window, which is what the printed verdict says. + pub fn open(&self) -> bool { + matches!((self.keep_min, self.loner_max), (Some(a), Some(b)) if a <= b) + } +} + +/// One bench's complete bake-off reading. +pub struct FoldBench { + pub corpus: &'static str, + pub anchors: Vec, + /// The headline rules: always-serve plus [`FOLD_MUTUAL_K`]'s detail depths. + pub readings: Vec, + /// `(k, reading)` across the whole swept range — what the window is derived + /// from, and printed as a compact table so the shape of the trade is visible + /// rather than asserted. + pub sweep: Vec<(usize, FoldReading)>, + pub window: FoldWindow, + /// Negative anchors on this bench (0 on the dense fixture, which has no + /// loner by construction). + pub neg_n: usize, + /// Labelled mates the served prefix never reaches at `limit` — always-serve's + /// own miss, identical under every rule. Printed beside the folded count so + /// no rule is charged for it. + pub mates_unserved: usize, + /// The median *full-depth* candidate pool an anchor here has — what a + /// reciprocity depth is a fraction OF. A `k` is only meaningful against it: + /// "top 7" of a 14-candidate pool and "top 7" of a 30-candidate one are + /// different claims, and the two corpora's windows below are only + /// comparable in this unit. + pub pool_median: usize, + /// Authored edges in the built vault — **candidate 2's** entire calibration + /// population ("what related looks like in this vault", read off the human's + /// own committed links). Both eval corpora are link-free by construction, so + /// this reads 0 and candidate 2 is *unpriceable here*: it is measured where + /// its population exists, by `make calibrate` on a real vault. + pub authored_edges: usize, +} + +/// Every note's own ranked candidate list as `path → rank`, read at full depth — +/// the reciprocity lookup candidate 1 is judged on. Read over **every note in +/// the vault**, not only the labelled anchors: reciprocity is decided by the +/// *candidate's* list, and most candidates are not anchors. +pub fn reciprocity_ranks( + vault: &Vault, +) -> Result>, Box> { + let mut out = HashMap::new(); + for note in vault.list_notes()? { + let ranks = vault + .similar(¬e.path, Z_SCAN_LIMIT)? + .into_iter() + .enumerate() + .map(|(i, c)| (c.path, i + 1)) + .collect::>(); + out.insert(note.path, ranks); + } + Ok(out) +} + +/// Score the fold bake-off on one built vault. `sweep_all` reads **every note** +/// as an anchor (the dense fixture's pane sweep, where the assertion is about +/// the surface rather than the labels); otherwise only the labelled anchors are +/// read, matching [`score_similar`](crate::discovery::score_similar)'s depth and population exactly. +pub fn score_fold( + vault: &Vault, + set: &SimilarSet, + corpus: &'static str, + sweep_all: bool, +) -> Result> { + let recip = reciprocity_ranks(vault)?; + + let mut anchors: Vec<(String, Option<&SimilarLabel>)> = Vec::new(); + if sweep_all { + for note in vault.list_notes()? { + let label = set + .anchors + .iter() + .find(|l| paths_match(¬e.path, &l.anchor)); + anchors.push((note.path, label)); + } + } else { + for label in &set.anchors { + anchors.push((label.anchor.clone(), Some(label))); + } + } + + // Outbound only. `neighbors` answers "what is 1 hop from here" and so returns + // an edge from *both* of its endpoints; summing the raw counts over every + // note would census edge **endpoints** and report candidate 2's population at + // twice its size (PR #204 review). Counting each edge once from its source is + // the number the rule would calibrate from. + let mut authored_edges = 0; + for note in vault.list_notes()? { + authored_edges += vault + .neighbors(¬e.path)? + .iter() + .filter(|n| n.direction == "outbound") + .count(); + } + + let mut rows_by_anchor: Vec = Vec::new(); + let mut neg_n = 0; + for (anchor, label) in anchors { + let expected: Vec = label.map(|l| l.expected.clone()).unwrap_or_default(); + // A *labelled* loner. An unlabelled anchor on the dense sweep is not a + // negative — that fixture carries no loner at all, and reading "no + // label" as "nothing relates" would invent the claim the bench exists + // to test. + let negative = label.is_some() && expected.is_empty(); + if negative { + neg_n += 1; + } + let rows = vault + .similar(&anchor, SIM_K)? + .into_iter() + .map(|c| FoldRow { + mate: expected.iter().any(|e| paths_match(&c.path, e)), + recip_rank: recip.get(&c.path).and_then(|ranks| { + ranks + .iter() + .find(|(p, _)| paths_match(p, &anchor)) + .map(|(_, r)| *r) + }), + cos: cosine_of(c.score), + path: c.path, + }) + .collect(); + rows_by_anchor.push(FoldAnchor { + anchor, + negative, + expected, + rows, + }); + } + + let mates_unserved = rows_by_anchor + .iter() + .map(|a| a.expected.len() - a.served_mates().len()) + .sum(); + let readings = std::iter::once(FoldRule::NoFold) + .chain(FOLD_MUTUAL_K.iter().map(|&k| FoldRule::Mutual(k))) + .map(|rule| read_fold(&rows_by_anchor, rule)) + .collect(); + let sweep: Vec<(usize, FoldReading)> = (1..=FOLD_K_SWEEP) + .map(|k| (k, read_fold(&rows_by_anchor, FoldRule::Mutual(k)))) + .collect(); + let window = FoldWindow { + keep_min: sweep + .iter() + .find(|(_, r)| r.mates_folded.is_empty() && r.dark_panes.is_empty()) + .map(|(k, _)| *k), + loner_max: (neg_n > 0) + .then(|| { + sweep + .iter() + .filter(|(_, r)| r.neg_empty == neg_n) + .map(|(k, _)| *k) + .next_back() + }) + .flatten(), + vacuous_at: sweep + .iter() + .find(|(_, r)| { + rows_by_anchor + .iter() + .all(|a| r.rule.fold(&a.rows) == a.rows.len()) + }) + .map(|(k, _)| *k), + loner_best: (neg_n > 0) + .then(|| { + let best = sweep.iter().map(|(_, r)| r.neg_empty).max()?; + let last = sweep + .iter() + .filter(|(_, r)| r.neg_empty == best) + .map(|(k, _)| *k) + .next_back()?; + Some((best, last)) + }) + .flatten(), + dark_below: sweep + .iter() + .filter(|(_, r)| !r.dark_panes.is_empty()) + .map(|(k, _)| *k) + .next_back(), + }; + + let pool_median = { + let mut sizes: Vec = recip.values().map(|r| r.len()).collect(); + sizes.sort_unstable(); + sizes.get(sizes.len() / 2).copied().unwrap_or(0) + }; + + Ok(FoldBench { + corpus, + pool_median, + anchors: rows_by_anchor, + readings, + sweep, + window, + neg_n, + mates_unserved, + authored_edges, + }) +} + +/// Judge one rule against one bench's rows. +pub fn read_fold(anchors: &[FoldAnchor], rule: FoldRule) -> FoldReading { + let mut reading = FoldReading { + rule, + mates_folded: Vec::new(), + mates_above: 0, + strangers_above: Vec::new(), + neg_empty: 0, + dark_panes: Vec::new(), + cards_above: 0, + }; + for a in anchors { + let fold = rule.fold(&a.rows); + reading.cards_above += fold; + if fold == 0 { + if a.negative { + reading.neg_empty += 1; + } else if !a.rows.is_empty() { + reading.dark_panes.push(a.anchor.clone()); + } + } + for (mate, rank) in a.served_mates() { + if rank <= fold { + reading.mates_above += 1; + } else { + reading.mates_folded.push((a.anchor.clone(), mate, rank)); + } + } + // The precision side, read above the fold: unlabelled cards the default + // view vouches for on a positive anchor. + if !a.negative && !a.expected.is_empty() { + for row in a.rows.iter().take(fold) { + if !row.mate { + reading + .strangers_above + .push((a.anchor.clone(), row.path.clone())); + } + } + } + } + reading +} + +/// The bake-off's printed readout: the headline rules side by side, the swept +/// `k` the window is derived from, and the per-anchor folds — which is the list +/// the verdict is argued from (process rule 1). +pub fn print_fold_bench(bench: &FoldBench) { + println!("\n{}", "=".repeat(78)); + println!( + "discovery fold bake-off — the default disclosure boundary on the {} corpus \ + (GH #200, Phase B; invariants.md D1)", + bench.corpus + ); + println!( + " {} anchors read at limit={SIM_K} over a median {}-candidate pool; a fold is a PREFIX of \ + the served order and everything below it stays served (D1)", + bench.anchors.len(), + bench.pool_median + ); + println!( + "\n {:<10} {:>11} {:>12} {:>15} {:>12} {:>10}", + "rule", "cards above", "mates folded", "strangers above", "loners empty", "dark panes" + ); + for r in &bench.readings { + println!( + " {:<10} {:>11} {:>12} {:>15} {:>12} {:>10}", + r.rule.label(), + r.cards_above, + r.mates_folded.len(), + r.strangers_above.len(), + format!("{}/{}", r.neg_empty, bench.neg_n), + r.dark_panes.len(), + ); + } + println!( + " (mates folded: served within limit={SIM_K} but below the fold — the fold's OWN cost, and \ + the quantity GH #200 judges a candidate at 0 on. Reported, not gated: nothing folds today, \ + and GH #202 shipped no fold either, so the exit gate has nothing here to watch until one \ + does. {} further labelled mate(s) \ + rank past limit={SIM_K} under every rule, always-serve included, so no fold is charged \ + for them.)", + bench.mates_unserved + ); + + // The named cost of every rule that has one — the list, not the count. + for r in &bench.readings { + if r.mates_folded.is_empty() { + continue; + } + println!(" {} folds labelled mates:", r.rule.label()); + for (anchor, mate, rank) in &r.mates_folded { + println!(" rank {rank} {anchor} → {mate}"); + } + } + + // The swept window — the #187 idiom on the disclosure axis: derive the + // rule's admissible range from THIS run rather than quote a constant. + println!( + "\n mutual-k sweep (k = 1..{FOLD_K_SWEEP}; the window a shippable k would have to sit in)" + ); + println!( + " {:>3} {:>11} {:>12} {:>15} {:>12} {:>10}", + "k", "cards above", "mates folded", "strangers above", "loners empty", "dark panes" + ); + for (k, r) in &bench.sweep { + println!( + " {k:>3} {:>11} {:>12} {:>15} {:>12} {:>10}", + r.cards_above, + r.mates_folded.len(), + r.strangers_above.len(), + format!("{}/{}", r.neg_empty, bench.neg_n), + r.dark_panes.len(), + ); + } + let frac = |k: usize| { + if bench.pool_median == 0 { + String::new() + } else { + format!( + " (= {:.2} of the {}-candidate pool)", + k as f64 / bench.pool_median as f64, + bench.pool_median + ) + } + }; + match bench.window.keep_min { + Some(k) => println!( + " k ≥ {k}{} folds no labelled mate and darkens no pane on this corpus", + frac(k) + ), + None => println!(" (no swept k folds zero mates with zero dark panes on this corpus)"), + } + match bench.window.loner_max { + Some(k) => println!( + " k ≤ {k}{} folds every labelled loner's default view to empty", + frac(k) + ), + None if bench.neg_n == 0 => { + println!( + " (no labelled loner on this corpus — the upper bound is the other bench's)" + ) + } + None => println!(" (no swept k empties every loner's default view on this corpus)"), + } + if let Some(k) = bench.window.vacuous_at { + println!( + " k ≥ {k} the fold equals always-serve on every anchor here — the rule stops \ + claiming anything" + ); + } + if let Some(k) = bench.window.dark_below { + println!( + " k ≤ {k} darkens at least one pane here — disqualified outright where the labels \ + say everything relates (D1's absolute)" + ); + } + if let Some((best, last)) = bench.window.loner_best { + println!( + " best loner claim: {best}/{} empty, holding to k = {last}", + bench.neg_n + ); + } + // The verdict names *which* bound failed. "No k works" and "this corpus + // cannot decide alone" are different sentences, and a bench with no loner + // can only ever supply the lower bound. + println!( + " → window {}", + match (bench.neg_n, bench.window.keep_min, bench.window.loner_max) { + (0, Some(k), _) => format!( + "UNDECIDABLE on this corpus alone — no loner here, so it supplies only the lower \ + bound (k ≥ {k}) and the absolute above; the upper bound is the other bench's" + ), + (0, None, _) => "UNDECIDABLE on this corpus alone — no loner here, and no swept k is \ + even clean on the labels" + .to_string(), + (_, _, None) => format!( + "EMPTY — no swept k empties every loner's default view, so the fold never fully \ + makes the claim it exists to make ({})", + bench + .window + .keep_min + .map(|k| format!("its lower bound here is k ≥ {k}")) + .unwrap_or_else(|| "and no swept k is clean on the labels either".into()) + ), + (_, Some(a), Some(b)) if a <= b => format!( + "OPEN on this corpus at k ∈ [{a}, {b}] — the other benches are the rest \ + of the claim" + ), + (_, Some(a), Some(b)) => format!( + "EMPTY — the k that stops folding labelled mates (≥ {a}) is past the k that still \ + empties every loner's view (≤ {b}), so no constant separates them" + ), + (_, None, Some(b)) => format!( + "EMPTY — no swept k is clean on the labels at all, while the loner claim holds \ + only to k ≤ {b}" + ), + } + ); + + // The per-anchor lists at the headline depths. + println!("\n per anchor — served / above the fold, by rule:"); + print!(" {:<40} {:>7}", "anchor", "served"); + for r in &bench.readings { + if r.rule != FoldRule::NoFold { + print!(" {:>9}", r.rule.label()); + } + } + println!(); + for a in &bench.anchors { + print!( + " {:<40} {:>7}", + format!( + "{}{}", + truncate(&a.anchor, 36), + if a.negative { " [loner]" } else { "" } + ), + a.rows.len() + ); + for r in &bench.readings { + if r.rule != FoldRule::NoFold { + let fold = r.rule.fold(&a.rows); + let lost = a + .served_mates() + .iter() + .filter(|(_, rank)| *rank > fold) + .count(); + print!( + " {:>9}", + if lost > 0 { + format!("{fold}(-{lost})") + } else { + fold.to_string() + } + ); + } + } + println!(); + } + + // Candidate 2's census — printed on every bench, because "we did not measure + // it" and "there was nothing to measure it on" are different sentences and + // only the second is an argument. + println!( + "\n candidate 2 (authored-edge reference bar): {} authored edges in this corpus — {}", + bench.authored_edges, + if bench.authored_edges == 0 { + "UNPRICEABLE here (a link-free corpus offers the rule no population to calibrate from; \ + it is measured where one exists, by `make calibrate` on a real vault)" + } else { + "priceable — see the calibrate replay" + } + ); +} + +/// The bake-off as one JSON subtree (`discovery_fold`): every rule's reading, +/// the swept window it was judged against, and the per-anchor rows both were +/// computed from — so a verdict cited from a row can be re-derived without +/// re-running the model, which is what `results.jsonl` is for. +pub fn fold_json(bench: &FoldBench) -> serde_json::Value { + let reading = |r: &FoldReading| { + serde_json::json!({ + "rule": r.rule.label(), + "cards_above": r.cards_above, + "mates_above": r.mates_above, + "mates_folded": r.mates_folded.iter().map(|(a, m, rank)| serde_json::json!({ + "anchor": a, "mate": m, "rank": rank, + })).collect::>(), + "strangers_above": r.strangers_above.iter().map(|(a, p)| serde_json::json!({ + "anchor": a, "path": p, + })).collect::>(), + "neg_empty": r.neg_empty, + "dark_panes": r.dark_panes, + }) + }; + serde_json::json!({ + "corpus": bench.corpus, + "limit": SIM_K, + "pool_median": bench.pool_median, + "neg_n": bench.neg_n, + // Always-serve's own miss at this limit, recorded apart from every + // rule's cost so the two can never be added together by a later reader. + "mates_unserved": bench.mates_unserved, + // Candidate 2's population, recorded even at zero: "unpriceable here" + // is a measurement, and a row that omitted it would read as untried. + "authored_edges": bench.authored_edges, + "rules": bench.readings.iter().map(reading).collect::>(), + "sweep": bench.sweep.iter().map(|(k, r)| serde_json::json!({ + "k": k, + "reading": reading(r), + })).collect::>(), + "window": { + "swept_to": FOLD_K_SWEEP, + "keep_min": bench.window.keep_min, + "loner_max": bench.window.loner_max, + "vacuous_at": bench.window.vacuous_at, + "dark_below": bench.window.dark_below, + "loner_best": bench.window.loner_best.map(|(empty, k)| serde_json::json!({ + "empty": empty, "of": bench.neg_n, "holds_to_k": k, + })), + "open": bench.window.open(), + }, + "anchors": bench.anchors.iter().map(|a| serde_json::json!({ + "anchor": a.anchor, + "negative": a.negative, + "served": a.rows.len(), + "folds": bench.readings.iter().map(|r| serde_json::json!({ + "rule": r.rule.label(), + "fold": r.rule.fold(&a.rows), + })).collect::>(), + "rows": a.rows.iter().map(|row| serde_json::json!({ + "path": row.path, + "cos": (row.cos * 1e4).round() / 1e4, + "mate": row.mate, + // The anchor's rank in THIS candidate's own list — the whole + // reciprocity signal, from which any k is re-derivable. + "recip_rank": row.recip_rank, + })).collect::>(), + })).collect::>(), + }) +} diff --git a/crates/b2-embed/examples/eval/gate.rs b/crates/b2-embed/examples/eval/gate.rs new file mode 100644 index 0000000..9f23312 --- /dev/null +++ b/crates/b2-embed/examples/eval/gate.rs @@ -0,0 +1,207 @@ +//! The exit gate: the assertions `make eval` exits 2 on, read off the **default** config's +//! passes (docs/evals.md, "The exit gate"). A manual quality gate, never a CI test. +//! +//! How each row is placed is the point, and the constants' own docs say why: a rank floor +//! sits *below* its reading (corpus-drift headroom, run noise being zero), while the +//! search-evidence rows sit *at* their structural zeros, because headroom there would read +//! as permission to serve a nonsense query or cut a real one. + +use crate::dense::DensePass; +use crate::discovery::SimilarPass; +use crate::evidence::{read_shipped_bar, SearchEvidence}; +use crate::retrieval::Pass; +use crate::SIM_K; + +/// The soft reference floor on the default config's hybrid note hit@1. +/// Untouched by the GH #197 re-derivation: retrieval never had a gate to +/// retire. +pub const FLOOR_HIT1: f64 = 0.75; + +/// The floor on **per-mate** discovery MRR@[`SIM_K`] (GH #188) — the non-saturating rank +/// metric, gated once a baseline existed to price it. Re-derived for always-serve +/// (ADR-0014): the reading moved 0.633 -> 0.650 when the existence gate retired. +/// +/// Placed **below** the reading, never at it: a gate pinned to today's number fails on the +/// first legitimate corpus edit, which trains the one habit this harness must never train +/// (process rule 2 — editing a *label* to get green). Sizing is measured, not guessed: five +/// consecutive runs on an unchanged corpus/model/build reproduce every rank exactly, so the +/// noise floor is 0 and the headroom exists for *corpus* drift. At n = 15 mates one mate +/// lost from rank 1 costs 1/15, and this floor sits ~2 such losses under the reading. +pub const FLOOR_MATE_MRR: f64 = 0.52; + +/// The floor on the **dense fixture's** per-mate MRR@[`SIM_K`] (ADR-0014, Phase 0b), gated +/// once its baseline existed — the same measure-then-calibrate order as [`FLOOR_MATE_MRR`]. +/// +/// The reading is 0.467 (n = 14): the model recovers the within-cluster labels and ranks the +/// three cross-cluster claims lower, which is headroom in both directions. (Its first +/// reading was 0.502; a one-word grammar fix moved one mate a rank — the worked example of +/// why these floors carry corpus-drift margin.) At n = 14 one mate lost from rank 1 costs +/// 1/14, and this sits ~2 such losses under. Process rule 2 binds hard here: in a corpus +/// where everything relates, relabelling toward the model's order would always look +/// plausible — a red reading argues about the notes. +pub const FLOOR_DENSE_MATE_MRR: f64 = 0.32; + +/// How many **labelled negative queries** the shipped evidence bar may still serve +/// (ADR-0015, GH #202). Zero: this is the defect the bar exists to fix. +/// +/// A floor at its measured value rather than below it, which is the deliberate exception to +/// the house sizing method — the "headroom" a floor normally carries would be *permission to +/// serve a nonsense query*. A new negative the bar serves is either a real regression or a +/// mislabelled query; both want a red reading. +pub const MAX_NEGATIVES_SERVED: usize = 0; + +/// How many **labelled relevant queries** the bar may cut (GH #202) — the search-side +/// tripwire ADR-0015 asserts at zero with no headroom, and the direction that costs a user +/// something real: a served nonsense row costs a little trust, a cut positive costs the +/// answer. Its precondition was met by GH #208, which labelled the date-shaped query pile. +/// The reading is 0 of 44. A nonzero value is never a calibration nudge: it means the *rule* +/// is wrong for a shape the corpus now carries. +pub const MAX_POSITIVES_CUT: usize = 0; + +/// How many of the **dense fixture's title-as-query probes** the bar may cut (GH #202). +/// Zero, and this is the assertion that would have caught the losing rule: a note's own +/// title names a note the vault demonstrably holds, so cutting one is indefensible whatever +/// a labelled corpus says. A third row rather than headroom on [`MAX_POSITIVES_CUT`] because +/// it gates a different *geometry*: the labelled corpus minimizes shared vocabulary by +/// construction, so topical concentration is only expressible here. Titles need no labels, +/// so nothing in this reading can be relabelled to clear it. +pub const MAX_DENSE_TITLES_CUT: usize = 0; + +/// Whether the default config clears every exit-gate row, printing a `[warn]` naming the +/// first row that fails (so a red run says which reading to argue with). +pub fn passes( + hybrid: &Pass, + similar: &SimilarPass, + dense: &DensePass, + evidence: &SearchEvidence, + model_id: &str, +) -> bool { + // The soft floors, on the DEFAULT config's passes — so this can double as + // a manual quality gate. Not a CI test. + if hybrid.note.hit1() < FLOOR_HIT1 { + eprintln!( + "\n[warn] hybrid hit@1 {:.2} is below the {FLOOR_HIT1} reference floor — inspect the misses above.", + hybrid.note.hit1() + ); + return false; + } + // The negatives' suppression assertion RETIRED with the gate it watched (ADR-0014): + // under always-serve a loner anchor serves its ranked nearest — that is the ruling, not + // a regression. The anchors stay labelled, the strangers instrument keeps counting, and + // what the served cards *claim* is the calibration block's band readout. (The + // pass-vs-pass suppression assertion retired later, for a different reason: it compared + // a call against itself and could never fire — GH #217. A returning existence gate is + // the dense zero-empty-panes assertion's and the per-mate floors' to catch.) + // + // Discovery **rank** (GH #188). The rank floor sits below its measured reading (corpus + // drift headroom, run noise being zero). + if similar.mate.mrr() < FLOOR_MATE_MRR { + eprintln!( + "\n[warn] per-mate MRR@{SIM_K} {:.3} is below the {FLOOR_MATE_MRR:.2} floor — discovery ranking regressed \ + (read the per-mate line's mates, not the aggregate; and do NOT relabel to clear this).", + similar.mate.mrr() + ); + return false; + } + // The dense fixture's existence assertion (GH #196/#197): every note in a + // corpus where everything relates must serve candidates. An empty pane here + // can only come from an anchor-local statistic claiming "nothing relates" + // on a vault where that is false — the exact failure GH #196 measured. + if !dense.empty_panes.is_empty() { + eprintln!( + "\n[warn] {} of {} dense-fixture notes serve an EMPTY pane ({}) — an existence \ + gate is refusing a vault whose every note genuinely relates (GH #196).", + dense.empty_panes.len(), + dense.notes, + dense.empty_panes.join(", ") + ); + return false; + } + // …and its rank floor, the dense sibling of FLOOR_MATE_MRR. + if dense.mate.mrr() < FLOOR_DENSE_MATE_MRR { + eprintln!( + "\n[warn] dense per-mate MRR@{SIM_K} {:.3} is below the {FLOOR_DENSE_MATE_MRR:.2} floor — \ + single-domain discovery ranking regressed (argue with the notes, not the labels).", + dense.mate.mrr() + ); + return false; + } + // The search-evidence rows (ADR-0015, GH #202), landed with the surfaces that consume + // the verdict. Everything above is discovery's and is deliberately UNCHANGED: search's + // bar moves no discovery rank and no reachability, so movement up there is a bug. + // + // All three sit at their structural zeros with no headroom — the exception to the house + // sizing method rather than an oversight, since headroom here would read as permission + // to serve a nonsense query or cut a real one. Skipped entirely when the model has no + // calibrated bar (ADR-0007): asserting the absence of a verdict would fail every run on + // a model the harness has simply not measured yet. + match read_shipped_bar(evidence, model_id) { + None => eprintln!( + "\n[note] no calibrated evidence bar for {model_id} — D2's exit-gate rows are not \ + asserted this run (M2)." + ), + Some(reading) => { + if reading.neg_served > MAX_NEGATIVES_SERVED { + eprintln!( + "\n[warn] the shipped evidence bar serves {} of {} labelled NEGATIVE queries where \ + D2 permits {MAX_NEGATIVES_SERVED} — a query the vault holds nothing for is being \ + answered with rows (read the per-query lines above, and do NOT relabel to clear \ + this).", + reading.neg_served, + evidence.negatives.len() + ); + return false; + } + // The tripwire, and the direction that costs a user the answer rather + // than a little trust. Its precondition is GH #208's date-shaped pile: + // the assertion is only worth what the query shapes behind it are. + if reading.pos_cut > MAX_POSITIVES_CUT { + eprintln!( + "\n[warn] the shipped evidence bar CUTS {} of {} labelled relevant queries where D2 \ + permits {MAX_POSITIVES_CUT} — a note the vault holds is unreachable for a query \ + naming it. Change the RULE, not the constant (the df ceiling died exactly here).", + reading.pos_cut, + evidence.positives.len() + ); + return false; + } + } + } + // The same two directions on the dense fixture — a different geometry, not a + // different threshold. Topical concentration is what killed the losing rule + // and is structurally inexpressible on the orthogonal corpus (process rule 2's + // token audit minimizes shared vocabulary), so these are their own rows. + // Titles carry no labels, so nothing here can be relabelled to clear it. + let titles_cut = dense + .search + .titles + .iter() + .filter(|p| p.vouched == Some(false)) + .count(); + if titles_cut > MAX_DENSE_TITLES_CUT { + eprintln!( + "\n[warn] the evidence bar cuts {titles_cut} of {} dense-fixture titles where \ + {MAX_DENSE_TITLES_CUT} is permitted — a note's own title is a query naming a note the \ + vault demonstrably holds, and the lexical half has gone inert on a single-subject vault \ + (GH #201's transfer check, as an assertion).", + dense.search.titles.len() + ); + return false; + } + let nonsense_served = dense + .search + .nonsense + .iter() + .filter(|p| p.vouched == Some(true)) + .count(); + if nonsense_served > MAX_NEGATIVES_SERVED { + eprintln!( + "\n[warn] the evidence bar serves {nonsense_served} of {} nonsense queries on the dense \ + fixture where {MAX_NEGATIVES_SERVED} is permitted — nonsense needs no token audit in any \ + vault, which is exactly why this reading transfers.", + dense.search.nonsense.len() + ); + return false; + } + true +} diff --git a/crates/b2-embed/examples/eval/instrument.rs b/crates/b2-embed/examples/eval/instrument.rs new file mode 100644 index 0000000..65b0f85 --- /dev/null +++ b/crates/b2-embed/examples/eval/instrument.rs @@ -0,0 +1,98 @@ +//! Instrument checks and plumbing the scored passes rest on: batch ≡ single embedding +//! faithfulness, the timed embed pass, and the candidate-width blindness warning (GH #141). + +use crate::K; +use b2_core::embed::Embedder; +use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, Vault}; +use b2_embed::LocalEmbedder; +use std::ops::ControlFlow; +use std::time::Instant; + +/// `LocalEmbedder::embed_batch` must be a faithful map of `embed`: right-padding short rows +/// to the batch's longest and masking them out has to leave each row's CLS vector unchanged. +/// The reindex path batches freely, so a regression here would silently corrupt every stored +/// vector — and every score this eval prints. +/// +/// It lives in the eval rather than `cargo test` because it needs the provisioned model, +/// which the fast suite deliberately never touches (ADR-0013). Running it here means it +/// actually runs, instead of sitting behind an `#[ignore]` nobody passes `--ignored` to. +pub fn check_batch_matches_single(model: &LocalEmbedder) -> Result<(), Box> { + // Deliberately varied lengths, so batching pads the short rows to the longest. + let texts = [ + "Spaced repetition schedules reviews at increasing intervals.", + "Sleep consolidates memory.", + "Short.", + "Focus and sustained attention shape what is later recalled from long-term memory across days.", + ]; + let refs: Vec<&str> = texts.to_vec(); + let batched = model.embed_batch(&refs)?; + if batched.len() != texts.len() { + return Err(format!( + "embed_batch returned {} rows for {} texts", + batched.len(), + texts.len() + ) + .into()); + } + let mut worst = f32::INFINITY; + for (text, batched_row) in texts.iter().zip(&batched) { + let single = model.embed(text)?; + if batched_row.len() != single.len() { + return Err(format!("batched/single dim mismatch for {text:?}").into()); + } + // Both rows are L2-normalized, so the dot product is cosine similarity; + // padding must not move it off ~1.0. Non-finite is checked first and + // explicitly: every comparison against a NaN is false, so `cos <= 0.9999` + // alone would wave a NaN row *through* the gate — the one failure mode a + // correctness check must not have. + let cos: f32 = batched_row.iter().zip(&single).map(|(a, b)| a * b).sum(); + if !cos.is_finite() { + return Err(format!("batched embedding is non-finite for {text:?}: {cos}").into()); + } + worst = worst.min(cos); + if cos <= 0.9999 { + return Err(format!( + "batched embedding differs from single for {text:?}: cosine {cos}" + ) + .into()); + } + } + eprintln!("[eval] batch ≡ single: worst-row cosine {worst:.6}\n"); + Ok(()) +} + +/// Run the embed pass, timing it and counting the chunks it filled. +pub fn timed_embed(vault: &Vault) -> Result<(usize, f64), Box> { + let mut chunks = 0usize; + let t0 = Instant::now(); + vault.embed(&mut |p| { + chunks = p.chunks_done; + ControlFlow::Continue(()) + })?; + Ok((chunks, t0.elapsed().as_secs_f64())) +} + +/// State the one thing this corpus **cannot** measure, on every run that can't. +/// +/// A corpus with no more chunks than a signal's candidate pool truncates *neither* list, so +/// both are already complete, widening cannot add a candidate, and every score above is +/// invariant under **candidate width**: a change to either view's headroom or to +/// `search::pool_size` prints bit-identical numbers here while genuinely reordering a real +/// vault (GH #141). Judged on the **narrower** of the two pools, since blindness is a claim +/// about every number the run prints. +/// +/// Scoped deliberately to width — `RRF_K` re-weights the *same* two lists, so it reorders +/// results on any corpus, and this eval sees that. A warning, not a gate: the point is that a +/// reader must not take an unmoved number as evidence of no change. The property itself is +/// measured by `--example stability`, on a vault big enough for the pool to bind. +pub fn warn_if_pool_blind(chunks: usize) { + let pool = chunk_candidate_pool(K).min(note_candidate_pool(K)); + if chunks <= pool { + eprintln!( + "[warn] {chunks} chunks ≤ {pool}-candidate pool — neither signal is truncated here, so a\n\ + \x20 candidate-width change (either hit pool, pool_size) cannot move any number in this run\n\ + \x20 (GH #141). `make stability` measures that property on a large vault. (RRF_K is\n\ + \x20 not in that set — it re-weights the same lists, and this corpus does see it.)\n" + ); + } +} diff --git a/crates/b2-embed/examples/eval/labels.rs b/crates/b2-embed/examples/eval/labels.rs new file mode 100644 index 0000000..f8e1dd1 --- /dev/null +++ b/crates/b2-embed/examples/eval/labels.rs @@ -0,0 +1,181 @@ +//! The labelled sets (`queries.json`, `similar.json`, `similar-dense.json`) and the lint +//! that refuses to score against labels the corpus cannot honour. + +use serde::Deserialize; +use std::collections::HashMap; +use std::path::Path; + +#[derive(Deserialize)] +pub struct QuerySet { + pub queries: Vec, +} + +#[derive(Deserialize)] +pub struct Labelled { + pub query: String, + /// The vault-relative path(s) that should rank first. **Empty = a negative + /// query** (invariants.md D2, GH #201): the labelled answer is "no + /// matches" — the vault holds no evidence for the query, so everything + /// served is junk by label, the query-side sibling of similar.json's + /// negative anchors. Negative queries are excluded from every rank + /// aggregate (adding one moves no pre-existing number) and are scored only + /// by the search evidence calibration. + pub relevant: Vec, + /// A short verbatim phrase from the target passage; when present the query is + /// also scored at chunk level (does a top-K chunk of a relevant note contain + /// it?). See queries.json's description for the labelling rules. + #[serde(default)] + pub passage: Option, + /// Notes beyond `relevant` that are honest evidence for the query (GH #206) — + /// the per-hit tail depth. The judgement is **exhaustive** for every positive + /// query: a served note in neither `relevant` nor here is irrelevant *by + /// label*, which is the statement the tail bake-off is judged on. Never enters + /// a rank aggregate — `relevant` alone says what should rank first. + #[serde(default)] + pub tail_relevant: Vec, +} + +#[derive(Deserialize)] +pub struct SimilarSet { + pub anchors: Vec, +} + +#[derive(Deserialize)] +pub struct SimilarLabel { + pub anchor: String, + /// Corpus notes a human says belong next to `anchor`. **Empty = a negative + /// anchor**: the labelled answer is "nothing relates", so the right result is + /// zero candidates and everything surfaced is junk by label (similar.json). + pub expected: Vec, +} + +/// Every note in one corpus dir, as `file name → lowercased content` — the +/// ground the label lint checks against. Lowercased once, so the passage check +/// matches the way [`score_pass`](crate::retrieval::score_pass) will (case-insensitive containment). +pub fn corpus_texts(dir: &Path) -> Result, Box> { + let mut out = HashMap::new(); + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + if entry.file_type()?.is_file() { + out.insert( + entry.file_name().to_string_lossy().into_owned(), + std::fs::read_to_string(entry.path())?.to_lowercase(), + ); + } + } + Ok(out) +} + +/// Refuse to score against labels the corpus cannot honour (see the call site). +/// +/// Checks: every labelled path (`relevant`, `tail_relevant`, `anchor`, +/// `expected`) names a note in its corpus; every `passage` occurs verbatim +/// (case-insensitively) in a note the query labels relevant — the containment +/// chunk scoring will test; and a negative query carries neither a `passage` +/// nor a `tail_relevant` (its whole list is junk by label already, per the +/// queries.json rules). Faults are all printed before the run refuses, so one +/// run names every problem rather than the first. +pub fn lint_labels( + corpus_dir: &Path, + dense_dir: &Path, + positives: &[Labelled], + negatives: &[Labelled], + sim_set: &SimilarSet, + dense_set: &SimilarSet, +) -> Result<(), Box> { + let corpus = corpus_texts(corpus_dir)?; + let dense = corpus_texts(dense_dir)?; + let mut faults: Vec = Vec::new(); + + for q in positives { + for (key, paths) in [ + ("relevant", &q.relevant), + ("tail_relevant", &q.tail_relevant), + ] { + for p in paths { + if !corpus.contains_key(p) { + faults.push(format!( + "queries.json: `{key}` names no corpus note: {p:?} (query {:?})", + q.query + )); + } + } + } + if let Some(passage) = &q.passage { + // A blank passage is the opposite defect from a typo'd one: every + // string contains "", so it would lint clean here and then "match" + // every top-K chunk of a relevant note in the chunk scoring — + // silently inflating chunk rank instead of reading a miss + // (PR #221 review). + if passage.trim().is_empty() { + faults.push(format!( + "queries.json: `passage` is blank (query {:?}) — it would match every chunk \ + and inflate chunk rank", + q.query + )); + } else { + let needle = passage.to_lowercase(); + let found = q + .relevant + .iter() + .filter_map(|p| corpus.get(p)) + .any(|text| text.contains(&needle)); + if !found { + faults.push(format!( + "queries.json: `passage` {passage:?} is not verbatim in any relevant note \ + (query {:?}) — chunk rank would read a permanent miss", + q.query + )); + } + } + } + } + for q in negatives { + if q.passage.is_some() { + faults.push(format!( + "queries.json: negative query {:?} carries a `passage` — negatives score no rank", + q.query + )); + } + if !q.tail_relevant.is_empty() { + faults.push(format!( + "queries.json: negative query {:?} carries `tail_relevant` — its whole served \ + list is junk by label", + q.query + )); + } + } + for (file, set, texts) in [ + ("similar.json", sim_set, &corpus), + ("similar-dense.json", dense_set, &dense), + ] { + for label in &set.anchors { + if !texts.contains_key(&label.anchor) { + faults.push(format!( + "{file}: `anchor` names no corpus note: {:?}", + label.anchor + )); + } + for e in &label.expected { + if !texts.contains_key(e) { + faults.push(format!( + "{file}: `expected` names no corpus note: {e:?} (anchor {:?})", + label.anchor + )); + } + } + } + } + if faults.is_empty() { + return Ok(()); + } + for f in &faults { + eprintln!("[lint] {f}"); + } + Err(format!( + "{} label lint fault(s) — a label the corpus cannot honour scores as a permanent miss \ + and reads as an engine regression; fix the label file (or the note) before scoring", + faults.len() + ) + .into()) +} diff --git a/crates/b2-embed/examples/eval/main.rs b/crates/b2-embed/examples/eval/main.rs new file mode 100644 index 0000000..45c1823 --- /dev/null +++ b/crates/b2-embed/examples/eval/main.rs @@ -0,0 +1,318 @@ +//! Semantic-retrieval + discovery eval — the pass that scores model quality **out of CI** +//! (ADR-0013). It lives as an *example*, not a test, so it never runs in the deterministic +//! `cargo test` suite and quality can never flake CI. +//! +//! ```console +//! cargo run -p b2-embed --example eval # score the configured model +//! cargo run -p b2-embed --example eval -- --sweep # + chunker A/B (the #44 gate) +//! cargo run -p b2-embed --example eval -- --stemmer # + FTS tokenizer A/B (the #157 gate) +//! ``` +//! +//! Any other argument is refused (exit 1) rather than ignored: a typo'd `--swep` would +//! otherwise score only the default config while the reader believes the A/B ran. +//! +//! **`docs/evals.md` is the notebook of record** — the corpus, what the exit code +//! enforces, every verdict this harness has ruled, and the process rules. Read it before +//! touching the corpus, the labels, or a constant here. What this comment carries is only +//! what a reader of *this file* needs: +//! +//! One run builds throwaway vaults from the labelled corpora in `evals/` and scores, through +//! the real pipeline: a **BM25 baseline** (after `project` only — the floor the model must +//! clear, since the labelled queries avoid their target's keywords); **hybrid retrieval** +//! plus a **vector-only ablation** (GH #158), whose delta is the measured value of the one +//! AI seam; **passage rank** at chunk level, which is where chunking levers show; and +//! **discovery**, scored per labelled mate rather than per anchor (GH #183 — the per-anchor +//! metric saturates), with the strangers a positive anchor serves counted, named, and +//! deliberately ungated, since the cheapest way to shrink that count is to label one. +//! +//! Two calibration blocks re-derive their windows **every run** rather than quoting a +//! reading: the discovery **z dump** (GH #187) and the **search evidence bake-off** +//! (ADR-0015, GH #201/#202). That is the house rule — constants in code, measurements in the +//! harness — and it exists because the GH #150 floors were frozen into a docstring and went +//! stale the first time the corpus grew a shape they were never read against. +//! +//! The **dense single-domain fixture** (`evals/corpus-dense/`) is scored in its own vault +//! and its own row, never averaged in: fifteen genuinely inter-related notes with no loner, +//! the geometry that broke every anchor-local existence gate (ADR-0014) and killed the first +//! lexical evidence rule (ADR-0015). The orthogonal corpus is structurally incapable of +//! expressing topical concentration, so a run that judges a bar only there judges it on the +//! geometry it survives. +//! +//! What this corpus mostly **cannot** score is *candidate width*: while its chunk count sits +//! at or under a signal's candidate pool, that list is never truncated, widening it cannot +//! add a candidate, and every number is invariant under that view's headroom and +//! `search::pool_size` (GH #141). The corpus has since grown a few chunks past the narrower +//! (passage-view) pool (GH #183), so the run reads the live counts and warns (`pool_blind`) +//! only when blindness actually holds; the property itself is measured by +//! `--example stability`, on a vault big enough for the pools to bind. `RRF_K` re-weights +//! the *same* lists, so it does move scores here and needs no separate instrument. +//! +//! `--sweep` re-chunks + re-embeds the same vault under variant [`ChunkConfig`]s. `--stemmer` +//! swaps `chunks_fts` between the shipped `porter unicode61` and the unstemmed ablation over +//! **identical** chunk rows and vectors, so every rank move is the tokenizer's alone. +//! +//! Every scored run appends one JSON line to `evals/results.jsonl` (gitignored), so runs +//! accumulate into a comparable dataset. +//! +//! This file is the run's order of operations and nothing else; each block of the report +//! lives in the module named for it — `labels` (the sets and their lint), `retrieval` +//! and `metrics` (the per-query passes), `discovery` (per-mate ranks, strangers, the z +//! dump), `fold` (the GH #200 bake-off), `evidence` (the D2 bar's calibration and +//! bake-off), `tail` (the GH #206 bake-off and its cross-bench join), `dense` (the +//! single-domain fixture), `ablation` (`--stemmer`/`--sweep`), `report` (the default +//! table and the JSONL row), `instrument` (the checks the numbers rest on), and `gate` +//! (the exit gate's constants and assertions). + +// `result_row`'s JSON literal (`report.rs`) is one `json!` expansion per key, and the row has +// grown a key per instrument (GH #158, #141, #183, #187, #188). Raising the +// limit keeps the row's shape legible in one place — the alternative is +// scattering its subtrees across helper functions to satisfy a macro, which +// costs the thing the row is for: a reader seeing every recorded field at once. +#![recursion_limit = "256"] + +mod ablation; +#[path = "../common/mod.rs"] +mod common; +mod dense; +mod discovery; +mod evidence; +mod fold; +mod gate; +mod instrument; +mod labels; +mod metrics; +mod report; +mod retrieval; +mod tail; + +use ablation::Baseline; +use b2_core::chunk::ChunkConfig; +use b2_core::db::FtsTokenizer; +use b2_core::embed::Embedder; +use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, Vault}; +use b2_embed::{provision, EmbedConfig, LocalEmbedder}; +use common::{append_result, git_short_sha, has_flag, reject_unknown_flags, ScratchVault}; +use dense::{dense_row, print_dense_report, score_dense}; +use discovery::{score_floor_z, score_similar}; +use evidence::{bake_off, print_search_bakeoff, print_search_evidence, score_search_evidence}; +use fold::{print_fold_bench, score_fold}; +use instrument::{check_batch_matches_single, timed_embed, warn_if_pool_blind}; +use labels::{lint_labels, Labelled, QuerySet, SimilarSet}; +use report::{print_default_report, result_row, Calibration, RowInputs, RunId}; +use retrieval::{score_pass, Retrieval}; +use std::path::Path; +use tail::{print_search_tail, print_tail_join, score_search_tail}; + +/// How deep we look for a relevant note/chunk when scoring. +const K: usize = 10; + +/// How many `similar` candidates we look at per anchor. +const SIM_K: usize = 5; + +/// How deep the z dump reads (GH #187) — every candidate note in a corpus this +/// size, since a threshold is calibrated against the whole population it has to +/// cut, not against the prefix a human-facing `limit` would show. +const Z_SCAN_LIMIT: usize = 500; + +fn main() { + match run() { + Err(e) => { + eprintln!("eval failed: {e}"); + std::process::exit(1); + } + Ok(passed) => { + if !passed { + std::process::exit(2); + } + } + } +} + +/// Returns whether the default config cleared every exit-gate row ([`gate::passes`]). +fn run() -> Result> { + // A bare `--` is dropped rather than refused (`make stability`'s posture: a pasted + // `cargo run … -- --sweep` must not fail on its own separator); anything else + // unrecognised is refused, since a typo'd flag would otherwise run the default + // measurement while the reader believes an A/B ran. + let args: Vec = std::env::args().skip(1).filter(|a| a != "--").collect(); + reject_unknown_flags(&args, &["--sweep", "--stemmer"], &[])?; + let sweep = has_flag(&args, "--sweep"); + let stemmer = has_flag(&args, "--stemmer"); + let evals_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("evals"); + let corpus_dir = evals_dir.join("corpus"); + let results_path = evals_dir.join("results.jsonl"); + + // Load the labelled sets. Queries split on their labels: positives carry + // rank labels; an empty `relevant` is a negative query (invariants.md D2, + // GH #201), scored only by the search evidence calibration so its presence + // moves no rank aggregate. + let set: QuerySet = + serde_json::from_str(&std::fs::read_to_string(evals_dir.join("queries.json"))?)?; + let (negatives, positives): (Vec, Vec) = + set.queries.into_iter().partition(|q| q.relevant.is_empty()); + let sim_set: SimilarSet = + serde_json::from_str(&std::fs::read_to_string(evals_dir.join("similar.json"))?)?; + let dense_set: SimilarSet = serde_json::from_str(&std::fs::read_to_string( + evals_dir.join("similar-dense.json"), + )?)?; + + // Lint the labels against the corpora they claim to describe, before + // anything expensive runs. A typo'd label path fails nothing on its own — + // it just reads as a permanent miss (a rank of `None`, a served row + // downgraded to filler by label) and gets chased as an engine regression — + // so the run refuses to score against labels the corpus cannot honour + // (exit 1: the run broke, not the gate). + lint_labels( + &corpus_dir, + &evals_dir.join("corpus-dense"), + &positives, + &negatives, + &sim_set, + &dense_set, + )?; + + // Ensure the model is available, then load it. (Provision is idempotent, so an + // already-installed model is a no-op; a missing one is fetched here.) + let config = EmbedConfig::load()?; + provision(&config, |line| eprintln!("[init] {line}"))?; + let embedder = LocalEmbedder::load(&config)?; + let model_id = embedder.model_id().to_string(); + let dim = embedder.dim(); + eprintln!("[eval] model = {model_id} (dim {dim})\n"); + + // A correctness gate, not a score: every number below is computed from batched + // embeddings, so they only mean anything if batching is faithful. + check_batch_matches_single(&embedder)?; + + // Build a throwaway vault from the corpus. + let scratch = ScratchVault::copy_flat(&corpus_dir)?; + let vault_root = scratch.root(); + let mut vault = Vault::open_with_embedder(vault_root, Box::new(embedder))?; + + // ---- Phase 1: projection only → the BM25-only baseline. ------------------ + // The vector space does not exist yet, so `search`/`search_chunks` run + // keyword-only (index-engine.md) — the ablation costs nothing + // extra: it is the same vault, paused between the two passes. + let report = vault.project(false)?; + let bm25 = score_pass(&vault, &positives, Retrieval::Fused)?; + eprintln!( + "[eval] projected {} notes; BM25-only baseline scored\n", + report.indexed + ); + + // The stemmer instrument's lexical arm is scored HERE, while the vault is + // still projected-but-unembedded, so `search` is honestly BM25-only under + // both tokenizers; the vault is handed back to the shipped default before + // anything embeds. + let bm25_unstemmed = if stemmer { + vault.rebuild_fts(FtsTokenizer::Unicode61)?; + let pass = score_pass(&vault, &positives, Retrieval::Fused)?; + vault.rebuild_fts(FtsTokenizer::PorterUnicode61)?; + Some(pass) + } else { + None + }; + + // ---- Phase 2: embed → vector-only ablation + hybrid + discovery. --------- + let (chunks, embed_secs) = timed_embed(&vault)?; + let vector = score_pass(&vault, &positives, Retrieval::VectorOnly)?; + let hybrid = score_pass(&vault, &positives, Retrieval::Fused)?; + let similar = score_similar(&vault, &sim_set)?; + // The fold bake-off (GH #200, Phase B) — the candidate default-disclosure + // rules judged on the same served lists `similar` above was scored from, so + // no rule is compared against a surface the others did not see. + let fold = score_fold(&vault, &sim_set, "orthogonal", false)?; + // The z calibration dump (GH #187) — the same shipped surface read deep, + // since the z travels ungated on it (GH #197). + let floor_z = score_floor_z(&vault, &sim_set)?; + // The search evidence dump (invariants.md D2, GH #201) — the query-side + // sibling of the z calibration, read over the same built vault. + let evidence = score_search_evidence(vault_root, &vault, &positives, &negatives)?; + eprintln!( + "[eval] embedded {chunks} chunks in {embed_secs:.1}s ({} candidates per signal at K={K}, \ + {} for the passage view)\n", + note_candidate_pool(K), + chunk_candidate_pool(K) + ); + warn_if_pool_blind(chunks); + + print_default_report(&positives, &bm25, &vector, &hybrid, &similar, &floor_z); + print_fold_bench(&fold); + print_search_evidence(&evidence); + print_search_bakeoff(&evidence, &bake_off(&evidence), &model_id); + // The per-hit tail bake-off (GH #206) — judged from the same served lists + // the evidence dump above recorded, against the tail_relevant keep-set. + let tail = score_search_tail(&evidence); + print_search_tail(&tail); + + let git = git_short_sha(); + let run_id = RunId { + git: git.as_deref(), + model: &model_id, + dim, + }; + append_result( + &results_path, + &result_row( + run_id, + RowInputs { + label: "default", + cfg: &ChunkConfig::default(), + tokenizer: FtsTokenizer::PorterUnicode61.sql(), + notes: report.indexed, + chunks, + embed_secs, + queries: &positives, + bm25: Some(&bm25), + vector: Some(&vector), + hybrid: &hybrid, + similar: Some(&similar), + calibration: Some(Calibration { + floor_z: &floor_z, + evidence: &evidence, + fold: &fold, + tail: &tail, + }), + }, + ), + )?; + + // ---- Phase 3: the dense single-domain fixture (GH #196/#197, Phase 0b). -- + // Its own throwaway vault, its own model load, its own results row (corpus + // id `dense`) — the fixture measures a *vault-level* geometry, so nothing + // about it may share state with the orthogonal corpus's run above. + let dense = score_dense(&evals_dir, &dense_set)?; + print_dense_report(&dense); + print_fold_bench(&dense.fold); + append_result(&results_path, &dense_row(run_id, &dense))?; + // The tail bake-off's cross-bench join (GH #206) — printable only here, + // where both corpora's readings exist in one run. + print_tail_join(&evidence, &tail, &dense.search.titles); + + // ---- Optional: the A/Bs, each against the default passes above. ---------- + let base = Baseline { + log: &results_path, + run: run_id, + notes: report.indexed, + chunks, + embed_secs, + queries: &positives, + bm25: &bm25, + vector: &vector, + hybrid: &hybrid, + }; + // Scored only under `--stemmer`, while the vault was still unembedded. + if let Some(bm25_unstemmed) = &bm25_unstemmed { + ablation::stemmer(&vault, &base, bm25_unstemmed)?; + } + if sweep { + ablation::sweep(&mut vault, &base, &sim_set)?; + } + + eprintln!("\n[eval] appended run to {}", results_path.display()); + + // The exit gate reads the DEFAULT config's passes only — never an A/B's. + Ok(gate::passes( + &hybrid, &similar, &dense, &evidence, &model_id, + )) +} diff --git a/crates/b2-embed/examples/eval/metrics.rs b/crates/b2-embed/examples/eval/metrics.rs new file mode 100644 index 0000000..bcfe107 --- /dev/null +++ b/crates/b2-embed/examples/eval/metrics.rs @@ -0,0 +1,88 @@ +//! Scoring primitives every block shares: the rank aggregate, the two-population +//! threshold window, rank notation, and label-path matching. + +use crate::K; + +/// Running hit@1 / hit@3 / MRR@K over a set of 1-based ranks. ("hit@k" — each +/// query has essentially one relevant target, so precision@k and recall@k +/// coincide with it.) +#[derive(Default)] +pub struct Agg { + pub n: usize, + pub hit1: usize, + pub hit3: usize, + pub rr: f64, +} + +impl Agg { + pub fn add(&mut self, rank: Option) { + self.n += 1; + if let Some(r) = rank { + self.rr += 1.0 / r as f64; + if r <= 1 { + self.hit1 += 1; + } + if r <= 3 { + self.hit3 += 1; + } + } + } + pub fn hit1(&self) -> f64 { + self.hit1 as f64 / self.n.max(1) as f64 + } + pub fn hit3(&self) -> f64 { + self.hit3 as f64 / self.n.max(1) as f64 + } + pub fn mrr(&self) -> f64 { + self.rr / self.n.max(1) as f64 + } +} + +/// The admissible interval for one threshold, read straight off two labelled +/// populations: a bar `t` keeps everything in `keep` iff `t ≤ min(keep)`, and +/// cuts everything in `cut` iff `t > max(cut)`, so every workable constant lies +/// in `(cut_max, keep_min]` — and **no constant works at all** when the two +/// populations overlap, which is a measured result rather than a failure to +/// search harder. +pub struct Window { + pub cut_max: f64, + pub keep_min: f64, +} + +impl Window { + /// Both edges, or `None` while either population is empty (nothing to + /// separate — an honest no-reading, not a wide-open window). + pub fn read(cut: &[f64], keep: &[f64]) -> Option { + let cut_max = cut.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let keep_min = keep.iter().copied().fold(f64::INFINITY, f64::min); + (cut_max.is_finite() && keep_min.is_finite()).then_some(Self { cut_max, keep_min }) + } + /// Whether any constant separates the two populations. + pub fn open(&self) -> bool { + self.cut_max < self.keep_min + } +} + +/// [`rank_str`] at a caller-named depth (the discovery metrics read at +/// [`SIM_K`](crate::SIM_K), not [`K`]). +pub fn rank_str_at(rank: Option, depth: usize) -> String { + match rank { + Some(1) => "✓1".to_string(), + Some(r) => format!("·{r}"), + None => format!("✗>{depth}"), + } +} + +/// Corpus notes are copied flat into the vault, so a result path equals (or ends +/// with) the labelled relevant path. +pub fn paths_match(result_path: &str, relevant: &str) -> bool { + result_path == relevant || result_path.ends_with(&format!("/{relevant}")) +} + +pub fn rank_str(rank: Option) -> String { + match rank { + Some(1) => "✓1".to_string(), + Some(r) => format!("·{r}"), + None => format!("✗>{K}"), + } +} diff --git a/crates/b2-embed/examples/eval/report.rs b/crates/b2-embed/examples/eval/report.rs new file mode 100644 index 0000000..e604b4c --- /dev/null +++ b/crates/b2-embed/examples/eval/report.rs @@ -0,0 +1,575 @@ +//! The default config's printed report (the per-query table, aggregates, discovery lines), +//! the paired per-query A/B diff, and the orthogonal corpus's JSONL row. + +use crate::common::{cosine_of, pile_stats, truncate}; +use crate::discovery::{print_floor_windows, FloorZ, SimilarPass}; +use crate::evidence::{bake_off, best_cos_pile, QueryEvidence, SearchEvidence}; +use crate::fold::{fold_json, FoldBench}; +use crate::gate::FLOOR_MATE_MRR; +use crate::labels::Labelled; +use crate::metrics::{rank_str, Agg, Window}; +use crate::retrieval::Pass; +use crate::tail::{tail_json, TailBench}; +use crate::{K, SIM_K}; +use b2_core::chunk::ChunkConfig; +use b2_core::search::EvidenceBar; +use b2_core::vault::{chunk_candidate_pool, note_candidate_pool}; + +pub fn print_default_report( + queries: &[Labelled], + bm25: &Pass, + vector: &Pass, + hybrid: &Pass, + similar: &SimilarPass, + floor_z: &FloorZ, +) { + println!( + "{:>5} {:>6} {:>6} {:>6} {:<40} top hybrid hit", + "bm25", "vec", "hybrid", "chunk", "query" + ); + println!("{}", "-".repeat(102)); + for (i, q) in queries.iter().enumerate() { + println!( + "{:>5} {:>6} {:>6} {:>6} {:<40} {}", + rank_str(bm25.scores[i].note), + rank_str(vector.scores[i].note), + rank_str(hybrid.scores[i].note), + match q.passage { + Some(_) => rank_str(hybrid.scores[i].chunk), + None => "".to_string(), + }, + truncate(&q.query, 40), + hybrid.scores[i].top, + ); + } + + println!("\n{}", "=".repeat(78)); + println!("note rank (n={}, K={K}):", queries.len()); + println!( + " bm25-only hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", + bm25.note.hit1(), + bm25.note.hit3(), + bm25.note.mrr() + ); + println!( + " vec-only hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", + vector.note.hit1(), + vector.note.hit3(), + vector.note.mrr() + ); + println!( + " hybrid hit@1={:.2} hit@3={:.2} MRR@{K}={:.3} semantic lift: {:+.2} hit@1", + hybrid.note.hit1(), + hybrid.note.hit3(), + hybrid.note.mrr(), + hybrid.note.hit1() - bm25.note.hit1(), + ); + // The standing form of the fusion finding (GH #158): every query + // where fusing the two signals ranked the labelled answer WORSE than the + // dense signal alone would have. RRF's consensus bias makes some of this + // inevitable; the point is that it is counted and named on every run instead + // of rediscovered by hand-decomposing scores. + let demoted: Vec = (0..queries.len()) + .filter(|&i| { + let Some(v) = vector.scores[i].note else { + return false; + }; + match hybrid.scores[i].note { + Some(h) => h > v, + None => true, + } + }) + .collect(); + if demoted.is_empty() { + println!(" fusion no query ranks worse under hybrid than under vector alone"); + } else { + println!( + " fusion {} quer{} rank worse under hybrid than under vector alone:", + demoted.len(), + if demoted.len() == 1 { "y" } else { "ies" } + ); + for &i in &demoted { + println!( + " vec {} → hybrid {} {}", + rank_str(vector.scores[i].note), + rank_str(hybrid.scores[i].note), + truncate(&queries[i].query, 48) + ); + } + } + if hybrid.chunk.n > 0 { + println!("chunk rank (passage-labelled, n={}):", hybrid.chunk.n); + println!( + " bm25-only hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", + bm25.chunk.hit1(), + bm25.chunk.hit3(), + bm25.chunk.mrr() + ); + println!( + " hybrid hit@1={:.2} hit@3={:.2} MRR@{K}={:.3}", + hybrid.chunk.hit1(), + hybrid.chunk.hit3(), + hybrid.chunk.mrr() + ); + } + println!( + "similar (n={} positive + {} negative, K={SIM_K}):", + similar.rank.n, similar.neg_n + ); + // The per-anchor metric (first mate found, then stop) is **no longer + // printed** (GH #188): it read 1.000 across every change it was meant to + // judge, and a line that cannot move is a line that trains skimming. It is + // still recorded — `results.jsonl`'s `"similar"` key is unchanged, so rows + // stay comparable back to the first run — so retiring the line costs the + // dataset nothing. + println!( + " per-mate hit@1={:.2} hit@3={:.2} MRR@{SIM_K}={:.3} (n={} mates, GATED at MRR@{SIM_K} ≥ {FLOOR_MATE_MRR:.2})", + similar.mate.hit1(), + similar.mate.hit3(), + similar.mate.mrr(), + similar.mate.n + ); + // Discovery's precision side (GH #188) — reported with names, never gated; + // see `SimilarPass::strangers` for why gating it would reward labelling. + println!( + " strangers {} card{} on {}/{} positive anchors (unlabelled notes served in the top-{SIM_K} — \ + a smoke alarm, not a gate: labels aren't exhaustive)", + similar.strangers.len(), + if similar.strangers.len() == 1 { "" } else { "s" }, + similar.stranger_anchors, + similar.rank.n + ); + for (anchor, path) in &similar.strangers { + println!(" {anchor} → {path}"); + } + if similar.neg_n > 0 { + // Under always-serve (GH #197) a negative anchor serves its ranked + // nearest like any other — that is the ruling, not a regression. What + // these anchors measure now is what the served cards *claim*: their + // bands (A2's readout, printed per leader in the calibration block). + println!( + " negatives {} loner anchors serve {} cards under always-serve (labels still say \ + \"nothing relates\"; the bands carry the honesty — leaders below)", + similar.neg_n, similar.neg_cards + ); + } + // The two calibration piles. If they separate, the gap IS the floor, read + // off measured data; if they overlap, no simple floor can hold and the + // escalation path (a discovery-side pair-scorer) is justified by data. + if let (Some((r_min, r_med, r_max)), Some((j_min, j_med, j_max))) = + (pile_stats(&similar.related), pile_stats(&similar.junk)) + { + println!( + " piles(cos) related n={:<3} min/med/max {:.3}/{:.3}/{:.3}", + similar.related.len(), + r_min, + r_med, + r_max + ); + println!( + " junk n={:<3} min/med/max {:.3}/{:.3}/{:.3}", + similar.junk.len(), + j_min, + j_med, + j_max + ); + let gap = r_min - j_max; + println!( + " related-min − junk-max = {:+.3} ({})", + gap, + if gap > 0.0 { + "piles separate — the gap is the floor" + } else { + "piles overlap — a simple floor cuts both" + } + ); + } + // The same question in the floor's own anchor-relative unit, where an + // answer is actionable: the piles are absolute cosines, and no constant in + // the code is ever compared against one. + print_floor_windows(floor_z); +} + +/// The paired per-query diff between two fused passes — what an A/B is actually +/// judged on. At this corpus's n, every aggregate delta is worth 1–2 queries, so +/// "hit@1 +0.05" and "these two queries flipped, this one broke" are the same +/// fact — but only the second form can be argued with, per-query, against the +/// labels (docs/evals.md, the process rules). Prints nothing but a +/// no-moves line when the variant reproduced the reference ranking exactly — +/// which, per the same rules, is itself a claim to verify against a +/// continuous quantity (the piles), never bare proof of "no effect". +pub fn print_rank_moves(queries: &[Labelled], reference: &Pass, variant: &Pass) { + let improved = |a: Option, b: Option| match (a, b) { + (None, Some(_)) => true, + (Some(x), Some(y)) => y < x, + _ => false, + }; + let mut note_up = 0usize; + let mut note_down = 0usize; + let mut lines = Vec::new(); + for (i, q) in queries.iter().enumerate() { + let (a, b) = (reference.scores[i].note, variant.scores[i].note); + if a != b { + if improved(a, b) { + note_up += 1; + } else { + note_down += 1; + } + lines.push(format!( + " Δ note {:>5} → {:<5} {}", + rank_str(a), + rank_str(b), + truncate(&q.query, 48) + )); + } + if q.passage.is_some() { + let (a, b) = (reference.scores[i].chunk, variant.scores[i].chunk); + if a != b { + lines.push(format!( + " Δ chunk {:>5} → {:<5} {}", + rank_str(a), + rank_str(b), + truncate(&q.query, 48) + )); + } + } + } + if lines.is_empty() { + println!(" Δ vs default: no per-query rank moved (verify against the piles before reading this as \"no effect\")"); + return; + } + println!(" Δ vs default — note ranks: {note_up} improved, {note_down} worsened",); + for line in lines { + println!("{line}"); + } +} + +/// Who scored a run and in which embedding space — the identity every row the run +/// appends carries (the orthogonal corpus's rows and the dense fixture's alike). +#[derive(Clone, Copy)] +pub struct RunId<'a> { + /// The repo's short commit, `None` outside a git checkout. + pub git: Option<&'a str>, + pub model: &'a str, + pub dim: usize, +} + +/// One scored configuration's inputs to [`result_row`]: a short-lived, read-only view +/// over passes `run` (or an A/B) owns, assembled at the call and consumed there. +#[derive(Clone, Copy)] +pub struct RowInputs<'a> { + /// The config's name in the row (`"default"`, `"unicode61"`, a sweep variant). + pub label: &'a str, + pub cfg: &'a ChunkConfig, + /// The `chunks_fts` tokenizer the row was scored under. + pub tokenizer: &'a str, + pub notes: usize, + pub chunks: usize, + pub embed_secs: f64, + /// The labelled positives every pass here was scored over. + pub queries: &'a [Labelled], + pub bm25: Option<&'a Pass>, + pub vector: Option<&'a Pass>, + pub hybrid: &'a Pass, + pub similar: Option<&'a SimilarPass>, + /// The calibration blocks, which only the default row re-derives; `None` on the + /// ablation and sweep rows, whose keys then record `null`. + pub calibration: Option>, +} + +/// The default row's calibration readings — present together or not at all. +#[derive(Clone, Copy)] +pub struct Calibration<'a> { + pub floor_z: &'a FloorZ, + pub evidence: &'a SearchEvidence, + pub fold: &'a FoldBench, + pub tail: &'a TailBench, +} + +/// Seconds since the Unix epoch, for a row's `ts` (0 on a clock before it). +pub fn unix_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// One appendable JSONL row for a scored configuration. +pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { + let RunId { git, model, dim } = run; + let RowInputs { + label, + cfg, + tokenizer, + notes, + chunks, + embed_secs, + queries, + bm25, + vector, + hybrid, + similar, + calibration, + } = row; + let floor_z = calibration.map(|c| c.floor_z); + let evidence = calibration.map(|c| c.evidence); + let fold = calibration.map(|c| c.fold); + let tail = calibration.map(|c| c.tail); + let ts = unix_secs(); + let agg = |a: &Agg| serde_json::json!({ "n": a.n, "hit1": a.hit1(), "hit3": a.hit3(), "mrr": a.mrr() }); + serde_json::json!({ + "ts": ts, + "git": git, + "model": model, + "dim": dim, + // NEW key (absent from rows before 2026-08-18): which corpus this row + // scored — the dense single-domain fixture appends its own `"dense"` + // rows (GH #196/#197), and rows must never average across corpora. + // Absent = this corpus, so every older row stays comparable unchanged. + "corpus": "orthogonal", + "config": { + "label": label, + "target_tokens": cfg.target_tokens, + "overlap_frac": cfg.overlap_frac, + "chars_per_token": cfg.chars_per_token, + "backscan_tokens": cfg.backscan_tokens, + "prepend_heading_path": cfg.prepend_heading_path, + }, + // NEW key (absent from rows before 2026-08-11, which were all scored + // under the then-default `unicode61`): the chunks_fts tokenizer this row + // was scored under (GH #157). Top-level rather than inside `config`, + // which stays the ChunkConfig alone. + "tokenizer": tokenizer, + "notes": notes, + "chunks": chunks, + // The caveat travels with the numbers: a row whose corpus fit inside the + // retrieval pool had both candidate lists complete, so no candidate-width + // change could have affected it and comparing it across one proves nothing + // (GH #141). Equality is blind too — a pool exactly the size of the corpus + // truncates nothing either. Both depths are recorded because since #142 the + // two views differ; the flat `pool` key of earlier rows is deliberately + // *not* reused, so a reader of a mixed file sees a missing field rather than + // one number silently meaning something narrower than it used to. + "pool_note": note_candidate_pool(K), + "pool_chunk": chunk_candidate_pool(K), + "pool_blind": chunks <= chunk_candidate_pool(K).min(note_candidate_pool(K)), + "embed_secs": embed_secs, + "note": { + "bm25": bm25.map(|p| agg(&p.note)), + // NEW key (absent from rows before 2026-08-10): the dense ablation — + // `Vault::search_vector_only`, the single-signal baseline fusion is + // judged against (GH #158). Same convention as pool_note/pool_chunk: + // a new key, never a redefined one. + "vector": vector.map(|p| agg(&p.note)), + "hybrid": agg(&hybrid.note), + }, + "chunk": { + "bm25": bm25.map(|p| agg(&p.chunk)), + "hybrid": agg(&hybrid.chunk), + }, + // "similar" keeps its pre-negative shape (the positive anchors' rank agg) + // so rows stay comparable across the change; the negative-anchor tally and + // the calibration piles are NEW keys, absent from older rows rather than + // redefining an existing one — the same convention as pool_note/pool_chunk. + "similar": similar.map(|s| agg(&s.rank)), + // NEW key (absent from rows before 2026-08-17): the per-mate, + // non-saturating companion to "similar" (GH #183). Same convention + // again — a new key, never a redefined one, so every older row stays + // comparable on "similar" itself. + "similar_per_mate": similar.map(|s| agg(&s.mate)), + // `similar_per_mate_raw` / `similar_mates_suppressed` retired with the + // pass-vs-pass tripwire (GH #217): under always-serve both passes read + // the one surface through the same call, so the diff was a tautology. + // Absent from newer rows, never redefined — the row conventions. + // NEW key (absent from rows before 2026-08-17): discovery's precision + // side — unlabelled notes served on positive anchors at the ranks' own + // depth (GH #188). Same convention as every key above: new, never a + // redefinition. Recorded with the pairs, because the count alone is + // unarguable and the pairs are what a reader checks against the notes. + "similar_strangers": similar.map(|s| serde_json::json!({ + "cards": s.strangers.len(), + "anchors": s.stranger_anchors, + "positives": s.rank.n, + // The depth the count is read at — the ranks' own top-K, so a + // future SIM_K change shows up in the row instead of silently + // redefining the number. + "depth": SIM_K, + // The pairs, because a bare count is unarguable: this metric is a + // smoke alarm whose correct answer is sometimes "that label is + // missing", and that argument needs the notes named. + "detail": s.strangers.iter().map(|(anchor, path)| serde_json::json!({ + "anchor": anchor, + "path": path, + })).collect::>(), + })), + "similar_negatives": similar.map(|s| serde_json::json!({ + "n": s.neg_n, "clean": s.neg_clean, "cards": s.neg_cards, + })), + // Cosine, 4 decimals: enough to place a floor, short enough to keep rows + // readable. Related = human-labelled matches; junk = everything else + // surfaced (see score_similar). + "similar_piles": similar.map(|s| serde_json::json!({ + "related": s.related.iter().map(|c| (c * 1e4).round() / 1e4).collect::>(), + "junk": s.junk.iter().map(|c| (c * 1e4).round() / 1e4).collect::>(), + })), + // The same scores with their per-anchor rank order kept: the relative + // drop-off cutoff is judged within one anchor's list, and tracing a pile + // value back to its pair needs the anchor. The piles above are this, + // flattened — kept anyway, because the flat distributions are what a + // quick jq/pandas histogram wants. + "similar_detail": similar.map(|s| s.detail.iter().map(|d| serde_json::json!({ + "anchor": d.anchor, + "negative": d.negative, + "candidates": d.candidates.iter().map(|(path, cos, related)| serde_json::json!({ + "path": path, + "cos": (cos * 1e4).round() / 1e4, + "related": related, + })).collect::>(), + })).collect::>()), + // NEW key (absent from rows before 2026-08-17): the floor's own + // calibration data in the floor's own unit — every candidate's ungated + // judge z, the populations the two constants answer to, and both + // re-derived windows (GH #187; the unit changed from stage-1 centroid z + // to stage-2 best-passage z with GH #192 — the row's "unit" field is + // what tells the two apart). Same convention as every key above: new, + // never a redefinition. This is the row the next recalibration reads, and + // the reason no window belongs in a doc comment — `null` on the ablation + // rows, which do not re-derive it. + "discovery_z": floor_z.map(|z| { + let (mates, strangers) = (z.mate_z(), z.stranger_z()); + let (neg_leaders, pos_leaders) = (z.neg_leader_z(), z.pos_leader_z()); + let win = |cut: &[f64], keep: &[f64]| Window::read(cut, keep).map(|w| serde_json::json!({ + "cut_max": (w.cut_max * 1e4).round() / 1e4, + "keep_min": (w.keep_min * 1e4).round() / 1e4, + "open": w.open(), + })); + let round = |v: &[f64]| v.iter().map(|z| (z * 1e4).round() / 1e4).collect::>(); + serde_json::json!({ + // The unit these piles/windows are measured in. Rows before + // GH #192 carried no key here and are stage-1 centroid z — a + // different unit; never compare across the flip. Since GH #197 + // the z gates nothing (the "shipped" and "replay_faults" keys + // of earlier rows retired with the gate — absent, per the + // convention, rather than redefined). + "unit": "stage2-best-passage", + "piles": { + "mates": round(&mates), + "strangers_on_positives": round(&strangers), + "negative_leaders": round(&neg_leaders), + "positive_leaders": round(&pos_leaders), + }, + "window_leader": win(&neg_leaders, &pos_leaders), + "window_member": win(&strangers, &mates), + "ungraded_anchors": z.ungraded, + // ~0 is the only trustworthy reading — see FloorZ::recheck_delta. + "z_recheck_max_delta": z.recheck_delta(), + // Per-anchor and per-candidate, because a window edge is only + // arguable once you can name the pair that set it. `z` is the + // stage-2 best-passage z, `cos` the same pair's cosine (the + // model-comparable unit). + "detail": z.anchors.iter().map(|a| serde_json::json!({ + "anchor": a.anchor, + "negative": a.negative, + "candidates": a.candidates.iter().map(|c| serde_json::json!({ + "path": c.path, + "z": (c.z * 1e4).round() / 1e4, + "cos": (cosine_of(c.score) * 1e4).round() / 1e4, + "mate": c.mate, + })).collect::>(), + })).collect::>(), + }) + }), + // NEW key (absent from rows before 2026-08-22): the search evidence + // dump (invariants.md D2, GH #201) — per labelled query, the absolute + // signals RRF discards (BM25 hit count + best score, dense top-1 + // cosine) and the shipped surface's served count, positives and + // negatives apart, with the would-be pure-cosine query window. Same + // convention as every key above: new, never a redefinition. `null` on + // ablation/sweep rows, which do not re-derive it. + "search_evidence": evidence.map(|e| { + let bar = EvidenceBar::for_model(model); + let row = |r: &QueryEvidence| serde_json::json!({ + "q": r.query, + "bm25_hits": r.bm25_hits, + "bm25_best": r.bm25_best.map(|b| (b * 1e4).round() / 1e4), + "best_cos": r.best_cos.map(|c| (c * 1e4).round() / 1e4), + "top": r.top, + "served": r.served(), + // NEW keys (absent from rows before 2026-08-22, GH #201 Phase C): + // the per-query evidence the bake-off is swept over, recorded raw + // so any (fraction, coverage, cos) cell is re-derivable from a row + // without re-running the model — the `discovery_fold` convention. + "dense_only": r.dense_only(), + // NEW key (absent from rows before GH #206): the served list + // itself, per row — path, per-hit provenance, and relevance by + // label — so any per-hit tail rule is re-derivable from a row + // without re-running the model, the `discovery_fold` convention + // at hit granularity. `keep` reads `relevant` ∪ `tail_relevant`. + "rows": r.rows.iter().map(|row| serde_json::json!({ + "path": row.path, + "bm25_rank": row.bm25_rank, + "cos": row.cos.map(|c| (c * 1e4).round() / 1e4), + "keep": row.keep, + })).collect::>(), + "chunk_total": r.chunk_total, + "terms": r.terms.iter().map(|(t, df)| serde_json::json!([t, df])) + .collect::>(), + // The ENGINE's verdict, matching the dense row's convention and + // what the exit gate asserts — recording the harness's + // restatement here instead would mask exactly the drift the + // [FAULT] check exists to surface (the restatement is + // re-derivable from `terms` + `best_cos` + `bar` regardless). + "vouched": r.vouched, + }); + let (pos_cos, neg_cos) = (best_cos_pile(&e.positives), best_cos_pile(&e.negatives)); + serde_json::json!({ + // The depth `served` is read at, so a future K change shows up + // in the row instead of silently redefining the number. + "k": K, + "positives": e.positives.iter().map(row).collect::>(), + "negatives": e.negatives.iter().map(row).collect::>(), + "window_best_cos": Window::read(&neg_cos, &pos_cos).map(|w| serde_json::json!({ + "cut_max": (w.cut_max * 1e4).round() / 1e4, + "keep_min": (w.keep_min * 1e4).round() / 1e4, + "open": w.open(), + })), + // NEW key (GH #201, Phase C): the whole bake-off grid, so the + // admissible window is re-derivable from the row — including the + // shipped bar's own three constants, which is what makes a later + // reader able to see that a bar has drifted out of the window it + // was read from. + "bar": bar.map(|b| serde_json::json!({ + "min_term_coverage": b.min_term_coverage, + "min_cos": b.min_cos, + })), + "bakeoff": bake_off(e).iter().map(|c| serde_json::json!({ + "min_term_coverage": c.coverage, + "pos_anchored": c.pos_anchored, + "neg_anchored": c.neg_anchored, + "cos_cut_floor": c.cut_floor().map(|v| (v * 1e4).round() / 1e4), + "cos_keep_ceiling": c.keep_ceiling().map(|v| (v * 1e4).round() / 1e4), + "admissible": c.admissible(), + })).collect::>(), + }) + }), + // The fold bake-off (GH #200, Phase B) — every candidate rule's reading + // on this run, with the per-anchor folds it was read from. Recorded on + // the default row only: the sweep's variants re-chunk the corpus, and a + // disclosure rule judged on a non-shipped chunker is a number about the + // chunker. + "discovery_fold": fold.map(fold_json), + // NEW key (absent from rows before GH #206): the per-hit tail bake-off + // — each family's re-derived constraint and edge payoff. Default row + // only, for the same reason as `discovery_fold`; the served rows it was + // read from are under `search_evidence`, so any other constant is + // re-derivable from the row. + "search_tail": tail.map(tail_json), + "queries": queries.iter().enumerate().map(|(i, q)| serde_json::json!({ + "q": q.query, + "bm25": bm25.map(|p| p.scores[i].note), + "vector": vector.map(|p| p.scores[i].note), + "hybrid": hybrid.scores[i].note, + "chunk": hybrid.scores[i].chunk, + })).collect::>(), + }) +} diff --git a/crates/b2-embed/examples/eval/retrieval.rs b/crates/b2-embed/examples/eval/retrieval.rs new file mode 100644 index 0000000..e3fb0da --- /dev/null +++ b/crates/b2-embed/examples/eval/retrieval.rs @@ -0,0 +1,89 @@ +//! Retrieval scoring: one pass over the labelled queries at note and chunk level, under +//! the shipped fusion or the dense-only ablation (GH #158). + +use crate::labels::Labelled; +use crate::metrics::{paths_match, Agg}; +use crate::K; +use b2_core::vault::Vault; + +/// One query's ranks in one retrieval mode: 1-based note rank, 1-based chunk rank +/// (only for passage-labelled queries), and the top note hit for display. +pub struct QueryScore { + pub note: Option, + pub chunk: Option, + pub top: String, +} + +/// A full pass over the query set in the vault's current state (keyword-only +/// before `embed`, hybrid after): per-query scores plus note- and chunk-level +/// aggregates. +pub struct Pass { + pub scores: Vec, + pub note: Agg, + pub chunk: Agg, +} + +/// Which retrieval a pass scores. `Fused` is the shipped path (`Vault::search` — +/// BM25-only before embed, hybrid after) plus chunk-level scoring for +/// passage-labelled queries; `VectorOnly` is the dense ablation +/// (`Vault::search_vector_only`, GH #158), note-level only — its chunk aggregate +/// stays empty. The ablation column is what lets a run say whether fusion paid +/// rent: the finding that RRF demotes dense rank-1 hits was established +/// by hand-decomposing fused scores once; this makes it a standing measurement. +#[derive(Clone, Copy, PartialEq)] +pub enum Retrieval { + Fused, + VectorOnly, +} + +/// Score every labelled query against the vault's current state: note rank via +/// the selected retrieval, and — for passage-labelled queries on the fused path — +/// chunk rank via `search_chunks` (the first top-K chunk that belongs to a +/// relevant note AND contains the labelled phrase, case-insensitively). +pub fn score_pass( + vault: &Vault, + queries: &[Labelled], + retrieval: Retrieval, +) -> Result> { + let mut scores = Vec::with_capacity(queries.len()); + let mut note_agg = Agg::default(); + let mut chunk_agg = Agg::default(); + for q in queries { + let results = match retrieval { + Retrieval::Fused => vault.search(&q.query, K)?, + Retrieval::VectorOnly => vault.search_vector_only(&q.query, K)?, + }; + let note = results + .iter() + .position(|r| q.relevant.iter().any(|rel| paths_match(&r.path, rel))) + .map(|p| p + 1); + let top = results + .first() + .map(|r| r.path.clone()) + .unwrap_or_else(|| "—".to_string()); + note_agg.add(note); + + let chunk = match (&q.passage, retrieval) { + (None, _) | (_, Retrieval::VectorOnly) => None, + (Some(passage), Retrieval::Fused) => { + let needle = passage.to_lowercase(); + let hits = vault.search_chunks(&q.query, K)?; + let rank = hits + .iter() + .position(|h| { + q.relevant.iter().any(|rel| paths_match(&h.path, rel)) + && h.text.to_lowercase().contains(&needle) + }) + .map(|p| p + 1); + chunk_agg.add(rank); + rank + } + }; + scores.push(QueryScore { note, chunk, top }); + } + Ok(Pass { + scores, + note: note_agg, + chunk: chunk_agg, + }) +} diff --git a/crates/b2-embed/examples/eval/tail.rs b/crates/b2-embed/examples/eval/tail.rs new file mode 100644 index 0000000..d05ea4d --- /dev/null +++ b/crates/b2-embed/examples/eval/tail.rs @@ -0,0 +1,668 @@ +//! The per-hit search tail bake-off (GH #206): the candidate prefix-cut families, their +//! constraints re-derived on the labelled corpus and on the dense fixture's title lists, +//! and the cross-bench join. The ruling of record is **no tail fold ships** +//! (docs/evals.md); every constraint is still re-derived each run. + +use crate::common::truncate; +use crate::dense::SearchProbe; +use crate::evidence::{QueryEvidence, SearchEvidence, ServedRow}; + +/// A candidate **per-hit tail rule** family (GH #206) — the fold where a real +/// query's evidence runs out, D2's per-hit half. Every family folds as a +/// **prefix cut** (invariants.md D1): the default view ends at the first served +/// row failing the family's test, and every row below folds with it — a passing +/// row under a failing one folds too, because a fold that punched holes in the +/// fused order would let row order and fold visibly disagree. +/// +/// The consequence that does all the work below: a filler row served *above* a +/// keep-labelled row must pass the test too, or the keep row under it folds. So +/// each family's constraint is read over the **keep-prefix** — every row at or +/// above a list's deepest keep row — not over the keep rows alone. +#[derive(Clone, Copy, PartialEq)] +pub enum TailRule { + /// Fold at the first row the lexical half never ranked (`bm25_rank: None`) + /// — the dense-only fold, the signal GH #201 measured (0 of 410 positive + /// rows vs 20 of 50 negative ones). Parameterless, so nothing + /// distributional to place — but **not** scale-free: `bm25_rank` is a rank + /// in a pool-truncated list (`search::pool_size`), so "never ranked" is + /// partly a fact about pool depth against vault size. `make calibrate + /// VAULT= ARGS=--search` measures that rather than assuming it + /// (process rule 5's posture, owed even without a constant). + Lexical, + /// Fold at the first row that is dense-only **and** under a per-hit cosine + /// bar — the shipped query rule's shape (lexical OR semantic) read per hit. + LexOrCos, + /// Fold at the first row under a per-hit cosine bar, lexical rank ignored + /// (a row the dense half never ranked fails at any bar). The single-signal + /// baseline the two-signal family must beat — the pure-cosine window at hit + /// granularity, included to be retired the way GH #201 retired its + /// query-level twin. + Cos, + /// Fold at the first row whose cosine sits more than δ under the list's own + /// best — the "drop-off" shape, scale-adaptive in the query and still + /// distributional in δ. + CosDrop, +} + +impl TailRule { + pub const ALL: [TailRule; 4] = [ + TailRule::Lexical, + TailRule::LexOrCos, + TailRule::Cos, + TailRule::CosDrop, + ]; + + pub fn label(self) -> &'static str { + match self { + TailRule::Lexical => "lexical (dense-only fold)", + TailRule::LexOrCos => "lex-or-cos ≥ c", + TailRule::Cos => "cos ≥ c", + TailRule::CosDrop => "cos ≥ best − δ", + } + } + + /// Whether `row` passes this family's per-hit test at constant `p` + /// (ignored by `Lexical`; `best` is the row's list's best served cosine, + /// read only by `CosDrop`). + pub fn passes(self, row: &ServedRow, p: f64, best: f64) -> bool { + match self { + TailRule::Lexical => row.bm25_rank.is_some(), + TailRule::LexOrCos => row.bm25_rank.is_some() || row.cos.is_some_and(|c| c >= p), + TailRule::Cos => row.cos.is_some_and(|c| c >= p), + TailRule::CosDrop => row.cos.is_some_and(|c| c >= best - p), + } + } + + /// Where this family folds `rows` at constant `p`: the index of the first + /// failing row, `rows.len()` when nothing folds. The prefix-cut definition + /// lives here and nowhere else. + pub fn fold(self, rows: &[ServedRow], p: f64) -> usize { + let best = best_served_cos(rows); + rows.iter() + .position(|r| !self.passes(r, p, best)) + .unwrap_or(rows.len()) + } +} + +/// The best served cosine of one list — [`TailRule::CosDrop`]'s reference +/// point. Read off the *served* rows, not the vault-wide dense top-1: the drop +/// rule is a claim about a list's own shape. +pub fn best_served_cos(rows: &[ServedRow]) -> f64 { + rows.iter() + .filter_map(|r| r.cos) + .fold(f64::NEG_INFINITY, f64::max) +} + +/// The keep-prefix of one served list: every row at or above its deepest +/// keep-labelled row. Empty when nothing served is keep-labelled — such a list +/// constrains no rule (there is nothing a fold could wrongly hide). +pub fn keep_prefix(rows: &[ServedRow]) -> &[ServedRow] { + match rows.iter().rposition(|r| r.keep) { + Some(last) => &rows[..=last], + None => &[], + } +} + +/// One family's constraint, re-derived from a set of served lists (GH #187's +/// idiom on the per-hit axis): the range of its constant, if any, at which no +/// keep-prefix row anywhere fails. Both edges carry the row that set them, +/// because a window edge is only arguable once the pair is named. +pub struct TailConstraint { + /// A keep-prefix row that cannot pass at **any** constant — the family is + /// dead on this bench (e.g. a row with no cosine under a cosine family). + pub dead: Option<(String, String)>, + /// `Cos`/`LexOrCos`: the highest admissible bar (the binding keep-prefix + /// row's own cosine). `CosDrop`: the lowest admissible δ (the binding + /// row's drop from its list's best). `None` = unconstrained — no + /// keep-prefix row engages the family's test at all. + pub edge: Option, + /// The (query-or-title, path) that set `edge`. + pub edge_row: Option<(String, String)>, + /// `Lexical` only: keep-prefix rows failing its fixed test. Nonzero = + /// inadmissible, and each failure is a keep row the fold would hide. + pub lexical_violations: usize, +} + +/// Read one family's [`TailConstraint`] over `(list name, full rows, keep-prefix +/// length)` triples, where every keep-prefix row must pass. The dense fixture +/// reuses this with whole lists as keep-prefixes: on a single-domain vault every +/// served row is a real match, so the absolute "truncate nothing" is the same +/// constraint with the keep-set saturated. The full list rides along because +/// [`TailRule::CosDrop`]'s reference is the **list's** best served cosine — +/// reading it off the prefix alone could understate a required δ when the best +/// row sits below the prefix, and the constraint must read exactly what the +/// fold would. +pub fn tail_constraint<'a>( + rule: TailRule, + lists: impl Iterator, +) -> TailConstraint { + let mut out = TailConstraint { + dead: None, + edge: None, + edge_row: None, + lexical_violations: 0, + }; + for (name, rows, prefix_len) in lists { + let best = best_served_cos(rows); + for row in &rows[..prefix_len] { + match rule { + TailRule::Lexical => { + if row.bm25_rank.is_none() { + out.lexical_violations += 1; + if out.dead.is_none() { + out.dead = Some((name.to_string(), row.path.clone())); + } + } + } + // The cosine arms read the row's cosine through a finiteness + // filter (PR #212 review): a NaN would sail through `c < e` + // comparisons as silently-true-nowhere — worse, `is_none_or` + // admits the FIRST row unconditionally, so a NaN first row + // would seed the edge. A non-finite reading is *no reading*, + // and the honest arm for that is `dead`, with the row named — + // never a silent skip that reports the family unconstrained. + TailRule::LexOrCos => { + if row.bm25_rank.is_none() { + match row.cos.filter(|c| c.is_finite()) { + // A served row is in at least one list, so a + // dense-only row carries a cosine unless it is + // non-finite — either way, no finite reading means + // this row can pass at no bar. + None => { + if out.dead.is_none() { + out.dead = Some((name.to_string(), row.path.clone())); + } + } + Some(c) => { + if out.edge.is_none_or(|e| c < e) { + out.edge = Some(c); + out.edge_row = Some((name.to_string(), row.path.clone())); + } + } + } + } + } + TailRule::Cos => match row.cos.filter(|c| c.is_finite()) { + None => { + if out.dead.is_none() { + out.dead = Some((name.to_string(), row.path.clone())); + } + } + Some(c) => { + if out.edge.is_none_or(|e| c < e) { + out.edge = Some(c); + out.edge_row = Some((name.to_string(), row.path.clone())); + } + } + }, + TailRule::CosDrop => match row.cos.filter(|c| c.is_finite()) { + None => { + if out.dead.is_none() { + out.dead = Some((name.to_string(), row.path.clone())); + } + } + Some(c) => { + let drop = best - c; + if out.edge.is_none_or(|e| drop > e) { + out.edge = Some(drop); + out.edge_row = Some((name.to_string(), row.path.clone())); + } + } + }, + } + } + } + out +} + +/// What a family buys at constant `p` on the labelled lists: rows cut, split by +/// label. At an admissible constant `kept_cut` is zero **by construction** — +/// the constraint above is exactly "every keep-prefix row passes" — so a +/// nonzero here is an arithmetic fault, printed as such rather than assumed +/// away. +pub struct TailPayoff { + pub filler_cut: usize, + pub kept_cut: usize, +} + +pub fn tail_payoff(rule: TailRule, lists: &[&QueryEvidence], p: f64) -> TailPayoff { + let mut out = TailPayoff { + filler_cut: 0, + kept_cut: 0, + }; + for q in lists { + let fold = rule.fold(&q.rows, p); + for row in &q.rows[fold..] { + if row.keep { + out.kept_cut += 1; + } else { + out.filler_cut += 1; + } + } + } + out +} + +/// The per-hit tail bake-off's labelled-corpus reading (GH #206): every +/// family's re-derived constraint plus its payoff at the constraint's own edge +/// — the most aggressive admissible point, since payoff is monotone in the +/// constant. Judged over the **positives**: the query-level bar already +/// answers the negatives whole (GH #201), so a tail rule ships riding on a +/// vouched verdict; the negatives' would-be reading is kept as context for +/// what the rule would buy where a query bar had missed. +pub struct TailBench { + pub keep_rows: usize, + pub filler_rows: usize, + /// Positives whose keep-prefix is the whole served list — a fold has + /// nothing to cut there under any admissible rule. + pub saturated: usize, + /// The **oracle** ceiling: rows below each list's last keep row — what a + /// perfect prefix fold (one placed by the labels themselves) would cut. + /// Every family's payoff is read against this, because "cuts N rows" means + /// nothing until the reachable maximum is beside it. + pub oracle: usize, + pub families: Vec, + /// Junk rows the parameterless lexical fold would cut on the negatives' + /// served lists, of their total — the GH #201 `dense_only` reading, priced + /// as a fold. + pub neg_lexical_cut: usize, + pub neg_rows: usize, +} + +pub struct TailFamilyReading { + pub rule: TailRule, + pub constraint: TailConstraint, + /// Payoff at the constraint's edge; `None` when the family is dead here or + /// (for the constant families) unconstrained-and-therefore-identical to a + /// simpler family. + pub payoff: Option, +} + +pub fn score_search_tail(ev: &SearchEvidence) -> TailBench { + let positives: Vec<&QueryEvidence> = ev.positives.iter().collect(); + let keep_rows = positives + .iter() + .flat_map(|q| q.rows.iter()) + .filter(|r| r.keep) + .count(); + let filler_rows = positives + .iter() + .flat_map(|q| q.rows.iter()) + .filter(|r| !r.keep) + .count(); + let saturated = positives + .iter() + .filter(|q| !q.rows.is_empty() && keep_prefix(&q.rows).len() == q.rows.len()) + .count(); + let oracle = positives + .iter() + .map(|q| q.rows.len() - keep_prefix(&q.rows).len()) + .sum(); + let families = TailRule::ALL + .iter() + .map(|&rule| { + let constraint = tail_constraint( + rule, + positives.iter().map(|q| { + ( + q.query.as_str(), + q.rows.as_slice(), + keep_prefix(&q.rows).len(), + ) + }), + ); + let payoff = match rule { + TailRule::Lexical => (constraint.lexical_violations == 0) + .then(|| tail_payoff(rule, &positives, f64::NAN)), + _ => match (&constraint.dead, constraint.edge) { + (None, Some(edge)) => Some(tail_payoff(rule, &positives, edge)), + _ => None, + }, + }; + TailFamilyReading { + rule, + constraint, + payoff, + } + }) + .collect(); + let neg_lexical_cut = ev + .negatives + .iter() + .map(|q| q.rows.len() - TailRule::Lexical.fold(&q.rows, f64::NAN)) + .sum(); + TailBench { + keep_rows, + filler_rows, + saturated, + oracle, + families, + neg_lexical_cut, + neg_rows: ev.negatives.iter().map(|q| q.rows.len()).sum(), + } +} + +pub fn print_search_tail(bench: &TailBench) { + println!( + " search tail bake-off (D2 per-hit — GH #206; the labels are GH #206's tail_relevant)" + ); + for line in [ + "the rule under audition: end the DEFAULT VIEW at the first served row failing a", + " per-hit evidence test — a PREFIX CUT (D1), so a filler row above a keep row", + " must pass too or the keep row folds with it. Constraint re-derived per run", + " over the keep-prefixes; payoff read at the constraint's own edge. \"No tail", + " fold\" is an admissible winner (the GH #200 outcome, on search's side).", + ] { + println!(" {line}"); + } + println!( + " served rows over the positives: {} keep / {} filler by label; {} list(s) saturated \ + (keep to the last served row)", + bench.keep_rows, bench.filler_rows, bench.saturated + ); + println!( + " oracle ceiling: a fold placed by the labels themselves (each list's last keep row) \ + would cut {} of {} — every payoff below is read against this", + bench.oracle, bench.filler_rows + ); + for f in &bench.families { + let constraint = match f.rule { + TailRule::Lexical => { + if f.constraint.lexical_violations == 0 { + "admissible (no dense-only keep-prefix row)".to_string() + } else { + let (q, p) = f.constraint.dead.as_ref().expect("violation names a row"); + format!( + "✗ {} keep-prefix row(s) are dense-only — first: {} → {}", + f.constraint.lexical_violations, + truncate(q, 28), + p + ) + } + } + _ => match (&f.constraint.dead, f.constraint.edge) { + (Some((q, p)), _) => { + format!( + "DEAD — {} → {} has no cosine to clear any bar", + truncate(q, 28), + p + ) + } + (None, None) => "unconstrained (no keep-prefix row engages the test)".to_string(), + (None, Some(edge)) => { + let (q, p) = f + .constraint + .edge_row + .as_ref() + .map(|(q, p)| (truncate(q, 28), p.clone())) + .unwrap_or_default(); + match f.rule { + TailRule::CosDrop => format!("δ ≥ {edge:.3} (set by {q} → {p})"), + _ => format!("c ≤ {edge:.3} (set by {q} → {p})"), + } + } + }, + }; + let payoff = match &f.payoff { + None => "—".to_string(), + Some(p) if p.kept_cut > 0 => format!( + "[FAULT] cuts {} keep row(s) at its own edge — the constraint arithmetic is wrong", + p.kept_cut + ), + Some(p) => format!("cuts {} of {} filler rows", p.filler_cut, bench.filler_rows), + }; + println!(" {:<24} {:<58} {}", f.rule.label(), constraint, payoff); + } + println!( + " negatives context: the lexical fold alone would cut {}/{} of their junk rows — the \ + query-level bar already cuts all of them (GH #201), so a tail rule is judged on what it \ + buys ABOVE that bar", + bench.neg_lexical_cut, bench.neg_rows + ); +} + +/// The tail bake-off's JSON (`search_tail` in the orthogonal row): each +/// family's re-derived constraint and edge payoff. The served rows it was read +/// from are already in `search_evidence` per query, so any other constant is +/// re-derivable from the row — the `discovery_fold` convention. +pub fn tail_json(bench: &TailBench) -> serde_json::Value { + let round = |v: f64| (v * 1e4).round() / 1e4; + serde_json::json!({ + "keep_rows": bench.keep_rows, + "filler_rows": bench.filler_rows, + "saturated_lists": bench.saturated, + "oracle": bench.oracle, + "neg_lexical_cut": bench.neg_lexical_cut, + "neg_rows": bench.neg_rows, + "families": bench.families.iter().map(|f| serde_json::json!({ + "rule": f.rule.label(), + "dead": f.constraint.dead.as_ref().map(|(q, p)| serde_json::json!([q, p])), + "edge": f.constraint.edge.map(round), + "edge_row": f.constraint.edge_row.as_ref().map(|(q, p)| serde_json::json!([q, p])), + "lexical_violations": f.constraint.lexical_violations, + "filler_cut_at_edge": f.payoff.as_ref().map(|p| p.filler_cut), + "kept_cut_at_edge": f.payoff.as_ref().map(|p| p.kept_cut), + })).collect::>(), + }) +} + +/// One family's dense-fixture reading (GH #206): the same [`TailConstraint`] +/// read over **whole** title lists — every served row is a real match by +/// geometry, so the keep-prefix is the entire list and the constraint *is* the +/// absolute ("truncate nothing"). `lexical_cut` is the parameterless family's +/// visible cost here: rows its fold would remove from title queries' views. +pub struct TailFamilyTransfer { + pub rule: TailRule, + pub constraint: TailConstraint, + pub lexical_cut: usize, +} + +pub fn dense_tail_families(titles: &[SearchProbe]) -> Vec { + TailRule::ALL + .iter() + .map(|&rule| TailFamilyTransfer { + rule, + constraint: tail_constraint( + rule, + titles + .iter() + .map(|t| (t.query.as_str(), t.rows.as_slice(), t.rows.len())), + ), + lexical_cut: match rule { + TailRule::Lexical => titles + .iter() + .map(|t| t.rows.len() - rule.fold(&t.rows, f64::NAN)) + .sum(), + _ => 0, + }, + }) + .collect() +} + +/// Print the dense fixture's tail-transfer reading (GH #206) — the per-hit +/// sibling of the query bar's `search_transfer` block, and the bench that +/// carries D1's absolute for this bake-off. +pub fn print_dense_tail(titles: &[SearchProbe]) { + println!( + " tail the per-hit tail bench on this geometry (GH #206): every served row of a" + ); + println!( + " title query is a real match, so a rule that folds one is disqualified —" + ); + println!(" the GH #200 absolute, on search's side"); + for f in dense_tail_families(titles) { + let reading = match f.rule { + TailRule::Lexical => { + if f.lexical_cut == 0 { + "cuts 0 title rows".to_string() + } else { + let (q, p) = f.constraint.dead.as_ref().expect("cut names a row"); + format!( + "✗ cuts {} title row(s) — first: {} → {}", + f.lexical_cut, + truncate(q, 24), + p + ) + } + } + _ => match (&f.constraint.dead, f.constraint.edge) { + (Some((q, p)), _) => { + format!("DEAD — {} → {} has no cosine", truncate(q, 24), p) + } + (None, None) => "unconstrained (no row engages the test)".to_string(), + (None, Some(edge)) => { + let (q, p) = f + .constraint + .edge_row + .as_ref() + .map(|(q, p)| (truncate(q, 24), p.clone())) + .unwrap_or_default(); + match f.rule { + TailRule::CosDrop => { + format!("needs δ ≥ {edge:.3} to fold nothing (set by {q} → {p})") + } + _ => format!("needs c ≤ {edge:.3} to fold nothing (set by {q} → {p})"), + } + } + }, + }; + println!(" {:<24} {}", f.rule.label(), reading); + } +} + +/// The dense tail-transfer reading as JSON, nested under the dense row's +/// `search_transfer` key (additive subkey, per the row conventions). +pub fn dense_tail_json(titles: &[SearchProbe]) -> serde_json::Value { + let round = |v: f64| (v * 1e4).round() / 1e4; + serde_json::json!(dense_tail_families(titles) + .iter() + .map(|f| serde_json::json!({ + "rule": f.rule.label(), + "dead": f.constraint.dead.as_ref().map(|(q, p)| serde_json::json!([q, p])), + "edge": f.constraint.edge.map(round), + "edge_row": f.constraint.edge_row.as_ref().map(|(q, p)| serde_json::json!([q, p])), + "lexical_cut": f.lexical_cut, + })) + .collect::>()) +} + +/// The tail bake-off's **cross-bench join** (GH #206), printed once both +/// corpora have been read in the same run: a family ships only if some +/// constant is admissible on the labelled corpus *and* folds nothing on the +/// dense fixture *and* still cuts labelled filler there — read at the joint +/// edge, the most aggressive point both benches allow. +pub fn print_tail_join(ev: &SearchEvidence, orth: &TailBench, titles: &[SearchProbe]) { + println!("\n{}", "=".repeat(78)); + println!("search tail — the cross-bench join (GH #206; both corpora, one run)"); + let positives: Vec<&QueryEvidence> = ev.positives.iter().collect(); + let dense = dense_tail_families(titles); + let mut winner = false; + for (orth_f, dense_f) in orth.families.iter().zip(&dense) { + let rule = orth_f.rule; + let verdict = + match rule { + TailRule::Lexical => { + if orth_f.constraint.lexical_violations > 0 { + format!( + "✗ inadmissible on the labelled corpus ({} keep-prefix row(s) dense-only)", + orth_f.constraint.lexical_violations + ) + } else if dense_f.lexical_cut > 0 { + format!( + "✗ disqualified on the dense fixture (cuts {} title rows)", + dense_f.lexical_cut + ) + } else { + let cut = orth_f.payoff.as_ref().map(|p| p.filler_cut).unwrap_or(0); + if cut == 0 { + "✓ admissible and VACUOUS — cuts 0 of the labelled filler, so it buys \ + nothing the query bar has not already bought" + .to_string() + } else { + winner = true; + format!( + "✓ admissible on both benches — cuts {cut} of the {} an oracle \ + fold reaches", + orth.oracle + ) + } + } + } + _ => { + let orth_dead = orth_f.constraint.dead.is_some(); + let dense_dead = dense_f.constraint.dead.is_some(); + if orth_dead || dense_dead { + format!( + "✗ DEAD on the {} bench (a required row has no cosine)", + if orth_dead { "labelled" } else { "dense" } + ) + } else { + // Joint edge: the tighter bench binds. `None` = that bench + // leaves the constant free. + let joint = match rule { + TailRule::CosDrop => { + match (orth_f.constraint.edge, dense_f.constraint.edge) { + (Some(a), Some(b)) => Some(a.max(b)), + (a, b) => a.or(b), + } + } + _ => match (orth_f.constraint.edge, dense_f.constraint.edge) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }, + }; + match joint { + None => { + "degenerates to the lexical fold (no row on either bench engages \ + the test) — see that family's verdict" + .to_string() + } + Some(edge) => { + let payoff = tail_payoff(rule, &positives, edge); + if payoff.kept_cut > 0 { + format!( + "[FAULT] cuts {} keep row(s) at the joint edge {edge:.3} — \ + the join arithmetic is wrong", + payoff.kept_cut + ) + } else if payoff.filler_cut == 0 { + format!( + "✓ admissible to {} {edge:.3} and VACUOUS — cuts 0 labelled \ + filler there", + if rule == TailRule::CosDrop { "δ ≥" } else { "c ≤" } + ) + } else { + winner = true; + format!( + "✓ admissible at {} {edge:.3} on both benches — cuts {} of \ + the {} an oracle fold reaches", + if rule == TailRule::CosDrop { "δ ≥" } else { "c ≤" }, + payoff.filler_cut, + orth.oracle + ) + } + } + } + } + } + }; + println!(" {:<24} {}", rule.label(), verdict); + } + if winner { + println!( + " → the ✓ family/families above survive both corpora at their joint edges. That is \ + ADMISSIBILITY, not a shipping order: a joint edge sits AT a bench's own binding row \ + (zero headroom — the constant placement the house sizing method forbids), each payoff \ + reads against the oracle ceiling above, and a shipped constant owes process rule 5's \ + real-vault reading besides (`make calibrate VAULT= ARGS=--search`, the tail block). The ruling of \ + record lives in docs/evals.md." + ); + } else { + println!( + " → NO family survives both benches with a nonzero payoff: the incumbent — no \ + per-hit tail fold — stands, the GH #200 outcome on search's side. The filler the \ + complaint names is above every admissible fold, so the honesty still rides on the \ + query-level bar and the copy." + ); + } +} diff --git a/docs/evals.md b/docs/evals.md index e37bc8f..4f2ad8e 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -174,8 +174,8 @@ as "no effect"; `make stability` is the instrument. ### The exit gate -`make eval` exits `0` only when the default config clears **all** assertions (the gate at the -end of `run()` in [`eval.rs`](../crates/b2-embed/examples/eval.rs)): +`make eval` exits `0` only when the default config clears **all** assertions (the gate is +`passes()` in [`eval/gate.rs`](../crates/b2-embed/examples/eval/gate.rs)): | Assertion | Constant | Direction | When it goes red, the fix is… | |---|---|---|---| From 16a71c7950bb4e7124b410b7a32a769425a9d49c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:59:09 +0000 Subject: [PATCH 24/30] test(core): shared fixtures and callbacks come from tests/common 19 inline copy-open(-reindex) sequences use common::opened_vault / reindexed_vault, and 11 inline opens of .b2/b2.sqlite use common::index_conn. The identical inbound() of mv.rs and props.rs, the near-identical write_note of discover.rs and why.rs, and the token callbacks (ask.rs stream_all, why.rs keep_streaming and the inline closures) move into common as inbound, write_note, keep_streaming and stream_into. No assertion changes. The Capture writers in logging.rs and discover_query_count.rs stay deliberate copies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/tests/ask.rs | 34 ++++++--------- crates/b2-core/tests/cancel.rs | 7 +--- crates/b2-core/tests/chunks.rs | 8 ++-- crates/b2-core/tests/common/mod.rs | 40 ++++++++++++++++++ crates/b2-core/tests/discover.rs | 10 +---- crates/b2-core/tests/embed.rs | 20 ++++----- crates/b2-core/tests/mv.rs | 16 +------ crates/b2-core/tests/project_embed.rs | 16 +++---- crates/b2-core/tests/props.rs | 19 ++------- crates/b2-core/tests/resources.rs | 4 +- crates/b2-core/tests/search.rs | 51 +++++++---------------- crates/b2-core/tests/vault.rs | 10 ++--- crates/b2-core/tests/why.rs | 24 ++--------- crates/b2-core/tests/write.rs | 6 +-- crates/b2-core/tests/write_frontmatter.rs | 7 +--- 15 files changed, 103 insertions(+), 169 deletions(-) diff --git a/crates/b2-core/tests/ask.rs b/crates/b2-core/tests/ask.rs index c1c3459..2e45008 100644 --- a/crates/b2-core/tests/ask.rs +++ b/crates/b2-core/tests/ask.rs @@ -13,20 +13,12 @@ use b2_core::llm::{ }; use b2_core::vault::Vault; use b2_core::Error; -use common::{opened_vault, reindexed_vault}; +use common::{keep_streaming, opened_vault, reindexed_vault, stream_into}; use std::cell::Cell; use std::fs; use std::ops::ControlFlow; use std::path::PathBuf; -/// A callback that keeps streaming: the plain, uncancelled ask. -fn stream_all(buf: &mut String) -> impl FnMut(&str) -> ControlFlow<()> + '_ { - move |tok: &str| { - buf.push_str(tok); - ControlFlow::Continue(()) - } -} - /// Wraps a provider and counts its calls — how the suite observes whether the /// condensation step ran without reaching into the orchestration. struct Counting<'a> { @@ -88,7 +80,7 @@ fn ask_grounds_the_answer_and_resolves_citations_end_to_end() { &FakeLlm, "forgetting curve", &[], - &mut stream_all(&mut streamed), + &mut stream_into(&mut streamed), ) .unwrap(); @@ -177,7 +169,7 @@ fn a_single_turn_ask_skips_condensation_and_a_follow_up_pays_for_it() { &counting, "forgetting curve", &[], - &mut stream_all(&mut sink), + &mut stream_into(&mut sink), ) .unwrap(); assert_eq!(counting.calls.get(), 1, "no condensation on a single turn"); @@ -194,7 +186,7 @@ fn a_single_turn_ask_skips_condensation_and_a_follow_up_pays_for_it() { &counting, "tell me more about that", &history, - &mut stream_all(&mut sink), + &mut stream_into(&mut sink), ) .unwrap(); assert_eq!(counting.calls.get(), 2, "condense, then answer"); @@ -248,7 +240,7 @@ fn condensation_failure_degrades_to_the_raw_question() { }, "quokka", &history, - &mut stream_all(&mut sink), + &mut stream_into(&mut sink), ) .unwrap(); assert!(!view.citations.is_empty()); @@ -267,7 +259,7 @@ fn condensation_failure_degrades_to_the_raw_question() { }, "quokka", &history, - &mut stream_all(&mut sink), + &mut stream_into(&mut sink), ) .unwrap(); assert_eq!( @@ -286,7 +278,7 @@ fn ask_answers_bm25_only_on_a_projected_but_unembedded_vault() { // grounding passages are the actual term matches, and chat still works. let mut streamed = String::new(); let view = vault - .ask(&FakeLlm, "forgetting", &[], &mut stream_all(&mut streamed)) + .ask(&FakeLlm, "forgetting", &[], &mut stream_into(&mut streamed)) .unwrap(); assert!(!view.cancelled); assert!(!view.citations.is_empty(), "chat keeps working unembedded"); @@ -342,7 +334,7 @@ fn a_hallucinated_marker_resolves_to_nothing_and_stays_in_the_text() { &Hallucinating, "forgetting curve", &[], - &mut stream_all(&mut sink), + &mut stream_into(&mut sink), ) .unwrap(); @@ -371,7 +363,7 @@ fn empty_retrieval_answers_no_evidence_rather_than_echoing() { &FakeLlm, "anything at all", &[], - &mut stream_all(&mut streamed), + &mut stream_into(&mut streamed), ) .unwrap(); @@ -417,7 +409,7 @@ fn a_failed_answer_call_surfaces_as_an_llm_error() { }, "forgetting curve", &[], - &mut |_| ControlFlow::Continue(()), + &mut keep_streaming(), ) .unwrap_err(); assert!(matches!(err, Error::Llm(_))); @@ -431,7 +423,7 @@ fn a_failed_answer_call_surfaces_as_an_llm_error() { }, "forgetting curve", &[], - &mut |_| ControlFlow::Continue(()), + &mut keep_streaming(), ) .unwrap_err(); assert!( @@ -455,9 +447,7 @@ fn ask_shares_searchs_model_mismatch_fail_fast() { let swapped = Vault::open_with_embedder(&root, Box::new(FakeEmbedder::new(128))).unwrap(); let err = swapped - .ask(&FakeLlm, "forgetting", &[], &mut |_| { - ControlFlow::Continue(()) - }) + .ask(&FakeLlm, "forgetting", &[], &mut keep_streaming()) .unwrap_err(); assert!(matches!(err, Error::ModelMismatch { .. })); } diff --git a/crates/b2-core/tests/cancel.rs b/crates/b2-core/tests/cancel.rs index 24ea96a..31e7ad5 100644 --- a/crates/b2-core/tests/cancel.rs +++ b/crates/b2-core/tests/cancel.rs @@ -11,8 +11,7 @@ use b2_core::chunk::ChunkConfig; use b2_core::embed::FakeEmbedder; use b2_core::ingest::{ingest_vault_with_progress, EmbedCtx, ProjectionCtx}; use b2_core::open; -use b2_core::vault::Vault; -use common::{count, golden_vault_copy}; +use common::{count, golden_vault_copy, opened_vault}; use std::ops::ControlFlow; #[test] @@ -66,9 +65,7 @@ fn cancel_after_first_batch_leaves_a_consistent_resumable_index() { #[test] fn facade_report_is_honest_about_a_cancelled_run() { let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, _) = opened_vault(tmp.path()); // Cancel at the first batch: fewer notes embed than are indexed, and `cancelled` // is set — the counts describe the partial work truthfully (§3). diff --git a/crates/b2-core/tests/chunks.rs b/crates/b2-core/tests/chunks.rs index 6d71289..65d9dd2 100644 --- a/crates/b2-core/tests/chunks.rs +++ b/crates/b2-core/tests/chunks.rs @@ -8,7 +8,7 @@ use b2_core::chunk::{chunk_body, BreakWeights, ChunkConfig}; use b2_core::embed::FakeEmbedder; use b2_core::ingest::ingest_vault; use b2_core::open; -use common::{golden_vault_copy, SRS_PATH}; +use common::{golden_vault_copy, index_conn, opened_vault, SRS_PATH}; #[test] fn chunks_are_projected_for_each_note() { @@ -76,13 +76,11 @@ fn vault_chunk_config_reaches_projection() { // `project(force)` on the same vault re-chunks under the new policy, so a // much finer target yields more chunks than the default did. Model-free. let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let mut vault = b2_core::Vault::open(&vault_dir).unwrap(); + let (mut vault, vault_dir) = opened_vault(tmp.path()); vault.project(false).unwrap(); let chunk_count = || -> i64 { - let conn = open(&vault_dir.join(".b2").join("b2.sqlite")).unwrap(); + let conn = index_conn(&vault_dir); conn.query_row("SELECT COUNT(*) FROM chunks", [], |r| r.get(0)) .unwrap() }; diff --git a/crates/b2-core/tests/common/mod.rs b/crates/b2-core/tests/common/mod.rs index b550b3a..8a0115f 100644 --- a/crates/b2-core/tests/common/mod.rs +++ b/crates/b2-core/tests/common/mod.rs @@ -14,6 +14,7 @@ use b2_core::vault::Vault; use b2_core::Result; use rusqlite::Connection; use std::fs; +use std::ops::ControlFlow; use std::path::{Path, PathBuf}; /// Vault-relative paths of the two golden-vault notes (data-model.md §8) — which @@ -83,6 +84,45 @@ pub fn count(conn: &Connection, table: &str) -> i64 { .unwrap() } +/// A note's inbound set as sortable `(label, src_path)` pairs — the shape the graph +/// exposes, and the thing a move must carry to the destination intact. +pub fn inbound(vault: &Vault, note_ref: &str) -> Vec<(String, String)> { + let mut ns: Vec<(String, String)> = vault + .neighbors(note_ref) + .unwrap() + .into_iter() + .filter(|n| n.direction == "inbound") + .map(|n| (n.label, n.path)) + .collect(); + ns.sort(); + ns +} + +// --- vault authoring and chat callbacks ------------------------------------------ + +/// Write a minimal note at `vault/name` with `body` under a small frontmatter. +pub fn write_note(vault: &Path, name: &str, body: &str) { + fs::write( + vault.join(name), + format!("---\ntype: note\ntitle: {name}\n---\n{body}\n"), + ) + .unwrap(); +} + +/// A token callback that keeps streaming and discards every token — the plain, +/// uncancelled run. +pub fn keep_streaming() -> impl FnMut(&str) -> ControlFlow<()> { + |_| ControlFlow::Continue(()) +} + +/// A token callback that keeps streaming and appends every token to `buf`. +pub fn stream_into(buf: &mut String) -> impl FnMut(&str) -> ControlFlow<()> + '_ { + move |tok: &str| { + buf.push_str(tok); + ControlFlow::Continue(()) + } +} + /// Ingest the golden vault into a standalone `dir/b2.sqlite`, for the module-level /// tests that drive `ingest_vault` directly instead of going through the façade. /// The embedder is explicit because the dimension is load-bearing in some suites. diff --git a/crates/b2-core/tests/discover.rs b/crates/b2-core/tests/discover.rs index f128c8e..2b3e9d1 100644 --- a/crates/b2-core/tests/discover.rs +++ b/crates/b2-core/tests/discover.rs @@ -15,7 +15,7 @@ use b2_core::discover::{self, CandidateNote}; use b2_core::embed::FakeEmbedder; use b2_core::ingest::ingest_vault; use b2_core::open; -use common::{ingest_golden, MEMORY_PATH, SRS_PATH}; +use common::{ingest_golden, write_note, MEMORY_PATH, SRS_PATH}; use rusqlite::Connection; use std::collections::BTreeSet; use std::fs; @@ -27,14 +27,6 @@ const B: &str = "b.md"; const C: &str = "c.md"; const E: &str = "e.md"; -fn write_note(vault: &Path, name: &str, body: &str) { - fs::write( - vault.join(name), - format!("---\ntype: note\ntitle: {name}\n---\n{body}\n"), - ) - .unwrap(); -} - /// a → b → e (a links b, b links e); c is disconnected. So within 1 hop of a is /// `{a, b}`; e is 2 hops (a triadic-closure candidate that must survive), c is far. fn linked_chain_vault(dir: &Path) -> Connection { diff --git a/crates/b2-core/tests/embed.rs b/crates/b2-core/tests/embed.rs index f255676..c9f9561 100644 --- a/crates/b2-core/tests/embed.rs +++ b/crates/b2-core/tests/embed.rs @@ -10,7 +10,9 @@ use b2_core::db; use b2_core::embed::{Embedder, FakeEmbedder}; use b2_core::ingest::ingest_vault; use b2_core::open; -use common::{count, golden_vault_copy, ingest_golden, SRS_PATH}; +use common::{ + count, golden_vault_copy, index_conn, ingest_golden, opened_vault, reindexed_vault, SRS_PATH, +}; use rusqlite::Connection; use std::ops::ControlFlow; @@ -111,9 +113,7 @@ fn reindex_is_incremental_and_force_reembeds_everything() { use b2_core::vault::Vault; let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, root) = opened_vault(tmp.path()); // First index: both notes are new → both embedded. let first = vault.reindex().unwrap(); @@ -187,15 +187,11 @@ fn ingest_populates_embeddings_and_records_meta() { #[test] fn centroids_track_the_stored_chunk_vectors() { use b2_core::embed::{centroid_of, pack_f32}; - use b2_core::vault::Vault; let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); - vault.reindex().unwrap(); + let (vault, root) = reindexed_vault(tmp.path()); - let conn = open(&root.join(".b2").join("b2.sqlite")).unwrap(); + let conn = index_conn(&root); let assert_centroids_current = |conn: &Connection| { let notes_with_vectors: i64 = conn .query_row( @@ -424,7 +420,7 @@ fn search_fails_fast_on_a_model_swap_and_a_reindex_heals_it() { // `open` left the stored vectors alone (so a misconfigured model can never wipe // a vault's embeddings) — the refusal is a query-time guard, not a migration. - let conn = open(&root.join(".b2").join("b2.sqlite")).unwrap(); + let conn = index_conn(&root); assert!(count(&conn, "embeddings") > 0, "vectors survive the reopen"); assert_eq!(meta(&conn, "embed_dim").as_deref(), Some("64")); drop(conn); @@ -432,6 +428,6 @@ fn search_fails_fast_on_a_model_swap_and_a_reindex_heals_it() { // The documented fix: reindex re-creates the space at the new dimension. swapped.reindex().unwrap(); assert!(!swapped.search("forgetting", 5).unwrap().is_empty()); - let conn = open(&root.join(".b2").join("b2.sqlite")).unwrap(); + let conn = index_conn(&root); assert_eq!(meta(&conn, "embed_dim").as_deref(), Some("128")); } diff --git a/crates/b2-core/tests/mv.rs b/crates/b2-core/tests/mv.rs index c304138..1a819e6 100644 --- a/crates/b2-core/tests/mv.rs +++ b/crates/b2-core/tests/mv.rs @@ -7,24 +7,10 @@ mod common; use b2_core::vault::Vault; use b2_core::Error; -use common::{reindexed_vault, MEMORY_PATH, SRS_PATH}; +use common::{inbound, reindexed_vault, MEMORY_PATH, SRS_PATH}; use std::fs; use std::path::Path; -/// The inbound set of a note, as sortable `(label, src_path)` pairs — the shape the -/// graph exposes and the thing a move must carry to the destination intact. -fn inbound(vault: &Vault, note_ref: &str) -> Vec<(String, String)> { - let mut ns: Vec<(String, String)> = vault - .neighbors(note_ref) - .unwrap() - .into_iter() - .filter(|n| n.direction == "inbound") - .map(|n| (n.label, n.path)) - .collect(); - ns.sort(); - ns -} - #[test] fn move_rewrites_inbound_links_and_the_graph_is_unchanged() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/crates/b2-core/tests/project_embed.rs b/crates/b2-core/tests/project_embed.rs index b72a405..c56b55d 100644 --- a/crates/b2-core/tests/project_embed.rs +++ b/crates/b2-core/tests/project_embed.rs @@ -14,7 +14,7 @@ use b2_core::ingest::{ }; use b2_core::open; use b2_core::vault::Vault; -use common::{count, golden_vault_copy}; +use common::{count, golden_vault_copy, index_conn, opened_vault}; use std::fs; use std::ops::ControlFlow; use std::path::Path; @@ -88,7 +88,7 @@ struct Observable { type EdgeKey = (String, String, Option, String, String, i64); fn observable_state(root: &Path) -> Observable { - let conn = open(&root.join(".b2").join("b2.sqlite")).unwrap(); + let conn = index_conn(root); let notes = count(&conn, "notes"); let chunk_texts = { let mut stmt = conn @@ -287,7 +287,7 @@ fn reindex_prunes_a_deleted_note_like_a_full_rebuild() { assert!(candidates.iter().all(|c| c.path != "bar.md")); drop(vault); - let conn = open(&root.join(".b2").join("b2.sqlite")).unwrap(); + let conn = index_conn(&root); // FTS stayed in lockstep through the cascade (no ghost text in the index)… assert_eq!(count(&conn, "chunks_fts"), count(&conn, "chunks")); // …and foo's `[[bar]]` re-dangled: phase 2 re-resolved it against the pruned @@ -400,9 +400,7 @@ fn reindex_completes_and_reports_skipped_files() { #[test] fn projected_vault_answers_keyword_search_and_similar_degrades_empty() { let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, _) = opened_vault(tmp.path()); vault.project(false).unwrap(); // Keyword search answers before any embedding — BM25-only, no model touched. @@ -429,9 +427,7 @@ fn projected_vault_answers_keyword_search_and_similar_degrades_empty() { #[test] fn embed_status_reports_the_coverage_fraction() { let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, root) = opened_vault(tmp.path()); // Projected but unembedded: every note counts toward the total, none is embedded, and // the embedding space doesn't exist yet — reads as 0/M, no error (the query short- @@ -503,7 +499,7 @@ fn a_note_with_no_body_counts_as_embedded_and_forecasts_no_work() { // Non-vacuity: they really do contribute no chunks, so this is the chunkless case and // not some other note quietly carrying the fraction. - let conn = open(&root.join(".b2/b2.sqlite")).unwrap(); + let conn = index_conn(&root); for path in ["stub.md", "blank.md"] { let chunks: i64 = conn .query_row( diff --git a/crates/b2-core/tests/props.rs b/crates/b2-core/tests/props.rs index e48e442..7e78306 100644 --- a/crates/b2-core/tests/props.rs +++ b/crates/b2-core/tests/props.rs @@ -14,8 +14,11 @@ //! run explores the identical case sequence and a failure reproduces exactly. To explore new //! ground change `SEED` locally; commit any find as a regular regression test. +mod common; + use b2_core::note::parse; use b2_core::vault::Vault; +use common::inbound; use proptest::prelude::*; use proptest::test_runner::{Config, RngAlgorithm, TestRng, TestRunner}; use rusqlite::Connection; @@ -431,7 +434,7 @@ fn apply(m: &Mutation, lv: &mut LiveVault) { /// here by their identity `(note, seq)`). Vectors and centroids are included — /// the FakeEmbedder is content-addressed, so they too must be pure projections. fn dump(root: &Path) -> Vec { - let conn = b2_core::open(&root.join(".b2").join("b2.sqlite")).unwrap(); + let conn = common::index_conn(root); let sections: &[(&str, &str, usize)] = &[ ( "note", @@ -561,20 +564,6 @@ fn incremental_reindex_equals_full_rebuild() { // --- invariant 3: rename keeps every backlink resolving -------------------------- -/// A note's inbound set as sortable `(label, src_path)` pairs — the thing a move -/// must leave unchanged (same helper shape as tests/mv.rs). -fn inbound(vault: &Vault, note_ref: &str) -> Vec<(String, String)> { - let mut ns: Vec<(String, String)> = vault - .neighbors(note_ref) - .unwrap() - .into_iter() - .filter(|n| n.direction == "inbound") - .map(|n| (n.label, n.path)) - .collect(); - ns.sort(); - ns -} - #[test] fn rename_keeps_every_backlink_resolving() { check( diff --git a/crates/b2-core/tests/resources.rs b/crates/b2-core/tests/resources.rs index a8f62d1..5711a73 100644 --- a/crates/b2-core/tests/resources.rs +++ b/crates/b2-core/tests/resources.rs @@ -15,7 +15,7 @@ mod common; /// `(path, class, size, content_hash)` rows, path-ordered — the comparable /// projection of `resources` (mtime/indexed_at are host state, not projection). fn resource_rows(root: &Path) -> Vec<(String, String, i64, String)> { - let conn = open(&root.join(".b2/b2.sqlite")).unwrap(); + let conn = common::index_conn(root); let mut stmt = conn .prepare("SELECT path, class, size, content_hash FROM resources ORDER BY path") .unwrap(); @@ -345,7 +345,7 @@ type EdgeTuple = ( Option, ); fn edges_from(root: &Path, src_path: &str) -> Vec { - let conn = open(&root.join(".b2/b2.sqlite")).unwrap(); + let conn = common::index_conn(root); let mut stmt = conn .prepare( "SELECT e.dst_path, e.dst_resource_path, e.dst_path_raw, e.type, e.embed, e.caption diff --git a/crates/b2-core/tests/search.rs b/crates/b2-core/tests/search.rs index 9cefbbb..4dd04a1 100644 --- a/crates/b2-core/tests/search.rs +++ b/crates/b2-core/tests/search.rs @@ -13,7 +13,10 @@ use b2_core::embed::FakeEmbedder; use b2_core::ingest::ingest_vault; use b2_core::search::{self, RRF_K}; use b2_core::{open, search::Hit}; -use common::{count, golden_vault_copy, index_conn, ingest_golden, MEMORY_PATH, SRS_PATH}; +use common::{ + count, golden_vault_copy, index_conn, ingest_golden, opened_vault, reindexed_vault, + MEMORY_PATH, SRS_PATH, +}; use std::fs; fn note_set(hits: &[Hit]) -> std::collections::BTreeSet { @@ -85,10 +88,7 @@ fn the_passage_view_retrieves_a_narrower_pool_than_the_note_view() { #[test] fn a_corpus_no_bigger_than_the_pool_ranks_the_same_at_any_depth() { let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, vault_dir) = reindexed_vault(tmp.path()); // The premise is about the *corpus*, not the result count, so it is counted on // the chunk rows themselves — 2 results would sit inside any pool regardless. @@ -123,10 +123,7 @@ fn an_absurd_limit_saturates_the_pool_instead_of_overflowing_it() { assert_eq!(b2_core::vault::chunk_candidate_pool(usize::MAX), usize::MAX); let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, vault_dir) = reindexed_vault(tmp.path()); // The whole corpus, once, rather than a panic or a truncated-by-wraparound page. let hits = vault.search("memory", usize::MAX).unwrap(); @@ -289,9 +286,7 @@ fn vector_only_search_is_the_dense_half_alone() { #[test] fn search_vector_only_dedups_and_refuses_to_impersonate_keywords() { let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); + let (vault, _) = opened_vault(tmp.path()); // Projection only: no embedding space yet. The hybrid view falls back to // BM25; the ablation view must return nothing rather than do the same. @@ -431,10 +426,7 @@ fn search_chunks_exposes_passage_level_hits() { // dedup, each hit resolved to its note path + heading breadcrumb + the chunk's // FULL text — containment-scorable, unlike `SearchResult`'s display snippet. let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, _) = reindexed_vault(tmp.path()); let hits = vault.search_chunks("forgetting curve", 10).unwrap(); assert!(!hits.is_empty()); @@ -521,15 +513,12 @@ fn search_chunks_still_fills_limit_when_a_ranked_chunk_is_dead() { const DEAD_CHUNK: i64 = 999_999; let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, vault_dir) = reindexed_vault(tmp.path()); let healthy = vault.search_chunks("memory", 2).unwrap(); assert_eq!(healthy.len(), 2); - let conn = open(&vault_dir.join(".b2/b2.sqlite")).unwrap(); + let conn = index_conn(&vault_dir); conn.execute( "INSERT INTO chunks_fts(rowid, text) VALUES (?1, 'memory')", rusqlite::params![DEAD_CHUNK], @@ -894,10 +883,7 @@ fn the_keyword_only_fallback_reports_no_dense_evidence() { #[test] fn search_evidence_serves_exactly_what_search_serves() { let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, _) = reindexed_vault(tmp.path()); let plain = vault.search("memory", 5).unwrap(); let view = vault.search_evidence("memory", 5).unwrap(); @@ -921,10 +907,7 @@ fn search_evidence_serves_exactly_what_search_serves() { #[test] fn a_zero_limit_evidence_read_still_reads_the_evidence() { let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, _) = reindexed_vault(tmp.path()); let view = vault.search_evidence("memory", 0).unwrap(); assert!(view.results.is_empty(), "a zero limit serves no rows"); @@ -942,10 +925,7 @@ fn a_zero_limit_evidence_read_still_reads_the_evidence() { #[test] fn excluding_a_served_note_backfills_from_the_same_ranking() { let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, _) = reindexed_vault(tmp.path()); // Both golden notes carry "memory", so the full read serves two rows and there // is a next-ranked note for the exclusion to backfill with. @@ -981,10 +961,7 @@ fn excluding_a_served_note_backfills_from_the_same_ranking() { #[test] fn exclusion_subtracts_rows_never_the_evidence() { let tmp = tempfile::TempDir::new().unwrap(); - let vault_dir = tmp.path().join("vault"); - golden_vault_copy(&vault_dir); - let vault = b2_core::Vault::open(&vault_dir).unwrap(); - vault.reindex().unwrap(); + let (vault, _) = reindexed_vault(tmp.path()); let full = vault.search_evidence("memory", 5).unwrap(); let excluded = vault diff --git a/crates/b2-core/tests/vault.rs b/crates/b2-core/tests/vault.rs index a71e1ef..64fdfbb 100644 --- a/crates/b2-core/tests/vault.rs +++ b/crates/b2-core/tests/vault.rs @@ -9,7 +9,7 @@ mod common; use b2_core::vault::Vault; use b2_core::Error; -use common::{golden_vault_copy, reindexed_vault, MEMORY_PATH, SRS_PATH}; +use common::{golden_vault_copy, opened_vault, reindexed_vault, MEMORY_PATH, SRS_PATH}; use std::fs; #[test] @@ -27,9 +27,7 @@ fn open_creates_the_b2_dir_and_index() { #[test] fn reindex_reports_counts_and_is_idempotent() { let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, root) = opened_vault(tmp.path()); let before: Vec = ["concepts/memory.md", "notes/spaced-repetition.md"] .iter() @@ -163,9 +161,7 @@ fn search_finds_the_note_with_a_snippet_and_is_note_level() { #[test] fn reads_before_reindex_are_empty_not_errors() { let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, _) = opened_vault(tmp.path()); assert!(vault.search("forgetting", 10).unwrap().is_empty()); assert!(vault.list_notes().unwrap().is_empty()); diff --git a/crates/b2-core/tests/why.rs b/crates/b2-core/tests/why.rs index 33c4723..ac29169 100644 --- a/crates/b2-core/tests/why.rs +++ b/crates/b2-core/tests/why.rs @@ -21,7 +21,7 @@ use b2_core::llm::{ }; use b2_core::vault::Vault; use b2_core::Error; -use common::index_conn; +use common::{index_conn, keep_streaming, stream_into, write_note}; use std::cell::RefCell; use std::collections::BTreeSet; use std::fs; @@ -33,14 +33,6 @@ const B: &str = "b.md"; const C: &str = "c.md"; const E: &str = "e.md"; -fn write_note(vault: &Path, name: &str, body: &str) { - fs::write( - vault.join(name), - format!("---\ntype: note\ntitle: Note {name}\n---\n{body}\n"), - ) - .unwrap(); -} - /// Several short sections, so a fine chunk target cuts each note into several passages /// and "the nearest pairs" is a real ranking rather than a single forced pair. fn sections(topic: &str) -> String { @@ -96,10 +88,6 @@ impl LlmProvider for Recording { } } -fn keep_streaming() -> impl FnMut(&str) -> ControlFlow<()> { - |_| ControlFlow::Continue(()) -} - // --- the evidence read (discover::passage_pairs) --------------------------------- #[test] @@ -277,10 +265,7 @@ fn a_model_that_ignores_its_tools_is_handed_the_evidence_and_its_first_try_is_ne let mut streamed = String::new(); let view = vault - .why_similar(&llm, A, C, 10, &mut |t| { - streamed.push_str(t); - ControlFlow::Continue(()) - }) + .why_similar(&llm, A, C, 10, &mut stream_into(&mut streamed)) .unwrap(); assert_eq!( @@ -567,10 +552,7 @@ fn citations_resolve_to_the_two_notes_and_the_answer_streams() { let mut streamed = String::new(); let view = vault - .why_similar(&FakeLlm, A, C, 10, &mut |tok| { - streamed.push_str(tok); - ControlFlow::Continue(()) - }) + .why_similar(&FakeLlm, A, C, 10, &mut stream_into(&mut streamed)) .unwrap(); assert_eq!(view.answer, streamed); diff --git a/crates/b2-core/tests/write.rs b/crates/b2-core/tests/write.rs index 7a839a8..b6846e9 100644 --- a/crates/b2-core/tests/write.rs +++ b/crates/b2-core/tests/write.rs @@ -10,7 +10,7 @@ mod common; use b2_core::db; use b2_core::vault::Vault; use b2_core::Error; -use common::{count, golden_vault_copy, index_conn, reindexed_vault}; +use common::{count, golden_vault_copy, index_conn, opened_vault, reindexed_vault}; use std::fs; use std::ops::ControlFlow; @@ -165,9 +165,7 @@ fn write_needs_no_embedding_space() { // A projected-only vault (no vector tables, no model anywhere): saving works — the // model-free proof (§7 invariant 4). let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, root) = opened_vault(tmp.path()); vault.project(false).unwrap(); let note = vault.read(SRS_PATH).unwrap(); diff --git a/crates/b2-core/tests/write_frontmatter.rs b/crates/b2-core/tests/write_frontmatter.rs index 6a92165..2b40b30 100644 --- a/crates/b2-core/tests/write_frontmatter.rs +++ b/crates/b2-core/tests/write_frontmatter.rs @@ -8,9 +8,8 @@ mod common; -use b2_core::vault::Vault; use b2_core::Error; -use common::{count, golden_vault_copy, index_conn, reindexed_vault}; +use common::{count, index_conn, opened_vault, reindexed_vault}; use rusqlite::Connection; use std::fs; @@ -210,9 +209,7 @@ fn needs_no_embedding_space() { // A projected-only vault (no vector tables, no model anywhere): the drawer // save works — the same model-free posture as `Vault::write`. let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path().join("vault"); - golden_vault_copy(&root); - let vault = Vault::open(&root).unwrap(); + let (vault, root) = opened_vault(tmp.path()); vault.project(false).unwrap(); let note = vault.read(SRS_PATH).unwrap(); From 76be7f16eb6d57ca0900d116fc545fc92ed912bc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:00:03 +0000 Subject: [PATCH 25/30] refactor(core): a folder move sorts its note set rather than trust the SQL order MoveSet looks moved notes up by old path; sorting in Rust keeps that invariant local instead of resting on SQLite's BINARY collation matching str ordering. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/mv.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/b2-core/src/mv.rs b/crates/b2-core/src/mv.rs index 603558b..30b5c4b 100644 --- a/crates/b2-core/src/mv.rs +++ b/crates/b2-core/src/mv.rs @@ -193,10 +193,11 @@ pub fn move_dir(ctx: EmbedCtx, from_input: &str, to_input: &str) -> Result = db::notes_under_dir(conn, &from)? + let mut notes: Vec<(String, String)> = db::notes_under_dir(conn, &from)? .into_iter() .filter_map(&rebased) .collect(); + notes.sort(); // `MoveSet::notes` is looked up by old path let resources: Vec = db::resources_under_dir(conn, &from)? .into_iter() .filter_map(&rebased) From 4eb86a18cebe15c21c8e4c2fe6492e3365602c3e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:21:10 +0000 Subject: [PATCH 26/30] =?UTF-8?q?refactor(core):=20wire=20the=20shared=20l?= =?UTF-8?q?ink/path=20helpers=20through=20the=20fa=C3=A7ade;=20drop=20the?= =?UTF-8?q?=20unbuilt=20graph=20join?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Vault::link renders its b2_relations entry with link::render_relation, the inverse of the parser that reads it back. - Every resource op checks the inventory in the façade (require_resource); rm::delete_resource no longer checks a second time. New Vault::resource_path gives the desktop's Open in system default one inventory-checked call instead of borrowing explain_resource's backlink query to validate. - ingest resolves a note-relative Markdown target with pathspec::parent_dir / pathspec::join_relative, next to relativize, its inverse (round-trip tested). - search::graph_filtered_search had no caller outside its tests: no command needs "nearest chunks within k hops", and discovery is its complement. It goes, with db::vector_search_all and db::chunk_note_map, which existed only for it. index-engine.md no longer calls it the substrate b2 similar runs on; it says what similar actually reads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/db.rs | 24 +------ crates/b2-core/src/discover.rs | 4 +- crates/b2-core/src/graph.rs | 5 +- crates/b2-core/src/ingest.rs | 28 +------- crates/b2-core/src/pathspec.rs | 45 +++++++++++++ crates/b2-core/src/rm.rs | 5 +- crates/b2-core/src/search.rs | 61 ++--------------- crates/b2-core/src/vault.rs | 19 ++++-- crates/b2-core/tests/discover.rs | 3 +- crates/b2-core/tests/search.rs | 81 ++--------------------- crates/b2-core/tests/vector_pool_scale.rs | 56 ++-------------- crates/b2-desktop/src/commands.rs | 15 ++--- docs/index-engine.md | 13 ++-- 13 files changed, 93 insertions(+), 266 deletions(-) diff --git a/crates/b2-core/src/db.rs b/crates/b2-core/src/db.rs index 23aab7f..55f4c70 100644 --- a/crates/b2-core/src/db.rs +++ b/crates/b2-core/src/db.rs @@ -22,7 +22,7 @@ use rusqlite::trace::{TraceEvent, TraceEventCodes}; use rusqlite::{ params, Connection, OptionalExtension, StatementStatus, Transaction, TransactionBehavior, }; -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashSet}; use std::path::Path; use std::sync::OnceLock; use std::time::Duration; @@ -1125,16 +1125,6 @@ pub fn note_for_chunk(conn: &Connection, chunk_id: i64) -> Result .optional()?) } -/// The whole `chunk_id -> note_path` map in one scan — the bulk form of -/// [`note_for_chunk`] for loops that resolve *many* hits (graph-filtered search walks -/// the full ranked space, where a per-hit lookup is the N+1 shape that once made -/// `b2 similar` a ~130s stall, #37). -pub fn chunk_note_map(conn: &Connection) -> Result> { - let mut stmt = conn.prepare("SELECT id, note_path FROM chunks")?; - let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))?; - Ok(rows.collect::>>()?) -} - /// A ranked chunk resolved for display in one read: its note, that note's title, and the /// chunk's heading breadcrumb and text. One statement, so the note and the chunk come /// from the same snapshot: a hit is either whole or `None` (GH #137). @@ -1498,8 +1488,7 @@ pub fn for_each_stored_vector(conn: &Connection, mut f: impl FnMut(i64, &[u8])) } /// Every chunk's squared-L2 distance to `query`, sorted nearest first (ties broken by -/// `chunk_id` for determinism) — the shared scan behind [`vector_search`] / -/// [`vector_search_all`], computed in-process over the [`for_each_stored_vector`] +/// `chunk_id` for determinism) — the scan behind [`vector_search`], computed in-process over the [`for_each_stored_vector`] /// stream: one sequential statement, one reused decode buffer (ADR-0006). fn scan_vector_distances(conn: &Connection, query: &[f32]) -> Result> { let mut out: Vec<(i64, f32)> = Vec::new(); @@ -1523,15 +1512,6 @@ pub fn vector_search(conn: &Connection, query: &[f32], k: usize) -> Result Result> { - let hits = scan_vector_distances(conn, query)?; - Ok(hits.into_iter().map(|(id, d)| (id, d.sqrt())).collect()) -} - // --------------------------------------------------------------------------- // edges // --------------------------------------------------------------------------- diff --git a/crates/b2-core/src/discover.rs b/crates/b2-core/src/discover.rs index e656536..c0bb293 100644 --- a/crates/b2-core/src/discover.rs +++ b/crates/b2-core/src/discover.rs @@ -4,9 +4,7 @@ //! that reads the graph. //! //! A candidate is a note **semantically near the anchor but not already connected** — -//! the *complement* of the graph, not the intersection (that is -//! [`crate::search::graph_filtered_search`], a scoped-traversal primitive and the wrong -//! tool here). Generation is deliberately **recall-oriented**, and the **ranked list is +//! the *complement* of the graph, not the intersection. Generation is deliberately **recall-oriented**, and the **ranked list is //! always served**: no statistic gates membership, and `limit` under-fills only for want //! of scorable notes. ADR-0014 carries the retired existence gate's whole evidence trail //! — including why an anchor-local statistic cannot tell *nothing is related* from diff --git a/crates/b2-core/src/graph.rs b/crates/b2-core/src/graph.rs index 3c7ca48..4e99af2 100644 --- a/crates/b2-core/src/graph.rs +++ b/crates/b2-core/src/graph.rs @@ -129,9 +129,8 @@ pub fn unresolved_outbound(conn: &Connection, note_path: &str) -> Result Result> { let mut seen = HashSet::from([anchor.to_string()]); let mut frontier = vec![anchor.to_string()]; diff --git a/crates/b2-core/src/ingest.rs b/crates/b2-core/src/ingest.rs index 8aa9112..db2703c 100644 --- a/crates/b2-core/src/ingest.rs +++ b/crates/b2-core/src/ingest.rs @@ -360,10 +360,7 @@ fn project_edges( // The source note's directory — the base for a Markdown-form relative target, read // straight off the path now that the path *is* the identity (ADR-0003). - let src_dir = src_path - .rsplit_once('/') - .map(|(dir, _)| dir) - .unwrap_or_default(); + let src_dir = crate::pathspec::parent_dir(src_path); // Resolve targets; record which (target, type) the frontmatter authors. let mut fm_keys: HashSet<(String, String)> = HashSet::new(); @@ -431,7 +428,7 @@ fn resolve_target( // wikilink habit); wikilinks are vault-root only, as today. let mut candidates: Vec = Vec::with_capacity(2); if link.md_form { - if let Some(joined) = join_vault_relative(src_dir, lookup) { + if let Some(joined) = crate::pathspec::join_relative(src_dir, lookup) { candidates.push(joined); } } @@ -454,27 +451,6 @@ fn resolve_target( Ok((None, None)) } -/// Join a relative `target` onto `base_dir` (both vault-relative, `/`-separated), -/// normalizing `.` and `..` segments. `None` when the target escapes the vault -/// root — such a path can never resolve, and the vault-root fallback still runs. -fn join_vault_relative(base_dir: &str, target: &str) -> Option { - let mut segments: Vec<&str> = if base_dir.is_empty() { - Vec::new() - } else { - base_dir.split('/').collect() - }; - for seg in target.split('/') { - match seg { - "" | "." => {} - ".." => { - segments.pop()?; - } - s => segments.push(s), - } - } - (!segments.is_empty()).then(|| segments.join("/")) -} - /// Deterministic id for an authored edge from its identity tuple (ADR-0010): stable /// across re-index, so the same body at the same path always yields the same id — and /// deliberately *not* stable across a move, since both ends are paths. Edge ids live diff --git a/crates/b2-core/src/pathspec.rs b/crates/b2-core/src/pathspec.rs index d973cd3..5d0edcf 100644 --- a/crates/b2-core/src/pathspec.rs +++ b/crates/b2-core/src/pathspec.rs @@ -107,6 +107,27 @@ pub(crate) fn rebase(path: &str, from: &str, to: &str) -> Option { Some(format!("{to}/{rest}")) } +/// Join a relative `target` onto `base_dir` (both vault-relative, `/`-separated), +/// normalizing `.` and `..` segments. `None` when the target escapes the vault +/// root — such a path can never resolve. The inverse of [`relativize`]. +pub(crate) fn join_relative(base_dir: &str, target: &str) -> Option { + let mut segments: Vec<&str> = if base_dir.is_empty() { + Vec::new() + } else { + base_dir.split('/').collect() + }; + for seg in target.split('/') { + match seg { + "" | "." => {} + ".." => { + segments.pop()?; + } + s => segments.push(s), + } + } + (!segments.is_empty()).then(|| segments.join("/")) +} + /// The relative path from the folder `base_dir` (`""` = the vault root) to the vault /// path `to_path`: the shared leading folders dropped, one `..` per remaining `base_dir` /// segment — the inverse of resolving a note-relative Markdown target. @@ -195,6 +216,30 @@ mod tests { assert_eq!(relativize("a/b", "c.png"), "../../c.png"); } + #[test] + fn join_relative_normalizes_and_refuses_to_escape_the_vault() { + assert_eq!( + join_relative("notes", "../docs/./plan.pdf").as_deref(), + Some("docs/plan.pdf") + ); + assert_eq!(join_relative("", "a/b.png").as_deref(), Some("a/b.png")); + assert_eq!(join_relative("", "../x.png"), None); + } + + #[test] + fn join_relative_inverts_relativize() { + for (base, to) in [ + ("", "a/b.png"), + ("notes", "docs/plan.pdf"), + ("a/b/c", "a/x.png"), + ] { + assert_eq!( + join_relative(base, &relativize(base, to)).as_deref(), + Some(to) + ); + } + } + #[test] fn trims_and_normalizes_separators() { assert_eq!(normalize_rel_md(" a\\b ").unwrap(), "a/b.md"); diff --git a/crates/b2-core/src/rm.rs b/crates/b2-core/src/rm.rs index d059a93..ee64874 100644 --- a/crates/b2-core/src/rm.rs +++ b/crates/b2-core/src/rm.rs @@ -75,10 +75,9 @@ pub fn delete_note(ctx: ProjectionCtx, rel: &str) -> Result { /// Delete the resource at `rel` — the note delete minus the identity step: file /// off disk, inventory row off the index (inbound edges' `dst_resource_path` is /// `ON DELETE SET NULL`), then re-project the inbound linkers so their edges -/// re-key to the raw-path (dangling) ids a rebuild derives. Errors with -/// [`Error::ResourceNotFound`] for a path not in the inventory. +/// re-key to the raw-path (dangling) ids a rebuild derives. The façade checked the +/// inventory, as it does for every resource op. pub fn delete_resource(ctx: ProjectionCtx, rel: &str) -> Result { - db::resource_detail(ctx.conn, rel)?.ok_or_else(|| Error::ResourceNotFound(rel.to_string()))?; let dangled = delete_set(ctx, &[], &[rel], || { remove_file_if_present(&ctx.root.join(rel)) })?; diff --git a/crates/b2-core/src/search.rs b/crates/b2-core/src/search.rs index e8f61a7..badfec4 100644 --- a/crates/b2-core/src/search.rs +++ b/crates/b2-core/src/search.rs @@ -2,14 +2,10 @@ //! retrieved in parallel and fused by RRF, then resolved up from chunks to notes //! (ADR-0008). A served result is a claim of evidence, which is what //! [`lexical_evidence`] and [`EvidenceBar`] are for (ADR-0015). -//! -//! [`graph_filtered_search`] is the vector-graph join B2 exists for: "nearest chunks -//! whose note is within k typed hops of note X". use crate::db; use crate::embed::Embedder; use crate::error::Result; -use crate::graph; use std::collections::HashMap; /// The RRF constant, k=60 (index-engine.md §1). @@ -481,7 +477,7 @@ pub fn keyword_only_search( /// Hybrid search: BM25 ⊕ vector(query) -> RRF -> top `limit`, resolved to notes /// (ADR-0008). A `limit` of 0 returns before the query is embedded — with the real model /// that call is the expensive part, and asking for no results should cost none. The same -/// guard opens [`keyword_only_search`] and [`graph_filtered_search`]. +/// guard opens [`keyword_only_search`]. pub fn hybrid_search( conn: &rusqlite::Connection, embedder: &dyn Embedder, @@ -542,7 +538,7 @@ pub fn cosine_of_distance(distance: f32) -> f64 { /// **An ablation instrument, not a product surface** (ADR-0013, GH #158): the eval scores /// it beside bm25-only and hybrid on every run, so fusion has a measured single-signal /// baseline to answer to. Scores are negated L2 distance (closer = higher), the same -/// convention as [`graph_filtered_search`] and **not** commensurable with RRF scores. A +/// convention as discovery's, and **not** commensurable with RRF scores. A /// vault with no embedding space yields no hits — there are no vectors to scan, and /// pretending otherwise would rank on nothing. pub fn vector_only_search( @@ -570,7 +566,7 @@ pub fn vector_only_search( resolve_hits(conn, ranked, &provenance, limit) } -/// The shared tail of the three retrieval entry points: resolve the ranked +/// The shared tail of the retrieval entry points: resolve the ranked /// `(chunk_id, score)` list to [`Hit`]s, best first, keeping the first `limit` **that /// still resolve** to a note. /// @@ -579,8 +575,7 @@ pub fn vector_only_search( /// when a concurrent reindex replaced its note's chunks mid-query, the posture C1 /// promises. Taking the top `limit` *first* would let a dead chunk hold a slot a live /// candidate could fill, so search would quietly under-fill during that window; steady -/// state is unchanged. Per-hit resolution is fine here because the loop stops at `limit` -/// — contrast [`graph_filtered_search`], which walks the full space and needs the bulk map. +/// state is unchanged. Per-hit resolution is fine here because the loop stops at `limit`. fn resolve_hits( conn: &rusqlite::Connection, fused: Vec<(i64, f64)>, @@ -606,51 +601,3 @@ fn resolve_hits( } Ok(hits) } - -/// Graph-filtered vector search: the `limit` nearest chunks whose note is within `hops` -/// typed hops of `anchor` — the vector-graph discovery join. Reachability is undirected -/// over the authored edges. Filtering scans the full ranked space and keeps reachable -/// notes, exact at vault scale; chunk-to-note resolution is one bulk map load, not a -/// per-row query, because the walk can visit the whole vault when a small neighbourhood -/// ranks deep — the N+1 shape that once stalled `b2 similar` (#37). -pub fn graph_filtered_search( - conn: &rusqlite::Connection, - embedder: &dyn Embedder, - query: &str, - anchor: &str, - hops: usize, - limit: usize, -) -> Result> { - if limit == 0 { - return Ok(Vec::new()); - } - let reachable = graph::reachable_within(conn, anchor, hops)?; - let chunk_note = db::chunk_note_map(conn)?; - - let mut hits = Vec::new(); - for (chunk_id, distance) in db::vector_search_all(conn, &embedder.embed_query(query)?)? { - // Before the push, not after: an after-the-push test can never fire at a - // limit the loop starts below, which is what the entry guard above covers. - if hits.len() == limit { - break; - } - let Some(note_path) = chunk_note.get(&chunk_id) else { - continue; - }; - if reachable.contains(note_path) { - hits.push(Hit { - chunk_id, - note_path: note_path.clone(), - score: -(distance as f64), // closer = higher - // One list walked in rank order, so `hits.len()` is this chunk's rank - // among the reachable ones — the only rank this path has. - provenance: HitProvenance { - bm25_rank: None, - vector_rank: Some(hits.len()), - distance: Some(distance), - }, - }); - } - } - Ok(hits) -} diff --git a/crates/b2-core/src/vault.rs b/crates/b2-core/src/vault.rs index 289b37e..0955449 100644 --- a/crates/b2-core/src/vault.rs +++ b/crates/b2-core/src/vault.rs @@ -18,6 +18,7 @@ use crate::embed::{Embedder, FakeEmbedder}; use crate::error::{Error, Result}; use crate::graph::{self, Direction}; use crate::import; +use crate::link; use crate::mv; use crate::rm; use crate::snippet::{query_snippet, snippet}; @@ -549,6 +550,17 @@ impl Vault { Ok(fs::read(self.root.join(path))?) } + /// The absolute path of an inventoried resource, for an adapter that hands the file + /// to the OS (*Open in system default*). Inventory-checked like + /// [`read_resource_bytes`](Self::read_resource_bytes), so a path a note authored can + /// never name a file outside the vault's resources. [`Error::ResourceNotFound`] + /// otherwise. + pub fn resource_path(&self, path: &str) -> Result { + let _op = tracing::debug_span!(target: "b2::vault", "resource_path", path).entered(); + self.require_resource(path)?; + Ok(self.root.join(path)) + } + /// The fallback card's data for one resource: inventory metadata plus the /// backlinks panel, straight off the materialized graph. `path` is vault-relative /// (the adapters dispatch here via [`crate::resource::doc_kind`]); errors with @@ -1055,11 +1067,7 @@ impl Vault { // The spec targets the dst's path. A note's title is its filename, so a bare // `[[path]]` already reads as the title — B2 writes no alias. - let link = format!("[[{dst_path}]]"); - let spec = match explanation { - Some(e) => format!("{edge_type} {link} — {e}"), - None => format!("{edge_type} {link}"), - }; + let spec = link::render_relation(edge_type, &dst_path, explanation); // 1. Markdown first: append to frontmatter b2_relations: (never the body, §0). let abs = self.root.join(&src_path); @@ -1127,6 +1135,7 @@ impl Vault { /// Errors with [`Error::ResourceNotFound`] for a path not in the inventory. pub fn delete_resource(&self, path: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "rm_resource", path).entered(); + self.require_resource(path)?; rm::delete_resource(self.ctx(), path) } diff --git a/crates/b2-core/tests/discover.rs b/crates/b2-core/tests/discover.rs index 2b3e9d1..a3871a0 100644 --- a/crates/b2-core/tests/discover.rs +++ b/crates/b2-core/tests/discover.rs @@ -175,10 +175,9 @@ fn two_stage_equals_exhaustive_max_sim_when_shortlist_covers() { // Exhaustive ground truth: every stored vector, min over the anchor's vectors, // best chunk per note (strictly-less keeps the first-seen chunk, as discover does). - let chunk_note = db::chunk_note_map(&conn).unwrap(); let mut best: HashMap = HashMap::new(); db::for_each_stored_vector(&conn, |chunk_id, blob| { - let note = &chunk_note[&chunk_id]; + let note = &db::note_for_chunk(&conn, chunk_id).unwrap().unwrap(); if note == anchor { return; // no links in this vault → the anchor is the whole exclusion set } diff --git a/crates/b2-core/tests/search.rs b/crates/b2-core/tests/search.rs index 4dd04a1..19b9f6c 100644 --- a/crates/b2-core/tests/search.rs +++ b/crates/b2-core/tests/search.rs @@ -1,10 +1,8 @@ -//! Step 5 — hybrid retrieval (index-engine.md): -//! BM25 ⊕ vector → RRF fusion (k=60), resolved to notes, plus the graph-filtered -//! vector⨝edge join (index-engine.md §3) — the substrate connection discovery -//! runs on. +//! Hybrid retrieval (flow ②, index-engine.md §4): BM25 ⊕ vector → RRF fusion (k=60), +//! resolved to notes, and the evidence readings RRF discards. //! //! Scope: with the deterministic *fake* embedder, vector ranking is not semantic, -//! so these prove the **plumbing** (fusion math + the join), not model quality — +//! so these prove the **plumbing** (fusion math + resolution), not model quality — //! that is the real-embedder eval suite (testability stack, point 5). mod common; @@ -266,7 +264,7 @@ fn vector_only_search_is_the_dense_half_alone() { search::vector_only_search(&conn, &FakeEmbedder::new(64), "forgetting curve", 5).unwrap(); assert!(!hits.is_empty()); assert!(hits.iter().all(|h| !h.note_path.is_empty())); - // Negated-distance scores, best first — the graph_filtered_search convention. + // Negated-distance scores, best first — discovery's convention. for w in hits.windows(2) { assert!(w[0].score >= w[1].score, "scores must be descending"); } @@ -311,55 +309,6 @@ fn search_vector_only_dedups_and_refuses_to_impersonate_keywords() { } } -#[test] -fn graph_filtered_search_restricts_to_reachable_notes() { - // A 3-note vault: a → b (linked), c disconnected, all share a keyword. - let tmp = tempfile::TempDir::new().unwrap(); - let vault = tmp.path().join("vault"); - fs::create_dir_all(&vault).unwrap(); - fs::write( - vault.join("a.md"), - "---\ntype: note\ntitle: A\n---\nshared topic alpha. See [[b]].\n", - ) - .unwrap(); - fs::write( - vault.join("b.md"), - "---\ntype: note\ntitle: B\n---\nshared topic beta.\n", - ) - .unwrap(); - fs::write( - vault.join("c.md"), - "---\ntype: note\ntitle: C\n---\nshared topic gamma.\n", - ) - .unwrap(); - - let conn = open(&tmp.path().join("b2.sqlite")).unwrap(); - ingest_vault(&conn, &vault, &FakeEmbedder::new(64)).unwrap(); - - // Within 1 hop of A: {A, B}. C is disconnected and must be excluded even - // though its text matches the query. - let hits = - search::graph_filtered_search(&conn, &FakeEmbedder::new(64), "shared topic", "a.md", 1, 10) - .unwrap(); - - let notes = note_set(&hits); - assert!(!notes.is_empty()); - assert!( - !notes.contains("c.md"), - "disconnected note must be filtered out" - ); - assert!(notes.iter().all(|n| n == "a.md" || n == "b.md")); - - // …and `limit` genuinely truncates that reachable set. This is the complement of - // tests/vector_pool_scale.rs, which pins the other side — that a limit *above* - // what is reachable returns everything rather than a silently capped prefix. - assert!(hits.len() > 1, "the fixture must have room to truncate"); - let capped = - search::graph_filtered_search(&conn, &FakeEmbedder::new(64), "shared topic", "a.md", 1, 1) - .unwrap(); - assert_eq!(capped.len(), 1, "the scan stops at the limit"); -} - /// A result's `snippet` must **window around the matched term**, not just show the /// chunk's head. Under qmd chunking (#19) a chunk is section-sized — far longer than /// the snippet budget — so a term buried past the head would otherwise never appear @@ -550,16 +499,6 @@ fn a_zero_limit_returns_no_hits() { .hits .is_empty() ); - assert!(search::graph_filtered_search( - &conn, - &FakeEmbedder::new(64), - "brain", - MEMORY_PATH, - 1, - 0 - ) - .unwrap() - .is_empty()); } /// …and it gets there without *doing* anything. The observable proof is the @@ -603,18 +542,6 @@ fn a_zero_limit_search_does_no_retrieval_work() { .is_empty()); } -#[test] -fn graph_filter_with_zero_hops_is_just_the_anchor() { - let tmp = tempfile::TempDir::new().unwrap(); - let conn = ingest_golden(tmp.path(), &FakeEmbedder::new(64)); - - // 0 hops from memory → only memory's own chunks are eligible. - let hits = - search::graph_filtered_search(&conn, &FakeEmbedder::new(64), "brain", MEMORY_PATH, 0, 10) - .unwrap(); - assert!(hits.iter().all(|h| h.note_path == MEMORY_PATH)); -} - // --------------------------------------------------------------------------- // Query evidence — the absolute signals RRF discards (invariants.md D2, GH #201) // --------------------------------------------------------------------------- diff --git a/crates/b2-core/tests/vector_pool_scale.rs b/crates/b2-core/tests/vector_pool_scale.rs index 8cbb3c2..4b5be9f 100644 --- a/crates/b2-core/tests/vector_pool_scale.rs +++ b/crates/b2-core/tests/vector_pool_scale.rs @@ -10,8 +10,8 @@ //! cannot recur without reintroducing `sqlite-vec`. //! //! So this asserts the property that actually matters — **exhaustiveness / no silent -//! truncation** — directly on the cap-bearing primitive, and proves the discovery and -//! graph-filtered paths return complete results on a modest, fast vault. Scope: the +//! truncation** — directly on the cap-bearing primitive, and proves discovery and hybrid +//! search return complete results on a modest, fast vault. Scope: the //! deterministic fake embedder, so this proves plumbing, not model quality. mod common; @@ -58,20 +58,10 @@ fn chunk_count(conn: &Connection) -> i64 { .unwrap() } -fn note_chunk_count(conn: &Connection, note_path: &str) -> i64 { - conn.query_row( - "SELECT COUNT(*) FROM chunks WHERE note_path = ?1", - [note_path], - |r| r.get(0), - ) - .unwrap() -} - /// The retired `vec0` store errored on `k > 4096`; the in-process scan cannot. Proven /// **directly on the primitive that would carry any such cap**, cheaply — with no -/// oversized fixture and no magic number: `vector_search_all` returns *every* stored -/// vector, and `vector_search(k)` returns exactly `min(k, N)` for `k` below, at, and -/// far past `N` (where the old store crashed). +/// oversized fixture and no magic number: `vector_search(k)` returns exactly `min(k, N)` +/// for `k` below, at, and far past `N` (where the old store crashed). #[test] fn vector_search_is_exhaustive_and_truncates_only_to_k() { let tmp = tempfile::TempDir::new().unwrap(); @@ -85,12 +75,6 @@ fn vector_search_is_exhaustive_and_truncates_only_to_k() { .embed_query("shared topic alpha") .unwrap(); - // No `k`: the scan ranks the ENTIRE space — one vector per stored chunk. - assert_eq!( - db::vector_search_all(&conn, &probe).unwrap().len(), - n, - "the whole-space scan returns every stored vector" - ); // `k` below N is honoured exactly… assert_eq!( db::vector_search(&conn, &probe, n / 2).unwrap().len(), @@ -135,38 +119,6 @@ fn similar_returns_the_full_candidate_set_without_a_silent_cap() { ); } -/// `graph_filtered_search` scans the *whole* ranked space and keeps the reachable -/// notes — so a `limit` above the reachable chunk count returns **every** reachable -/// chunk, wherever it ranks, un-truncated (and nothing unreachable). -#[test] -fn graph_filtered_search_returns_every_reachable_chunk_without_a_silent_cap() { - let tmp = tempfile::TempDir::new().unwrap(); - let (conn, ids) = big_vault(tmp.path(), 50, 90); - - // No links: within any hop count the anchor reaches only itself. Its chunks all - // match the query; asking for more than it has must return all of them — proof - // the scan considers the entire ranked space, not just a capped prefix. - let anchor_chunks = note_chunk_count(&conn, &ids[0]); - let hits = search::graph_filtered_search( - &conn, - &FakeEmbedder::new(64), - "shared topic", - &ids[0], - 1, - (anchor_chunks as usize) + 100, - ) - .unwrap(); - assert_eq!( - hits.len() as i64, - anchor_chunks, - "every one of the disconnected anchor's chunks comes back, un-truncated" - ); - assert!( - hits.iter().all(|h| h.note_path == ids[0]), - "only the (disconnected) anchor is reachable" - ); -} - /// A large `--limit` blows the fused pool (`limit × 5`) well past the old `vec0` cap; /// this crashed on any embedded vault, independent of size. A modest vault proves the /// fix: the scan honours the full limit (here the whole vault, < limit) rather than diff --git a/crates/b2-desktop/src/commands.rs b/crates/b2-desktop/src/commands.rs index 1b0e221..6f07d7e 100644 --- a/crates/b2-desktop/src/commands.rs +++ b/crates/b2-desktop/src/commands.rs @@ -16,7 +16,7 @@ use crate::chat::ChatPrefs; use crate::error::CmdError; use crate::watch::VaultWatcher; -use crate::{open_read, open_semantic, open_vault, open_vault_at, AppState}; +use crate::{open_read, open_semantic, open_vault, AppState}; use b2_core::add::AddReport; use b2_core::ingest::ReindexProgress; use b2_core::llm::{ChatTurn, LlmProvider}; @@ -176,16 +176,13 @@ pub fn read_resource(state: State<'_, AppState>, path: String) -> Result, path: String) -> Result<(), CmdError> { - let root = state.current_root().ok_or(CmdError::VaultRequired)?; - let (vault, _) = open_vault_at(&root, false)?; - vault.explain_resource(&path)?; // inventory check: unknown paths refuse, never open - tauri_plugin_opener::open_path(root.join(&path), None::<&str>) + // Inventory-checked by the façade: an unknown path refuses, never opens. + let abs = read_op(state.inner(), |v| v.resource_path(&path))?; + tauri_plugin_opener::open_path(abs, None::<&str>) .map_err(|e| CmdError::OpenFailed(e.to_string())) } diff --git a/docs/index-engine.md b/docs/index-engine.md index 3c6b73d..ba23c43 100644 --- a/docs/index-engine.md +++ b/docs/index-engine.md @@ -110,9 +110,8 @@ Why this shape fits B2: There is no `status` column and no suggestion queue: `b2 link` appends a typed-link string to the source note's frontmatter and re-projects that note. Committing is the projection of an authored line, not an in-place index write. -- **Hybrid retrieval and graph queries compose in one query.** "Semantic-nearest chunks whose - note is within 2 typed hops of note X" is a join across `embeddings`, `chunks`, and - `edges`. This is the substrate `b2 similar` runs on. +- **Vectors and the graph live in one database.** `b2 similar` reads both in one pass: + nearest notes by stored vectors, minus the anchor's 1-hop neighbours from `edges`. - **Deterministic seams for tests.** A fake embedder writes to `embeddings`, so the whole pipeline is assertable with no live model (E2). @@ -401,10 +400,10 @@ prefix is the wrong side of the space). The stages: list at a nonzero `limit` means only "nothing to compare": no unlinked note has stored vectors yet, or the space is not semantic. The CLI's two empty states say exactly that. -**`graph_filtered_search`** is the near-neighbor of both flows that is neither: the -vector⨝graph scoped-traversal primitive, "nearest chunks whose note is within k typed hops of -an anchor" (near ∩ connected). Discovery is its complement (near ∖ connected). -`vector_only_search` is the eval harness's ablation instrument, never an adapter surface. +Discovery is near ∖ connected. Its intersection (near ∩ connected, a graph-scoped vector +search) is not built: no command needs it, and it would arrive as a façade operation when one +does. `vector_only_search` is the eval harness's ablation instrument, never an adapter +surface. ### Does brute force scale to B2? From 50d805c73e32b0d36f7cfb2741683cadcee91960 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:24:05 +0000 Subject: [PATCH 27/30] refactor(core)!: project only the note columns something reads (schema 7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit notes.type, notes.description, notes.updated (+ notes_type_idx) and the whole note_aliases table were written on every projection and read by nothing: NoteView takes type/updated straight from the parsed file, the resolver never consulted aliases, and no embed prompt used description. They go, with the NoteRow/NoteFields fields that fed them and the test that pinned the alias table. SCHEMA_VERSION 6 -> 7, so an existing index rebuilds on open and re-embeds (S5: a version bump plus a rebuild). The spec said more than the code did, so it now says what is true: data-model.md §1 no longer claims description feeds the embedding prompt or that aliases resolve links (both round-trip, neither is read yet), and type no longer "defaults to note". index-engine.md's schema sketch matches the DDL. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-core/src/db.rs | 62 +++++-------------------- crates/b2-core/src/ingest.rs | 8 +--- crates/b2-core/src/note.rs | 15 ++---- crates/b2-core/tests/embed.rs | 4 +- crates/b2-core/tests/ingest_resolver.rs | 35 ++------------ crates/b2-core/tests/props.rs | 10 +--- crates/b2-core/tests/resources.rs | 8 ++-- crates/b2-core/tests/substrate.rs | 15 +++--- docs/data-model.md | 7 +-- docs/index-engine.md | 7 ++- 10 files changed, 44 insertions(+), 127 deletions(-) diff --git a/crates/b2-core/src/db.rs b/crates/b2-core/src/db.rs index 55f4c70..bd4995c 100644 --- a/crates/b2-core/src/db.rs +++ b/crates/b2-core/src/db.rs @@ -1,5 +1,5 @@ //! Opening the index, the schema migration, and the projection helpers for the -//! Markdown-derived tiers: `notes`/`note_aliases`, `chunks` (+FTS5), the +//! Markdown-derived tiers: `notes`, `chunks` (+FTS5), the //! `embeddings`/`note_centroids` vector tables, and the typed `edges` graph. Every //! table here is a derived projection of Markdown — nothing is a source of truth //! (ADR-0002). @@ -40,7 +40,7 @@ use std::time::Duration; /// inventory and widened `edges` with resource targets. **5** switched `chunks_fts` /// to `porter unicode61` (the GH #157 A/B's verdict). **6** re-keyed the whole index /// on the vault-relative path and made `embeddings` content-addressed (GH #170). -pub const SCHEMA_VERSION: i64 = 6; +pub const SCHEMA_VERSION: i64 = 7; /// Statements at or over this take the slow-query WARN path (`B2_SLOW_QUERY_MS` /// overrides; see [`slow_query_threshold`]). @@ -239,10 +239,9 @@ fn is_locked(err: &Error) -> bool { /// concurrent `DROP TABLE` (#114), and dropping a table takes its indexes and triggers /// with it, so a missing table is the visible edge of every partial rebuild. The unit /// test at the foot of this file pins the list to what the DDL actually creates. -const SCHEMA_TABLES: [&str; 7] = [ +const SCHEMA_TABLES: [&str; 6] = [ "meta", "notes", - "note_aliases", "chunks", "chunks_fts", "resources", @@ -393,31 +392,19 @@ fn apply_schema(conn: &Connection) -> Result<()> { DROP TABLE IF EXISTS embeddings; DROP TABLE IF EXISTS chunks_fts; DROP TABLE IF EXISTS chunks; - DROP TABLE IF EXISTS note_aliases; + DROP TABLE IF EXISTS note_aliases; -- schema <= 6 DROP TABLE IF EXISTS notes; DELETE FROM meta;", )?; conn.execute_batch( "CREATE TABLE IF NOT EXISTS notes ( path TEXT PRIMARY KEY, - type TEXT NOT NULL, title TEXT, - description TEXT, created TEXT, - updated TEXT, body_hash TEXT NOT NULL, mtime INTEGER, indexed_at TEXT NOT NULL ); - CREATE INDEX IF NOT EXISTS notes_type_idx ON notes(type); - - CREATE TABLE IF NOT EXISTS note_aliases ( - note_path TEXT NOT NULL - REFERENCES notes(path) ON DELETE CASCADE ON UPDATE CASCADE, - alias TEXT NOT NULL, - PRIMARY KEY (note_path, alias) - ); - CREATE INDEX IF NOT EXISTS note_aliases_alias_idx ON note_aliases(alias); CREATE TABLE IF NOT EXISTS chunks ( id INTEGER PRIMARY KEY, @@ -496,25 +483,21 @@ fn apply_schema(conn: &Connection) -> Result<()> { } // --------------------------------------------------------------------------- -// notes + aliases +// notes // --------------------------------------------------------------------------- -/// One note's projection into `notes` (+ its `aliases`). Borrowed view so callers +/// One note's projection into `notes`. Borrowed view so callers /// pass slices of an already-parsed note without extra allocation. #[derive(Debug)] pub struct NoteRow<'a> { pub path: &'a str, - pub r#type: &'a str, pub title: Option<&'a str>, - pub description: Option<&'a str>, pub created: Option<&'a str>, - pub updated: Option<&'a str>, pub body_hash: &'a str, pub mtime: Option, - pub aliases: &'a [String], } -/// Upsert a note keyed by its vault-relative `path` and replace its aliases. +/// Upsert a note keyed by its vault-relative `path`. /// `indexed_at` is set by SQLite so the projection needs no wall-clock from Rust. /// /// `ON CONFLICT(path)` is the *whole* of path reconciliation, which is the point of @@ -522,37 +505,16 @@ pub struct NoteRow<'a> { /// so a note deleted and recreated there is simply that path's note now. pub fn upsert_note(conn: &Connection, row: &NoteRow) -> Result<()> { conn.execute( - "INSERT INTO notes - (path, type, title, description, created, updated, body_hash, mtime, indexed_at) - VALUES - (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, strftime('%Y-%m-%dT%H:%M:%SZ','now')) + "INSERT INTO notes (path, title, created, body_hash, mtime, indexed_at) + VALUES (?1, ?2, ?3, ?4, ?5, strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT(path) DO UPDATE SET - type = excluded.type, title = excluded.title, - description = excluded.description, created = excluded.created, - updated = excluded.updated, body_hash = excluded.body_hash, mtime = excluded.mtime, indexed_at = excluded.indexed_at", - params![ - row.path, - row.r#type, - row.title, - row.description, - row.created, - row.updated, - row.body_hash, - row.mtime, - ], + params![row.path, row.title, row.created, row.body_hash, row.mtime], )?; - conn.execute("DELETE FROM note_aliases WHERE note_path = ?1", [row.path])?; - for alias in row.aliases { - conn.execute( - "INSERT OR IGNORE INTO note_aliases(note_path, alias) VALUES (?1, ?2)", - params![row.path, alias], - )?; - } Ok(()) } @@ -563,7 +525,7 @@ pub fn upsert_note(conn: &Connection, row: &NoteRow) -> Result<()> { /// `similar` and the graph keep serving, so an incremental reindex diverges from a /// from-scratch rebuild (S3). /// -/// Aliases, chunks (FTS in lockstep via the `chunks_ad` trigger), centroid and +/// Chunks (FTS in lockstep via the `chunks_ad` trigger), centroid and /// **outgoing** edges cascade with the row. Vectors no longer do — they are /// content-addressed and may be shared, so [`prune_orphan_vectors`] collects them. /// **Inbound** edges are the caller's concern: `edges.dst_path` carries no FK (it must @@ -1680,7 +1642,7 @@ pub fn resources_under_dir(conn: &Connection, dir: &str) -> Result> /// Re-key a note from `old_path` to `new_path` — **the** index-side move (ADR-0003). /// -/// One statement, and the FK graph does the rest: `note_aliases`, `chunks`, +/// One statement, and the FK graph does the rest: `chunks`, /// `note_centroids` and `edges.src_path` all declare `ON UPDATE CASCADE`, so every /// derived row travels with the note atomically and its chunk *vectors* are never /// touched at all — they are content-addressed, so they belong to the text, not the diff --git a/crates/b2-core/src/ingest.rs b/crates/b2-core/src/ingest.rs index db2703c..0990e5c 100644 --- a/crates/b2-core/src/ingest.rs +++ b/crates/b2-core/src/ingest.rs @@ -1,4 +1,4 @@ -//! Ingest (flow ①): parse -> project into `notes`/`note_aliases`, `chunks` (+FTS) and +//! Ingest (flow ①): parse -> project into `notes`, `chunks` (+FTS) and //! the typed `edges` graph, all keyed by the note's vault-relative path (ADR-0003). //! **Ingest writes nothing to the vault** (ADR-0004) — it is a pure read of it. //! @@ -202,16 +202,10 @@ fn project_note_and_chunks( conn, &NoteRow { path: rel_path, - // `type` is required by the model; default the projection to "note" - // for the rare untyped file (the file itself is never modified). - r#type: fields.r#type.as_deref().unwrap_or("note"), title: Some(title.as_str()), - description: fields.description.as_deref(), created: fields.created.as_deref(), - updated: fields.updated.as_deref(), body_hash: &body_hash, mtime, - aliases: &fields.aliases, }, )?; diff --git a/crates/b2-core/src/note.rs b/crates/b2-core/src/note.rs index 98ecdca..c0e9239 100644 --- a/crates/b2-core/src/note.rs +++ b/crates/b2-core/src/note.rs @@ -12,19 +12,16 @@ use crate::error::{Error, Result}; use yaml_rust2::{Yaml, YamlLoader}; -/// The frontmatter fields B2 projects into the `notes` table. Extraction is -/// best-effort: unparseable frontmatter still round-trips (raw is preserved); the -/// fields just come back empty. +/// The frontmatter fields B2 reads: the ones a reader displays (`type`, `created`, +/// `updated`, `tags`) and B2's own `b2_relations:`. Extraction is best-effort: +/// unparseable frontmatter still round-trips (raw is preserved); the fields just come +/// back empty. Every other key — `title`, `description`, `aliases`, your own — is +/// round-tripped verbatim and read by nothing (data-model.md §1). #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct NoteFields { pub r#type: Option, - // No `title`: a note's display title is its filename ([`display_title`]), and a - // frontmatter `title:` key is inert (data-model.md §1) — round-tripped verbatim - // with the raw bytes like any key B2 does not project. - pub description: Option, pub created: Option, pub updated: Option, - pub aliases: Vec, pub tags: Vec, /// Raw `b2_relations:` entries (typed-link strings, §2) — B2's namespaced /// frontmatter home for typed edges, the only place a verb/explanation lives. @@ -247,10 +244,8 @@ fn extract_fields(yaml: &str) -> (NoteFields, bool) { Some(doc) => { readable = doc.as_hash().is_some() || matches!(doc, Yaml::Null); f.r#type = doc["type"].as_str().map(str::to_string); - f.description = doc["description"].as_str().map(str::to_string); f.created = scalar_to_string(&doc["created"]); f.updated = scalar_to_string(&doc["updated"]); - f.aliases = string_list(&doc["aliases"]); f.tags = string_list(&doc["tags"]); f.relations = string_list(&doc["b2_relations"]); } diff --git a/crates/b2-core/tests/embed.rs b/crates/b2-core/tests/embed.rs index b8fa2ba..5e29d4f 100644 --- a/crates/b2-core/tests/embed.rs +++ b/crates/b2-core/tests/embed.rs @@ -341,8 +341,8 @@ fn concurrent_embed_passes_leave_one_intact_vector_space() { { let conn = open(&db_path).unwrap(); conn.execute( - "INSERT INTO notes(path, type, body_hash, indexed_at) - VALUES ('n.md', 'note', 'hash', '2026-07-26T00:00:00Z')", + "INSERT INTO notes(path, body_hash, indexed_at) + VALUES ('n.md', 'hash', '2026-07-26T00:00:00Z')", [], ) .unwrap(); diff --git a/crates/b2-core/tests/ingest_resolver.rs b/crates/b2-core/tests/ingest_resolver.rs index ea155f0..baa6484 100644 --- a/crates/b2-core/tests/ingest_resolver.rs +++ b/crates/b2-core/tests/ingest_resolver.rs @@ -1,7 +1,6 @@ -//! Step 1 — ingest into `notes`/`note_aliases` and the link resolver -//! (index-engine.md): ingest the golden vault, resolve a wikilink in both the -//! extensionless and `.md` forms, then prove `aliases:` projects into the alias -//! table. +//! Step 1 — ingest into `notes` and the link resolver +//! (index-engine.md): ingest the golden vault, then resolve a wikilink in both the +//! extensionless and `.md` forms. //! //! Since GH #170 the resolver is one-way and one-step: a note's identity **is** its //! vault-relative path, so there is nothing to translate — `resolve_link_target` @@ -10,11 +9,9 @@ mod common; +use b2_core::db; use b2_core::embed::FakeEmbedder; -use b2_core::ingest::ingest_vault; -use b2_core::{db, open}; use common::{ingest_golden, MEMORY_PATH}; -use std::fs; #[test] fn ingests_golden_vault_and_resolves_a_link_in_both_authored_forms() { @@ -43,27 +40,3 @@ fn ingests_golden_vault_and_resolves_a_link_in_both_authored_forms() { // both golden notes landed assert_eq!(common::count(&conn, "notes"), 2); } - -#[test] -fn aliases_are_projected_and_searchable() { - let tmp = tempfile::TempDir::new().unwrap(); - let vault = tmp.path().join("vault"); - fs::create_dir_all(&vault).unwrap(); - fs::write( - vault.join("srs.md"), - "---\ntype: concept\ntitle: \"Spaced repetition\"\naliases: [SRS, spacing-effect]\n---\nBody.\n", - ) - .unwrap(); - - let conn = open(&tmp.path().join("b2.sqlite")).unwrap(); - ingest_vault(&conn, &vault, &FakeEmbedder::default()).unwrap(); - - let alias_hit: String = conn - .query_row( - "SELECT note_path FROM note_aliases WHERE alias = 'SRS'", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(alias_hit, "srs.md", "aliases key on the note's path (L1)"); -} diff --git a/crates/b2-core/tests/props.rs b/crates/b2-core/tests/props.rs index 7e78306..32b26db 100644 --- a/crates/b2-core/tests/props.rs +++ b/crates/b2-core/tests/props.rs @@ -438,15 +438,9 @@ fn dump(root: &Path) -> Vec { let sections: &[(&str, &str, usize)] = &[ ( "note", - "SELECT path, type, ifnull(title,''), ifnull(description,''), - ifnull(created,''), ifnull(updated,''), body_hash + "SELECT path, ifnull(title,''), ifnull(created,''), body_hash FROM notes ORDER BY path", - 7, - ), - ( - "alias", - "SELECT note_path, alias FROM note_aliases ORDER BY note_path, alias", - 2, + 4, ), ( "chunk", diff --git a/crates/b2-core/tests/resources.rs b/crates/b2-core/tests/resources.rs index 5711a73..6ba68b6 100644 --- a/crates/b2-core/tests/resources.rs +++ b/crates/b2-core/tests/resources.rs @@ -84,8 +84,8 @@ fn v3_index_is_dropped_and_rebuilt_at_v4() { { let conn = open(&db_path).unwrap(); conn.execute_batch( - "INSERT INTO notes(path, type, body_hash, indexed_at) - VALUES ('a.md', 'note', 'h', 'now'); + "INSERT INTO notes(path, body_hash, indexed_at) + VALUES ('a.md', 'h', 'now'); INSERT INTO resources(path, class, size, content_hash, indexed_at) VALUES ('img.png', 'image', 3, 'h', 'now');", ) @@ -125,8 +125,8 @@ fn resource_edges_are_fk_checked_and_redangle_on_prune() { let conn = open(&tmp.path().join("b2.sqlite")).unwrap(); conn.execute_batch( - "INSERT INTO notes(path, type, body_hash, indexed_at) - VALUES ('a.md', 'note', 'h', 'now');", + "INSERT INTO notes(path, body_hash, indexed_at) + VALUES ('a.md', 'h', 'now');", ) .unwrap(); diff --git a/crates/b2-core/tests/substrate.rs b/crates/b2-core/tests/substrate.rs index 969dac1..bb2dc1d 100644 --- a/crates/b2-core/tests/substrate.rs +++ b/crates/b2-core/tests/substrate.rs @@ -21,10 +21,9 @@ use std::time::Duration; /// and doesn't need to be. A table added there and forgotten here only makes the /// assertions in *this* file weaker, never wrong — the completeness check that actually /// guards the index reads the pinned list, not this one. -const SCHEMA_TABLES: [&str; 7] = [ +const SCHEMA_TABLES: [&str; 6] = [ "meta", "notes", - "note_aliases", "chunks", "chunks_fts", "resources", @@ -286,8 +285,8 @@ fn an_index_stamped_current_but_missing_a_table_is_rebuilt() { let conn = open(&db_path).unwrap(); // A note the index believes is projected — the row that must not outlive the repair. conn.execute( - "INSERT INTO notes(path, type, body_hash, indexed_at) - VALUES ('kept.md', 'note', 'hash', '2026-07-26T00:00:00Z')", + "INSERT INTO notes(path, body_hash, indexed_at) + VALUES ('kept.md', 'hash', '2026-07-26T00:00:00Z')", [], ) .unwrap(); @@ -325,8 +324,8 @@ fn an_index_with_its_schema_stamp_missing_is_rebuilt() { { let conn = open(&db_path).unwrap(); conn.execute( - "INSERT INTO notes(path, type, body_hash, indexed_at) - VALUES ('kept.md', 'note', 'hash', '2026-07-26T00:00:00Z')", + "INSERT INTO notes(path, body_hash, indexed_at) + VALUES ('kept.md', 'hash', '2026-07-26T00:00:00Z')", [], ) .unwrap(); @@ -411,8 +410,8 @@ fn an_index_from_a_newer_b2_is_refused_not_rebuilt() { { let conn = open(&db_path).unwrap(); conn.execute( - "INSERT INTO notes(path, type, body_hash, indexed_at) - VALUES ('kept.md', 'note', 'hash', '2026-08-28T00:00:00Z')", + "INSERT INTO notes(path, body_hash, indexed_at) + VALUES ('kept.md', 'hash', '2026-08-28T00:00:00Z')", [], ) .unwrap(); diff --git a/docs/data-model.md b/docs/data-model.md index a67afa9..5e95eb0 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -148,18 +148,19 @@ Optional keys B2 recognizes: - **`type`**: what kind of note this is (`note`, `concept`, `source`, `person`, `daily`, …). Controlled but extensible; unknown values are tolerated; the OKF entity discriminator (§5). - Defaults to `note`. Its only consumer is display, so nothing keys on its presence and the + Absent means untyped. Its only consumer is display, so nothing keys on its presence and the new-note template does not seed it (GH #80: the template stamps only what cannot be reconstructed later). Not `b2`-namespaced on purpose: a courtesy you own, not a key B2 machines on. - **`title`**: recognized but inert (L2). A note's title *is* its filename (basename minus `.md`). The key is parsed and round-tripped losslessly like any other, and never drives display, link aliases, or search. -- **`description`**: a one-line summary. Feeds the embedding prompt and OKF export. +- **`description`**: a one-line summary. Round-tripped; nothing in B2 reads it yet. - **`tags`**: a list of strings. - **`created` / `updated`**: ISO-8601 date or datetime. `created` is set by B2 at creation (`b2 add`); `updated` is yours (or another tool's) to maintain. B2 does not stamp it. -- **`aliases`**: Obsidian-native additional titles; treated as alternate link aliases. +- **`aliases`**: Obsidian-native additional titles. Round-tripped; B2 does not yet resolve a + link written against an alias (a `[[SRS]]` naming `srs.md` by its alias dangles, G5). - **`provenance`**: optional, opt-in note-level authorship: `{by: human | agent:, source?, confidence?}`. Absent means `{by: human}`. B2 neither requires nor manages it. Edges carry no provenance (§4). diff --git a/docs/index-engine.md b/docs/index-engine.md index ba23c43..580fa0c 100644 --- a/docs/index-engine.md +++ b/docs/index-engine.md @@ -69,9 +69,8 @@ The precise DDL and build order live in `crates/b2-core/src/db.rs` (schema) and b2.sqlite — DISPOSABLE CACHE (= projection of the vault directory; drop & rebuild any time) ├── MIRROR OF THE VAULT (lets us diff vs. disk) │ ├── meta(key, value) -- schema_version, embed_model_id, embed_dim -│ ├── notes(path PK, type, title, description, -- the path IS the identity (L1) -│ │ created, updated, body_hash, mtime, indexed_at) -│ ├── note_aliases(note_path, alias) -- frontmatter `aliases:` +│ ├── notes(path PK, title, created, -- the path IS the identity (L1) +│ │ body_hash, mtime, indexed_at) │ └── resources(path PK, class, size, mtime, -- non-.md peers; class by extension (§10 dm) │ content_hash, indexed_at) │ @@ -93,7 +92,7 @@ vault has an embedding space" signal the BM25-only fallbacks key on (M4, ADR-000 Why this shape fits B2: - **Everything keys on the vault-relative path** (L1, ADR-0003). `notes.path` is the primary - key; `chunks.note_path`, `note_aliases.note_path`, `note_centroids.note_path`, and + key; `chunks.note_path`, `note_centroids.note_path`, and `edges.src_path` are `REFERENCES notes(path) ON DELETE CASCADE ON UPDATE CASCADE`. That makes a B2-performed move a path re-key rather than a rebuild: `UPDATE notes SET path = …` cascades through every child in one statement, after the inbound link-text rewrite and From 8a5b232faa83d6f4863f81c762c1d387e18a5d1e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:32:42 +0000 Subject: [PATCH 28/30] fix(ui): a moved image re-points state before its bytes load; comments name the split modules executeMove adopted the moved resource only after its picture's bytes arrived, so the watcher pulse the move itself causes could find state still naming the old path and warn "no longer on disk" about B2's own move. The path is adopted first; the picture follows if the resource is still open. Comments that pointed at render.ts for code that now lives in markdown.ts / settingsview.ts / keysview.ts / chatview.ts / graphview.ts / explainview.ts / widgets.ts name the module it moved to, including the sanitizer's trust-boundary header. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- ui/src/livepreview.ts | 8 ++++---- ui/src/main.ts | 32 ++++++++++++++++++-------------- ui/src/sanitize.ts | 4 ++-- ui/src/settingstabs.ts | 4 ++-- ui/style.css | 12 ++++++------ 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/ui/src/livepreview.ts b/ui/src/livepreview.ts index b2f57d8..3593419 100644 --- a/ui/src/livepreview.ts +++ b/ui/src/livepreview.ts @@ -51,7 +51,7 @@ import { renderMarkdown } from "./markdown.ts"; // --- the wikilink tree extension (spec §4, insight §2.3) -------------------------- // // A `[[target]]` / `[[target|label]]` inline node — the same grammar as the reading -// view's `marked` tokenizer (render.ts): target = one-or-more chars that aren't `]` or +// view's `marked` tokenizer (markdown.ts): target = one-or-more chars that aren't `]` or // `|`, optional `|label` where label is one-or-more non-`]` chars. Giving the tree a // `Wikilink` node lets the *one* decoration engine style wikilinks uniformly with every // other construct, instead of a bolt-on. Positions are document-relative throughout. @@ -111,7 +111,7 @@ export const WIKILINK_RE = WIKILINK_EXACT; // --- the note's pictures (the `![[image.png]]` embed) -------------------------------- // // An embed's bytes arrive over IPC long after the editor mounted, and they arrive for -// the *document*, not for the editor — the reading view draws the same map (render.ts). +// the *document*, not for the editor — the reading view draws the same map (markdown.ts). // So they enter the editor as ordinary editor state: a field main.ts writes with an // effect, which every decoration pass then reads. That is what keeps `inlineDecorations` // a pure function of `EditorState` (the property the whole suite leans on) instead of a @@ -279,7 +279,7 @@ class TableWidget extends WidgetType { } // The picture of an `![[image.png]]` embed, drawn in the buffer's place — the editor's -// half of the reading view's inline image (render.ts). An inline replace, not a block +// half of the reading view's inline image (markdown.ts). An inline replace, not a block // widget: it spans no line break, so it belongs to the ViewPlugin with every other // inline conceal, and an embed written mid-sentence stays mid-sentence. // @@ -419,7 +419,7 @@ function handleNode( // plain text to the grammar (the tree has no node for it), so the construct's real // span starts at `node.from - 1` and every offset below is taken from that. Three // things follow from the marker, and all three are the reading view's rules — - // read and edit must not disagree about what a note says (render.ts): + // read and edit must not disagree about what a note says (markdown.ts): // // • the `|`-part is a display **width**, not a label, so an embed shows its // *target* where a plain wikilink shows its label (never a bare "500"); diff --git a/ui/src/main.ts b/ui/src/main.ts index da720e5..8b660a8 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -216,7 +216,7 @@ function capturePaneFocus(pane: HTMLElement): (() => void) | null { * a watcher pulse, or the dialog's own state change, and WebKit drops focus to ``. * * Restored by **`id`**: a modal control's id is the identity that outlives the swap - * (render.ts's Settings builder says so out loud, which is why every control in there + * (settingsview.ts's Settings builder says so out loud, which is why every control in there * carries one). Null when the keyboard was somewhere else entirely, or on a control with * no id — a repaint must only ever *give back* focus, never take it, and guessing a * replacement for an unidentifiable control is taking it. @@ -478,11 +478,11 @@ async function paintCodeHighlights(): Promise { // // There are two meters and one painter. The shell's lives in the top bar; Settings → // Index paints a second while a run is live, because Settings took the whole window and a -// meter behind an opaque surface is no meter (render.ts). So this walks *every* +// meter behind an opaque surface is no meter (widgets.ts). So this walks *every* // `.reindex-progress` on screen and writes the same values into each — one computation, // so the two can't disagree about a run, and adding a third costs nothing here. // -// The button does not: it is Settings → Index's now (render.ts `indexPanelHtml`), so it +// The button does not: it is Settings → Index's now (settingsview.ts `indexPanelHtml`), so it // exists only while that dialog is open on that section — hence the null-tolerant lookup // rather than `el`. `settingsPanelHtml` paints it in the right state to begin with; this // keeps it there through the runs that *don't* full-render, which is every auto-index @@ -705,7 +705,7 @@ async function loadResourceImage(r: ResourceExplainView): Promise // --- the note's inline pictures (`![[image.png]]`) --------------------------------- // -// An embed draws the file it names (render.ts, livepreview.ts), and the bytes for that +// An embed draws the file it names (markdown.ts, livepreview.ts), and the bytes for that // come over the same `read_resource` command the resource card uses. What is *here* is // the reconciliation: which pictures the open document should be holding, and the reads // that close the gap. @@ -1110,7 +1110,7 @@ let holdTimer: number | null = null; /** The ⌘ sheet's own paint. No memo and no focus dance: nothing in it is focusable or * typed into, so a rewrite costs one innerHTML of static markup and can't take anything - * away from the user (render.ts's `cmdSheetHtml` says why it is deliberately not a + * away from the user (keysview.ts's `cmdSheetHtml` says why it is deliberately not a * dialog). * * Called by `render()` as well, so an unrelated repaint can't leave the layer behind — @@ -1194,7 +1194,7 @@ const FOCUSABLE = function overlayFocusables(): HTMLElement[] { // `[role="dialog"]`, not a class: the overlay layer has two shapes now — the `.modal` // box the link/move/delete dialogs paint into, and Settings' full-window - // `.settings-screen` (render.ts) — and what they have in common is the semantics the + // `.settings-screen` (settingsview.ts) — and what they have in common is the semantics the // trap exists to serve, not the chrome. const modal = document.querySelector('#modal-root [role="dialog"]'); // The `tabIndex >= 0` filter is what makes a **roving tabstop inside a modal** work: @@ -1961,8 +1961,12 @@ async function executeMove(node: TreeNodeRef, to: string): Promise { state.current = await api.readNote(openNotePath); } if (openResourcePath !== null) { + // The new path is adopted before the picture's bytes are read: the watcher pulse + // this move causes must never find state still naming the old, vanished path. const moved = await api.explainResource(openResourcePath); - adoptResource(moved, await loadResourceImage(moved)); + adoptResource(moved, null); + const picture = await loadResourceImage(moved); + if (state.currentResource === moved) state.resourceImage = picture; } await loadNotes(); if (openNotePath !== null) await refreshDiscovery(); // backlinks may show new paths @@ -2254,7 +2258,7 @@ function clearSearch(): void { // --- chat (flow ④, GH #151/#153/#155) ----------------------------------------------- // -// The wiring; the paint is render.ts's `chatPaneHtml` and the pure logic is chat.ts. +// The wiring; the paint is chatview.ts's `chatPaneHtml` and the pure logic is chat.ts. // What lives here is what only the running app can own: the streaming turn, its // cancellation, and the focus/repaint discipline a token-by-token surface demands. // @@ -2711,7 +2715,7 @@ async function commitLink(): Promise { // (appearance), Embedding (the model picker — selecting one persists to the shared config // the CLI also reads, and a real switch is completed by the user with b2 init + Reindex, // which the flashed guidance names), and Keyboard (K1's discoverable half — the table -// lives in shortcuts.ts). This is the wiring; the paint is render.ts. +// lives in shortcuts.ts). This is the wiring; the paint is settingsview.ts. /** Open Settings, optionally jumping straight to a section — `?` lands on Keyboard, the * "semantic search is off" banner lands on Embedding where its Download button is. @@ -3671,7 +3675,7 @@ function mountEditor(body: string): void { } /** The editor chip's tooltip. Its chord comes out of the live registry rather than being - * spelled here, for `graphToggleHtml`'s reason (render.ts): ⇧⌘E is rebindable (#121), so + * spelled here, for `graphToggleHtml`'s reason (graphview.ts): ⇧⌘E is rebindable (#121), so * a tooltip naming the shipped default would be wrong for the user who moved it. Off * "live preview" rather than the reading bar's "rendered Markdown" — one sticky flag, * two surfaces, and each names what *it* shows when the flag is off. */ @@ -4373,7 +4377,7 @@ function buildShell(): void { floating at the far end of the bar. Hidden between runs, so this is just the path almost all of the time. The Reindex button that used to stand here has moved into Settings → Index — indexing is automatic now, and permanent chrome - for an exception trains the eye to skip the bar (render.ts, indexPanelHtml). + for an exception trains the eye to skip the bar (settingsview.ts, indexPanelHtml). What stays is the live meter and the Cancel that belongs with it — visible wherever you are in the app, except behind Settings, which covers the bar and so paints a second meter of its own. --> @@ -4466,7 +4470,7 @@ function settingsClick(target: HTMLElement): void { void provisionModel(); return; } - // Settings → Chat. The Local/Cloud segments are a *view* of the endpoint (render.ts + // Settings → Chat. The Local/Cloud segments are a *view* of the endpoint (settingsview.ts // says why), so pressing one rewrites the URL field to that configuration's starting // point and shows or hides the key + its privacy copy — the consent moment is the // configuration moment (M5). @@ -4479,7 +4483,7 @@ function settingsClick(target: HTMLElement): void { void saveChatConfig(); return; } - // The Model field's two shapes (render.ts's `chatModelFieldHtml`). Neither saves: + // The Model field's two shapes (settingsview.ts's `chatModelFieldHtml`). Neither saves: // this only decides whether the field is a list of what the daemon has or a box for // a name it doesn't have yet. if (target.closest("[data-chat-model-custom]")) { @@ -4695,7 +4699,7 @@ function wireClicks(): void { // Settings: a rail tab, the Download button (in-app `b2 init`), else the Done button // closes it. Checked before the link-modal backdrop branch so settings wins when it's // up. There is no click-outside to close on any more — the surface is the whole window - // (render.ts) — so the ways out are Done and Escape. + // (settingsview.ts) — so the ways out are Done and Escape. if (state.settingsOpen) { settingsClick(target); return; // clicks inside Settings do nothing else diff --git a/ui/src/sanitize.ts b/ui/src/sanitize.ts index 24da867..80a6761 100644 --- a/ui/src/sanitize.ts +++ b/ui/src/sanitize.ts @@ -2,7 +2,7 @@ // // Note content is **untrusted input**: authorship is not trust, and a `.md` can arrive // from a shared vault, a download, or a web clip. So every byte of note-derived HTML -// crosses this seam before it can reach `innerHTML`. render.ts wires it as `marked`'s +// crosses this seam before it can reach `innerHTML`. markdown.ts wires it as `marked`'s // `postprocess` hook, which is what makes it *unavoidable* rather than remembered — the // reading view, the live-preview table widget, and any future call site are covered by // construction, not by each caller choosing to sanitize. @@ -30,7 +30,7 @@ import DOMPurify, { * - `ALLOW_DATA_ATTR: false` + `ADD_ATTR: ["data-target"]` — B2 delegates clicks off * `data-*` hooks, so leaving the default (every `data-*` allowed) would let a note mint * the app's own handles. `data-target` is the one exception because it *is* the - * wikilink contract: render.ts emits `` and both + * wikilink contract: markdown.ts emits `` and both * the reading view's follow handler and livepreview's `TableWidget` read it back. A * note forging one gains nothing — it can write `[[target]]` and get the same anchor. * - `FORBID_TAGS: ["form"]` — the CSP hole this issue named: `form-action` does *not* diff --git a/ui/src/settingstabs.ts b/ui/src/settingstabs.ts index 9ca3567..8c8df42 100644 --- a/ui/src/settingstabs.ts +++ b/ui/src/settingstabs.ts @@ -12,7 +12,7 @@ // not a failure mode, not a moment you'd open the dialog. And the keyboard reference is a // *reference*, read start to finish, which is the one thing a settings column must never // make you scroll past. The rail is where expansion lands (vault prefs, editor prefs, -// diagnostics): a new tab is a row here plus a panel in render.ts, and nothing else moves. +// diagnostics): a new tab is a row here plus a panel in settingsview.ts, and nothing else moves. // Index is the worked example — it arrived as one button that used to live in the top bar. // // Invariant K1 (docs/invariants.md, GH #78) governs the rail like every other @@ -38,7 +38,7 @@ export interface SettingsTab { } /** The rail, in paint order. Adding a section means adding a row here and a panel in - * render.ts's `settingsPanelHtml` — the navigation, the roving tabstop, and the wrap + * settingsview.ts's `settingsPanelHtml` — the navigation, the roving tabstop, and the wrap * all follow from this list. */ export const SETTINGS_TABS: SettingsTab[] = [ { id: "general", label: "General", hint: "Appearance and app-wide preferences" }, diff --git a/ui/style.css b/ui/style.css index 0550659..5ec93d6 100644 --- a/ui/style.css +++ b/ui/style.css @@ -1112,7 +1112,7 @@ body.is-resizing { background: var(--accent-soft); } -/* The inline image an `![[picture.png]]` embed draws (render.ts). The embed is still a +/* The inline image an `![[picture.png]]` embed draws (markdown.ts). The embed is still a link — clicking it opens the resource card — but the picture *is* the label now, so the link's dashed underline and hover wash come off: they would frame the image rather than mark a word. The accent outline on hover is the same affordance said in @@ -2265,7 +2265,7 @@ body.is-loading { text. Inline it sat exactly where a chord's last glyph goes, so "⌘4 •" read as a chord with a dot in it — the one thing a keyboard reference can't be ambiguous about. A badge hung off the corner is the editor-tab idiom for "modified", and can't be misread as a - key. The `title` is what says it in words (render.ts), for the reader and the screen + key. The `title` is what says it in words (keysview.ts), for the reader and the screen reader both. */ .kbd-changed::after { content: ""; @@ -2612,7 +2612,7 @@ button.move-dest:hover { /* --- settings: the full-window surface ------------------------------------------- Settings outgrew a single scrolling column, then it outgrew the box that replaced it - (render.ts's `settingsScreenHtml` argues the case), so it takes the window: the app is + (settingsview.ts's `settingsScreenHtml` argues the case), so it takes the window: the app is *behind* it, not dimmed under it. No backdrop — there is no outside left to click — which is what lets the surface be opaque and the panel be as tall as the screen. @@ -2706,7 +2706,7 @@ button.move-dest:hover { box-shadow: 0 0 0 3px var(--accent-soft); } -/* The panel is focusable (`tabindex="0"` in render.ts) because it is the scroll +/* The panel is focusable (`tabindex="0"` in settingsview.ts) because it is the scroll container: a region you can't focus is a region you can't scroll without the mouse, and the Keyboard section is a page of table with no controls of its own. Hence a ring it can actually show — inset, so it reads as "this region has the keyboard" rather @@ -2739,7 +2739,7 @@ button.move-dest:hover { /* One exception, and it is a different *kind* of content: the chord sheet is a reference read in columns, not prose read in lines, and the width is the whole reason it packs into two columns instead of scrolling as one. Set on the wrapper rather than the grid, - because a child cannot outgrow the measure that clips it (render.ts picks the class by + because a child cannot outgrow the measure that clips it (settingsview.ts picks the class by section). */ .settings-measure-wide { max-width: 1120px; @@ -3593,7 +3593,7 @@ button.move-dest:hover { } /* --- Explain: the Compare view for one Similar card (GH #236) --------------------- - The centre pane's fourth mode (render.ts `explainPaneHtml`). A reading column like a + The centre pane's fourth mode (explainview.ts `explainPaneHtml`). A reading column like a note's, under a sticky bar with the one way back. Passages are note text shown as text, so they keep their line breaks and scroll inside a bounded box rather than letting one 11k-token passage push the rest of the view off the page. */ From cc5a67c81281cc9890ee91698bba10b52a71b992 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:47:08 +0000 Subject: [PATCH 29/30] fix: restore the Metal-fallback notice; mv's docs and tests say what is true From review of the cleanup: - b2-embed's Metal-to-CPU fallback had become a tracing event, which neither adapter shows unless logging is asked for. A fallback changes the embedding space's identity (no @metal tag), so a Metal-built index then reads as a model change; the stderr notice explaining why is back, and the tracing dependency it pulled in goes. - mv.rs claimed a move rewrites "exactly the links that point at it". It rewrites by written text within an inbound file, so a second link with the same text that resolves elsewhere is rewritten too (true before this cleanup as well). The docs now name that gap instead of denying it; the move_note doc no longer promises a Markdown-form link always dangles. - The three a_*_move_leaves_the_index_a_rebuild_would_project tests compared only edges and resources; projection() now compares notes and chunks too, so the names hold. - Stale mentions of the dropped note_aliases table (mv, rm, index-engine.md) and a SCHEMA_VERSION doc entry for 7. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- Cargo.lock | 1 - crates/b2-core/src/db.rs | 4 +++- crates/b2-core/src/mv.rs | 16 +++++++++------ crates/b2-core/src/rm.rs | 2 +- crates/b2-core/tests/mv.rs | 39 +++++++++++++++++++++++++++++++++--- crates/b2-embed/Cargo.toml | 3 --- crates/b2-embed/src/model.rs | 14 ++++++------- docs/index-engine.md | 2 +- 8 files changed, 58 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 771cd90..179359d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -402,7 +402,6 @@ dependencies = [ "thiserror 2.0.18", "tokenizers", "toml 0.8.23", - "tracing", ] [[package]] diff --git a/crates/b2-core/src/db.rs b/crates/b2-core/src/db.rs index bd4995c..3296ae8 100644 --- a/crates/b2-core/src/db.rs +++ b/crates/b2-core/src/db.rs @@ -39,7 +39,9 @@ use std::time::Duration; /// because its module is no longer linked to drop it. **4** added the `resources` /// inventory and widened `edges` with resource targets. **5** switched `chunks_fts` /// to `porter unicode61` (the GH #157 A/B's verdict). **6** re-keyed the whole index -/// on the vault-relative path and made `embeddings` content-addressed (GH #170). +/// on the vault-relative path and made `embeddings` content-addressed (GH #170). **7** +/// dropped the columns nothing read (`notes.type`, `description`, `updated`) and the +/// `note_aliases` table. pub const SCHEMA_VERSION: i64 = 7; /// Statements at or over this take the slow-query WARN path (`B2_SLOW_QUERY_MS` diff --git a/crates/b2-core/src/mv.rs b/crates/b2-core/src/mv.rs index 30b5c4b..e7d29e7 100644 --- a/crates/b2-core/src/mv.rs +++ b/crates/b2-core/src/mv.rs @@ -6,7 +6,7 @@ //! bounded by the moved set's backlink count: the human-facing copy — the link text in //! every file linking *at* a moved member, rewritten in place — and the index: each moved //! note re-keyed by one [`db::repoint_note_path`] whose `ON UPDATE CASCADE` FKs carry -//! chunks, aliases, centroid and outbound edges atomically, each moved resource by one +//! chunks, centroid and outbound edges atomically, each moved resource by one //! [`db::repoint_resource`], then a re-projection of the inbound sources so their edges //! (`edges.dst_path` has no FK — it must be free to dangle) point at the new paths. A //! moved note's **vectors are not touched at all**: content-addressed (ADR-0006), they @@ -18,7 +18,10 @@ //! rename on disk, *then* re-project from the now-current Markdown. And bounded, not a //! scan: [`db::inbound_edges_of`] names exactly the files to touch, so the cost is //! O(inbound links). The link text is found by [`crate::link`]'s own scanner, so a move -//! rewrites exactly the links ingest projected. +//! reads links exactly as ingest does. Known gap: within one inbound file the rewrite is +//! keyed by a link's *written* text, so a second link with the same text that resolves +//! elsewhere (a note-relative Markdown link and a vault-root wikilink both written +//! `img.png`, naming two different files) is rewritten too. //! //! **The vault half is all or nothing** (GH #230). Reindexing can't repair rewritten //! link text (it projects whatever the Markdown says), so a failed move must leave every @@ -73,8 +76,8 @@ pub struct MoveReport { /// /// Only the `[[…]]` form is rewritten at a note. A Markdown-form link at one /// (`[x](notes/a.md)`) keeps its text, but its source is re-projected like every inbound -/// source, so the edge dangles exactly as a rebuild would project it rather than keep -/// naming a path that no longer exists. +/// source, so its edge is exactly what a rebuild would project — dangling, or resolved to +/// whatever note that text now names — rather than keep naming a path that is gone. pub fn move_note(ctx: EmbedCtx, old_rel: &str, new_rel_input: &str) -> Result { let new_rel = pathspec::normalize_rel_md(new_rel_input).map_err(Error::MoveDestination)?; refuse_same_path(&new_rel, old_rel, "note")?; @@ -493,8 +496,9 @@ fn plan_inbound(vault_root: &Path, wiki: &ByFile, md: &ByFile) -> Result { /// Only the target token changes: every other byte (the brackets, the `|alias`, the /// link text, whitespace around the target) is preserved, and a target merely sharing a /// prefix with a key is never touched. The links are the ones [`link::link_spans`] -/// finds — the scanner ingest projects edges from — so a move rewrites exactly the links -/// that point at it. Returns the rewritten text and the count of targets replaced. +/// finds — the scanner ingest projects edges from — matched by their written text (see +/// the module doc's known gap). Returns the rewritten text and the count of targets +/// replaced. fn rewrite_targets(raw: &str, wiki: &Targets, md: &Targets) -> (String, usize) { let mut out = String::with_capacity(raw.len()); let (mut copied, mut count) = (0usize, 0usize); diff --git a/crates/b2-core/src/rm.rs b/crates/b2-core/src/rm.rs index ee64874..03c64ea 100644 --- a/crates/b2-core/src/rm.rs +++ b/crates/b2-core/src/rm.rs @@ -58,7 +58,7 @@ pub struct DirDeleteReport { } /// Delete the note at `rel`: file off disk, projection rows off the index -/// (chunks/FTS/centroid/aliases/outbound edges cascade with the `notes` row), then +/// (chunks/FTS/centroid/outbound edges cascade with the `notes` row), then /// re-project the inbound linkers so their edges re-dangle. The façade resolved the ref. /// The note's chunk **vectors** deliberately do not cascade — content-addressed and /// possibly shared (ADR-0006), they are collected by the whole-vault pass. diff --git a/crates/b2-core/tests/mv.rs b/crates/b2-core/tests/mv.rs index 1a819e6..2e1d957 100644 --- a/crates/b2-core/tests/mv.rs +++ b/crates/b2-core/tests/mv.rs @@ -791,9 +791,9 @@ fn re_running_a_move_interrupted_before_its_rename_finishes_it() { // one pipeline, so what a move repairs is what ingest projected, and each kind leaves // the index a rebuild would produce. -/// Every projected edge and inventory row, sorted — the index state a move must leave -/// equal to a from-scratch rebuild (S3). Edge ids are derived from the resolved -/// target, so a wrongly (un)resolved edge shows up here too. +/// Every projected note, chunk, edge and inventory row, sorted — the index state a move +/// must leave equal to a from-scratch rebuild (S3). Edge ids are derived from the +/// resolved target, so a wrongly (un)resolved edge shows up here too. fn projection(root: &Path) -> Vec { let conn = common::index_conn(root); let mut rows: Vec = Vec::new(); @@ -819,6 +819,39 @@ fn projection(root: &Path) -> Vec { .unwrap() .map(Result::unwrap), ); + let mut notes = conn + .prepare("SELECT path, ifnull(title, ''), ifnull(created, ''), body_hash FROM notes") + .unwrap(); + rows.extend( + notes + .query_map([], |r| { + Ok(format!( + "note {} {} {} {}", + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + )) + }) + .unwrap() + .map(Result::unwrap), + ); + let mut chunks = conn + .prepare("SELECT note_path, seq, text_hash FROM chunks") + .unwrap(); + rows.extend( + chunks + .query_map([], |r| { + Ok(format!( + "chunk {} {} {}", + r.get::<_, String>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, String>(2)?, + )) + }) + .unwrap() + .map(Result::unwrap), + ); let mut resources = conn .prepare("SELECT path, class, size, content_hash FROM resources") .unwrap(); diff --git a/crates/b2-embed/Cargo.toml b/crates/b2-embed/Cargo.toml index f3a9c3f..2e415d0 100644 --- a/crates/b2-embed/Cargo.toml +++ b/crates/b2-embed/Cargo.toml @@ -33,9 +33,6 @@ serde_json = "1.0.150" toml = "0.8" dirs = "6.0" thiserror = "2.0.18" -# The Metal-fallback warning, on the `b2::embed` target: a library reports through the -# adapter's subscriber rather than printing into its output. -tracing = "0.1.41" # On macOS, link Apple's Accelerate (vDSP/BLAS) for candle's CPU matmuls. Without # it candle falls back to a naive kernel that makes bge-base inference ~10-100× diff --git a/crates/b2-embed/src/model.rs b/crates/b2-embed/src/model.rs index 39b5434..f2f8b64 100644 --- a/crates/b2-embed/src/model.rs +++ b/crates/b2-embed/src/model.rs @@ -214,18 +214,18 @@ fn l2_normalize(v: &[f32]) -> Vec { /// Try to open the Metal GPU — only when compiled `--features metal` (candle's /// `metal_is_available()` is literally `cfg!(feature = "metal")`), and never a hard /// requirement: any failure returns `None` and the caller uses the CPU (GH #40). `announce` -/// gates the fallback warning (a `b2::embed` tracing event) so the load path can warn while -/// the cheap capability probe ([`active_device_label`]) stays silent. No `unwrap`: a failed `new_metal` is a soft +/// gates the fallback notice so the load path can say it while the cheap capability probe +/// ([`active_device_label`]) stays silent. The notice goes to stderr rather than a log +/// event on purpose: it explains a user-visible consequence (a CPU-tagged embedding space, +/// so a Metal-built index reads as a model change), and a log is off unless asked for. No `unwrap`: a failed `new_metal` is a soft /// degrade (no-panic rule), not a load error. fn open_metal(announce: bool) -> Option { if candle_core::utils::metal_is_available() { match Device::new_metal(0) { Ok(d) => return Some(d), - Err(e) if announce => tracing::warn!( - target: "b2::embed", - error = %e, - "Metal GPU unavailable; embedding on CPU" - ), + Err(e) if announce => { + eprintln!("note: Metal GPU unavailable ({e}); embedding on CPU") + } Err(_) => {} } } diff --git a/docs/index-engine.md b/docs/index-engine.md index 580fa0c..dd674c8 100644 --- a/docs/index-engine.md +++ b/docs/index-engine.md @@ -625,7 +625,7 @@ be budgeted, tested, and watched. `db::upsert_note`'s `ON CONFLICT(path)` is the whole of the reconciliation. A note deleted with no replacement is reconciled by the whole-vault pass ([GH #31](https://github.com/AlteredCraft/B2/issues/31)): `project_vault` prunes every - `notes` row whose path the walk did not see this run (aliases, chunks, FTS, centroid, and + `notes` row whose path the walk did not see this run (chunks, FTS, centroid, and outgoing edges cascade; inbound links re-dangle when phase 2 re-derives edges against the pruned resolver), *except* rows whose file was skipped as unreadable: the walk *saw* that file, so evicting it would lie. Single-note ingest (`add`/`mv`/`write`) touches one note From e51a12b67bf00ba1ed4d62e020a9f44fcabb649c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 23:18:30 +0000 Subject: [PATCH 30/30] refactor(eval): load the model once per run A warm eval loaded LocalEmbedder up to three times: inside provision()'s "already installed" check, again right after, and a third time for the dense fixture. examples/common gains load_or_provision (load first, provision only if that fails); eval shares one Rc-backed SharedEmbedder across its fixtures, and score_dense takes the embedder instead of loading its own. stability and groundedness use load_or_provision too. Verified on full --stemmer --sweep runs: identical stdout and results rows against the pre-change binary. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GCwoP7o76vxQ8rEQq7zM8w --- crates/b2-embed/examples/common/mod.rs | 17 +++++++++ crates/b2-embed/examples/eval/dense.rs | 11 +++--- crates/b2-embed/examples/eval/instrument.rs | 40 ++++++++++++++++++++- crates/b2-embed/examples/eval/main.rs | 21 +++++------ crates/b2-embed/examples/stability.rs | 7 ++-- crates/b2-llm/examples/groundedness.rs | 7 ++-- 6 files changed, 78 insertions(+), 25 deletions(-) diff --git a/crates/b2-embed/examples/common/mod.rs b/crates/b2-embed/examples/common/mod.rs index 6743c10..d3e82a0 100644 --- a/crates/b2-embed/examples/common/mod.rs +++ b/crates/b2-embed/examples/common/mod.rs @@ -15,6 +15,7 @@ #![allow(dead_code)] use b2_core::vault::{NoteSummary, SearchEvidenceView}; +use b2_embed::{provision, EmbedConfig, LocalEmbedder}; use std::error::Error; use std::path::{Path, PathBuf}; @@ -164,6 +165,22 @@ impl ScratchVault { } } +/// Load the configured embedding model, provisioning it first only when it is +/// not already loadable. +/// +/// `provision`'s idempotent fast path *is* a full model load, so calling it and +/// then loading again pays for the model twice on every warm run. Loading first +/// is the same decision in the other order: a model that loads is exactly what +/// the fast path would have accepted, and one that doesn't goes through +/// `provision` (fetch + verify) before the load that follows. +pub fn load_or_provision(config: &EmbedConfig) -> Result> { + if let Ok(embedder) = LocalEmbedder::load(config) { + return Ok(embedder); + } + provision(config, |line| eprintln!("[init] {line}"))?; + Ok(LocalEmbedder::load(config)?) +} + /// Whether `name` was passed as a bare flag. pub fn has_flag(args: &[String], name: &str) -> bool { args.iter().any(|a| a == name) diff --git a/crates/b2-embed/examples/eval/dense.rs b/crates/b2-embed/examples/eval/dense.rs index 32e2227..32823cc 100644 --- a/crates/b2-embed/examples/eval/dense.rs +++ b/crates/b2-embed/examples/eval/dense.rs @@ -7,7 +7,7 @@ use crate::discovery::AnchorDetail; use crate::evidence::ServedRow; use crate::fold::{fold_json, score_fold, FoldBench}; use crate::gate::FLOOR_DENSE_MATE_MRR; -use crate::instrument::timed_embed; +use crate::instrument::{timed_embed, SharedEmbedder}; use crate::labels::SimilarSet; use crate::metrics::{paths_match, rank_str_at, Agg}; use crate::report::{unix_secs, RunId}; @@ -15,7 +15,6 @@ use crate::tail::{dense_tail_json, print_dense_tail}; use crate::{K, SIM_K}; use b2_core::embed::Embedder; use b2_core::vault::Vault; -use b2_embed::{EmbedConfig, LocalEmbedder}; use std::path::Path; /// Score the dense single-domain fixture (GH #196/#197, Phase 0b) in a throwaway @@ -26,12 +25,12 @@ use std::path::Path; pub fn score_dense( evals_dir: &Path, set: &SimilarSet, + embedder: SharedEmbedder, ) -> Result> { let corpus_dir = evals_dir.join("corpus-dense"); - // A second model load rather than sharing the first vault's: the embedder was - // moved into that vault, and the fixture's whole point is an isolated run. - let config = EmbedConfig::load()?; - let embedder = LocalEmbedder::load(&config)?; + // The run's one loaded model, shared rather than reloaded: the fixture's whole + // point is an isolated run, and that isolation is the vault's — its own + // throwaway copy, its own index — not a second copy of the same weights. let model_id = embedder.model_id().to_string(); let scratch = ScratchVault::copy_flat(&corpus_dir)?; let vault = Vault::open_with_embedder(scratch.root(), Box::new(embedder))?; diff --git a/crates/b2-embed/examples/eval/instrument.rs b/crates/b2-embed/examples/eval/instrument.rs index 65b0f85..c5fe193 100644 --- a/crates/b2-embed/examples/eval/instrument.rs +++ b/crates/b2-embed/examples/eval/instrument.rs @@ -6,8 +6,46 @@ use b2_core::embed::Embedder; use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, Vault}; use b2_embed::LocalEmbedder; use std::ops::ControlFlow; +use std::rc::Rc; use std::time::Instant; +/// One loaded model serving every throwaway vault of the run. +/// +/// `Vault::open_with_embedder` takes its embedder by value, so without this each vault +/// (the orthogonal corpus's, the dense fixture's) loaded its own copy of the same +/// weights — a full model load apiece. The model is read-only and deterministic +/// (`&self` throughout), so sharing it costs the fixture none of its isolation: that +/// is a property of the *vault*, which stays its own. Every [`Embedder`] method +/// forwards, including the ones with trait defaults `LocalEmbedder` overrides (the +/// query prefix, the batched forward pass) — a method added to the trait later must +/// be forwarded here too, or the shared model silently runs the default instead. +#[derive(Clone)] +pub struct SharedEmbedder(Rc); + +impl SharedEmbedder { + pub fn new(model: LocalEmbedder) -> Self { + Self(Rc::new(model)) + } +} + +impl Embedder for SharedEmbedder { + fn model_id(&self) -> &str { + self.0.model_id() + } + fn dim(&self) -> usize { + self.0.dim() + } + fn embed(&self, text: &str) -> b2_core::Result> { + self.0.embed(text) + } + fn embed_query(&self, text: &str) -> b2_core::Result> { + self.0.embed_query(text) + } + fn embed_batch(&self, texts: &[&str]) -> b2_core::Result>> { + self.0.embed_batch(texts) + } +} + /// `LocalEmbedder::embed_batch` must be a faithful map of `embed`: right-padding short rows /// to the batch's longest and masking them out has to leave each row's CLS vector unchanged. /// The reindex path batches freely, so a regression here would silently corrupt every stored @@ -16,7 +54,7 @@ use std::time::Instant; /// It lives in the eval rather than `cargo test` because it needs the provisioned model, /// which the fast suite deliberately never touches (ADR-0013). Running it here means it /// actually runs, instead of sitting behind an `#[ignore]` nobody passes `--ignored` to. -pub fn check_batch_matches_single(model: &LocalEmbedder) -> Result<(), Box> { +pub fn check_batch_matches_single(model: &dyn Embedder) -> Result<(), Box> { // Deliberately varied lengths, so batching pads the short rows to the longest. let texts = [ "Spaced repetition schedules reviews at increasing intervals.", diff --git a/crates/b2-embed/examples/eval/main.rs b/crates/b2-embed/examples/eval/main.rs index 45c1823..11c55d1 100644 --- a/crates/b2-embed/examples/eval/main.rs +++ b/crates/b2-embed/examples/eval/main.rs @@ -90,13 +90,15 @@ use b2_core::chunk::ChunkConfig; use b2_core::db::FtsTokenizer; use b2_core::embed::Embedder; use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, Vault}; -use b2_embed::{provision, EmbedConfig, LocalEmbedder}; -use common::{append_result, git_short_sha, has_flag, reject_unknown_flags, ScratchVault}; +use b2_embed::EmbedConfig; +use common::{ + append_result, git_short_sha, has_flag, load_or_provision, reject_unknown_flags, ScratchVault, +}; use dense::{dense_row, print_dense_report, score_dense}; use discovery::{score_floor_z, score_similar}; use evidence::{bake_off, print_search_bakeoff, print_search_evidence, score_search_evidence}; use fold::{print_fold_bench, score_fold}; -use instrument::{check_batch_matches_single, timed_embed, warn_if_pool_blind}; +use instrument::{check_batch_matches_single, timed_embed, warn_if_pool_blind, SharedEmbedder}; use labels::{lint_labels, Labelled, QuerySet, SimilarSet}; use report::{print_default_report, result_row, Calibration, RowInputs, RunId}; use retrieval::{score_pass, Retrieval}; @@ -171,11 +173,10 @@ fn run() -> Result> { &dense_set, )?; - // Ensure the model is available, then load it. (Provision is idempotent, so an - // already-installed model is a no-op; a missing one is fetched here.) + // Ensure the model is available and load it — once for the whole run: both + // corpora's vaults share this one copy (a missing model is fetched here first). let config = EmbedConfig::load()?; - provision(&config, |line| eprintln!("[init] {line}"))?; - let embedder = LocalEmbedder::load(&config)?; + let embedder = SharedEmbedder::new(load_or_provision(&config)?); let model_id = embedder.model_id().to_string(); let dim = embedder.dim(); eprintln!("[eval] model = {model_id} (dim {dim})\n"); @@ -187,7 +188,7 @@ fn run() -> Result> { // Build a throwaway vault from the corpus. let scratch = ScratchVault::copy_flat(&corpus_dir)?; let vault_root = scratch.root(); - let mut vault = Vault::open_with_embedder(vault_root, Box::new(embedder))?; + let mut vault = Vault::open_with_embedder(vault_root, Box::new(embedder.clone()))?; // ---- Phase 1: projection only → the BM25-only baseline. ------------------ // The vector space does not exist yet, so `search`/`search_chunks` run @@ -278,10 +279,10 @@ fn run() -> Result> { )?; // ---- Phase 3: the dense single-domain fixture (GH #196/#197, Phase 0b). -- - // Its own throwaway vault, its own model load, its own results row (corpus + // Its own throwaway vault (sharing the loaded model), its own results row (corpus // id `dense`) — the fixture measures a *vault-level* geometry, so nothing // about it may share state with the orthogonal corpus's run above. - let dense = score_dense(&evals_dir, &dense_set)?; + let dense = score_dense(&evals_dir, &dense_set, embedder)?; print_dense_report(&dense); print_fold_bench(&dense.fold); append_result(&results_path, &dense_row(run_id, &dense))?; diff --git a/crates/b2-embed/examples/stability.rs b/crates/b2-embed/examples/stability.rs index 8c98b06..923f628 100644 --- a/crates/b2-embed/examples/stability.rs +++ b/crates/b2-embed/examples/stability.rs @@ -51,8 +51,8 @@ mod common; use b2_core::embed::Embedder; use b2_core::vault::{chunk_candidate_pool, note_candidate_pool, ChunkSearchResult, Vault}; -use b2_embed::{provision, EmbedConfig, LocalEmbedder}; -use common::{git_short_sha, has_flag, reject_unknown_flags, truncate}; +use b2_embed::EmbedConfig; +use common::{git_short_sha, has_flag, load_or_provision, reject_unknown_flags, truncate}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::ops::ControlFlow; @@ -173,8 +173,7 @@ fn run() -> Result<(), Box> { let (vault, embedder_label) = if real_model { let config = EmbedConfig::load()?; - provision(&config, |line| eprintln!("[init] {line}"))?; - let embedder = LocalEmbedder::load(&config)?; + let embedder = load_or_provision(&config)?; let label = embedder.model_id().to_string(); (Vault::open_with_embedder(&root, Box::new(embedder))?, label) } else { diff --git a/crates/b2-llm/examples/groundedness.rs b/crates/b2-llm/examples/groundedness.rs index 0afe084..2ec11cb 100644 --- a/crates/b2-llm/examples/groundedness.rs +++ b/crates/b2-llm/examples/groundedness.rs @@ -41,9 +41,9 @@ use b2_core::chat::{cited_markers, ASK_PASSAGES}; use b2_core::embed::Embedder; use b2_core::llm::{LlmProvider, NO_EVIDENCE_ANSWER}; use b2_core::vault::Vault; -use b2_embed::{provision, EmbedConfig, LocalEmbedder}; +use b2_embed::EmbedConfig; use b2_llm::{LlmConfig, OpenAiCompatProvider}; -use common::{append_result, git_short_sha, truncate, ScratchVault}; +use common::{append_result, git_short_sha, load_or_provision, truncate, ScratchVault}; use serde::Deserialize; use std::error::Error; use std::ops::ControlFlow; @@ -128,8 +128,7 @@ fn run() -> Result> { ); let embed_config = EmbedConfig::load()?; - provision(&embed_config, |line| eprintln!("[init] {line}"))?; - let embedder = LocalEmbedder::load(&embed_config)?; + let embedder = load_or_provision(&embed_config)?; let embed_model = embedder.model_id().to_string(); eprintln!("[eval] embedder = {embed_model}");