From bb0503b28caf870560feac0e2cd40032589e68f2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 19:15:54 +0000 Subject: [PATCH 1/6] Add an opt-in relevance filter before generateDiff filterRelevant asks a cheap model, per file, whether that file matters to the task. Only yes files are included when buildEnvironment is given those decisions. The patchling.app playground exposes the pass with a toggle, a decision-model picker, and bundled sample projects. Co-authored-by: ntc-ai --- CHANGELOG.md | 2 + README.md | 36 ++- landing/index.html | 411 ++++++++++++++++++++++++++++++++- landing/lib/env.js | 40 ++++ landing/lib/filter.js | 412 ++++++++++++++++++++++++++++++++++ landing/lib/llm.js | 296 ++++++++++++++++++++++++ landing/lib/text.js | 218 ++++++++++++++++++ landing/relevance.js | 16 ++ package.json | 1 + src/filter.d.ts | 106 +++++++++ src/filter.js | 412 ++++++++++++++++++++++++++++++++++ src/gptdiff.d.ts | 19 +- src/gptdiff.js | 35 ++- src/index.d.ts | 4 + src/index.js | 14 ++ test/filter_relevant.test.js | 352 +++++++++++++++++++++++++++++ test/landing_lib_sync.test.js | 17 ++ test/live-filter.test.js | 46 ++++ 18 files changed, 2427 insertions(+), 10 deletions(-) create mode 100644 landing/lib/env.js create mode 100644 landing/lib/filter.js create mode 100644 landing/lib/llm.js create mode 100644 landing/lib/text.js create mode 100644 landing/relevance.js create mode 100644 src/filter.d.ts create mode 100644 src/filter.js create mode 100644 test/filter_relevant.test.js create mode 100644 test/landing_lib_sync.test.js create mode 100644 test/live-filter.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index e74e037..7444ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### API +- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. Bounded concurrency, a pluggable cache keyed by model + task hash + content hash (in-memory by default), and a skip for binary and oversized files. Ambiguous answers and errors include the file and are reported on the decision. `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. - Renamed the package from `gptdiff-js` to `patchling` (npm name `patchling`; an intermediate publish used `gptdiff`). - `generateDiff` and `smartapply` stream tokens through `opts.onToken`. `generateDiff` reports token usage through `opts.onUsage`. - Ship JSDoc-generated `.d.ts` files, with `types` conditions on the package exports. @@ -26,6 +27,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### Site - `patchling.app` landing page: in-browser playground (canned demo and live NanoGPT), model picker, and Open Graph card. +- The playground can run `filterRelevant` before the diff: a toggle, a decision-model picker, and a per-file yes/no view (reasons, tokens, skips). Bundled sample projects exercise it without leaving the page. - README links that 404'd after the repo rename now point at patchling.app and the Python package. ### CI diff --git a/README.md b/README.md index a89ebc7..18045d7 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Configuration comes from environment variables (Node) or `setEnv(...)` overrides | `GPTDIFF_LLM_API_KEY` | NanoGPT API key (`sk-nano-…`) | — (required for live calls) | | `GPTDIFF_LLM_BASE_URL` | OpenAI-compatible base URL | `https://nano-gpt.com/api/v1/` | | `GPTDIFF_MODEL` | Model id | `xiaomi/mimo-v2.5-pro-ultraspeed` | +| `GPTDIFF_FILTER_MODEL` | Cheap model for `filterRelevant` | `xiaomi/mimo-v2.5-pro-ultraspeed` | ```js import { setEnv } from 'patchling'; @@ -101,6 +102,39 @@ Builds the prompt, calls the LLM, and returns the unified diff extracted from th - `opts.callLlm` — inject a custom/mock completion client (used heavily in tests). - `opts.onUsage` — optional callback invoked with `{ promptTokens, completionTokens, totalTokens }` after the LLM call completes, so callers can surface cost/usage without changing the `Promise` return type. +### `filterRelevant(files, task, opts?) → Promise` +Opt-in pass for a large file map. Each candidate file is sent, in full, with the task to a cheap model that answers `is this file relevant to the task? yes/no`. Only `yes` files should be handed to `generateDiff`. `smartapply` still takes the full map, so files the diff never mentions are left as they were. + +```js +import { filterRelevant, buildEnvironment, generateDiff, smartapply } from 'patchling'; + +const decisions = await filterRelevant(files, task, { + model: 'xiaomi/mimo-v2.5-pro-ultraspeed', // GPTDIFF_FILTER_MODEL, else this cheap default + concurrency: 8, // max in-flight calls + // cache: null, // default is an in-memory cache + // maxBytes: 262144, // skip larger files; null disables + onDecision(d) { console.log(d.path, d.decision); }, +}); +const diff = await generateDiff(buildEnvironment(files, { decisions }), task, { model }); +const next = await smartapply(diff, files); +``` + +- `opts.model`, `opts.apiKey`, `opts.baseUrl` — same key and base URL resolution as `generateDiff` (`GPTDIFF_LLM_API_KEY`, `GPTDIFF_LLM_BASE_URL`). +- `opts.concurrency` — at most this many classifications in flight (default 8). +- `opts.cache` — `get(key)` / `set(key, value)`, sync or async. The key is `model`, SHA-256 of the task, and SHA-256 of the contents, separated by NUL. Omit it for a process-wide in-memory cache (`createMemoryCache()` builds another). `null` turns caching off. Definitive yes/no answers are stored; errors and ambiguous answers are not. +- `opts.maxBytes` — UTF-8 size cap (default 262144). Larger files, and files that look binary (a NUL, or a dense run of control bytes), are not sent. +- `opts.onDecision` — called as each file finishes, in completion order. The returned array stays in input order. +- `opts.callLlm` — inject a mock client. + +Each decision is `{ path, decision, raw, tokens, cached, skippedReason, error, reason }`. + +- `decision` is `"yes"`, `"no"`, or `"skip"`. +- `skippedReason` is `"binary"`, `"too-large"`, or `null`. +- Ambiguous text (after `` blocks are stripped) and a thrown call become `"yes"` with `error` set, so a bad answer cannot drop a file. `error` is `"ambiguous answer"` or the thrown message. +- `buildEnvironment(files, { decisions })` keeps only `"yes"` paths. Passing the decisions array itself does the same. With no second argument, every file is included. + +Python (`filter_relevant` in [patchling-py](https://github.com/255BITS/patchling-py)) takes the same arguments in snake_case: `max_bytes`, `on_decision`, and `skipped_reason` on each decision. `build_environment(files, decisions=decisions)` keeps the `"yes"` files. + ### `smartapply(diffText, files, opts?) → Promise<{ path: content }>` Applies a diff to an in-memory file map with per-file, LLM-assisted conflict resolution (runs files concurrently). Handles creation, modification, and deletion; `…` and reasoning preambles are stripped automatically. Returns a **new** map; deleted files are omitted. @@ -112,7 +146,7 @@ Applies a diff to an in-memory file map with per-file, LLM-assisted conflict res Deterministic, no-LLM patch application (the strict "basic" applier). `changed` is `true` iff something was created, modified, or deleted; `files` is the updated map (equal to the input on failure — no partial writes). ### Other exports -`parseDiffPerFile`, `buildEnvironment`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. +`parseDiffPerFile`, `buildEnvironment`, `filterRelevant`, `createMemoryCache`, `parseRelevanceAnswer`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. ## Differences from the Python package diff --git a/landing/index.html b/landing/index.html index 89804da..9a1596e 100644 --- a/landing/index.html +++ b/landing/index.html @@ -240,6 +240,34 @@ #modellist .price { color: var(--muted); white-space: nowrap; } #modellist .divider { font-size: .68rem; color: var(--muted); text-align: center; margin: .35rem 0; } + .filterbar { display: flex; flex-wrap: wrap; gap: .45rem .8rem; align-items: center; margin-top: .65rem; font-size: .82rem; } + .filterbar label { display: inline-flex; gap: .4rem; align-items: center; font-weight: 600; cursor: pointer; } + .filterbar .hint { color: var(--muted); font-size: .75rem; } + #filtermodelrow { margin-top: .15rem; } + #filtermodelpicker { display: none; margin-top: .45rem; border: 1px solid var(--line); border-radius: .55rem; padding: .6rem; background: var(--panel2); } + #filtermodelsearch { width: 100%; font: inherit; font-size: .82rem; padding: .4rem .6rem; border-radius: .45rem; border: 1px solid var(--line); background: var(--panel); color: var(--ink); } + #filtermodelsearch:focus { outline: 2px solid var(--accent-soft); border-color: var(--accent); } + #filtermodellist { max-height: 200px; overflow: auto; margin-top: .4rem; } + #filtermodellist button { + display: flex; justify-content: space-between; gap: 1rem; width: 100%; text-align: left; + font: inherit; font-size: .78rem; padding: .3rem .5rem; border: 0; border-radius: .35rem; + background: none; color: var(--ink); cursor: pointer; + } + #filtermodellist button:hover, #filtermodellist button.sel { background: var(--accent-soft); } + #filtermodellist button.sel { font-weight: 700; } + #filtermodellist .price { color: var(--muted); white-space: nowrap; } + #decisionwrap { display: none; margin-top: .7rem; } + #decisionwrap .dlabel { font-size: .72rem; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); font-weight: 700; display: block; margin-bottom: .15rem; } + #decisionwrap .dlabel span { text-transform: none; letter-spacing: 0; font-weight: 400; } + .drow { display: flex; flex-wrap: wrap; gap: .28rem .55rem; align-items: baseline; padding: .32rem 0; border-bottom: 1px solid var(--line); font-size: .78rem; } + .drow .why { color: var(--muted); flex: 1 1 12rem; } + .badge.yes { background: var(--add-bg); color: var(--add); } + .badge.no { background: var(--del-bg); color: var(--del); } + .badge.skip { background: var(--code-bg); color: var(--muted); } + button[data-sample] { white-space: normal; text-align: left; line-height: 1.35; width: 100%; justify-content: flex-start; } + button[data-sample].on { border-color: var(--accent); color: var(--accent); background: var(--accent-soft); } + #usage pre { white-space: pre-wrap; } + /* ---------- built with it ---------- */ .gallery { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: .9rem; margin-top: 1.5rem; } .gcard { @@ -320,6 +348,7 @@ Demo + Samples Built with it How it works Any model @@ -352,7 +381,26 @@

Give your app a “describe the change” box.

+
+ + Before the diff, a cheap model answers yes or no for each file. Binary and oversized files are skipped. +
+ +
+ +
+

· Enter uses the exact text

+

◐ working…

+
+ which files are relevant +
+

Demo mode replays a pre-recorded reply, so the files and goal were reset to match. Use your own key below to transform your own edits.

the diff the model wrote @@ -482,6 +530,89 @@

smartapply(diff, files)

+
+

Sample projects

+

A bigger directory, a smaller prompt.

+

These three projects are bundled in this page — nothing to fetch. Each one has a suggested task and a recorded relevance pass, so you can see yes, no, and skipped files before the diff runs. More projects are landing in patchling-examples (live demos).

+
+
+

Greeting app

+

Six files. The task only changes the greeting. History is over the sample size cap; the logo is binary.

+ greet.py · main.py · math_utils.py · README.md · assets/logo.bin · vendor/HISTORY.txt +

+
+
+

Small website

+

The colors live in one stylesheet. The script and the license are noise.

+ index.html · styles.css · app.js · LICENSE +

+
+
+

Lab character

+

Mood and memory are the scene. A menu and a grocery list are not.

+ soul.md · mood.md · memory.md · menu.md · todo.txt +

+
+
+
+ +
+

Usage

+

The same filter in JavaScript and Python.

+

Only the yes files are passed to generateDiff. smartapply still receives the full map, so everything the diff never mentions stays put. Ambiguous answers and errors include the file and show up on the decision.

+
+
+
filter.js
+ 
import { filterRelevant, buildEnvironment, generateDiff, smartapply } from "patchling";
+
+const decisions = await filterRelevant(files, task, {
+  model: "xiaomi/mimo-v2.5-pro-ultraspeed",
+  concurrency: 8,
+  onDecision(d) { console.log(d.path, d.decision, d.reason || d.skippedReason || d.error); },
+});
+
+const diff = await generateDiff(buildEnvironment(files, { decisions }), task, { apiKey, model });
+const next = await smartapply(diff, files, { apiKey, model });
+
+
+
filter.py
+ 
from patchling import filter_relevant, build_environment, generate_diff, smartapply
+
+decisions = filter_relevant(
+    files,
+    task,
+    model="xiaomi/mimo-v2.5-pro-ultraspeed",
+    concurrency=8,
+)
+
+env = build_environment(files, decisions=decisions)
+diff = generate_diff(env, task)
+updated = smartapply(diff, files)
+
+
+

Each decision is { path, decision, raw, tokens, cached, skippedReason, error, reason }. decision is yes, no, or skip. skippedReason is binary or too-large. The cache key is the model, a hash of the task, and a hash of the file contents. Python uses the same fields in snake_case (skipped_reason, max_bytes, on_decision).

+
+

Why it doesn't flake

Why diffs — and why yours will apply.

@@ -605,6 +736,7 @@

Python

from "https://esm.sh/patchling"; import { registerClient, beginSignIn, completeSignIn } from "https://esm.sh/patchling/oauth"; +import { filterRelevant, DEFAULT_FILTER_MODEL } from "./relevance.js"; const $ = (id) => document.getElementById(id); const DEFAULT_MODEL = "zai-org/glm-5.2:thinking"; @@ -662,6 +794,10 @@

Python

}, preview: (files) => `${files["scene.svg"] || ""}`, + decisions: { + "scene.svg": { decision: "yes", reason: "the picture the sentence restyles", tokens: 44, raw: "yes — the picture the sentence restyles" }, + "style.css": { decision: "yes", reason: "the colors the sentence restyles", tokens: 39, raw: "yes — the colors the sentence restyles" }, + }, }, character: { label: "Direct a character", @@ -691,6 +827,11 @@

Python

p{margin:.4em 0}.dir{color:#93a39d;font-style:italic;text-align:center;font-size:12px}.you{text-align:right;color:#9ecbff}.bot{color:#e4ece9} mood: ${esc(mood)}${rows}`; }, + decisions: { + "soul.md": { decision: "yes", reason: "who is speaking", tokens: 28, raw: "yes — who is speaking" }, + "mood.md": { decision: "yes", reason: "the beat the task changes", tokens: 31, raw: "yes — the beat the task changes" }, + "chat.md": { decision: "yes", reason: "the scene continues here", tokens: 33, raw: "yes — the scene continues here" }, + }, }, comic: { label: "Add a comic panel", @@ -726,6 +867,11 @@

Python

.sc{margin:2px 0;color:#b8c0c8;font-style:italic}.dl{margin:0;font-weight:700}

${esc(title)}

${cards}`; }, + decisions: { + "config.json": { decision: "yes", reason: "page title and cast", tokens: 22, raw: "yes — page title and cast" }, + "panels/1.json": { decision: "no", reason: "the opening panel stays", tokens: 24, raw: "no — the opening panel stays" }, + "panels/2.json": { decision: "yes", reason: "the beat the cliffhanger follows", tokens: 30, raw: "yes — the beat the cliffhanger follows" }, + }, }, goodbye: { label: "And yes — code", @@ -740,6 +886,91 @@

${esc(title)}

${cards}`; applied: { "main.py": 'from greet import farewell\n\ndef main():\n farewell("World")\n\nif __name__ == "__main__":\n main()\n', }, + decisions: { + "greet.py": { decision: "yes", reason: "defines the greeting", tokens: 26, raw: "yes — defines the greeting" }, + "main.py": { decision: "yes", reason: "calls the greeting", tokens: 24, raw: "yes — calls the greeting" }, + }, + }, + "proj-greet": { + sample: true, + filterDefault: true, + maxBytes: 400, + label: "Greeting app", + desc: "Six files, one greeting. The recorded pass keeps greet.py, drops the helpers, and skips a binary logo and an oversized history (this sample caps files at 400 bytes).", + goal: "Say goodbye instead of hello", + files: { + "greet.py": 'def greet(name):\n print("Hello, " + name + "!")\n', + "main.py": 'from greet import greet\n\ndef main():\n greet("World")\n', + "math_utils.py": 'def add(a, b):\n return a + b\n', + "README.md": 'A tiny greeting program. The words live in greet.py.\n', + "assets/logo.bin": "\u0000PNG logo", + "vendor/HISTORY.txt": "Internal release notes. Not part of the greeting.\n".repeat(12), + }, + decisions: { + "greet.py": { decision: "yes", reason: "prints the greeting the task changes", tokens: 48, raw: "yes — prints the greeting the task changes" }, + "main.py": { decision: "no", reason: "calls greet, but the words are not here", tokens: 41, raw: "no — calls greet, but the words are not here" }, + "math_utils.py": { decision: "no", reason: "arithmetic helper", tokens: 22, raw: "no — arithmetic helper" }, + "README.md": { decision: "no", reason: "docs, not the string to edit", tokens: 27, raw: "no — docs, not the string to edit" }, + "assets/logo.bin": { decision: "skip", skippedReason: "binary", tokens: null, raw: null }, + "vendor/HISTORY.txt": { decision: "skip", skippedReason: "too-large", tokens: null, raw: null }, + }, + response: 'The greeting string is only in greet.py.\n\n```diff\n--- a/greet.py\n+++ b/greet.py\n@@ -1,2 +1,2 @@\n def greet(name):\n- print("Hello, " + name + "!")\n+ print("Goodbye, " + name + "!")\n```\n', + applied: {}, + }, + "proj-page": { + sample: true, + filterDefault: true, + label: "Small website", + desc: "Four files. Navy and the title size live in styles.css. The markup, the boot script, and the license stay out of the diff.", + goal: "Make the page background navy and the title larger", + files: { + "index.html": '\n\nWelcome\n\n

Welcome

\n

A small page.

\n\n\n', + "styles.css": 'body { background: #ffffff; color: #222222; font-family: sans-serif; }\nh1 { font-size: 16px; }\n', + "app.js": 'console.log("ready");\n', + "LICENSE": 'MIT. Copyright the author.\n', + }, + decisions: { + "index.html": { decision: "no", reason: "markup does not set the color or the size", tokens: 38, raw: "no — markup does not set the color or the size" }, + "styles.css": { decision: "yes", reason: "background and title size", tokens: 29, raw: "yes — background and title size" }, + "app.js": { decision: "no", reason: "boot log", tokens: 14, raw: "no — boot log" }, + "LICENSE": { decision: "no", reason: "license text", tokens: 12, raw: "no — license text" }, + }, + response: 'Only the stylesheet sets the background and the title size.\n\n```diff\n--- a/styles.css\n+++ b/styles.css\n@@ -1,2 +1,2 @@\n-body { background: #ffffff; color: #222222; font-family: sans-serif; }\n-h1 { font-size: 16px; }\n+body { background: #0b1026; color: #e8eefc; font-family: sans-serif; }\n+h1 { font-size: 28px; }\n```\n', + applied: {}, + previewHeight: 180, + preview: (files) => (files["index.html"] || "").replace("", ``), + }, + "proj-lab": { + sample: true, + filterDefault: true, + label: "Lab character", + desc: "The power cut belongs in mood and memory. The soul, the menu, and the grocery list are not that scene.", + goal: "The lab loses power", + files: { + "soul.md": "MARVIS — a sardonic lab assistant. Loyal, theatrical, allergic to being thanked.\n", + "mood.md": "smug - the fusion demo went perfectly.\n", + "memory.md": "The fusion demo held for nine minutes.\n", + "menu.md": "Tuesday special: mushroom risotto.\n", + "todo.txt": "- oat milk\n- batteries\n", + }, + decisions: { + "soul.md": { decision: "no", reason: "personality is unchanged", tokens: 24, raw: "no — personality is unchanged" }, + "mood.md": { decision: "yes", reason: "the beat the task changes", tokens: 30, raw: "yes — the beat the task changes" }, + "memory.md": { decision: "yes", reason: "what just happened", tokens: 26, raw: "yes — what just happened" }, + "menu.md": { decision: "no", reason: "a restaurant menu", tokens: 18, raw: "no — a restaurant menu" }, + "todo.txt": { decision: "no", reason: "a grocery list", tokens: 16, raw: "no — a grocery list" }, + }, + response: 'The power cut lands in mood and memory. The rest of the folder is someone else\'s day.\n\n```diff\n--- a/mood.md\n+++ b/mood.md\n@@ -1 +1 @@\n-smug - the fusion demo went perfectly.\n+rattled - on backup batteries, pretending otherwise.\n--- a/memory.md\n+++ b/memory.md\n@@ -1 +1,2 @@\n The fusion demo held for nine minutes.\n+Then the lights died.\n```\n', + applied: {}, + previewHeight: 220, + preview: (files) => { + const mood = ((files["mood.md"] || "").split("\n")[0] || "").trim(); + const memory = (files["memory.md"] || "").trim().split("\n").filter(Boolean) + .map((l) => `

${esc(l)}

`).join(""); + return `mood: ${esc(mood)}${memory}`; + }, }, }; /* SCENARIOS:END */ @@ -807,6 +1038,7 @@

${esc(title)}

${cards}`; computeTop(); allModels.sort((a, b) => (rank(a.id) - rank(b.id)) || a.id.localeCompare(b.id)); renderModelRow(); + renderFilterModelRow(); return allModels; } function modelButton(m) { @@ -850,6 +1082,65 @@

${esc(title)}

${cards}`; }); renderModelRow(); +/* ---------- decision model (cheap yes/no pass) ---------- */ +let FILTER_MODEL = localStorage.getItem("pl_filter_model") || DEFAULT_FILTER_MODEL; +function setFilterModel(id) { + FILTER_MODEL = id; + localStorage.setItem("pl_filter_model", id); + renderFilterModelRow(); +} +function renderFilterModelRow() { + const m = priceOf(FILTER_MODEL) || {}; + $("filtermodelname").textContent = FILTER_MODEL; + $("filtermodelprice").textContent = m.prompt != null ? `(${px(m.prompt)} / ${px(m.completion)} per 1M)` : ""; +} +function filterModelButton(m) { + const b = document.createElement("button"); + b.type = "button"; + if (m.id === FILTER_MODEL) b.className = "sel"; + const name = document.createElement("span"); name.className = "mono"; name.textContent = m.id; + const price = document.createElement("span"); price.className = "price mono"; + price.textContent = m.prompt != null ? `${px(m.prompt)} / ${px(m.completion)}` : "—"; + b.append(name, price); + b.addEventListener("click", () => { setFilterModel(m.id); $("filtermodelpicker").style.display = "none"; }); + return b; +} +function renderFilterModelList() { + const q = $("filtermodelsearch").value.trim().toLowerCase(); + const matches = (allModels || []).filter((m) => m.id.toLowerCase().includes(q)); + $("filtermodellist").innerHTML = ""; + for (const m of matches) $("filtermodellist").appendChild(filterModelButton(m)); + $("filtermodelcount").textContent = `${matches.length} match${matches.length === 1 ? "" : "es"}`; +} +function renderFilterChrome() { + const on = $("filteron").checked; + $("filtermodelrow").style.display = on ? "flex" : "none"; + if (!on) $("filtermodelpicker").style.display = "none"; + localStorage.setItem("pl_filter", on ? "1" : "0"); +} +$("filteron").checked = localStorage.getItem("pl_filter") === "1"; +$("filteron").addEventListener("change", renderFilterChrome); +$("filtermodelchange").addEventListener("click", async () => { + const open = $("filtermodelpicker").style.display === "block"; + $("filtermodelpicker").style.display = open ? "none" : "block"; + if (!open) { + $("filtermodelsearch").value = ""; + $("filtermodellist").innerHTML = "
loading…
"; + await loadModels(); + renderFilterModelList(); + $("filtermodelsearch").focus(); + } +}); +$("filtermodelsearch").addEventListener("input", renderFilterModelList); +$("filtermodelsearch").addEventListener("keydown", (e) => { + if (e.key === "Enter") { + const v = $("filtermodelsearch").value.trim(); + if (v) { setFilterModel(v); $("filtermodelpicker").style.display = "none"; } + } +}); +renderFilterModelRow(); +renderFilterChrome(); + /* ---------- OAuth (Sign in with NanoGPT, PKCE) ---------- */ // NanoGPT canonicalizes the loopback host to "localhost"; hop there so the // redirect returns to the same origin. @@ -916,6 +1207,7 @@

${esc(title)}

${cards}`; const wrap = $("scenarios"); wrap.innerHTML = ""; for (const [key, s] of Object.entries(SCENARIOS)) { + if (s.sample) continue; const b = document.createElement("button"); b.textContent = s.label; b.classList.toggle("on", key === scenarioKey); @@ -927,14 +1219,26 @@

${esc(title)}

${cards}`; function selectScenario(key) { scenarioKey = key; - files = { ...SCENARIOS[key].files }; - $("goal").value = SCENARIOS[key].goal; + const scn = SCENARIOS[key]; + files = { ...scn.files }; + $("goal").value = scn.goal; + if (scn.filterDefault) { + $("filteron").checked = true; + renderFilterChrome(); + } renderScenarios(); + renderSampleSelection(); renderEditors(); renderPreview(); clearOutputs(); } +function renderSampleSelection() { + for (const btn of document.querySelectorAll("[data-sample]")) { + btn.classList.toggle("on", btn.dataset.sample === scenarioKey); + } +} + function renderEditors() { const wrap = $("editors"); wrap.innerHTML = ""; @@ -967,6 +1271,8 @@

${esc(title)}

${cards}`; $("diffwrap").style.display = "none"; $("diffpane").textContent = ""; $("diffsrc").textContent = ""; + $("decisionwrap").style.display = "none"; + $("decisionlist").innerHTML = ""; $("results").innerHTML = ""; $("verdict").style.display = "none"; $("verdict").innerHTML = ""; @@ -1124,13 +1430,15 @@

${esc(title)}

${cards}`; running = true; renderRun(); $("upsell").style.display = "none"; - $("diffsrc").textContent = mode === "demo" ? "· pre-recorded reply, real engine" : "· live from NanoGPT"; $("diffpane").textContent = ""; - $("diffwrap").style.display = "block"; + $("diffwrap").style.display = "none"; + $("decisionlist").innerHTML = ""; + $("decisionwrap").style.display = "none"; $("results").innerHTML = ""; $("status").style.display = "block"; document.querySelector("#status .spin").style.display = ""; - $("phase").textContent = "generating diff…"; + const filtering = $("filteron").checked; + $("phase").textContent = filtering ? "checking which files are relevant…" : "generating diff…"; const diffPane = $("diffpane"); const onDiffToken = (t) => { @@ -1140,11 +1448,27 @@

${esc(title)}

${cards}`; try { const key = getKey(); + let decisions = null; + if (filtering) { + $("decisionwrap").style.display = "block"; + $("filtersrc").textContent = mode === "demo" ? "· pre-recorded decisions" : "· live, cheap model"; + decisions = await collectDecisions(inputFiles, goal); + if (!decisions.some((d) => d.decision === "yes")) { + $("phase").textContent = "nothing relevant — diff skipped"; + document.querySelector("#status .spin").style.display = "none"; + resultMsg("No file was marked relevant, so generateDiff did not run."); + return; + } + } + + $("phase").textContent = "generating diff…"; + $("diffsrc").textContent = mode === "demo" ? "· pre-recorded reply, real engine" : "· live from NanoGPT"; + $("diffwrap").style.display = "block"; const genOpts = mode === "demo" ? { callLlm: makeFakeCallLlm(scn), onToken: onDiffToken } : { apiKey: key, model: MODEL, onToken: onDiffToken }; - const diff = await generateDiff(buildEnvironment(inputFiles), goal, genOpts); + const diff = await generateDiff(buildEnvironment(filesForDiff(inputFiles, decisions)), goal, genOpts); renderDiff(diff); const perFile = parseDiffPerFile(diff); @@ -1225,6 +1549,81 @@

${esc(title)}

${cards}`; } } +function filesForDiff(map, decisions) { + if (!decisions) return map; + const yes = new Set(decisions.filter((d) => d.decision === "yes").map((d) => d.path)); + const out = {}; + for (const [path, content] of Object.entries(map)) { + if (yes.has(path)) out[path] = content; + } + return out; +} + +function renderDecision(d) { + const row = document.createElement("div"); + row.className = "drow"; + const badge = document.createElement("span"); + badge.className = "badge " + (d.decision === "yes" ? "yes" : d.decision === "no" ? "no" : "skip"); + badge.textContent = d.decision === "skip" ? "skipped" : d.decision; + const path = document.createElement("span"); + path.className = "mono"; + path.textContent = d.path; + const why = document.createElement("span"); + why.className = "why"; + const bits = []; + if (d.skippedReason) bits.push(d.skippedReason === "too-large" ? "too large" : d.skippedReason); + if (d.reason) bits.push(d.reason); + if (d.error) bits.push(d.error); + if (d.cached) bits.push("cached"); + if (d.tokens != null) bits.push(d.tokens + " tokens"); + why.textContent = bits.join(" · "); + row.append(badge, path, why); + $("decisionlist").appendChild(row); +} + +async function collectDecisions(inputFiles, goal) { + if (mode === "demo") { + const canned = SCENARIOS[scenarioKey].decisions || {}; + const out = []; + for (const path of Object.keys(inputFiles)) { + const src = canned[path] || { + decision: "yes", + reason: "included — this demo has no recorded decision for the file", + tokens: 0, + raw: "yes", + }; + const d = { + path, + decision: src.decision, + raw: src.raw ?? null, + tokens: src.tokens ?? null, + cached: false, + skippedReason: src.skippedReason ?? null, + error: src.error ?? null, + reason: src.reason ?? null, + }; + out.push(d); + renderDecision(d); + await sleep(50); + } + return out; + } + $("decisionlist").innerHTML = ""; + return filterRelevant(inputFiles, goal, { + apiKey: getKey(), + model: FILTER_MODEL, + maxBytes: SCENARIOS[scenarioKey].maxBytes, + onDecision: renderDecision, + }); +} + +for (const btn of document.querySelectorAll("[data-sample]")) { + btn.addEventListener("click", () => { + selectScenario(btn.dataset.sample); + $("playground").scrollIntoView({ behavior: "smooth", block: "nearest" }); + }); +} + $("run").addEventListener("click", run); $("goal").addEventListener("keydown", (e) => { if (e.key === "Enter" && !$("run").disabled) { e.preventDefault(); run(); } diff --git a/landing/lib/env.js b/landing/lib/env.js new file mode 100644 index 0000000..2728268 --- /dev/null +++ b/landing/lib/env.js @@ -0,0 +1,40 @@ +/** + * Environment access that works in both Node and the browser. + * + * In Node, reads from `process.env`. In the browser there is no `process`, so + * callers can register overrides (e.g. values obtained via OAuth sign-in) + * through `setEnv`, which are consulted first. + */ + +const overrides = Object.create(null); + +/** + * Set an environment-style override (used in the browser where there is no + * `process.env`). Pass `undefined` to clear an override. + * @param {string} name + * @param {string | undefined} value + */ +export function setEnv(name, value) { + if (value === undefined) { + delete overrides[name]; + } else { + overrides[name] = value; + } +} + +/** + * Read an environment variable. Overrides take precedence over `process.env`. + * @param {string} name + * @param {string} [fallback] + * @returns {string | undefined} + */ +export function getEnv(name, fallback = undefined) { + if (name in overrides) return overrides[name]; + if (typeof process !== 'undefined' && process.env && name in process.env) { + return process.env[name]; + } + return fallback; +} + +export const DEFAULT_MODEL = 'xiaomi/mimo-v2.5-pro-ultraspeed'; +export const DEFAULT_BASE_URL = 'https://nano-gpt.com/api/v1/'; diff --git a/landing/lib/filter.js b/landing/lib/filter.js new file mode 100644 index 0000000..a88b17d --- /dev/null +++ b/landing/lib/filter.js @@ -0,0 +1,412 @@ +/** + * Opt-in relevance filter for large file maps. + * + * For each candidate file, a cheap model sees the full contents plus the task + * and answers "is this file relevant to the task? yes/no". Only `yes` + * decisions should be passed to `generateDiff` (see `buildEnvironment`'s + * `decisions` option). `smartapply` still wants the full map so files the + * diff never mentions stay put. + * + * Mirrors the Python package's `filter_relevant` (255BITS/patchling-py): + * + * filter_relevant(files, task, *, model=None, concurrency=8, cache=None, + * max_bytes=262144, on_decision=None, api_key=None, + * base_url=None) -> list[decision] + * + * decision: path, decision ("yes" | "no" | "skip"), raw, tokens, cached, + * skipped_reason ("binary" | "too-large" | None), error, reason + * + * JS uses camelCase (`maxBytes`, `onDecision`, `skippedReason`) like the rest + * of this package. Ambiguous answers and per-file errors include the file + * (`decision: "yes"`) and surface the problem on `error` instead of throwing. + */ + +import { stripThinkTags } from './text.js'; +import { callLlm as defaultCallLlm, resolveApiKey, resolveBaseUrl } from './llm.js'; +import { getEnv } from './env.js'; + +/** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ +export const DEFAULT_FILTER_MODEL = 'xiaomi/mimo-v2.5-pro-ultraspeed'; + +/** Files larger than this (UTF-8 bytes) are skipped unless `maxBytes` is set. */ +export const DEFAULT_FILTER_MAX_BYTES = 262144; + +/** In-flight classifications per call when `concurrency` is omitted. */ +export const DEFAULT_FILTER_CONCURRENCY = 8; + +const FILTER_SYSTEM_PROMPT = `You decide if one file is relevant to a coding task. Reply with "yes" or "no" as the first word. You may add one short reason after it. Do not quote the file.`; + +const AMBIGUOUS = 'ambiguous answer'; + +/** + * @typedef {object} RelevanceDecision + * @property {string} path + * @property {'yes' | 'no' | 'skip'} decision `yes` files go on to generateDiff. + * Ambiguous answers and errors are `yes` so a bad classifier cannot drop a + * file. `skip` files were not sent to the model. + * @property {string | null} raw model text, unchanged (think tags included) + * @property {number | null} tokens total tokens for the call; null when skipped or the call failed + * @property {boolean} cached true when this answer was read from the cache + * @property {'binary' | 'too-large' | null} skippedReason + * @property {string | null} error set for thrown calls and ambiguous answers + * @property {string | null} reason short reason after the yes/no, when the model gave one + */ + +/** + * @typedef {object} RelevanceCacheEntry + * @property {'yes' | 'no'} decision + * @property {string | null} raw + * @property {number | null} tokens + * @property {string | null} reason + */ + +/** + * A cache is any object with `get(key)` and `set(key, value)`. Either method + * may be sync or async. Keys are `${model}\\0${taskHash}\\0${contentHash}` + * (SHA-256 hex). Pass `cache: null` to disable caching. + * + * @typedef {object} RelevanceCache + * @property {(key: string) => RelevanceCacheEntry | undefined | Promise} get + * @property {(key: string, value: RelevanceCacheEntry) => void | Promise} set + */ + +let sharedMemoryCache; + +/** + * Process-wide in-memory cache used when `filterRelevant` is called without + * a `cache` option. Pass the result as `cache` to share or inspect it. + * @returns {RelevanceCache} + */ +export function createMemoryCache() { + /** @type {Map} */ + const store = new Map(); + return { + get(key) { + return store.has(key) ? store.get(key) : undefined; + }, + set(key, value) { + store.set(key, value); + }, + }; +} + +function memoryCache() { + if (!sharedMemoryCache) sharedMemoryCache = createMemoryCache(); + return sharedMemoryCache; +} + +/** + * @param {string} text + * @returns {Promise} hex SHA-256 + */ +async function sha256(text) { + const data = new TextEncoder().encode(text); + const digest = await crypto.subtle.digest('SHA-256', data); + let hex = ''; + for (const byte of new Uint8Array(digest)) hex += byte.toString(16).padStart(2, '0'); + return hex; +} + +/** + * @param {string} model + * @param {string} taskHash + * @param {string} contentHash + * @returns {string} + */ +function cacheKey(model, taskHash, contentHash) { + return `${model}\0${taskHash}\0${contentHash}`; +} + +function resolveFilterModel(model) { + return model || getEnv('GPTDIFF_FILTER_MODEL') || DEFAULT_FILTER_MODEL; +} + +/** + * @param {unknown} content + * @returns {boolean} + */ +function looksBinary(content) { + if (typeof content !== 'string') return true; + const n = Math.min(content.length, 8192); + let controls = 0; + for (let i = 0; i < n; i++) { + const c = content.charCodeAt(i); + if (c === 0) return true; + // C0 controls other than tab, LF, and CR. A dense run is not text. + if (c < 32 && c !== 9 && c !== 10 && c !== 13) controls++; + } + return n > 0 && controls / n > 0.3; +} + +/** + * UTF-8 byte length, stopping once `maxBytes` is exceeded. + * @param {string} content + * @param {number} maxBytes + * @returns {boolean} + */ +function exceedsMaxBytes(content, maxBytes) { + if (!Number.isFinite(maxBytes)) return false; + let bytes = 0; + for (let i = 0; i < content.length; i++) { + const c = content.charCodeAt(i); + if (c < 0x80) bytes += 1; + else if (c < 0x800) bytes += 2; + else if (c >= 0xd800 && c <= 0xdbff) { + bytes += 4; + i += 1; + } else bytes += 3; + if (bytes > maxBytes) return true; + } + return false; +} + +/** + * @param {unknown} response + * @returns {number} + */ +function totalTokens(response) { + const usage = response && /** @type {{ usage?: Record }} */ (response).usage; + if (!usage) return 0; + const prompt = usage.prompt_tokens ?? usage.input_tokens ?? 0; + const completion = usage.completion_tokens ?? usage.output_tokens ?? 0; + return usage.total_tokens ?? prompt + completion; +} + +/** + * Parse a yes/no classification. Think tags and "> Reasoning" preambles are + * removed first. The first word wins when it is yes or no. If both words + * appear, or neither does, the answer is ambiguous and the caller includes + * the file. + * + * @param {unknown} raw + * @returns {{ decision: 'yes' | 'no', reason: string | null, ambiguous: boolean }} + */ +export function parseRelevanceAnswer(raw) { + const stripped = stripThinkTags(raw == null ? '' : String(raw)).trim(); + const unfenced = stripped.replace(/^```[a-z0-9]*\s*/i, '').replace(/```\s*$/i, '').trim(); + const lines = unfenced.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); + const first = (lines[0] || '').replace(/^[*_>"'`]+/, '').replace(/[*_"`']+$/g, '').trim(); + const lead = first.match(/^(yes|no)\b[\s.:,\-—!]*(.*)$/i); + + if (lead) { + const decision = /** @type {'yes' | 'no'} */ (lead[1].toLowerCase()); + const sameLine = lead[2].trim(); + const reason = [sameLine, lines.slice(1).join(' ')].filter(Boolean).join(' ').trim() || null; + const firstHasYes = /\byes\b/i.test(first); + const firstHasNo = /\bno\b/i.test(first); + if (firstHasYes && firstHasNo) { + return { decision: 'yes', reason: unfenced || null, ambiguous: true }; + } + return { decision, reason, ambiguous: false }; + } + + const hasYes = /\byes\b/i.test(unfenced); + const hasNo = /\bno\b/i.test(unfenced); + if (hasYes && !hasNo) return { decision: 'yes', reason: unfenced || null, ambiguous: false }; + if (hasNo && !hasYes) return { decision: 'no', reason: unfenced || null, ambiguous: false }; + return { decision: 'yes', reason: unfenced || null, ambiguous: true }; +} + +/** + * @param {Array<[string, string]>} items + * @param {number} limit + * @param {(item: [string, string], index: number) => Promise} fn + * @returns {Promise} + */ +async function mapLimit(items, limit, fn) { + /** @type {RelevanceDecision[]} */ + const results = new Array(items.length); + if (!items.length) return results; + let cursor = 0; + const workers = Math.min(limit, items.length); + const run = async () => { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + results[index] = await fn(items[index], index); + } + }; + await Promise.all(Array.from({ length: workers }, () => run())); + return results; +} + +/** + * @param {object} args + * @param {string} args.path + * @param {unknown} args.content + * @param {string} args.task + * @param {string} args.taskHash + * @param {string} args.model + * @param {RelevanceCache | null} args.store + * @param {number} args.maxBytes + * @param {string | undefined} args.apiKey + * @param {string | undefined} args.baseUrl + * @param {Function} args.callLlm + * @param {number} args.maxTokens + * @param {number} args.temperature + * @returns {Promise} + */ +async function classifyFile(args) { + const { path, content, task, taskHash, model, store, maxBytes, apiKey, baseUrl, callLlm, maxTokens, temperature } = args; + + if (looksBinary(content)) { + return { + path, + decision: 'skip', + raw: null, + tokens: null, + cached: false, + skippedReason: 'binary', + error: null, + reason: null, + }; + } + + const text = /** @type {string} */ (content); + if (exceedsMaxBytes(text, maxBytes)) { + return { + path, + decision: 'skip', + raw: null, + tokens: null, + cached: false, + skippedReason: 'too-large', + error: null, + reason: null, + }; + } + + const contentHash = await sha256(text); + const key = cacheKey(model, taskHash, contentHash); + if (store) { + const hit = await store.get(key); + if (hit && (hit.decision === 'yes' || hit.decision === 'no')) { + return { + path, + decision: hit.decision, + raw: hit.raw ?? null, + tokens: hit.tokens ?? null, + cached: true, + skippedReason: null, + error: null, + reason: hit.reason ?? null, + }; + } + } + + const userContent = `Task:\n${task}\n\nFile: ${path}\n${text}\n\nis this file relevant to the task? yes/no`; + try { + const response = await callLlm({ + apiKey: resolveApiKey(apiKey), + baseUrl: resolveBaseUrl(baseUrl), + model, + messages: [ + { role: 'system', content: FILTER_SYSTEM_PROMPT }, + { role: 'user', content: userContent }, + ], + maxTokens, + temperature, + }); + const raw = response?.choices?.[0]?.message?.content ?? ''; + const parsed = parseRelevanceAnswer(raw); + /** @type {RelevanceDecision} */ + const decision = { + path, + decision: parsed.decision, + raw: raw == null ? '' : String(raw), + tokens: totalTokens(response), + cached: false, + skippedReason: null, + error: parsed.ambiguous ? AMBIGUOUS : null, + reason: parsed.ambiguous ? (parsed.reason || null) : parsed.reason, + }; + // Keep failures and ambiguous answers out of the cache so a later call + // can try again. Definitive yes/no answers are stable for this content. + if (store && !parsed.ambiguous) { + await store.set(key, { + decision: parsed.decision, + raw: decision.raw, + tokens: decision.tokens, + reason: decision.reason, + }); + } + return decision; + } catch (err) { + const message = err && /** @type {Error} */ (err).message ? /** @type {Error} */ (err).message : String(err); + return { + path, + decision: 'yes', + raw: null, + tokens: null, + cached: false, + skippedReason: null, + error: message, + reason: null, + }; + } +} + +/** + * Ask a cheap model which files are relevant to `task`. + * + * Files are classified with at most `concurrency` calls in flight. Binary + * files and files over `maxBytes` are not sent. The cache key is + * `(model, sha256(task), sha256(content))`. Omitting `cache` uses an + * in-memory cache shared by later calls; `cache: null` disables it. + * + * @param {Record} files + * @param {string} task + * @param {object} [opts] + * @param {string} [opts.model] cheap classifier; `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} + * @param {number} [opts.concurrency] max in-flight calls (default 8) + * @param {RelevanceCache | null} [opts.cache] pluggable cache; null disables; default is in-memory + * @param {number | null} [opts.maxBytes] UTF-8 size cap; null disables; default {@link DEFAULT_FILTER_MAX_BYTES} + * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) + * @param {string} [opts.apiKey] same resolution as the rest of the package (`GPTDIFF_LLM_API_KEY`) + * @param {string} [opts.baseUrl] same resolution as the rest of the package (`GPTDIFF_LLM_BASE_URL`) + * @param {Function} [opts.callLlm] injectable client (tests) + * @param {number} [opts.maxTokens] + * @param {number} [opts.temperature] + * @returns {Promise} one decision per file, in input order + */ +export async function filterRelevant(files, task, opts = {}) { + const { + model, + concurrency = DEFAULT_FILTER_CONCURRENCY, + cache, + maxBytes = DEFAULT_FILTER_MAX_BYTES, + onDecision, + apiKey, + baseUrl, + callLlm = defaultCallLlm, + maxTokens = 128, + temperature = 0, + } = opts; + + const resolvedModel = resolveFilterModel(model); + const store = cache === undefined ? memoryCache() : cache; + const cap = maxBytes == null ? Infinity : maxBytes; + const limit = Math.max(1, Number.isFinite(concurrency) ? Math.floor(concurrency) : DEFAULT_FILTER_CONCURRENCY); + const taskText = task == null ? '' : String(task); + const taskHash = await sha256(taskText); + const entries = Object.entries(files || {}); + + return mapLimit(entries, limit, async ([path, content]) => { + const decision = await classifyFile({ + path, + content, + task: taskText, + taskHash, + model: resolvedModel, + store, + maxBytes: cap, + apiKey, + baseUrl, + callLlm, + maxTokens, + temperature, + }); + if (typeof onDecision === 'function') onDecision(decision); + return decision; + }); +} diff --git a/landing/lib/llm.js b/landing/lib/llm.js new file mode 100644 index 0000000..6e6f80f --- /dev/null +++ b/landing/lib/llm.js @@ -0,0 +1,296 @@ +/** + * LLM completion client for gptdiff-js. + * + * Speaks the OpenAI-compatible chat-completions protocol (used by NanoGPT) + * over `fetch`, so it runs unchanged in the browser and in Node 18+. An + * Anthropic branch is included for parity with the Python implementation. + */ + +import { getEnv, DEFAULT_BASE_URL } from './env.js'; + +/** + * Normalize a base URL so it has no trailing slash, then append `path`. + */ +function joinUrl(baseUrl, path) { + const base = baseUrl.replace(/\/+$/, ''); + return `${base}${path}`; +} + +/** + * @param {string} baseUrl + * @returns {string} + */ +function domainForUrl(baseUrl) { + try { + return new URL(baseUrl).host || baseUrl; + } catch { + return baseUrl; + } +} + +/** + * Convert an OpenAI-style message (possibly with image_url blocks) to the + * Anthropic content-block shape. + */ +function convertOpenAiMessageToAnthropic(message) { + if (!Array.isArray(message.content)) return message; + const converted = []; + for (const block of message.content) { + if (!block || typeof block !== 'object') continue; + if (block.type === 'text') { + converted.push({ type: 'text', text: block.text || '' }); + } else if (block.type === 'image_url') { + const url = block.image_url?.url || ''; + if (url.startsWith('data:') && url.includes(';base64,')) { + const [header, data] = url.split(',', 2); + const mediaType = header.split(':', 2)[1].split(';', 1)[0]; + converted.push({ + type: 'image', + source: { type: 'base64', media_type: mediaType, data }, + }); + } + } else { + converted.push(block); + } + } + return { ...message, content: converted }; +} + +/** + * Consume an OpenAI-compatible SSE stream (`stream: true`) from a fetch + * Response, invoking `onToken(text)` for every content delta. + * + * Handles `data:` events split across network chunks, multi-line `data:` + * events (lines are joined with `\n` per the SSE spec), and the terminal + * `data: [DONE]` sentinel. Accumulates the full content and the final + * `usage` object (requested via `stream_options: { include_usage: true }`) + * and returns the same shape as a non-streaming chat completion. + * + * @param {Response} resp + * @param {(text: string) => void} onToken + * @returns {Promise<{choices: Array<{message: {content: string}}>, usage: object}>} + */ +async function consumeSseStream(resp, onToken) { + if (!resp.body || typeof resp.body.getReader !== 'function') { + throw new Error('Streaming requested but the response body is not a readable stream.'); + } + const reader = resp.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + let content = ''; + let usage = null; + let dataLines = []; + + // Dispatch one SSE event; returns true when the [DONE] sentinel is seen. + const dispatch = () => { + if (!dataLines.length) return false; + const payload = dataLines.join('\n'); + dataLines = []; + if (payload.trim() === '[DONE]') return true; + let json; + try { + json = JSON.parse(payload); + } catch { + return false; // ignore keep-alives / unparseable fragments + } + if (json.error) { + throw new Error(`LLM API error (stream): ${JSON.stringify(json.error)}`); + } + const text = json.choices?.[0]?.delta?.content; + if (typeof text === 'string' && text.length) { + content += text; + onToken(text); + } + if (json.usage) usage = json.usage; + return false; + }; + + // Process one line; returns true when the stream is logically finished. + const handleLine = (rawLine) => { + const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine; + if (line === '') return dispatch(); // blank line terminates an event + if (line.startsWith('data:')) { + dataLines.push(line.startsWith('data: ') ? line.slice(6) : line.slice(5)); + } + // `event:`, `id:` and `:` comment lines are ignored. + return false; + }; + + let finished = false; + while (!finished) { + const { value, done } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + let idx; + while ((idx = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, idx); + buffer = buffer.slice(idx + 1); + if (handleLine(line)) { + finished = true; + break; + } + } + } + // Flush any trailing bytes / final event lacking a terminating blank line. + buffer += decoder.decode(); + if (!finished && buffer) handleLine(buffer); + if (!finished) dispatch(); + + return { choices: [{ message: { content } }], usage }; +} + +/** + * Call a chat-completions endpoint. + * + * @param {object} opts + * @param {string} [opts.apiKey] + * @param {string} [opts.baseUrl] + * @param {string} opts.model + * @param {Array} opts.messages + * @param {number} [opts.maxTokens] + * @param {number} [opts.temperature] + * @param {number} [opts.budgetTokens] Anthropic extended-thinking budget + * @param {(text: string) => void} [opts.onToken] when provided, the request is + * made with `stream: true` and this is called once per content delta as + * tokens arrive. The resolved value keeps the exact non-streaming shape + * (`{ choices: [{ message: { content } }], usage }`), so callers work + * unchanged. Ignored on the Anthropic branch (which stays non-streaming). + * @param {typeof fetch} [opts.fetchImpl] override for testing + * @param {number} [opts.maxRetries] retries on transient 429/5xx (default 3) + * @param {number} [opts.retryBaseMs] base backoff in ms (default 1500, exponential, capped 30s) + * @returns {Promise<{choices: Array<{message: {content: string}}>, usage: object}>} + */ +export async function callLlm({ + apiKey, + baseUrl, + model, + messages, + maxTokens = null, + temperature = 1.0, + budgetTokens = null, + onToken = null, + fetchImpl, + maxRetries = 3, + retryBaseMs = 1500, +}) { + const doFetch = fetchImpl || (typeof fetch !== 'undefined' ? fetch : null); + if (!doFetch) { + throw new Error('No fetch implementation available. Pass opts.fetchImpl in this environment.'); + } + const url = baseUrl || DEFAULT_BASE_URL; + + if (url.includes('api.anthropic.com')) { + const anthropicUrl = 'https://api.anthropic.com/v1/messages'; + let systemMessage = null; + const filtered = []; + for (const m of messages) { + if (m.role === 'system') systemMessage = m.content; + else filtered.push(convertOpenAiMessageToAnthropic(m)); + } + // Anthropic's API requires max_tokens, so fall back to a generous default + // only on this path; callers can still pass an explicit cap. + const data = { model, messages: filtered, max_tokens: maxTokens ?? 32000, temperature }; + if (systemMessage) data.system = systemMessage; + if (budgetTokens) { + data.temperature = 1; + data.thinking = { budget_tokens: budgetTokens, type: 'enabled' }; + } + const resp = await doFetch(anthropicUrl, { + method: 'POST', + headers: { + 'x-api-key': apiKey, + 'Content-Type': 'application/json', + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify(data), + }); + const responseData = await resp.json(); + if (responseData.error) { + throw new Error(`Error from Anthropic API: ${JSON.stringify(responseData)}`); + } + const thinkingItems = responseData.content.filter((c) => c.type === 'thinking').map((c) => c.thinking); + const textItems = responseData.content.filter((c) => c.type === 'text').map((c) => c.text); + if (!textItems.length) throw new Error("No 'text' type found in response content"); + let messageContent = textItems[0]; + if (thinkingItems.length) { + messageContent = `${thinkingItems[0]}\n${messageContent}`; + } + const inputTokens = responseData.usage.input_tokens; + const outputTokens = responseData.usage.output_tokens; + return { + choices: [{ message: { content: messageContent } }], + usage: { + prompt_tokens: inputTokens, + completion_tokens: outputTokens, + total_tokens: inputTokens + outputTokens, + }, + }; + } + + // OpenAI-compatible (NanoGPT) path, with backoff on transient errors. + const endpoint = joinUrl(url, '/chat/completions'); + // Only send max_tokens when explicitly capped; otherwise let the model emit its full output. + const requestBody = JSON.stringify({ + model, + messages, + temperature, + ...(maxTokens ? { max_tokens: maxTokens } : {}), + // Streaming: ask for SSE deltas plus a final usage chunk. + ...(onToken ? { stream: true, stream_options: { include_usage: true } } : {}), + }); + let resp; + for (let attempt = 0; ; attempt++) { + resp = await doFetch(endpoint, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: requestBody, + }); + // Retry transient overload / rate-limit / server errors with backoff. + const transient = resp.status === 429 || resp.status >= 500; + if (resp.ok || !transient || attempt >= maxRetries) break; + const retryAfter = parseFloat(resp.headers.get?.('retry-after')); + const waitMs = Number.isFinite(retryAfter) + ? Math.min(retryAfter * 1000, 30000) + : Math.min(retryBaseMs * 2 ** attempt, 30000); + await new Promise((r) => setTimeout(r, waitMs)); + } + if (!resp.ok) { + const body = await resp.text().catch(() => ''); + throw new Error(`LLM request failed (${resp.status}) from ${domainForUrl(url)}: ${body}`); + } + if (onToken) { + // Retry/backoff above covered the initial request; an error mid-stream + // (after tokens have flowed) simply propagates to the caller. + return consumeSseStream(resp, onToken); + } + const json = await resp.json(); + if (json.error) { + throw new Error(`LLM API error from ${domainForUrl(url)}: ${JSON.stringify(json.error)}`); + } + return json; +} + +/** + * Resolve the effective API key, preferring an explicit override then the + * GPTDIFF_LLM_API_KEY environment variable. + * @param {string} [apiKey] + * @returns {string | undefined} + */ +export function resolveApiKey(apiKey) { + return apiKey || getEnv('GPTDIFF_LLM_API_KEY'); +} + +/** + * Resolve the effective base URL, preferring an explicit override then the + * GPTDIFF_LLM_BASE_URL environment variable, then the NanoGPT default. + * @param {string} [baseUrl] + * @returns {string} + */ +export function resolveBaseUrl(baseUrl) { + return baseUrl || getEnv('GPTDIFF_LLM_BASE_URL', DEFAULT_BASE_URL) || DEFAULT_BASE_URL; +} + +export { domainForUrl }; diff --git a/landing/lib/text.js b/landing/lib/text.js new file mode 100644 index 0000000..27f53d0 --- /dev/null +++ b/landing/lib/text.js @@ -0,0 +1,218 @@ +/** + * Pure text utilities ported from gptdiff's Python implementation. + * + * These have no I/O and run identically in the browser and Node. + */ + +/** + * Split a string into lines while keeping the line-ending characters attached, + * mirroring Python's `str.splitlines(keepends=True)` for `\n`, `\r`, `\r\n`. + * + * @param {string} s + * @returns {string[]} + */ +export function splitLinesKeepEnds(s) { + if (s === '') return []; + const lines = []; + let i = 0; + let start = 0; + while (i < s.length) { + const ch = s[i]; + if (ch === '\n') { + lines.push(s.slice(start, i + 1)); + i += 1; + start = i; + } else if (ch === '\r') { + if (s[i + 1] === '\n') { + lines.push(s.slice(start, i + 2)); + i += 2; + } else { + lines.push(s.slice(start, i + 1)); + i += 1; + } + start = i; + } else { + i += 1; + } + } + if (start < s.length) lines.push(s.slice(start)); + return lines; +} + +/** + * Split a string into lines WITHOUT keeping line endings, mirroring Python's + * `str.splitlines()` — notably, a single trailing newline does not produce a + * trailing empty element. + * + * @param {string} s + * @returns {string[]} + */ +export function splitLines(s) { + if (s === '') return []; + const parts = s.split(/\r\n|\r|\n/); + if (parts.length && parts[parts.length - 1] === '') parts.pop(); + return parts; +} + +/** + * Color-code a diff for terminal output: lines starting with '-' in red and + * lines starting with '+' in green. Other lines are untouched. + * + * @param {string} diffText + * @returns {string} + */ +export function colorCodeDiff(diffText) { + const red = '\x1b[31m'; + const green = '\x1b[32m'; + const reset = '\x1b[0m'; + + return diffText + .split('\n') + .map((line) => { + if (line.startsWith('-')) return `${red}${line}${reset}`; + if (line.startsWith('+')) return `${green}${line}${reset}`; + return line; + }) + .join('\n'); +} + +/** + * Extract and remove the chain-of-thought reasoning block from an LLM response. + * + * The reasoning block begins with a line containing "> Reasoning" and ends with + * a "Reasoned ... seconds" marker. Multiple blocks are supported. + * + * @param {string} fullResponse + * @returns {[string, string]} tuple of [finalContent, reasoning] + */ +export function swallowReasoning(fullResponse) { + const pattern = />\s*Reasoning[\s\S]*?Reasoned[\s\S]*?seconds/g; + const reasoningList = []; + + const finalContent = fullResponse.replace(pattern, (match) => { + const reasoning = match + .split(/\r\n|\r|\n/) + .map((line) => line.replace(/^\++/, '').trim()) + .join('\n') + .trim(); + reasoningList.push(reasoning); + return ''; + }); + + return [finalContent.trim(), reasoningList.join('\n')]; +} + +/** + * If the LLM wrapped the file content in a Markdown code fence, unwrap it. + * + * Uses line-based fence detection (open on the first ``` line, close on the + * LAST ``` line) so inner triple-backticks inside the file content do not + * truncate the result. File content — including a trailing newline — is + * preserved verbatim. + * + * @param {string} updated + * @param {string} original + * @returns {string} + */ +export function stripBadOutput(updated, original) { + // If the file itself starts with a code fence, don't try to unwrap. + if (original.replace(/^\s+/, '').startsWith('```')) { + return updated; + } + + const lines = splitLinesKeepEnds(updated); + + // Find the first line that opens a fence (possibly after a preamble). + let openIdx = null; + for (let i = 0; i < lines.length; i++) { + if (lines[i].replace(/^\s+/, '').startsWith('```')) { + openIdx = i; + break; + } + } + if (openIdx === null) return updated; + + // Find the LAST line that is exactly a closing fence after the opener. + let closeIdx = null; + for (let i = lines.length - 1; i > openIdx; i--) { + if (lines[i].trim() === '```') { + closeIdx = i; + break; + } + } + if (closeIdx === null) return updated; + + let inner = lines.slice(openIdx + 1, closeIdx).join(''); + + // If the opener was a bare ``` and the model put the language on the next + // line instead (e.g. "```\npython\n..."), drop that stray language tag. + if (lines[openIdx].trim() === '```') { + const innerLines = splitLinesKeepEnds(inner); + if (innerLines.length > 1) { + const firstInner = innerLines[0].trim(); + if (firstInner && !firstInner.includes(' ') && firstInner.toLowerCase() !== 'diff') { + inner = innerLines.slice(1).join(''); + } + } + } + + return inner; +} + +/** + * Extract the bodies of all ```diff fenced code blocks from `text`. + * Mirrors the MarkdownParser + "diff" tool collection in the Python source. + * + * @param {string} text + * @returns {string[]} block bodies in document order + */ +export function extractDiffBlocks(text) { + const blocks = []; + const lines = text.split(/\r\n|\r|\n/); + let inBlock = false; + let current = []; + const flush = () => { + // Mirror the Markdown tool parser: the captured block body keeps the + // newline that precedes the closing fence. + blocks.push(current.length ? current.join('\n') + '\n' : ''); + }; + for (const line of lines) { + const stripped = line.trim(); + if (!inBlock) { + // Only fences whose language is exactly "diff" open a collected block, + // matching the Python source (which routes solely the "diff" tool). + if (stripped.startsWith('```') && stripped.slice(3).trim() === 'diff') { + inBlock = true; + current = []; + } + continue; + } + if (stripped.startsWith('```')) { + flush(); + inBlock = false; + current = []; + continue; + } + current.push(line); + } + // Force-close an unterminated block (the parser keeps the trailing buffer). + if (inBlock) flush(); + return blocks; +} + +/** + * Remove `...` blocks from a response, returning the content + * outside any think tool. Mirrors call_llm_for_apply_with_think_tool_available. + * + * @param {string} response + * @returns {string} + */ +export function stripThinkTags(response) { + const [withoutReasoning] = swallowReasoning(response); + let withoutThink = withoutReasoning.replace(/[\s\S]*?<\/think>/g, ''); + // Force-close a dangling, unterminated tag (mirrors the XML parser, + // which swallows everything after an unclosed tag) so chain-of-thought is not + // written into the file. + withoutThink = withoutThink.replace(/[\s\S]*$/, ''); + return withoutThink.trim(); +} diff --git a/landing/relevance.js b/landing/relevance.js new file mode 100644 index 0000000..d91b0b0 --- /dev/null +++ b/landing/relevance.js @@ -0,0 +1,16 @@ +/** + * Browser entry for the relevance filter on patchling.app. + * + * Cloudflare Pages publishes `landing/` only, so this re-exports the package + * sources vendored in `./lib` (kept identical to `src/` — see + * test/landing_lib_sync.test.js). The rest of the playground still calls + * generateDiff / smartapply from the published package. + */ +export { + filterRelevant, + createMemoryCache, + parseRelevanceAnswer, + DEFAULT_FILTER_MODEL, + DEFAULT_FILTER_MAX_BYTES, + DEFAULT_FILTER_CONCURRENCY, +} from './lib/filter.js'; diff --git a/package.json b/package.json index 3b6e021..58ab7ee 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "test": "node --test", "test:live": "GPTDIFF_LIVE=1 node --test test/live.test.js", "test:live:llm": "LIVE_LLM=1 node --test test/live-smartapply.test.js", + "test:live:filter": "LIVE_LLM=1 node --test test/live-filter.test.js", "benchmark": "node benchmark/run.js", "types": "rm -f src/*.d.ts && tsc -p tsconfig.json" }, diff --git a/src/filter.d.ts b/src/filter.d.ts new file mode 100644 index 0000000..e141360 --- /dev/null +++ b/src/filter.d.ts @@ -0,0 +1,106 @@ +/** + * Process-wide in-memory cache used when `filterRelevant` is called without + * a `cache` option. Pass the result as `cache` to share or inspect it. + * @returns {RelevanceCache} + */ +export function createMemoryCache(): RelevanceCache; +/** + * Parse a yes/no classification. Think tags and "> Reasoning" preambles are + * removed first. The first word wins when it is yes or no. If both words + * appear, or neither does, the answer is ambiguous and the caller includes + * the file. + * + * @param {unknown} raw + * @returns {{ decision: 'yes' | 'no', reason: string | null, ambiguous: boolean }} + */ +export function parseRelevanceAnswer(raw: unknown): { + decision: "yes" | "no"; + reason: string | null; + ambiguous: boolean; +}; +/** + * Ask a cheap model which files are relevant to `task`. + * + * Files are classified with at most `concurrency` calls in flight. Binary + * files and files over `maxBytes` are not sent. The cache key is + * `(model, sha256(task), sha256(content))`. Omitting `cache` uses an + * in-memory cache shared by later calls; `cache: null` disables it. + * + * @param {Record} files + * @param {string} task + * @param {object} [opts] + * @param {string} [opts.model] cheap classifier; `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} + * @param {number} [opts.concurrency] max in-flight calls (default 8) + * @param {RelevanceCache | null} [opts.cache] pluggable cache; null disables; default is in-memory + * @param {number | null} [opts.maxBytes] UTF-8 size cap; null disables; default {@link DEFAULT_FILTER_MAX_BYTES} + * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) + * @param {string} [opts.apiKey] same resolution as the rest of the package (`GPTDIFF_LLM_API_KEY`) + * @param {string} [opts.baseUrl] same resolution as the rest of the package (`GPTDIFF_LLM_BASE_URL`) + * @param {Function} [opts.callLlm] injectable client (tests) + * @param {number} [opts.maxTokens] + * @param {number} [opts.temperature] + * @returns {Promise} one decision per file, in input order + */ +export function filterRelevant(files: Record, task: string, opts?: { + model?: string; + concurrency?: number; + cache?: RelevanceCache | null; + maxBytes?: number | null; + onDecision?: (decision: RelevanceDecision) => void; + apiKey?: string; + baseUrl?: string; + callLlm?: Function; + maxTokens?: number; + temperature?: number; +}): Promise; +/** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ +export const DEFAULT_FILTER_MODEL: "xiaomi/mimo-v2.5-pro-ultraspeed"; +/** Files larger than this (UTF-8 bytes) are skipped unless `maxBytes` is set. */ +export const DEFAULT_FILTER_MAX_BYTES: 262144; +/** In-flight classifications per call when `concurrency` is omitted. */ +export const DEFAULT_FILTER_CONCURRENCY: 8; +export type RelevanceDecision = { + path: string; + /** + * `yes` files go on to generateDiff. + * Ambiguous answers and errors are `yes` so a bad classifier cannot drop a + * file. `skip` files were not sent to the model. + */ + decision: "yes" | "no" | "skip"; + /** + * model text, unchanged (think tags included) + */ + raw: string | null; + /** + * total tokens for the call; null when skipped or the call failed + */ + tokens: number | null; + /** + * true when this answer was read from the cache + */ + cached: boolean; + skippedReason: "binary" | "too-large" | null; + /** + * set for thrown calls and ambiguous answers + */ + error: string | null; + /** + * short reason after the yes/no, when the model gave one + */ + reason: string | null; +}; +export type RelevanceCacheEntry = { + decision: "yes" | "no"; + raw: string | null; + tokens: number | null; + reason: string | null; +}; +/** + * A cache is any object with `get(key)` and `set(key, value)`. Either method + * may be sync or async. Keys are `${model}\\0${taskHash}\\0${contentHash}` + * (SHA-256 hex). Pass `cache: null` to disable caching. + */ +export type RelevanceCache = { + get: (key: string) => RelevanceCacheEntry | undefined | Promise; + set: (key: string, value: RelevanceCacheEntry) => void | Promise; +}; diff --git a/src/filter.js b/src/filter.js new file mode 100644 index 0000000..a88b17d --- /dev/null +++ b/src/filter.js @@ -0,0 +1,412 @@ +/** + * Opt-in relevance filter for large file maps. + * + * For each candidate file, a cheap model sees the full contents plus the task + * and answers "is this file relevant to the task? yes/no". Only `yes` + * decisions should be passed to `generateDiff` (see `buildEnvironment`'s + * `decisions` option). `smartapply` still wants the full map so files the + * diff never mentions stay put. + * + * Mirrors the Python package's `filter_relevant` (255BITS/patchling-py): + * + * filter_relevant(files, task, *, model=None, concurrency=8, cache=None, + * max_bytes=262144, on_decision=None, api_key=None, + * base_url=None) -> list[decision] + * + * decision: path, decision ("yes" | "no" | "skip"), raw, tokens, cached, + * skipped_reason ("binary" | "too-large" | None), error, reason + * + * JS uses camelCase (`maxBytes`, `onDecision`, `skippedReason`) like the rest + * of this package. Ambiguous answers and per-file errors include the file + * (`decision: "yes"`) and surface the problem on `error` instead of throwing. + */ + +import { stripThinkTags } from './text.js'; +import { callLlm as defaultCallLlm, resolveApiKey, resolveBaseUrl } from './llm.js'; +import { getEnv } from './env.js'; + +/** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ +export const DEFAULT_FILTER_MODEL = 'xiaomi/mimo-v2.5-pro-ultraspeed'; + +/** Files larger than this (UTF-8 bytes) are skipped unless `maxBytes` is set. */ +export const DEFAULT_FILTER_MAX_BYTES = 262144; + +/** In-flight classifications per call when `concurrency` is omitted. */ +export const DEFAULT_FILTER_CONCURRENCY = 8; + +const FILTER_SYSTEM_PROMPT = `You decide if one file is relevant to a coding task. Reply with "yes" or "no" as the first word. You may add one short reason after it. Do not quote the file.`; + +const AMBIGUOUS = 'ambiguous answer'; + +/** + * @typedef {object} RelevanceDecision + * @property {string} path + * @property {'yes' | 'no' | 'skip'} decision `yes` files go on to generateDiff. + * Ambiguous answers and errors are `yes` so a bad classifier cannot drop a + * file. `skip` files were not sent to the model. + * @property {string | null} raw model text, unchanged (think tags included) + * @property {number | null} tokens total tokens for the call; null when skipped or the call failed + * @property {boolean} cached true when this answer was read from the cache + * @property {'binary' | 'too-large' | null} skippedReason + * @property {string | null} error set for thrown calls and ambiguous answers + * @property {string | null} reason short reason after the yes/no, when the model gave one + */ + +/** + * @typedef {object} RelevanceCacheEntry + * @property {'yes' | 'no'} decision + * @property {string | null} raw + * @property {number | null} tokens + * @property {string | null} reason + */ + +/** + * A cache is any object with `get(key)` and `set(key, value)`. Either method + * may be sync or async. Keys are `${model}\\0${taskHash}\\0${contentHash}` + * (SHA-256 hex). Pass `cache: null` to disable caching. + * + * @typedef {object} RelevanceCache + * @property {(key: string) => RelevanceCacheEntry | undefined | Promise} get + * @property {(key: string, value: RelevanceCacheEntry) => void | Promise} set + */ + +let sharedMemoryCache; + +/** + * Process-wide in-memory cache used when `filterRelevant` is called without + * a `cache` option. Pass the result as `cache` to share or inspect it. + * @returns {RelevanceCache} + */ +export function createMemoryCache() { + /** @type {Map} */ + const store = new Map(); + return { + get(key) { + return store.has(key) ? store.get(key) : undefined; + }, + set(key, value) { + store.set(key, value); + }, + }; +} + +function memoryCache() { + if (!sharedMemoryCache) sharedMemoryCache = createMemoryCache(); + return sharedMemoryCache; +} + +/** + * @param {string} text + * @returns {Promise} hex SHA-256 + */ +async function sha256(text) { + const data = new TextEncoder().encode(text); + const digest = await crypto.subtle.digest('SHA-256', data); + let hex = ''; + for (const byte of new Uint8Array(digest)) hex += byte.toString(16).padStart(2, '0'); + return hex; +} + +/** + * @param {string} model + * @param {string} taskHash + * @param {string} contentHash + * @returns {string} + */ +function cacheKey(model, taskHash, contentHash) { + return `${model}\0${taskHash}\0${contentHash}`; +} + +function resolveFilterModel(model) { + return model || getEnv('GPTDIFF_FILTER_MODEL') || DEFAULT_FILTER_MODEL; +} + +/** + * @param {unknown} content + * @returns {boolean} + */ +function looksBinary(content) { + if (typeof content !== 'string') return true; + const n = Math.min(content.length, 8192); + let controls = 0; + for (let i = 0; i < n; i++) { + const c = content.charCodeAt(i); + if (c === 0) return true; + // C0 controls other than tab, LF, and CR. A dense run is not text. + if (c < 32 && c !== 9 && c !== 10 && c !== 13) controls++; + } + return n > 0 && controls / n > 0.3; +} + +/** + * UTF-8 byte length, stopping once `maxBytes` is exceeded. + * @param {string} content + * @param {number} maxBytes + * @returns {boolean} + */ +function exceedsMaxBytes(content, maxBytes) { + if (!Number.isFinite(maxBytes)) return false; + let bytes = 0; + for (let i = 0; i < content.length; i++) { + const c = content.charCodeAt(i); + if (c < 0x80) bytes += 1; + else if (c < 0x800) bytes += 2; + else if (c >= 0xd800 && c <= 0xdbff) { + bytes += 4; + i += 1; + } else bytes += 3; + if (bytes > maxBytes) return true; + } + return false; +} + +/** + * @param {unknown} response + * @returns {number} + */ +function totalTokens(response) { + const usage = response && /** @type {{ usage?: Record }} */ (response).usage; + if (!usage) return 0; + const prompt = usage.prompt_tokens ?? usage.input_tokens ?? 0; + const completion = usage.completion_tokens ?? usage.output_tokens ?? 0; + return usage.total_tokens ?? prompt + completion; +} + +/** + * Parse a yes/no classification. Think tags and "> Reasoning" preambles are + * removed first. The first word wins when it is yes or no. If both words + * appear, or neither does, the answer is ambiguous and the caller includes + * the file. + * + * @param {unknown} raw + * @returns {{ decision: 'yes' | 'no', reason: string | null, ambiguous: boolean }} + */ +export function parseRelevanceAnswer(raw) { + const stripped = stripThinkTags(raw == null ? '' : String(raw)).trim(); + const unfenced = stripped.replace(/^```[a-z0-9]*\s*/i, '').replace(/```\s*$/i, '').trim(); + const lines = unfenced.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); + const first = (lines[0] || '').replace(/^[*_>"'`]+/, '').replace(/[*_"`']+$/g, '').trim(); + const lead = first.match(/^(yes|no)\b[\s.:,\-—!]*(.*)$/i); + + if (lead) { + const decision = /** @type {'yes' | 'no'} */ (lead[1].toLowerCase()); + const sameLine = lead[2].trim(); + const reason = [sameLine, lines.slice(1).join(' ')].filter(Boolean).join(' ').trim() || null; + const firstHasYes = /\byes\b/i.test(first); + const firstHasNo = /\bno\b/i.test(first); + if (firstHasYes && firstHasNo) { + return { decision: 'yes', reason: unfenced || null, ambiguous: true }; + } + return { decision, reason, ambiguous: false }; + } + + const hasYes = /\byes\b/i.test(unfenced); + const hasNo = /\bno\b/i.test(unfenced); + if (hasYes && !hasNo) return { decision: 'yes', reason: unfenced || null, ambiguous: false }; + if (hasNo && !hasYes) return { decision: 'no', reason: unfenced || null, ambiguous: false }; + return { decision: 'yes', reason: unfenced || null, ambiguous: true }; +} + +/** + * @param {Array<[string, string]>} items + * @param {number} limit + * @param {(item: [string, string], index: number) => Promise} fn + * @returns {Promise} + */ +async function mapLimit(items, limit, fn) { + /** @type {RelevanceDecision[]} */ + const results = new Array(items.length); + if (!items.length) return results; + let cursor = 0; + const workers = Math.min(limit, items.length); + const run = async () => { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + results[index] = await fn(items[index], index); + } + }; + await Promise.all(Array.from({ length: workers }, () => run())); + return results; +} + +/** + * @param {object} args + * @param {string} args.path + * @param {unknown} args.content + * @param {string} args.task + * @param {string} args.taskHash + * @param {string} args.model + * @param {RelevanceCache | null} args.store + * @param {number} args.maxBytes + * @param {string | undefined} args.apiKey + * @param {string | undefined} args.baseUrl + * @param {Function} args.callLlm + * @param {number} args.maxTokens + * @param {number} args.temperature + * @returns {Promise} + */ +async function classifyFile(args) { + const { path, content, task, taskHash, model, store, maxBytes, apiKey, baseUrl, callLlm, maxTokens, temperature } = args; + + if (looksBinary(content)) { + return { + path, + decision: 'skip', + raw: null, + tokens: null, + cached: false, + skippedReason: 'binary', + error: null, + reason: null, + }; + } + + const text = /** @type {string} */ (content); + if (exceedsMaxBytes(text, maxBytes)) { + return { + path, + decision: 'skip', + raw: null, + tokens: null, + cached: false, + skippedReason: 'too-large', + error: null, + reason: null, + }; + } + + const contentHash = await sha256(text); + const key = cacheKey(model, taskHash, contentHash); + if (store) { + const hit = await store.get(key); + if (hit && (hit.decision === 'yes' || hit.decision === 'no')) { + return { + path, + decision: hit.decision, + raw: hit.raw ?? null, + tokens: hit.tokens ?? null, + cached: true, + skippedReason: null, + error: null, + reason: hit.reason ?? null, + }; + } + } + + const userContent = `Task:\n${task}\n\nFile: ${path}\n${text}\n\nis this file relevant to the task? yes/no`; + try { + const response = await callLlm({ + apiKey: resolveApiKey(apiKey), + baseUrl: resolveBaseUrl(baseUrl), + model, + messages: [ + { role: 'system', content: FILTER_SYSTEM_PROMPT }, + { role: 'user', content: userContent }, + ], + maxTokens, + temperature, + }); + const raw = response?.choices?.[0]?.message?.content ?? ''; + const parsed = parseRelevanceAnswer(raw); + /** @type {RelevanceDecision} */ + const decision = { + path, + decision: parsed.decision, + raw: raw == null ? '' : String(raw), + tokens: totalTokens(response), + cached: false, + skippedReason: null, + error: parsed.ambiguous ? AMBIGUOUS : null, + reason: parsed.ambiguous ? (parsed.reason || null) : parsed.reason, + }; + // Keep failures and ambiguous answers out of the cache so a later call + // can try again. Definitive yes/no answers are stable for this content. + if (store && !parsed.ambiguous) { + await store.set(key, { + decision: parsed.decision, + raw: decision.raw, + tokens: decision.tokens, + reason: decision.reason, + }); + } + return decision; + } catch (err) { + const message = err && /** @type {Error} */ (err).message ? /** @type {Error} */ (err).message : String(err); + return { + path, + decision: 'yes', + raw: null, + tokens: null, + cached: false, + skippedReason: null, + error: message, + reason: null, + }; + } +} + +/** + * Ask a cheap model which files are relevant to `task`. + * + * Files are classified with at most `concurrency` calls in flight. Binary + * files and files over `maxBytes` are not sent. The cache key is + * `(model, sha256(task), sha256(content))`. Omitting `cache` uses an + * in-memory cache shared by later calls; `cache: null` disables it. + * + * @param {Record} files + * @param {string} task + * @param {object} [opts] + * @param {string} [opts.model] cheap classifier; `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} + * @param {number} [opts.concurrency] max in-flight calls (default 8) + * @param {RelevanceCache | null} [opts.cache] pluggable cache; null disables; default is in-memory + * @param {number | null} [opts.maxBytes] UTF-8 size cap; null disables; default {@link DEFAULT_FILTER_MAX_BYTES} + * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) + * @param {string} [opts.apiKey] same resolution as the rest of the package (`GPTDIFF_LLM_API_KEY`) + * @param {string} [opts.baseUrl] same resolution as the rest of the package (`GPTDIFF_LLM_BASE_URL`) + * @param {Function} [opts.callLlm] injectable client (tests) + * @param {number} [opts.maxTokens] + * @param {number} [opts.temperature] + * @returns {Promise} one decision per file, in input order + */ +export async function filterRelevant(files, task, opts = {}) { + const { + model, + concurrency = DEFAULT_FILTER_CONCURRENCY, + cache, + maxBytes = DEFAULT_FILTER_MAX_BYTES, + onDecision, + apiKey, + baseUrl, + callLlm = defaultCallLlm, + maxTokens = 128, + temperature = 0, + } = opts; + + const resolvedModel = resolveFilterModel(model); + const store = cache === undefined ? memoryCache() : cache; + const cap = maxBytes == null ? Infinity : maxBytes; + const limit = Math.max(1, Number.isFinite(concurrency) ? Math.floor(concurrency) : DEFAULT_FILTER_CONCURRENCY); + const taskText = task == null ? '' : String(task); + const taskHash = await sha256(taskText); + const entries = Object.entries(files || {}); + + return mapLimit(entries, limit, async ([path, content]) => { + const decision = await classifyFile({ + path, + content, + task: taskText, + taskHash, + model: resolvedModel, + store, + maxBytes: cap, + apiKey, + baseUrl, + callLlm, + maxTokens, + temperature, + }); + if (typeof onDecision === 'function') onDecision(decision); + return decision; + }); +} diff --git a/src/gptdiff.d.ts b/src/gptdiff.d.ts index 76ac5f6..2880528 100644 --- a/src/gptdiff.d.ts +++ b/src/gptdiff.d.ts @@ -1,9 +1,26 @@ /** * Rebuild the environment string from a `{ path: content }` map. + * + * Opt-in relevance filter: pass `{ decisions }` from `filterRelevant` and + * only files whose decision is `"yes"` are included. An array of those + * decisions is also accepted as the second argument. Omit it and every file + * is included, in insertion order. Skipped and `"no"` files stay out of the + * string that `generateDiff` sees. + * * @param {Record} filesDict + * @param {{ decisions?: Array<{ path?: string, decision?: string }> } | Array<{ path?: string, decision?: string }>} [opts] + * `{ decisions }` from `filterRelevant`, or that array itself. * @returns {string} */ -export function buildEnvironment(filesDict: Record): string; +export function buildEnvironment(filesDict: Record, opts?: { + decisions?: Array<{ + path?: string; + decision?: string; + }>; +} | Array<{ + path?: string; + decision?: string; +}>): string; /** * Build the prompt, call the LLM, and extract the unified diff from the * ```diff fenced blocks of the response. diff --git a/src/gptdiff.js b/src/gptdiff.js index 386f569..4474061 100644 --- a/src/gptdiff.js +++ b/src/gptdiff.js @@ -47,14 +47,45 @@ const APPLY_SYSTEM_PROMPT = `Please apply the diff to this file. Return the full 4. Only return the final file content. Do not add any additional markup and do not add a code block. 5. You must return the entire file. It overwrites the existing file.`; +/** + * Decisions from `filterRelevant`, or a bare array of them. + * `null` means "include every file" (the default). + * @param {object | Array<{ path?: string, decision?: string }> | undefined} opts + * @returns {Array<{ path?: string, decision?: string }> | null} + */ +function decisionsFrom(opts) { + if (!opts) return null; + if (Array.isArray(opts)) return opts; + if (Array.isArray(opts.decisions)) return opts.decisions; + return null; +} + /** * Rebuild the environment string from a `{ path: content }` map. + * + * Opt-in relevance filter: pass `{ decisions }` from `filterRelevant` and + * only files whose decision is `"yes"` are included. An array of those + * decisions is also accepted as the second argument. Omit it and every file + * is included, in insertion order. Skipped and `"no"` files stay out of the + * string that `generateDiff` sees. + * * @param {Record} filesDict + * @param {{ decisions?: Array<{ path?: string, decision?: string }> } | Array<{ path?: string, decision?: string }>} [opts] + * `{ decisions }` from `filterRelevant`, or that array itself. * @returns {string} */ -export function buildEnvironment(filesDict) { +export function buildEnvironment(filesDict, opts) { + let entries = Object.entries(filesDict || {}); + const decisions = decisionsFrom(opts); + if (decisions) { + const yes = new Set(); + for (const decision of decisions) { + if (decision && decision.decision === 'yes' && decision.path) yes.add(decision.path); + } + entries = entries.filter(([path]) => yes.has(path)); + } const env = []; - for (const [path, content] of Object.entries(filesDict)) { + for (const [path, content] of entries) { env.push(`File: ${path}`); env.push('Content:'); env.push(content); diff --git a/src/index.d.ts b/src/index.d.ts index d5c8158..f8234b0 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -1,5 +1,9 @@ export * as oauth from "./oauth.js"; +export type RelevanceDecision = import("./filter.js").RelevanceDecision; +export type RelevanceCache = import("./filter.js").RelevanceCache; +export type RelevanceCacheEntry = import("./filter.js").RelevanceCacheEntry; export { generateDiff, smartapply, buildEnvironment, callLlmForDiff, callLlmForApply, callLlmForApplyWithThink } from "./gptdiff.js"; +export { filterRelevant, createMemoryCache, parseRelevanceAnswer, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_CONCURRENCY } from "./filter.js"; export { parseDiffPerFile, applyDiff, applyPatchToFile } from "./applydiff.js"; export { colorCodeDiff, swallowReasoning, stripBadOutput, extractDiffBlocks, stripThinkTags } from "./text.js"; export { callLlm, resolveApiKey, resolveBaseUrl } from "./llm.js"; diff --git a/src/index.js b/src/index.js index b2ca763..ae45636 100644 --- a/src/index.js +++ b/src/index.js @@ -6,6 +6,20 @@ */ export { generateDiff, smartapply, buildEnvironment, callLlmForDiff, callLlmForApply, callLlmForApplyWithThink } from './gptdiff.js'; +export { + filterRelevant, + createMemoryCache, + parseRelevanceAnswer, + DEFAULT_FILTER_MODEL, + DEFAULT_FILTER_MAX_BYTES, + DEFAULT_FILTER_CONCURRENCY, +} from './filter.js'; + +/** + * @typedef {import('./filter.js').RelevanceDecision} RelevanceDecision + * @typedef {import('./filter.js').RelevanceCache} RelevanceCache + * @typedef {import('./filter.js').RelevanceCacheEntry} RelevanceCacheEntry + */ export { parseDiffPerFile, applyDiff, applyPatchToFile } from './applydiff.js'; export { colorCodeDiff, swallowReasoning, stripBadOutput, extractDiffBlocks, stripThinkTags } from './text.js'; export { callLlm, resolveApiKey, resolveBaseUrl } from './llm.js'; diff --git a/test/filter_relevant.test.js b/test/filter_relevant.test.js new file mode 100644 index 0000000..714250c --- /dev/null +++ b/test/filter_relevant.test.js @@ -0,0 +1,352 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + filterRelevant, + createMemoryCache, + parseRelevanceAnswer, + buildEnvironment, + DEFAULT_FILTER_MODEL, + DEFAULT_FILTER_MAX_BYTES, +} from '../src/index.js'; +import { setEnv } from '../src/env.js'; + +function llm(replies) { + const calls = []; + let inFlight = 0; + let maxInFlight = 0; + const fn = async (req) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + const index = calls.length; + calls.push(req); + await new Promise((r) => setTimeout(r, 15)); + inFlight -= 1; + const next = typeof replies === 'function' ? replies(req, index) : replies[index]; + if (next instanceof Error) throw next; + const content = typeof next === 'string' || next == null ? (next ?? 'yes') : next.content; + const tokens = typeof next === 'object' && next && 'tokens' in next ? next.tokens : 11; + return { + choices: [{ message: { content } }], + usage: { prompt_tokens: 7, completion_tokens: Math.max(0, tokens - 7), total_tokens: tokens }, + }; + }; + return { fn, calls, stats: () => ({ maxInFlight, count: calls.length }) }; +} + +function replyByPath(map) { + return (req) => { + const path = req.messages[1].content.match(/File: ([^\n]+)/)[1]; + if (!Object.prototype.hasOwnProperty.call(map, path)) throw new Error(`no reply for ${path}`); + return map[path]; + }; +} + +const QUESTION = 'is this file relevant to the task? yes/no'; + +test('parseRelevanceAnswer strips think tags and accepts yes/no with a reason', () => { + assert.deepEqual(parseRelevanceAnswer('yes'), { decision: 'yes', reason: null, ambiguous: false }); + assert.deepEqual(parseRelevanceAnswer(' YES. '), { decision: 'yes', reason: null, ambiguous: false }); + assert.deepEqual(parseRelevanceAnswer('No'), { decision: 'no', reason: null, ambiguous: false }); + assert.deepEqual(parseRelevanceAnswer('no — arithmetic helpers'), { + decision: 'no', + reason: 'arithmetic helpers', + ambiguous: false, + }); + assert.deepEqual(parseRelevanceAnswer('the greeting lives here\n\nno'), { + decision: 'no', + reason: null, + ambiguous: false, + }); + assert.deepEqual(parseRelevanceAnswer('**yes**\ndefines greet'), { + decision: 'yes', + reason: 'defines greet', + ambiguous: false, + }); + assert.equal(parseRelevanceAnswer('The answer is no.').decision, 'no'); + const both = parseRelevanceAnswer('Yes and no'); + assert.equal(both.decision, 'yes'); + assert.equal(both.ambiguous, true); + const maybe = parseRelevanceAnswer('maybe later'); + assert.equal(maybe.decision, 'yes'); + assert.equal(maybe.ambiguous, true); + const empty = parseRelevanceAnswer('only thoughts'); + assert.equal(empty.decision, 'yes'); + assert.equal(empty.ambiguous, true); +}); + +test('filterRelevant sends full contents and the task, and returns per-file decisions', async () => { + const mock = llm(replyByPath({ + 'greet.py': 'yes — defines the greeting', + 'math_utils.py': 'no', + })); + const files = { + 'greet.py': 'def greet():\n print("hello")\n', + 'math_utils.py': 'def add(a, b):\n return a + b\n', + }; + const seen = []; + const decisions = await filterRelevant(files, 'Say goodbye instead of hello', { + model: 'cheap-test', + apiKey: 'sk-test', + baseUrl: 'https://example.test/v1', + cache: createMemoryCache(), + callLlm: mock.fn, + onDecision: (d) => seen.push(d.path), + }); + + assert.equal(decisions.length, 2); + assert.equal(decisions[0].path, 'greet.py'); + assert.equal(decisions[0].decision, 'yes'); + assert.equal(decisions[0].reason, 'defines the greeting'); + assert.equal(decisions[0].raw, 'yes — defines the greeting'); + assert.equal(decisions[0].tokens, 11); + assert.equal(decisions[0].cached, false); + assert.equal(decisions[0].skippedReason, null); + assert.equal(decisions[0].error, null); + assert.equal(decisions[1].decision, 'no'); + assert.deepEqual(seen.sort(), ['greet.py', 'math_utils.py']); + + assert.equal(mock.calls.length, 2); + for (const [path, content] of Object.entries(files)) { + const call = mock.calls.find((c) => c.messages[1].content.includes(`File: ${path}\n`)); + assert.ok(call, path); + assert.equal(call.model, 'cheap-test'); + assert.equal(call.apiKey, 'sk-test'); + assert.equal(call.baseUrl, 'https://example.test/v1'); + assert.equal(call.temperature, 0); + assert.match(call.messages[1].content, /Task:\nSay goodbye instead of hello/); + assert.ok(call.messages[1].content.includes(content)); + assert.ok(call.messages[1].content.includes(QUESTION)); + } +}); + +test('ambiguous answers and thrown errors include the file and surface the error', async () => { + const mock = llm(replyByPath({ + a: 'maybe', + b: new Error('connection reset'), + c: 'no', + })); + const decisions = await filterRelevant( + { a: 'aaa', b: 'bbb', c: 'ccc' }, + 'touch a', + { model: 'cheap-test', cache: null, callLlm: mock.fn }, + ); + assert.equal(decisions[0].decision, 'yes'); + assert.equal(decisions[0].error, 'ambiguous answer'); + assert.equal(decisions[0].raw, 'maybe'); + assert.equal(decisions[1].decision, 'yes'); + assert.equal(decisions[1].error, 'connection reset'); + assert.equal(decisions[1].raw, null); + assert.equal(decisions[1].tokens, null); + assert.equal(decisions[2].decision, 'no'); + assert.equal(decisions[2].error, null); +}); + +test('binary and huge files are skipped and never sent', async () => { + const mock = llm(['yes']); + const huge = 'x'.repeat(50); + const files = { + 'keep.py': 'print(1)\n', + 'logo.bin': 'PNG\0bytes', + 'notes.txt': huge, + 'odd.dat': '\u0001'.repeat(20), + }; + const decisions = await filterRelevant(files, 'print', { + model: 'cheap-test', + cache: null, + maxBytes: 40, + callLlm: mock.fn, + }); + const by = Object.fromEntries(decisions.map((d) => [d.path, d])); + assert.equal(by['keep.py'].decision, 'yes'); + assert.equal(by['logo.bin'].decision, 'skip'); + assert.equal(by['logo.bin'].skippedReason, 'binary'); + assert.equal(by['logo.bin'].tokens, null); + assert.equal(by['notes.txt'].decision, 'skip'); + assert.equal(by['notes.txt'].skippedReason, 'too-large'); + assert.equal(by['odd.dat'].skippedReason, 'binary'); + assert.equal(mock.stats().count, 1); + assert.ok(!mock.calls[0].messages[1].content.includes('PNG')); +}); + +test('maxBytes counts UTF-8 bytes and can be raised or disabled', async () => { + const mock = llm(['yes', 'yes', 'no']); + const accented = 'é'; // 1 UTF-16 unit, 2 UTF-8 bytes + const files = { a: accented, b: 'b' }; + const skipped = await filterRelevant(files, 't', { + model: 'cheap-test', + cache: null, + maxBytes: 1, + callLlm: mock.fn, + }); + assert.equal(skipped[0].skippedReason, 'too-large'); + assert.equal(skipped[1].decision, 'yes'); + + const sent = await filterRelevant({ a: accented }, 't', { + model: 'cheap-test', + cache: null, + maxBytes: 2, + callLlm: mock.fn, + }); + assert.equal(sent[0].decision, 'yes'); + assert.equal(sent[0].skippedReason, null); + + const open = await filterRelevant({ a: 'zzzz'.repeat(100) }, 't', { + model: 'cheap-test', + cache: null, + maxBytes: null, + callLlm: mock.fn, + }); + assert.equal(open[0].decision, 'no'); +}); + +test('a file of exactly maxBytes is sent', async () => { + const mock = llm(['yes']); + const content = 'abcd'; + const decisions = await filterRelevant({ a: content }, 't', { + model: 'cheap-test', + cache: null, + maxBytes: 4, + callLlm: mock.fn, + }); + assert.equal(decisions[0].decision, 'yes'); + assert.equal(mock.stats().count, 1); +}); + +test('concurrency stays bounded and result order follows the input', async () => { + const names = ['a', 'b', 'c', 'd', 'e', 'f']; + const files = Object.fromEntries(names.map((n) => [n, `body ${n}`])); + const mock = llm(names.map(() => 'yes')); + const decisions = await filterRelevant(files, 'all', { + model: 'cheap-test', + concurrency: 2, + cache: null, + callLlm: mock.fn, + }); + assert.deepEqual(decisions.map((d) => d.path), names); + assert.equal(mock.stats().maxInFlight, 2); + assert.equal(mock.stats().count, 6); +}); + +test('cache is keyed by model, task hash, and content hash', async () => { + const cache = createMemoryCache(); + const first = llm(['yes — greeting']); + const files = { 'greet.py': 'print("hello")\n' }; + const task = 'change the greeting'; + await filterRelevant(files, task, { model: 'cheap-a', cache, callLlm: first.fn }); + + const second = llm(['no']); + const again = await filterRelevant({ 'other.py': 'print("hello")\n' }, task, { + model: 'cheap-a', + cache, + callLlm: second.fn, + }); + assert.equal(second.stats().count, 0); + assert.equal(again[0].cached, true); + assert.equal(again[0].path, 'other.py'); + assert.equal(again[0].decision, 'yes'); + assert.equal(again[0].reason, 'greeting'); + assert.equal(again[0].tokens, 11); + + const otherTask = llm(['no']); + const moved = await filterRelevant(files, 'rename the function', { + model: 'cheap-a', + cache, + callLlm: otherTask.fn, + }); + assert.equal(moved[0].cached, false); + assert.equal(moved[0].decision, 'no'); + + const otherModel = llm(['no']); + const remodeled = await filterRelevant(files, task, { + model: 'cheap-b', + cache, + callLlm: otherModel.fn, + }); + assert.equal(remodeled[0].cached, false); + assert.equal(otherModel.stats().count, 1); + + const stored = []; + const custom = { + async get(key) { + stored.push(['get', key]); + return undefined; + }, + async set(key, value) { + stored.push(['set', key, value.decision]); + }, + }; + await filterRelevant({ p: 'body' }, 'task', { model: 'm', cache: custom, callLlm: llm(['yes']).fn }); + const set = stored.find((row) => row[0] === 'set'); + assert.ok(set); + const parts = set[1].split('\0'); + assert.equal(parts.length, 3); + assert.equal(parts[0], 'm'); + assert.match(parts[1], /^[0-9a-f]{64}$/); + assert.match(parts[2], /^[0-9a-f]{64}$/); + assert.notEqual(parts[1], parts[2]); +}); + +test('ambiguous answers are not cached', async () => { + const cache = createMemoryCache(); + const first = llm(['maybe']); + await filterRelevant({ a: 'body' }, 'task', { model: 'm', cache, callLlm: first.fn }); + const second = llm(['no']); + const decisions = await filterRelevant({ a: 'body' }, 'task', { model: 'm', cache, callLlm: second.fn }); + assert.equal(second.stats().count, 1); + assert.equal(decisions[0].decision, 'no'); + assert.equal(decisions[0].cached, false); +}); + +test('default model and the shared memory cache', async () => { + const prev = process.env.GPTDIFF_FILTER_MODEL; + setEnv('GPTDIFF_FILTER_MODEL', 'from-env'); + try { + const mock = llm(['yes']); + const task = `env-model-${Date.now()}`; + await filterRelevant({ a: `unique-${task}` }, task, { cache: undefined, callLlm: mock.fn }); + assert.equal(mock.calls[0].model, 'from-env'); + const again = llm(['no']); + const hit = await filterRelevant({ b: `unique-${task}` }, task, { callLlm: again.fn }); + assert.equal(again.stats().count, 0); + assert.equal(hit[0].cached, true); + } finally { + setEnv('GPTDIFF_FILTER_MODEL', undefined); + if (prev === undefined) delete process.env.GPTDIFF_FILTER_MODEL; + else process.env.GPTDIFF_FILTER_MODEL = prev; + } + assert.equal(typeof DEFAULT_FILTER_MODEL, 'string'); + assert.equal(DEFAULT_FILTER_MAX_BYTES, 262144); +}); + +test('buildEnvironment includes only yes files when decisions are passed', () => { + const files = { + 'greet.py': 'hello\n', + 'main.py': 'main\n', + 'math_utils.py': 'add\n', + 'logo.bin': 'PNG', + }; + const untouched = buildEnvironment(files); + assert.match(untouched, /File: greet\.py/); + assert.match(untouched, /File: math_utils\.py/); + assert.match(untouched, /File: logo\.bin/); + + const decisions = [ + { path: 'greet.py', decision: 'yes' }, + { path: 'main.py', decision: 'no' }, + { path: 'math_utils.py', decision: 'skip', skippedReason: 'binary' }, + { path: 'logo.bin', decision: 'skip', skippedReason: 'too-large' }, + ]; + const filtered = buildEnvironment(files, { decisions }); + assert.match(filtered, /File: greet\.py/); + assert.match(filtered, /hello/); + assert.doesNotMatch(filtered, /math_utils/); + assert.doesNotMatch(filtered, /main/); + assert.doesNotMatch(filtered, /logo/); + assert.equal(filtered, buildEnvironment(files, decisions)); + + const includedByError = buildEnvironment(files, { + decisions: [{ path: 'main.py', decision: 'yes', error: 'ambiguous answer' }], + }); + assert.match(includedByError, /File: main\.py/); + assert.doesNotMatch(includedByError, /greet/); +}); diff --git a/test/landing_lib_sync.test.js b/test/landing_lib_sync.test.js new file mode 100644 index 0000000..1d30bca --- /dev/null +++ b/test/landing_lib_sync.test.js @@ -0,0 +1,17 @@ +/** + * patchling.app is deployed from landing/ alone, so the filter the page + * imports has to live there. These copies must stay identical to src/. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +const FILES = ['filter.js', 'text.js', 'llm.js', 'env.js']; + +for (const name of FILES) { + test(`landing/lib/${name} matches src/${name}`, () => { + const src = readFileSync(new URL(`../src/${name}`, import.meta.url), 'utf8'); + const copy = readFileSync(new URL(`../landing/lib/${name}`, import.meta.url), 'utf8'); + assert.equal(copy, src); + }); +} diff --git a/test/live-filter.test.js b/test/live-filter.test.js new file mode 100644 index 0000000..122738a --- /dev/null +++ b/test/live-filter.test.js @@ -0,0 +1,46 @@ +/** + * Opt-in relevance filter against a live cheap model. + * + * Skipped unless both of these are set: + * LIVE_LLM=1 + * GPTDIFF_LLM_API_KEY= + * + * `npm test` / CI does not set LIVE_LLM, so this file makes no network calls + * there. Run it with: npm run test:live:filter + * + * Model defaults to the cheap filter model (override with GPTDIFF_FILTER_MODEL). + * Base URL is the package default (override GPTDIFF_LLM_BASE_URL). + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { filterRelevant, buildEnvironment, DEFAULT_FILTER_MODEL } from '../src/index.js'; + +const apiKey = process.env.GPTDIFF_LLM_API_KEY; +const LIVE = process.env.LIVE_LLM === '1' && !!apiKey; +const MODEL = process.env.GPTDIFF_FILTER_MODEL || DEFAULT_FILTER_MODEL; + +test( + 'live: filterRelevant keeps the greeting file and drops the unrelated one', + { + skip: LIVE ? false : 'set LIVE_LLM=1 and GPTDIFF_LLM_API_KEY to run', + timeout: 120_000, + }, + async () => { + const files = { + 'greet.py': 'def greet():\n print("hello")\n', + 'math_utils.py': 'def add(a, b):\n return a + b\n', + }; + const task = 'Change the greeting from hello to goodbye. Do not edit unrelated helpers.'; + const decisions = await filterRelevant(files, task, { model: MODEL, apiKey }); + const by = Object.fromEntries(decisions.map((d) => [d.path, d])); + + assert.equal(by['greet.py'].decision, 'yes', JSON.stringify(by['greet.py'])); + assert.equal(by['math_utils.py'].decision, 'no', JSON.stringify(by['math_utils.py'])); + assert.equal(by['greet.py'].skippedReason, null); + assert.ok(by['greet.py'].raw); + + const env = buildEnvironment(files, { decisions }); + assert.match(env, /File: greet\.py/); + assert.doesNotMatch(env, /math_utils/); + }, +); From a16c8ae892236c4297374b7528795cbf1ee08c56 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 19:32:02 +0000 Subject: [PATCH 2/6] Align the relevance filter with the Python API Use gemini-2.0-flash, the same size and binary policies, and camelCase decision fields. A truncated yes still forwards the original file, and a path can persist the cache as versioned JSON in Node. Co-authored-by: ntc-ai --- CHANGELOG.md | 4 +- README.md | 45 +- landing/index.html | 51 +- landing/lib/filter.js | 1171 +++++++++++++++++++++++++--------- src/filter.d.ts | 277 ++++++-- src/filter.js | 1171 +++++++++++++++++++++++++--------- src/gptdiff.d.ts | 2 +- src/gptdiff.js | 2 +- src/index.d.ts | 5 +- src/index.js | 10 +- test/filter_relevant.test.js | 790 ++++++++++++++++------- test/live-filter.test.js | 8 +- 12 files changed, 2584 insertions(+), 952 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7444ff1..4421e92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### API -- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. Bounded concurrency, a pluggable cache keyed by model + task hash + content hash (in-memory by default), and a skip for binary and oversized files. Ambiguous answers and errors include the file and are reported on the decision. `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. +- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `gemini-2.0-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file. The cache key adds a truncation tag. Pass a path for an optional Node JSON cache (`{ version: 1, entries }`, corrupt files ignored). Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`). Ambiguous answers and thrown calls include the file; only a thrown call sets `error`. `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. - Renamed the package from `gptdiff-js` to `patchling` (npm name `patchling`; an intermediate publish used `gptdiff`). - `generateDiff` and `smartapply` stream tokens through `opts.onToken`. `generateDiff` reports token usage through `opts.onUsage`. - Ship JSDoc-generated `.d.ts` files, with `types` conditions on the package exports. @@ -27,7 +27,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### Site - `patchling.app` landing page: in-browser playground (canned demo and live NanoGPT), model picker, and Open Graph card. -- The playground can run `filterRelevant` before the diff: a toggle, a decision-model picker, and a per-file yes/no view (reasons, tokens, skips). Bundled sample projects exercise it without leaving the page. +- The playground can run `filterRelevant` before the diff: a toggle, a decision-model picker (default `gemini-2.0-flash`), and a per-file yes/no view (reasons, tokens, binary and oversize skips, truncated previews). Bundled sample projects exercise it without leaving the page. - README links that 404'd after the repo rename now point at patchling.app and the Python package. ### CI diff --git a/README.md b/README.md index 18045d7..11c85b6 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Configuration comes from environment variables (Node) or `setEnv(...)` overrides | `GPTDIFF_LLM_API_KEY` | NanoGPT API key (`sk-nano-…`) | — (required for live calls) | | `GPTDIFF_LLM_BASE_URL` | OpenAI-compatible base URL | `https://nano-gpt.com/api/v1/` | | `GPTDIFF_MODEL` | Model id | `xiaomi/mimo-v2.5-pro-ultraspeed` | -| `GPTDIFF_FILTER_MODEL` | Cheap model for `filterRelevant` | `xiaomi/mimo-v2.5-pro-ultraspeed` | +| `GPTDIFF_FILTER_MODEL` | Cheap model for `filterRelevant` | `gemini-2.0-flash` | ```js import { setEnv } from 'patchling'; @@ -102,38 +102,43 @@ Builds the prompt, calls the LLM, and returns the unified diff extracted from th - `opts.callLlm` — inject a custom/mock completion client (used heavily in tests). - `opts.onUsage` — optional callback invoked with `{ promptTokens, completionTokens, totalTokens }` after the LLM call completes, so callers can surface cost/usage without changing the `Promise` return type. -### `filterRelevant(files, task, opts?) → Promise` -Opt-in pass for a large file map. Each candidate file is sent, in full, with the task to a cheap model that answers `is this file relevant to the task? yes/no`. Only `yes` files should be handed to `generateDiff`. `smartapply` still takes the full map, so files the diff never mentions are left as they were. +### `filterRelevant(files, task, opts?) → Promise` +Opt-in pass for a large file map. Each candidate file is sent with the task to a cheap model that answers `is this file relevant to the task? yes/no`. `result.files` is the `yes` map, in input order, with the original contents. Hand that to `generateDiff`. `smartapply` still takes the full map, so files the diff never mentions are left as they were. ```js import { filterRelevant, buildEnvironment, generateDiff, smartapply } from 'patchling'; -const decisions = await filterRelevant(files, task, { - model: 'xiaomi/mimo-v2.5-pro-ultraspeed', // GPTDIFF_FILTER_MODEL, else this cheap default +const result = await filterRelevant(files, task, { + // model: 'gemini-2.0-flash', // GPTDIFF_FILTER_MODEL, else this default concurrency: 8, // max in-flight calls - // cache: null, // default is an in-memory cache - // maxBytes: 262144, // skip larger files; null disables + // cache: null, // omit for a fresh in-memory cache; a path string persists it + // maxBytes: 200000, // 0 or null disables + // maxTokens: 12000, // input-token cap; 0 or null disables + // onOversize: 'truncate', // include | exclude | truncate + // onBinary: 'exclude', // include | exclude | truncate onDecision(d) { console.log(d.path, d.decision); }, }); -const diff = await generateDiff(buildEnvironment(files, { decisions }), task, { model }); +const diff = await generateDiff(buildEnvironment(result.files), task, { model }); const next = await smartapply(diff, files); ``` -- `opts.model`, `opts.apiKey`, `opts.baseUrl` — same key and base URL resolution as `generateDiff` (`GPTDIFF_LLM_API_KEY`, `GPTDIFF_LLM_BASE_URL`). +- `opts.model`, `opts.apiKey`, `opts.baseUrl` — same key and base URL resolution as `generateDiff` (`GPTDIFF_LLM_API_KEY`, `GPTDIFF_LLM_BASE_URL`). The model falls back to `GPTDIFF_FILTER_MODEL`, then `gemini-2.0-flash`. - `opts.concurrency` — at most this many classifications in flight (default 8). -- `opts.cache` — `get(key)` / `set(key, value)`, sync or async. The key is `model`, SHA-256 of the task, and SHA-256 of the contents, separated by NUL. Omit it for a process-wide in-memory cache (`createMemoryCache()` builds another). `null` turns caching off. Definitive yes/no answers are stored; errors and ambiguous answers are not. -- `opts.maxBytes` — UTF-8 size cap (default 262144). Larger files, and files that look binary (a NUL, or a dense run of control bytes), are not sent. -- `opts.onDecision` — called as each file finishes, in completion order. The returned array stays in input order. +- `opts.cache` — a `RelevanceCache`, a `{ get, set }` object, or a filesystem path (Node writes `{ version: 1, entries }` atomically; a corrupt file is ignored). Omit it, or pass `true`, for a fresh in-memory cache used only by this call. `null` and `false` turn caching off. `createMemoryCache()` builds another memory cache you can share. The key is the model, SHA-256 of the task, SHA-256 of the original contents, and a variant tag (`full`, `binary-preview`, or `truncated::`), separated by newlines. Thrown errors are not stored. +- `opts.maxBytes` — UTF-8 size cap (default 200000). `opts.maxTokens` — input-token cap (default 12000; estimated as characters / 4). `0` or `null` disables either limit. +- `opts.onOversize` and `opts.onBinary` — `include` (keep, no model call), `exclude` (drop, no model call), or `truncate` (ask about a preview). Defaults are `truncate` and `exclude`. A `yes` on a truncated preview still forwards the original text in `result.files`. +- `opts.onDecision` — called as each file finishes, in completion order. `result.decisions` stays in input order. - `opts.callLlm` — inject a mock client. -Each decision is `{ path, decision, raw, tokens, cached, skippedReason, error, reason }`. +Each decision is `{ path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error }`. -- `decision` is `"yes"`, `"no"`, or `"skip"`. -- `skippedReason` is `"binary"`, `"too-large"`, or `null`. -- Ambiguous text (after `` blocks are stripped) and a thrown call become `"yes"` with `error` set, so a bad answer cannot drop a file. `error` is `"ambiguous answer"` or the thrown message. -- `buildEnvironment(files, { decisions })` keeps only `"yes"` paths. Passing the decisions array itself does the same. With no second argument, every file is included. +- `decision` is `"yes"` or `"no"`. Ambiguous text (after `` / `` blocks are stripped) and a thrown call become `"yes"`, so a bad answer cannot drop a file. `error` is set only when the call throws. +- `skippedReason` is `"binary"`, `"oversize"`, `"truncated"`, or `null`. +- `tokens`, `promptTokens`, and `completionTokens` are `0` when no model call was made. `reason` is a string. +- `result.billedTokens` sums token counts for decisions that were not cache hits. Identical file bodies share one request; the later paths are `cached: true`. +- `buildEnvironment(files, { decisions })` keeps only `"yes"` paths. Passing the decisions array itself does the same. With no second argument, every file is included. `buildEnvironment(result.files)` is the same set. -Python (`filter_relevant` in [patchling-py](https://github.com/255BITS/patchling-py)) takes the same arguments in snake_case: `max_bytes`, `on_decision`, and `skipped_reason` on each decision. `build_environment(files, decisions=decisions)` keeps the `"yes"` files. +Python (`filter_relevant` in [patchling-py](https://github.com/255BITS/patchling-py)) returns the same result. Its fields are snake_case: `max_bytes`, `max_tokens`, `on_oversize`, `on_binary`, `skipped_reason`, `prompt_tokens`, `completion_tokens`. ### `smartapply(diffText, files, opts?) → Promise<{ path: content }>` Applies a diff to an in-memory file map with per-file, LLM-assisted conflict resolution (runs files concurrently). Handles creation, modification, and deletion; `…` and reasoning preambles are stripped automatically. Returns a **new** map; deleted files are omitted. @@ -146,14 +151,14 @@ Applies a diff to an in-memory file map with per-file, LLM-assisted conflict res Deterministic, no-LLM patch application (the strict "basic" applier). `changed` is `true` iff something was created, modified, or deleted; `files` is the updated map (equal to the input on failure — no partial writes). ### Other exports -`parseDiffPerFile`, `buildEnvironment`, `filterRelevant`, `createMemoryCache`, `parseRelevanceAnswer`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. +`parseDiffPerFile`, `buildEnvironment`, `filterRelevant`, `RelevanceCache`, `createMemoryCache`, `parseRelevanceAnswer`, `isBinaryContent`, `countTokens`, `formatFilterSummary`, `dumpDecisions`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. ## Differences from the Python package - **No filesystem.** `applyDiff`/`smartapply` take and return a `{ path: content }` map rather than reading/writing a project directory. - **Scope.** Only `generateDiff` + `smartapply` (and their dependencies) are ported — not the Python package's CLIs (`patchling`, `patchling-apply`). - **Dependency injection** replaces Python's `monkeypatch`: pass `callLlm` / `callLlmForApply` to test without a network. -- **`fetch` + Web Crypto** replace `openai`/`requests`/`tiktoken`. +- **`fetch` + Web Crypto** replace `openai`/`requests`/`tiktoken`. Relevance token counts use the same characters/4 estimate Python uses when tiktoken is absent. A disk-backed `RelevanceCache` is Node-only; the browser keeps the in-memory cache. ## License diff --git a/landing/index.html b/landing/index.html index 9a1596e..638cb8f 100644 --- a/landing/index.html +++ b/landing/index.html @@ -383,7 +383,7 @@

Give your app a “describe the change” box.

- Before the diff, a cheap model answers yes or no for each file. Binary and oversized files are skipped. + Before the diff, a cheap model answers yes or no for each file. Binary files are excluded. Oversized files are truncated for the check; a yes still sends the original text.
-

Each decision is { path, decision, raw, tokens, cached, skippedReason, error, reason }. decision is yes, no, or skip. skippedReason is binary or too-large. The cache key is the model, a hash of the task, and a hash of the file contents. Python uses the same fields in snake_case (skipped_reason, max_bytes, on_decision).

+

Each decision is { path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error }. decision is yes or no. skippedReason is binary, oversize, truncated, or null. The default model is gemini-2.0-flash (GPTDIFF_FILTER_MODEL). The cache key is the model, a hash of the task, a hash of the original contents, and a truncation tag. Python uses the same fields in snake_case (skipped_reason, prompt_tokens, max_bytes, max_tokens, on_oversize, on_binary).

@@ -895,6 +891,7 @@

${esc(title)}

${cards}`; sample: true, filterDefault: true, maxBytes: 400, + onOversize: "exclude", label: "Greeting app", desc: "Six files, one greeting. The recorded pass keeps greet.py, drops the helpers, and skips a binary logo and an oversized history (this sample caps files at 400 bytes).", goal: "Say goodbye instead of hello", @@ -911,8 +908,8 @@

${esc(title)}

${cards}`; "main.py": { decision: "no", reason: "calls greet, but the words are not here", tokens: 41, raw: "no — calls greet, but the words are not here" }, "math_utils.py": { decision: "no", reason: "arithmetic helper", tokens: 22, raw: "no — arithmetic helper" }, "README.md": { decision: "no", reason: "docs, not the string to edit", tokens: 27, raw: "no — docs, not the string to edit" }, - "assets/logo.bin": { decision: "skip", skippedReason: "binary", tokens: null, raw: null }, - "vendor/HISTORY.txt": { decision: "skip", skippedReason: "too-large", tokens: null, raw: null }, + "assets/logo.bin": { decision: "no", skippedReason: "binary", reason: "binary file; excluded without a model call", tokens: 0, raw: null }, + "vendor/HISTORY.txt": { decision: "no", skippedReason: "oversize", reason: "over the 400-byte sample cap", tokens: 0, raw: null }, }, response: 'The greeting string is only in greet.py.\n\n```diff\n--- a/greet.py\n+++ b/greet.py\n@@ -1,2 +1,2 @@\n def greet(name):\n- print("Hello, " + name + "!")\n+ print("Goodbye, " + name + "!")\n```\n', applied: {}, @@ -1562,20 +1559,21 @@

${esc(title)}

${cards}`; function renderDecision(d) { const row = document.createElement("div"); row.className = "drow"; + const skipped = d.skippedReason === "binary" || d.skippedReason === "oversize"; const badge = document.createElement("span"); - badge.className = "badge " + (d.decision === "yes" ? "yes" : d.decision === "no" ? "no" : "skip"); - badge.textContent = d.decision === "skip" ? "skipped" : d.decision; + badge.className = "badge " + (skipped ? "skip" : d.decision === "yes" ? "yes" : "no"); + badge.textContent = skipped ? "skipped" : d.decision; const path = document.createElement("span"); path.className = "mono"; path.textContent = d.path; const why = document.createElement("span"); why.className = "why"; const bits = []; - if (d.skippedReason) bits.push(d.skippedReason === "too-large" ? "too large" : d.skippedReason); + if (d.skippedReason) bits.push(d.skippedReason); if (d.reason) bits.push(d.reason); if (d.error) bits.push(d.error); if (d.cached) bits.push("cached"); - if (d.tokens != null) bits.push(d.tokens + " tokens"); + if (d.tokens) bits.push(d.tokens + " tokens"); why.textContent = bits.join(" · "); row.append(badge, path, why); $("decisionlist").appendChild(row); @@ -1609,12 +1607,17 @@

${esc(title)}

${cards}`; return out; } $("decisionlist").innerHTML = ""; - return filterRelevant(inputFiles, goal, { + const scn = SCENARIOS[scenarioKey]; + const result = await filterRelevant(inputFiles, goal, { apiKey: getKey(), model: FILTER_MODEL, - maxBytes: SCENARIOS[scenarioKey].maxBytes, + maxBytes: scn.maxBytes, + maxTokens: scn.maxTokens, + onOversize: scn.onOversize, + onBinary: scn.onBinary, onDecision: renderDecision, }); + return result.decisions; } for (const btn of document.querySelectorAll("[data-sample]")) { diff --git a/landing/lib/filter.js b/landing/lib/filter.js index a88b17d..84ae6c3 100644 --- a/landing/lib/filter.js +++ b/landing/lib/filter.js @@ -1,98 +1,220 @@ /** * Opt-in relevance filter for large file maps. * - * For each candidate file, a cheap model sees the full contents plus the task - * and answers "is this file relevant to the task? yes/no". Only `yes` - * decisions should be passed to `generateDiff` (see `buildEnvironment`'s - * `decisions` option). `smartapply` still wants the full map so files the - * diff never mentions stay put. + * For each candidate file, a cheap model sees the contents plus the task and + * answers "is this file relevant to the task? yes/no". Only `yes` files are + * returned for `generateDiff`. `smartapply` still wants the full map so files + * the diff never mentions stay put. * - * Mirrors the Python package's `filter_relevant` (255BITS/patchling-py): + * Mirrors `filter_relevant` in 255BITS/patchling-py (`cursor/relevance-filter-0192`). + * Field names are camelCase. Ambiguous answers and per-file errors include the + * file. An error is attached only when the call throws. * - * filter_relevant(files, task, *, model=None, concurrency=8, cache=None, - * max_bytes=262144, on_decision=None, api_key=None, - * base_url=None) -> list[decision] - * - * decision: path, decision ("yes" | "no" | "skip"), raw, tokens, cached, - * skipped_reason ("binary" | "too-large" | None), error, reason - * - * JS uses camelCase (`maxBytes`, `onDecision`, `skippedReason`) like the rest - * of this package. Ambiguous answers and per-file errors include the file - * (`decision: "yes"`) and surface the problem on `error` instead of throwing. + * The disk cache loads `node:fs` on demand so this module stays browser-safe. */ -import { stripThinkTags } from './text.js'; +import { swallowReasoning } from './text.js'; import { callLlm as defaultCallLlm, resolveApiKey, resolveBaseUrl } from './llm.js'; import { getEnv } from './env.js'; /** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ -export const DEFAULT_FILTER_MODEL = 'xiaomi/mimo-v2.5-pro-ultraspeed'; - -/** Files larger than this (UTF-8 bytes) are skipped unless `maxBytes` is set. */ -export const DEFAULT_FILTER_MAX_BYTES = 262144; +export const DEFAULT_FILTER_MODEL = 'gemini-2.0-flash'; /** In-flight classifications per call when `concurrency` is omitted. */ export const DEFAULT_FILTER_CONCURRENCY = 8; -const FILTER_SYSTEM_PROMPT = `You decide if one file is relevant to a coding task. Reply with "yes" or "no" as the first word. You may add one short reason after it. Do not quote the file.`; +/** UTF-8 size cap. `0` or `null` disables it. */ +export const DEFAULT_FILTER_MAX_BYTES = 200_000; + +/** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ +export const DEFAULT_FILTER_MAX_TOKENS = 12_000; -const AMBIGUOUS = 'ambiguous answer'; +/** Completion cap for the yes/no call. */ +export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 256; + +const POLICIES = ['include', 'exclude', 'truncate']; + +const ANSWER_PREFIX = /^(?:answer|relevance|relevant|decision)\s*[:\-]\s*/i; +const FENCE_LINE = /^```[a-zA-Z0-9_-]*\s*$/; + +const SYSTEM_PROMPT = ( + 'You are a relevance filter for a code-editing task. ' + + 'Decide whether the given file must be read or edited to accomplish the task, ' + + 'or is necessary context for that edit. ' + + 'Reply with a single line that starts with exactly "yes" or exactly "no", ' + + 'followed by a short reason. ' + + 'Answer "yes" when the file is likely to need changes or is required context. ' + + 'Answer "no" when it is unrelated. Do not write a diff.' +); /** * @typedef {object} RelevanceDecision * @property {string} path - * @property {'yes' | 'no' | 'skip'} decision `yes` files go on to generateDiff. - * Ambiguous answers and errors are `yes` so a bad classifier cannot drop a - * file. `skip` files were not sent to the model. - * @property {string | null} raw model text, unchanged (think tags included) - * @property {number | null} tokens total tokens for the call; null when skipped or the call failed - * @property {boolean} cached true when this answer was read from the cache - * @property {'binary' | 'too-large' | null} skippedReason - * @property {string | null} error set for thrown calls and ambiguous answers - * @property {string | null} reason short reason after the yes/no, when the model gave one + * @property {'yes' | 'no'} decision `yes` files go on to generateDiff. + * Ambiguous answers and errors are `yes`. + * @property {string} reason short explanation; the cleaned model text when a call was made + * @property {string | null} raw model text before parsing, or null when no call was made + * @property {number} tokens total tokens for the call; 0 when no call was made + * @property {number} promptTokens + * @property {number} completionTokens + * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) + * @property {'binary' | 'oversize' | 'truncated' | null} skippedReason + * set when the file was not sent in full + * @property {string | null} error set when the call threw; the file is still included */ /** - * @typedef {object} RelevanceCacheEntry + * @typedef {object} FilterRelevantResult + * @property {Record} files `yes` files in input order, original contents + * @property {RelevanceDecision[]} decisions one per input file, in input order + * @property {string} model + * @property {RelevanceDecision[]} errors decisions whose call threw + * @property {number} billedTokens sum of `tokens` on decisions that were not cache hits + * @property {number} billedPromptTokens + * @property {number} billedCompletionTokens + */ + +/** + * Record stored in a cache. Snake_case so a JSON file matches the Python cache. + * @typedef {object} RelevanceCacheRecord * @property {'yes' | 'no'} decision - * @property {string | null} raw - * @property {number | null} tokens - * @property {string | null} reason + * @property {string} [reason] + * @property {string | null} [raw] + * @property {number} [tokens] + * @property {number} [prompt_tokens] + * @property {number} [completion_tokens] + * @property {'binary' | 'oversize' | 'truncated' | null} [skipped_reason] + * @property {string | null} [error] + * @property {string} [cache_key] */ /** - * A cache is any object with `get(key)` and `set(key, value)`. Either method - * may be sync or async. Keys are `${model}\\0${taskHash}\\0${contentHash}` - * (SHA-256 hex). Pass `cache: null` to disable caching. + * In-memory relevance cache with an optional on-disk JSON file. * - * @typedef {object} RelevanceCache - * @property {(key: string) => RelevanceCacheEntry | undefined | Promise} get - * @property {(key: string, value: RelevanceCacheEntry) => void | Promise} set + * Keys are `model`, SHA-256 of the task, SHA-256 of the original contents, and + * a variant tag (`full`, `binary-preview`, or `truncated::`), + * separated by newlines. A judgment on a truncated body is not reused as a + * full-content judgment. */ +export class RelevanceCache { + /** + * @param {string} [path] JSON file. Omit for memory only. + * Loaded on the first read. Corrupt files are ignored. + */ + constructor(path) { + this.path = path ? String(path) : null; + /** @type {Record} */ + this._entries = Object.create(null); + /** @type {Promise | null} */ + this._loadPromise = null; + } + + /** + * @param {string} model + * @param {string} task + * @param {string} content original file text + * @param {string} [variant] + * @returns {Promise} + */ + static async makeKey(model, task, content, variant = 'full') { + const taskHash = await sha256(task); + const contentHash = await sha256(content); + return `${model}\n${taskHash}\n${contentHash}\n${variant}`; + } + + /** + * @param {string} key + * @returns {Promise} + */ + async get(key) { + await this.load(); + const record = this._entries[key]; + return record ? { ...record } : undefined; + } -let sharedMemoryCache; + /** + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ + async put(key, record) { + await this.load(); + this._entries[key] = { ...record }; + } + + /** + * Alias of {@link put} so a `{ get, set }` cache and a `RelevanceCache` share a shape. + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ + async set(key, record) { + await this.put(key, record); + } + + /** Write `{ version: 1, entries }` atomically. No-op without a path. */ + async flush() { + if (!this.path) return; + await this.load(); + const fs = await import('node:fs/promises'); + const pathMod = await import('node:path'); + const dir = pathMod.dirname(pathMod.resolve(this.path)); + await fs.mkdir(dir, { recursive: true }); + const tmp = pathMod.join( + dir, + `.relevance-cache-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.tmp`, + ); + const payload = JSON.stringify({ version: 1, entries: this._entries }); + try { + await fs.writeFile(tmp, payload, 'utf8'); + await fs.rename(tmp, this.path); + } catch (err) { + await fs.unlink(tmp).catch(() => {}); + throw err; + } + } + + /** @returns {Promise} */ + async load() { + if (!this.path) return; + if (!this._loadPromise) this._loadPromise = this._load(); + await this._loadPromise; + } + + /** @returns {Promise} */ + async _load() { + let fs; + try { + fs = await import('node:fs/promises'); + } catch { + throw new Error('A disk-backed relevance cache requires Node (node:fs).'); + } + let text; + try { + text = await fs.readFile(this.path, 'utf8'); + } catch (err) { + if (err && /** @type {NodeJS.ErrnoException} */ (err).code === 'ENOENT') return; + return; + } + let payload; + try { + payload = JSON.parse(text); + } catch { + return; + } + const entries = payload && typeof payload === 'object' ? payload.entries : null; + if (entries && typeof entries === 'object' && !Array.isArray(entries)) { + this._entries = entries; + } + } +} /** - * Process-wide in-memory cache used when `filterRelevant` is called without - * a `cache` option. Pass the result as `cache` to share or inspect it. + * Fresh in-memory cache. Pass one instance as `cache` to share it across calls. * @returns {RelevanceCache} */ export function createMemoryCache() { - /** @type {Map} */ - const store = new Map(); - return { - get(key) { - return store.has(key) ? store.get(key) : undefined; - }, - set(key, value) { - store.set(key, value); - }, - }; -} - -function memoryCache() { - if (!sharedMemoryCache) sharedMemoryCache = createMemoryCache(); - return sharedMemoryCache; + return new RelevanceCache(); } /** @@ -108,305 +230,754 @@ async function sha256(text) { } /** - * @param {string} model - * @param {string} taskHash - * @param {string} contentHash + * Drop `` / `` blocks, including an unclosed tail. + * @param {string} text * @returns {string} */ -function cacheKey(model, taskHash, contentHash) { - return `${model}\0${taskHash}\0${contentHash}`; +function stripFilterThinkTags(text) { + if (!text) return ''; + let cleaned = text.replace(/]*>[\s\S]*?<\/think>/gi, ''); + cleaned = cleaned.replace(/]*>[\s\S]*?<\/thinking>/gi, ''); + cleaned = cleaned.replace(/]*>[\s\S]*$/gi, ''); + cleaned = cleaned.replace(/]*>[\s\S]*$/gi, ''); + cleaned = cleaned.replace(/<\/think>/gi, ''); + cleaned = cleaned.replace(/<\/thinking>/gi, ''); + return cleaned.trim(); +} + +/** + * Parse a yes/no relevance answer. + * + * Think tags and chain-of-thought reasoning blocks are removed before the + * answer is read, so a "no" inside a think tag cannot override a final "yes". + * A single consistent yes or no wins, including "Answer: yes" and a fenced + * "NO". Both words, or neither, is ambiguous — callers include the file. + * + * @param {unknown} raw + * @returns {{ decision: 'yes' | 'no' | null, cleaned: string }} + */ +export function parseRelevanceAnswer(raw) { + let text = stripFilterThinkTags(raw == null ? '' : String(raw)); + const [swallowed] = swallowReasoning(text); + text = swallowed.trim(); + const fenceLines = text.split(/\r\n|\r|\n/); + if ( + fenceLines.length >= 2 + && FENCE_LINE.test(fenceLines[0].trim()) + && fenceLines[fenceLines.length - 1].trim() === '```' + ) { + text = fenceLines.slice(1, -1).join('\n').trim(); + } + const cleaned = text.trim().replace(/^`+|`+$/g, '').trim(); + const contentLines = cleaned.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); + if (!contentLines.length) return { decision: null, cleaned }; + + let answerText = contentLines[0].replace(ANSWER_PREFIX, '').trim(); + if (contentLines.length > 1) { + answerText = `${answerText}\n${contentLines.slice(1).join('\n')}`; + } + const words = answerText.match(/\b(yes|no)\b/gi) || []; + const lowered = words.map((word) => word.toLowerCase()); + if (lowered.length && new Set(lowered).size === 1) { + return { decision: /** @type {'yes' | 'no'} */ (lowered[0]), cleaned }; + } + return { decision: null, cleaned }; +} + +/** + * @param {string} name + * @param {unknown} value + * @returns {'include' | 'exclude' | 'truncate'} + */ +function checkPolicy(name, value) { + if (!POLICIES.includes(/** @type {string} */ (value))) { + throw new Error(`${name} must be one of ${POLICIES.join(', ')}; got ${JSON.stringify(value)}`); + } + return /** @type {'include' | 'exclude' | 'truncate'} */ (value); } +/** + * @param {string | undefined} model + * @returns {string} + */ function resolveFilterModel(model) { return model || getEnv('GPTDIFF_FILTER_MODEL') || DEFAULT_FILTER_MODEL; } /** - * @param {unknown} content - * @returns {boolean} + * `null` and `false` disable the cache. Omit or pass `true` for a fresh + * in-memory cache. A string is a JSON file. A `RelevanceCache` is used as-is. + * An object with `get` and `put`/`set` is a pluggable cache. + * @param {unknown} cache + * @returns {RelevanceCache | { get: Function, put?: Function, set?: Function, load?: Function, flush?: Function } | null} */ -function looksBinary(content) { - if (typeof content !== 'string') return true; - const n = Math.min(content.length, 8192); - let controls = 0; - for (let i = 0; i < n; i++) { - const c = content.charCodeAt(i); - if (c === 0) return true; - // C0 controls other than tab, LF, and CR. A dense run is not text. - if (c < 32 && c !== 9 && c !== 10 && c !== 13) controls++; +function resolveCache(cache) { + if (cache === false || cache === null) return null; + if (cache === undefined || cache === true) return new RelevanceCache(); + if (cache instanceof RelevanceCache) return cache; + if (typeof cache === 'string') return new RelevanceCache(cache); + if ( + cache + && typeof cache === 'object' + && typeof /** @type {{ get?: unknown }} */ (cache).get === 'function' + && ( + typeof /** @type {{ put?: unknown }} */ (cache).put === 'function' + || typeof /** @type {{ set?: unknown }} */ (cache).set === 'function' + ) + ) { + return /** @type {{ get: Function, put?: Function, set?: Function }} */ (cache); } - return n > 0 && controls / n > 0.3; + throw new TypeError('cache must be a RelevanceCache, a filesystem path, null, or false'); } /** - * UTF-8 byte length, stopping once `maxBytes` is exceeded. - * @param {string} content + * Character-length stand-in for a tokenizer (Python's fallback when tiktoken + * is absent). Empty text is 0; anything else is at least 1. + * @param {string} text + * @returns {number} + */ +export function countTokens(text) { + if (!text) return 0; + return Math.max(1, Math.floor(String(text).length / 4)); +} + +/** + * @param {string} text + * @param {number | null} maxBytes + * @param {number | null} maxTokens + * @returns {string} + */ +function truncateText(text, maxBytes, maxTokens) { + let out = text; + if (maxTokens && countTokens(out) > maxTokens) { + out = out.slice(0, maxTokens * 4); + } + if (maxBytes) out = truncateUtf8(out, maxBytes); + return out; +} + +/** + * UTF-8 prefix. An incomplete trailing character is dropped, matching + * Python's `errors="ignore"` on a cut byte string. + * @param {string} text * @param {number} maxBytes - * @returns {boolean} + * @returns {string} */ -function exceedsMaxBytes(content, maxBytes) { - if (!Number.isFinite(maxBytes)) return false; - let bytes = 0; - for (let i = 0; i < content.length; i++) { - const c = content.charCodeAt(i); - if (c < 0x80) bytes += 1; - else if (c < 0x800) bytes += 2; - else if (c >= 0xd800 && c <= 0xdbff) { - bytes += 4; - i += 1; - } else bytes += 3; - if (bytes > maxBytes) return true; - } - return false; +function truncateUtf8(text, maxBytes) { + const bytes = new TextEncoder().encode(text); + if (bytes.length <= maxBytes) return text; + let end = maxBytes; + while (end > 0 && (bytes[end - 1] & 0xc0) === 0x80) end -= 1; + if (end > 0) { + const lead = bytes[end - 1]; + const need = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4; + if (end - 1 + need > maxBytes) end -= 1; + } + return new TextDecoder().decode(bytes.subarray(0, end)); } /** - * @param {unknown} response + * @param {string} text * @returns {number} */ -function totalTokens(response) { - const usage = response && /** @type {{ usage?: Record }} */ (response).usage; - if (!usage) return 0; - const prompt = usage.prompt_tokens ?? usage.input_tokens ?? 0; - const completion = usage.completion_tokens ?? usage.output_tokens ?? 0; - return usage.total_tokens ?? prompt + completion; +function utf8ByteLength(text) { + return new TextEncoder().encode(text).length; } /** - * Parse a yes/no classification. Think tags and "> Reasoning" preambles are - * removed first. The first word wins when it is yes or no. If both words - * appear, or neither does, the answer is ambiguous and the caller includes - * the file. - * - * @param {unknown} raw - * @returns {{ decision: 'yes' | 'no', reason: string | null, ambiguous: boolean }} + * True when `content` looks like a binary file (NUL or a dense run of + * non-text controls). Non-text values other than byte arrays are not binary; + * {@link filterRelevant} rejects them when it coerces the file. + * @param {unknown} content + * @returns {boolean} */ -export function parseRelevanceAnswer(raw) { - const stripped = stripThinkTags(raw == null ? '' : String(raw)).trim(); - const unfenced = stripped.replace(/^```[a-z0-9]*\s*/i, '').replace(/```\s*$/i, '').trim(); - const lines = unfenced.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); - const first = (lines[0] || '').replace(/^[*_>"'`]+/, '').replace(/[*_"`']+$/g, '').trim(); - const lead = first.match(/^(yes|no)\b[\s.:,\-—!]*(.*)$/i); - - if (lead) { - const decision = /** @type {'yes' | 'no'} */ (lead[1].toLowerCase()); - const sameLine = lead[2].trim(); - const reason = [sameLine, lines.slice(1).join(' ')].filter(Boolean).join(' ').trim() || null; - const firstHasYes = /\byes\b/i.test(first); - const firstHasNo = /\bno\b/i.test(first); - if (firstHasYes && firstHasNo) { - return { decision: 'yes', reason: unfenced || null, ambiguous: true }; - } - return { decision, reason, ambiguous: false }; +export function isBinaryContent(content) { + if (content instanceof Uint8Array) return binaryBytes(content); + if (typeof content !== 'string') return false; + return textLooksBinary(content.slice(0, 8192)); +} + +/** + * @param {Uint8Array} bytes + * @returns {boolean} + */ +function binaryBytes(bytes) { + const sample = bytes.subarray(0, 8192); + for (let i = 0; i < sample.length; i++) { + if (sample[i] === 0) return true; + } + try { + const text = new TextDecoder('utf-8', { fatal: true }).decode(sample); + return textLooksBinary(text); + } catch { + return true; } +} - const hasYes = /\byes\b/i.test(unfenced); - const hasNo = /\bno\b/i.test(unfenced); - if (hasYes && !hasNo) return { decision: 'yes', reason: unfenced || null, ambiguous: false }; - if (hasNo && !hasYes) return { decision: 'no', reason: unfenced || null, ambiguous: false }; - return { decision: 'yes', reason: unfenced || null, ambiguous: true }; +/** + * @param {string} sample + * @returns {boolean} + */ +function textLooksBinary(sample) { + if (!sample) return false; + if (sample.includes('\0')) return true; + let controls = 0; + for (let i = 0; i < sample.length; i++) { + const c = sample.charCodeAt(i); + if (c < 32 && c !== 9 && c !== 10 && c !== 13 && c !== 12) controls += 1; + } + return controls / sample.length > 0.3; } /** - * @param {Array<[string, string]>} items + * @param {unknown} content + * @returns {string} + */ +function coerceText(content) { + if (typeof content === 'string') return content; + if (content instanceof Uint8Array) { + return new TextDecoder('utf-8', { fatal: false }).decode(content); + } + const name = content === null ? 'null' : typeof content; + throw new TypeError(`file content must be str or bytes, got ${name}`); +} + +/** + * @param {unknown} value + * @param {number} fallback + * @returns {number | null} null disables the limit (`0` or `null`) + */ +function resolveLimit(value, fallback) { + const chosen = value === undefined ? fallback : value; + if (chosen === null || chosen === 0) return null; + const n = Number(chosen); + if (!Number.isFinite(n) || n <= 0) return null; + return n; +} + +/** + * @param {unknown} value + * @returns {string} + */ +/** + * Tag a limit the way Python's f-string does: omitted uses the default number, + * `null` is `None`, and `0` stays `0`. + * @param {unknown} value + * @param {number} fallback + * @returns {string} + */ +function rawLimitTag(value, fallback) { + const chosen = value === undefined ? fallback : value; + if (chosen === null || chosen === undefined) return 'None'; + return String(chosen); +} + +/** + * @param {object} fields + * @returns {RelevanceDecision} + */ +function decisionObject(fields) { + return { + path: fields.path, + decision: fields.decision, + reason: fields.reason ?? '', + raw: fields.raw ?? null, + tokens: fields.tokens ?? 0, + promptTokens: fields.promptTokens ?? 0, + completionTokens: fields.completionTokens ?? 0, + cached: !!fields.cached, + skippedReason: fields.skippedReason ?? null, + error: fields.error ?? null, + }; +} + +/** + * @param {string} path + * @param {RelevanceCacheRecord} record + * @param {boolean} cached + * @returns {RelevanceDecision} + */ +function decisionFromRecord(path, record, cached) { + const rec = /** @type {Record} */ (record); + return decisionObject({ + path, + decision: /** @type {'yes' | 'no'} */ (rec.decision), + reason: rec.reason || '', + raw: rec.raw ?? null, + tokens: Number(rec.tokens || 0), + promptTokens: Number(rec.prompt_tokens ?? rec.promptTokens ?? 0), + completionTokens: Number(rec.completion_tokens ?? rec.completionTokens ?? 0), + cached, + skippedReason: rec.skipped_reason ?? rec.skippedReason ?? null, + error: rec.error ?? null, + }); +} + +/** + * @param {string} path + * @param {'yes' | 'no'} decision + * @param {'binary' | 'oversize'} skippedReason + * @param {string} reason + * @returns {RelevanceDecision} + */ +function skipDecision(path, decision, skippedReason, reason) { + return decisionObject({ + path, + decision, + reason, + raw: null, + tokens: 0, + cached: false, + skippedReason, + error: null, + }); +} + +/** + * @param {string} task + * @param {string[]} paths + * @param {string} body + * @param {string | null} note + * @returns {string} + */ +function buildUserPrompt(task, paths, body, note) { + const label = paths.length === 1 ? 'File path' : 'File paths'; + const parts = [`Task:\n${task}`, `${label}:\n${paths.join('\n')}`]; + if (note) parts.push(note); + parts.push(`File contents:\n${body}`); + parts.push('Is this file relevant to the task? yes/no'); + return parts.join('\n\n'); +} + +/** + * @param {unknown} response + * @returns {{ raw: string, promptTokens: number, completionTokens: number, tokens: number }} + */ +function usageAndText(response) { + const message = /** @type {{ choices?: Array<{ message?: { content?: string | null } }>, usage?: Record }} */ (response); + const content = message?.choices?.[0]?.message?.content ?? ''; + const usage = message?.usage || {}; + const promptTokens = Number(usage.prompt_tokens ?? usage.input_tokens ?? 0); + const completionTokens = Number(usage.completion_tokens ?? usage.output_tokens ?? 0); + const tokens = Number(usage.total_tokens ?? (promptTokens + completionTokens)); + return { + raw: content || '', + promptTokens, + completionTokens, + tokens, + }; +} + +/** + * @param {unknown} err + * @returns {string} + */ +function errorMessage(err) { + if (err instanceof Error) return `${err.name}: ${err.message}`; + return String(err); +} + +/** + * @param {Array} items * @param {number} limit - * @param {(item: [string, string], index: number) => Promise} fn - * @returns {Promise} + * @param {(item: string) => Promise} fn + * @returns {Promise} */ async function mapLimit(items, limit, fn) { - /** @type {RelevanceDecision[]} */ - const results = new Array(items.length); - if (!items.length) return results; + if (!items.length) return; let cursor = 0; const workers = Math.min(limit, items.length); const run = async () => { while (cursor < items.length) { const index = cursor; cursor += 1; - results[index] = await fn(items[index], index); + await fn(items[index]); } }; await Promise.all(Array.from({ length: workers }, () => run())); - return results; } /** - * @param {object} args - * @param {string} args.path - * @param {unknown} args.content - * @param {string} args.task - * @param {string} args.taskHash - * @param {string} args.model - * @param {RelevanceCache | null} args.store - * @param {number} args.maxBytes - * @param {string | undefined} args.apiKey - * @param {string | undefined} args.baseUrl - * @param {Function} args.callLlm - * @param {number} args.maxTokens - * @param {number} args.temperature - * @returns {Promise} - */ -async function classifyFile(args) { - const { path, content, task, taskHash, model, store, maxBytes, apiKey, baseUrl, callLlm, maxTokens, temperature } = args; - - if (looksBinary(content)) { - return { - path, - decision: 'skip', - raw: null, - tokens: null, - cached: false, - skippedReason: 'binary', - error: null, - reason: null, - }; + * @param {{ get: Function, put?: Function, set?: Function }} store + * @param {string} key + * @returns {Promise} + */ +async function cacheGet(store, key) { + const hit = await store.get(key); + return hit || undefined; +} + +/** + * @param {{ put?: Function, set?: Function }} store + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ +async function cachePut(store, key, record) { + if (typeof store.put === 'function') await store.put(key, record); + else await store.set(key, record); +} + +/** + * Ask a cheap model which files are relevant to `task`. + * + * `files` is a `{ path: content }` map (string or byte array). The returned + * `files` map contains only the `yes` files, in input order, with their + * original contents — a truncated preview is what the model saw, not what is + * forwarded. `decisions` has one entry per input file. + * + * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that + * JSON file), `true` or omitted (a fresh in-memory cache for this call), or + * `null` / `false` (disabled). The cache key is the model, a hash of the task, + * a hash of the original file contents, and a truncation variant. + * + * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. + * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` + * (drop, no model call), or `truncate` (ask about a truncated or redacted + * preview). Pass `0` or `null` for `maxBytes` or `maxTokens` to disable that + * limit. Credentials resolve the same way as the rest of the library. + * + * @param {Record} files + * @param {string} task + * @param {object} [opts] + * @param {string} [opts.model] `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} + * @param {number} [opts.concurrency] max in-flight calls (default {@link DEFAULT_FILTER_CONCURRENCY}) + * @param {RelevanceCache | string | boolean | null | { get: Function, put?: Function, set?: Function }} [opts.cache] + * @param {number | null} [opts.maxBytes] UTF-8 cap; `0` or `null` disables; default {@link DEFAULT_FILTER_MAX_BYTES} + * @param {number | null} [opts.maxTokens] input-token cap; `0` or `null` disables; default {@link DEFAULT_FILTER_MAX_TOKENS} + * @param {'include' | 'exclude' | 'truncate'} [opts.onOversize] default `truncate` + * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` + * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) + * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) + * @param {string} [opts.apiKey] `GPTDIFF_LLM_API_KEY` when omitted + * @param {string} [opts.baseUrl] `GPTDIFF_LLM_BASE_URL` when omitted + * @param {Function} [opts.callLlm] injectable client (tests) + * @returns {Promise} + */ +export async function filterRelevant(files, task, opts = {}) { + if (files == null || typeof files !== 'object' || Array.isArray(files)) { + throw new TypeError('files must be an object of path -> content'); } + if (typeof task !== 'string' || task.trim() === '') { + throw new Error('task is required'); + } + const concurrency = Math.trunc(Number(opts.concurrency === undefined ? DEFAULT_FILTER_CONCURRENCY : opts.concurrency)); + if (!Number.isFinite(concurrency) || concurrency < 1) { + throw new Error('concurrency must be >= 1'); + } + const onOversize = checkPolicy('onOversize', opts.onOversize === undefined ? 'truncate' : opts.onOversize); + const onBinary = checkPolicy('onBinary', opts.onBinary === undefined ? 'exclude' : opts.onBinary); + const maxBytes = resolveLimit(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES); + const maxTokens = resolveLimit(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS); + const maxOutputTokens = opts.maxOutputTokens === undefined + ? DEFAULT_FILTER_MAX_OUTPUT_TOKENS + : opts.maxOutputTokens; - const text = /** @type {string} */ (content); - if (exceedsMaxBytes(text, maxBytes)) { - return { - path, - decision: 'skip', - raw: null, - tokens: null, - cached: false, - skippedReason: 'too-large', - error: null, - reason: null, - }; + const model = resolveFilterModel(opts.model); + const store = resolveCache(opts.cache); + const callLlm = opts.callLlm || defaultCallLlm; + const apiKey = opts.apiKey == null ? resolveApiKey(undefined) : opts.apiKey; + const baseUrl = resolveBaseUrl(opts.baseUrl); + const onDecision = opts.onDecision; + + /** @type {Map} */ + const byPath = new Map(); + /** @type {Map>} */ + const groups = new Map(); + /** @type {string[]} */ + const groupOrder = []; + const entries = Object.entries(files); + + /** + * @param {string} key + * @param {{ path: string, body: string, note: string | null, skippedReason: 'truncated' | null }} item + */ + function enqueue(key, item) { + if (!groups.has(key)) { + groups.set(key, []); + groupOrder.push(key); + } + groups.get(key).push(item); } - const contentHash = await sha256(text); - const key = cacheKey(model, taskHash, contentHash); - if (store) { - const hit = await store.get(key); - if (hit && (hit.decision === 'yes' || hit.decision === 'no')) { - return { - path, - decision: hit.decision, - raw: hit.raw ?? null, - tokens: hit.tokens ?? null, - cached: true, - skippedReason: null, - error: null, - reason: hit.reason ?? null, - }; + /** + * @param {RelevanceDecision} decision + * @returns {RelevanceDecision} + */ + function emit(decision) { + if (typeof onDecision === 'function') onDecision(decision); + return decision; + } + + for (const [path, content] of entries) { + const binary = isBinaryContent(content); + const text = coerceText(content); + if (binary) { + if (onBinary === 'truncate') { + const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); + const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; + const body = `[binary file, ${nbytes} bytes, contents omitted]`; + const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview'); + enqueue(key, { path, body, note, skippedReason: 'truncated' }); + } else { + const decision = onBinary === 'include' ? 'yes' : 'no'; + const verb = decision === 'yes' ? 'included' : 'excluded'; + const made = skipDecision(path, decision, 'binary', `binary file; ${verb} without a model call`); + byPath.set(path, made); + emit(made); + } + continue; + } + + const nBytes = utf8ByteLength(text); + const nTokens = countTokens(text); + const overBytes = !!maxBytes && nBytes > maxBytes; + const overTokens = !!maxTokens && nTokens > maxTokens; + if (overBytes || overTokens) { + if (onOversize === 'truncate') { + const body = truncateText(text, maxBytes, maxTokens); + const note = ( + 'Note: this file was truncated to fit the relevance check ' + + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` + ); + const variant = `truncated:${rawLimitTag(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS)}:${rawLimitTag(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES)}`; + const key = await RelevanceCache.makeKey(model, task, text, variant); + enqueue(key, { path, body, note, skippedReason: 'truncated' }); + } else { + const decision = onOversize === 'include' ? 'yes' : 'no'; + const verb = decision === 'yes' ? 'included' : 'excluded'; + const made = skipDecision( + path, + decision, + 'oversize', + `over size threshold (${nBytes} bytes, ${nTokens} tokens); ${verb} without a model call`, + ); + byPath.set(path, made); + emit(made); + } + continue; } + + const key = await RelevanceCache.makeKey(model, task, text, 'full'); + enqueue(key, { path, body: text, note: null, skippedReason: null }); } - const userContent = `Task:\n${task}\n\nFile: ${path}\n${text}\n\nis this file relevant to the task? yes/no`; - try { - const response = await callLlm({ - apiKey: resolveApiKey(apiKey), - baseUrl: resolveBaseUrl(baseUrl), - model, - messages: [ - { role: 'system', content: FILTER_SYSTEM_PROMPT }, - { role: 'user', content: userContent }, - ], - maxTokens, - temperature, - }); - const raw = response?.choices?.[0]?.message?.content ?? ''; + if (store && typeof store.load === 'function') await store.load(); + + /** + * @param {string} key + * @returns {Promise} + */ + async function cached(key) { + if (!store) return false; + const hit = await cacheGet(store, key); + return !!(hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error); + } + + let needsCall = false; + for (const key of groupOrder) { + if (!(await cached(key))) { + needsCall = true; + break; + } + } + if (needsCall && !apiKey) { + throw new Error( + 'GPTDIFF_LLM_API_KEY is required for the relevance filter. ' + + 'Set it in the environment or pass apiKey.', + ); + } + + /** + * @param {string} key + * @returns {Promise} + */ + async function runGroup(key) { + const items = groups.get(key); + if (store) { + const hit = await cacheGet(store, key); + if (hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error) { + for (const item of items) { + const decision = decisionFromRecord(item.path, { ...hit, cache_key: key }, true); + byPath.set(item.path, decision); + emit(decision); + } + return; + } + } + + const sample = items[0]; + let raw; + let promptTokens; + let completionTokens; + let totalTokens; + try { + const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); + let userContent = userPrompt; + if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; + const response = await callLlm({ + apiKey, + baseUrl, + model, + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: userContent }, + ], + maxTokens: maxOutputTokens, + temperature: 0, + budgetTokens: null, + }); + ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); + } catch (err) { + const message = errorMessage(err); + for (const item of items) { + const decision = decisionObject({ + path: item.path, + decision: 'yes', + reason: `error; included by default: ${message}`, + raw: null, + tokens: 0, + cached: false, + skippedReason: item.skippedReason, + error: message, + }); + byPath.set(item.path, decision); + emit(decision); + } + return; + } + const parsed = parseRelevanceAnswer(raw); - /** @type {RelevanceDecision} */ - const decision = { - path, - decision: parsed.decision, - raw: raw == null ? '' : String(raw), - tokens: totalTokens(response), - cached: false, - skippedReason: null, - error: parsed.ambiguous ? AMBIGUOUS : null, - reason: parsed.ambiguous ? (parsed.reason || null) : parsed.reason, + /** @type {'yes' | 'no'} */ + let decisionName; + /** @type {string} */ + let reason; + if (parsed.decision == null) { + decisionName = 'yes'; + reason = parsed.cleaned + ? `ambiguous answer; included by default: ${parsed.cleaned}` + : 'ambiguous answer; included by default'; + } else { + decisionName = parsed.decision; + reason = parsed.cleaned; + } + /** @type {RelevanceCacheRecord} */ + const record = { + decision: decisionName, + reason, + raw, + tokens: totalTokens, + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + skipped_reason: sample.skippedReason, + error: null, + cache_key: key, }; - // Keep failures and ambiguous answers out of the cache so a later call - // can try again. Definitive yes/no answers are stable for this content. - if (store && !parsed.ambiguous) { - await store.set(key, { - decision: parsed.decision, - raw: decision.raw, - tokens: decision.tokens, - reason: decision.reason, + if (store) await cachePut(store, key, record); + items.forEach((item, index) => { + const decision = decisionObject({ + path: item.path, + decision: decisionName, + reason, + raw, + tokens: totalTokens, + promptTokens, + completionTokens, + cached: index > 0, + skippedReason: item.skippedReason, + error: null, }); + byPath.set(item.path, decision); + emit(decision); + }); + } + + try { + await mapLimit(groupOrder, concurrency, runGroup); + } finally { + if (store && typeof store.flush === 'function') await store.flush(); + } + + /** @type {RelevanceDecision[]} */ + const decisions = entries.map(([path]) => byPath.get(path)); + /** @type {Record} */ + const included = {}; + for (const [path, content] of entries) { + if (byPath.get(path).decision === 'yes') { + included[path] = typeof content === 'string' ? content : coerceText(content); } - return decision; - } catch (err) { - const message = err && /** @type {Error} */ (err).message ? /** @type {Error} */ (err).message : String(err); - return { - path, - decision: 'yes', - raw: null, - tokens: null, - cached: false, - skippedReason: null, - error: message, - reason: null, - }; } + const errors = decisions.filter((decision) => decision.error); + let billedTokens = 0; + let billedPromptTokens = 0; + let billedCompletionTokens = 0; + for (const decision of decisions) { + if (decision.cached) continue; + billedTokens += decision.tokens; + billedPromptTokens += decision.promptTokens; + billedCompletionTokens += decision.completionTokens; + } + return { + files: included, + decisions, + model, + errors, + billedTokens, + billedPromptTokens, + billedCompletionTokens, + }; } /** - * Ask a cheap model which files are relevant to `task`. - * - * Files are classified with at most `concurrency` calls in flight. Binary - * files and files over `maxBytes` are not sent. The cache key is - * `(model, sha256(task), sha256(content))`. Omitting `cache` uses an - * in-memory cache shared by later calls; `cache: null` disables it. - * - * @param {Record} files - * @param {string} task - * @param {object} [opts] - * @param {string} [opts.model] cheap classifier; `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} - * @param {number} [opts.concurrency] max in-flight calls (default 8) - * @param {RelevanceCache | null} [opts.cache] pluggable cache; null disables; default is in-memory - * @param {number | null} [opts.maxBytes] UTF-8 size cap; null disables; default {@link DEFAULT_FILTER_MAX_BYTES} - * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) - * @param {string} [opts.apiKey] same resolution as the rest of the package (`GPTDIFF_LLM_API_KEY`) - * @param {string} [opts.baseUrl] same resolution as the rest of the package (`GPTDIFF_LLM_BASE_URL`) - * @param {Function} [opts.callLlm] injectable client (tests) - * @param {number} [opts.maxTokens] - * @param {number} [opts.temperature] - * @returns {Promise} one decision per file, in input order + * One-screen summary of a filter pass. Per-file lines when `verbose`. + * @param {FilterRelevantResult} result + * @param {boolean} [verbose] + * @returns {string} */ -export async function filterRelevant(files, task, opts = {}) { - const { - model, - concurrency = DEFAULT_FILTER_CONCURRENCY, - cache, - maxBytes = DEFAULT_FILTER_MAX_BYTES, - onDecision, - apiKey, - baseUrl, - callLlm = defaultCallLlm, - maxTokens = 128, - temperature = 0, - } = opts; - - const resolvedModel = resolveFilterModel(model); - const store = cache === undefined ? memoryCache() : cache; - const cap = maxBytes == null ? Infinity : maxBytes; - const limit = Math.max(1, Number.isFinite(concurrency) ? Math.floor(concurrency) : DEFAULT_FILTER_CONCURRENCY); - const taskText = task == null ? '' : String(task); - const taskHash = await sha256(taskText); - const entries = Object.entries(files || {}); - - return mapLimit(entries, limit, async ([path, content]) => { - const decision = await classifyFile({ - path, - content, - task: taskText, - taskHash, - model: resolvedModel, - store, - maxBytes: cap, - apiKey, - baseUrl, - callLlm, - maxTokens, - temperature, - }); - if (typeof onDecision === 'function') onDecision(decision); - return decision; - }); +export function formatFilterSummary(result, verbose = false) { + const total = result.decisions.length; + const kept = Object.keys(result.files).length; + const errorCount = result.errors.length; + let line = `Relevance filter (${result.model}): kept ${kept}/${total} files, ${result.billedTokens} tokens`; + if (errorCount) line += `, ${errorCount} error${errorCount === 1 ? '' : 's'}`; + const lines = [line]; + for (const decision of result.decisions) { + if (decision.error) { + lines.push(` error ${decision.path}: ${decision.error} (included by default)`); + } else if (verbose) { + const flags = []; + if (decision.cached) flags.push('cached'); + if (decision.skippedReason) flags.push(decision.skippedReason); + const suffix = flags.length ? ` (${flags.join(', ')})` : ''; + lines.push(` ${decision.decision.padEnd(3, ' ')} ${decision.path}${suffix}`); + } else if (decision.skippedReason && decision.decision === 'no') { + lines.push(` skip ${decision.path} (${decision.skippedReason})`); + } + } + return lines.join('\n'); +} + +/** + * Write the filter result as JSON so callers can inspect every decision. + * Decision fields stay camelCase. `files` is the list of kept paths. + * @param {FilterRelevantResult} result + * @param {string} path + * @returns {Promise} + */ +export async function dumpDecisions(result, path) { + const fs = await import('node:fs/promises'); + const pathMod = await import('node:path'); + const destination = String(path); + const directory = pathMod.dirname(pathMod.resolve(destination)); + if (directory) await fs.mkdir(directory, { recursive: true }); + const payload = { + model: result.model, + files: Object.keys(result.files), + decisions: result.decisions, + billedTokens: result.billedTokens, + billedPromptTokens: result.billedPromptTokens, + billedCompletionTokens: result.billedCompletionTokens, + errors: result.errors, + }; + await fs.writeFile(destination, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } diff --git a/src/filter.d.ts b/src/filter.d.ts index e141360..5bfe9aa 100644 --- a/src/filter.d.ts +++ b/src/filter.d.ts @@ -1,106 +1,273 @@ /** - * Process-wide in-memory cache used when `filterRelevant` is called without - * a `cache` option. Pass the result as `cache` to share or inspect it. + * Fresh in-memory cache. Pass one instance as `cache` to share it across calls. * @returns {RelevanceCache} */ export function createMemoryCache(): RelevanceCache; /** - * Parse a yes/no classification. Think tags and "> Reasoning" preambles are - * removed first. The first word wins when it is yes or no. If both words - * appear, or neither does, the answer is ambiguous and the caller includes - * the file. + * Parse a yes/no relevance answer. + * + * Think tags and chain-of-thought reasoning blocks are removed before the + * answer is read, so a "no" inside a think tag cannot override a final "yes". + * A single consistent yes or no wins, including "Answer: yes" and a fenced + * "NO". Both words, or neither, is ambiguous — callers include the file. * * @param {unknown} raw - * @returns {{ decision: 'yes' | 'no', reason: string | null, ambiguous: boolean }} + * @returns {{ decision: 'yes' | 'no' | null, cleaned: string }} */ export function parseRelevanceAnswer(raw: unknown): { - decision: "yes" | "no"; - reason: string | null; - ambiguous: boolean; + decision: "yes" | "no" | null; + cleaned: string; }; +/** + * Character-length stand-in for a tokenizer (Python's fallback when tiktoken + * is absent). Empty text is 0; anything else is at least 1. + * @param {string} text + * @returns {number} + */ +export function countTokens(text: string): number; +/** + * True when `content` looks like a binary file (NUL or a dense run of + * non-text controls). Non-text values other than byte arrays are not binary; + * {@link filterRelevant} rejects them when it coerces the file. + * @param {unknown} content + * @returns {boolean} + */ +export function isBinaryContent(content: unknown): boolean; /** * Ask a cheap model which files are relevant to `task`. * - * Files are classified with at most `concurrency` calls in flight. Binary - * files and files over `maxBytes` are not sent. The cache key is - * `(model, sha256(task), sha256(content))`. Omitting `cache` uses an - * in-memory cache shared by later calls; `cache: null` disables it. + * `files` is a `{ path: content }` map (string or byte array). The returned + * `files` map contains only the `yes` files, in input order, with their + * original contents — a truncated preview is what the model saw, not what is + * forwarded. `decisions` has one entry per input file. * - * @param {Record} files + * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that + * JSON file), `true` or omitted (a fresh in-memory cache for this call), or + * `null` / `false` (disabled). The cache key is the model, a hash of the task, + * a hash of the original file contents, and a truncation variant. + * + * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. + * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` + * (drop, no model call), or `truncate` (ask about a truncated or redacted + * preview). Pass `0` or `null` for `maxBytes` or `maxTokens` to disable that + * limit. Credentials resolve the same way as the rest of the library. + * + * @param {Record} files * @param {string} task * @param {object} [opts] - * @param {string} [opts.model] cheap classifier; `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} - * @param {number} [opts.concurrency] max in-flight calls (default 8) - * @param {RelevanceCache | null} [opts.cache] pluggable cache; null disables; default is in-memory - * @param {number | null} [opts.maxBytes] UTF-8 size cap; null disables; default {@link DEFAULT_FILTER_MAX_BYTES} + * @param {string} [opts.model] `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} + * @param {number} [opts.concurrency] max in-flight calls (default {@link DEFAULT_FILTER_CONCURRENCY}) + * @param {RelevanceCache | string | boolean | null | { get: Function, put?: Function, set?: Function }} [opts.cache] + * @param {number | null} [opts.maxBytes] UTF-8 cap; `0` or `null` disables; default {@link DEFAULT_FILTER_MAX_BYTES} + * @param {number | null} [opts.maxTokens] input-token cap; `0` or `null` disables; default {@link DEFAULT_FILTER_MAX_TOKENS} + * @param {'include' | 'exclude' | 'truncate'} [opts.onOversize] default `truncate` + * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` + * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) - * @param {string} [opts.apiKey] same resolution as the rest of the package (`GPTDIFF_LLM_API_KEY`) - * @param {string} [opts.baseUrl] same resolution as the rest of the package (`GPTDIFF_LLM_BASE_URL`) + * @param {string} [opts.apiKey] `GPTDIFF_LLM_API_KEY` when omitted + * @param {string} [opts.baseUrl] `GPTDIFF_LLM_BASE_URL` when omitted * @param {Function} [opts.callLlm] injectable client (tests) - * @param {number} [opts.maxTokens] - * @param {number} [opts.temperature] - * @returns {Promise} one decision per file, in input order + * @returns {Promise} */ -export function filterRelevant(files: Record, task: string, opts?: { +export function filterRelevant(files: Record, task: string, opts?: { model?: string; concurrency?: number; - cache?: RelevanceCache | null; + cache?: RelevanceCache | string | boolean | null | { + get: Function; + put?: Function; + set?: Function; + }; maxBytes?: number | null; + maxTokens?: number | null; + onOversize?: "include" | "exclude" | "truncate"; + onBinary?: "include" | "exclude" | "truncate"; + maxOutputTokens?: number; onDecision?: (decision: RelevanceDecision) => void; apiKey?: string; baseUrl?: string; callLlm?: Function; - maxTokens?: number; - temperature?: number; -}): Promise; +}): Promise; +/** + * One-screen summary of a filter pass. Per-file lines when `verbose`. + * @param {FilterRelevantResult} result + * @param {boolean} [verbose] + * @returns {string} + */ +export function formatFilterSummary(result: FilterRelevantResult, verbose?: boolean): string; +/** + * Write the filter result as JSON so callers can inspect every decision. + * Decision fields stay camelCase. `files` is the list of kept paths. + * @param {FilterRelevantResult} result + * @param {string} path + * @returns {Promise} + */ +export function dumpDecisions(result: FilterRelevantResult, path: string): Promise; /** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ -export const DEFAULT_FILTER_MODEL: "xiaomi/mimo-v2.5-pro-ultraspeed"; -/** Files larger than this (UTF-8 bytes) are skipped unless `maxBytes` is set. */ -export const DEFAULT_FILTER_MAX_BYTES: 262144; +export const DEFAULT_FILTER_MODEL: "gemini-2.0-flash"; /** In-flight classifications per call when `concurrency` is omitted. */ export const DEFAULT_FILTER_CONCURRENCY: 8; +/** UTF-8 size cap. `0` or `null` disables it. */ +export const DEFAULT_FILTER_MAX_BYTES: 200000; +/** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ +export const DEFAULT_FILTER_MAX_TOKENS: 12000; +/** Completion cap for the yes/no call. */ +export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS: 256; +/** + * @typedef {object} RelevanceDecision + * @property {string} path + * @property {'yes' | 'no'} decision `yes` files go on to generateDiff. + * Ambiguous answers and errors are `yes`. + * @property {string} reason short explanation; the cleaned model text when a call was made + * @property {string | null} raw model text before parsing, or null when no call was made + * @property {number} tokens total tokens for the call; 0 when no call was made + * @property {number} promptTokens + * @property {number} completionTokens + * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) + * @property {'binary' | 'oversize' | 'truncated' | null} skippedReason + * set when the file was not sent in full + * @property {string | null} error set when the call threw; the file is still included + */ +/** + * @typedef {object} FilterRelevantResult + * @property {Record} files `yes` files in input order, original contents + * @property {RelevanceDecision[]} decisions one per input file, in input order + * @property {string} model + * @property {RelevanceDecision[]} errors decisions whose call threw + * @property {number} billedTokens sum of `tokens` on decisions that were not cache hits + * @property {number} billedPromptTokens + * @property {number} billedCompletionTokens + */ +/** + * Record stored in a cache. Snake_case so a JSON file matches the Python cache. + * @typedef {object} RelevanceCacheRecord + * @property {'yes' | 'no'} decision + * @property {string} [reason] + * @property {string | null} [raw] + * @property {number} [tokens] + * @property {number} [prompt_tokens] + * @property {number} [completion_tokens] + * @property {'binary' | 'oversize' | 'truncated' | null} [skipped_reason] + * @property {string | null} [error] + * @property {string} [cache_key] + */ +/** + * In-memory relevance cache with an optional on-disk JSON file. + * + * Keys are `model`, SHA-256 of the task, SHA-256 of the original contents, and + * a variant tag (`full`, `binary-preview`, or `truncated::`), + * separated by newlines. A judgment on a truncated body is not reused as a + * full-content judgment. + */ +export class RelevanceCache { + /** + * @param {string} model + * @param {string} task + * @param {string} content original file text + * @param {string} [variant] + * @returns {Promise} + */ + static makeKey(model: string, task: string, content: string, variant?: string): Promise; + /** + * @param {string} [path] JSON file. Omit for memory only. + * Loaded on the first read. Corrupt files are ignored. + */ + constructor(path?: string); + path: string; + /** @type {Record} */ + _entries: Record; + /** @type {Promise | null} */ + _loadPromise: Promise | null; + /** + * @param {string} key + * @returns {Promise} + */ + get(key: string): Promise; + /** + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ + put(key: string, record: RelevanceCacheRecord): Promise; + /** + * Alias of {@link put} so a `{ get, set }` cache and a `RelevanceCache` share a shape. + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ + set(key: string, record: RelevanceCacheRecord): Promise; + /** Write `{ version: 1, entries }` atomically. No-op without a path. */ + flush(): Promise; + /** @returns {Promise} */ + load(): Promise; + /** @returns {Promise} */ + _load(): Promise; +} export type RelevanceDecision = { path: string; /** * `yes` files go on to generateDiff. - * Ambiguous answers and errors are `yes` so a bad classifier cannot drop a - * file. `skip` files were not sent to the model. + * Ambiguous answers and errors are `yes`. + */ + decision: "yes" | "no"; + /** + * short explanation; the cleaned model text when a call was made */ - decision: "yes" | "no" | "skip"; + reason: string; /** - * model text, unchanged (think tags included) + * model text before parsing, or null when no call was made */ raw: string | null; /** - * total tokens for the call; null when skipped or the call failed + * total tokens for the call; 0 when no call was made */ - tokens: number | null; + tokens: number; + promptTokens: number; + completionTokens: number; /** - * true when this answer was read from the cache + * true when this judgment was reused (a cache hit, or a later path in a shared request) */ cached: boolean; - skippedReason: "binary" | "too-large" | null; /** - * set for thrown calls and ambiguous answers + * set when the file was not sent in full */ - error: string | null; + skippedReason: "binary" | "oversize" | "truncated" | null; /** - * short reason after the yes/no, when the model gave one + * set when the call threw; the file is still included */ - reason: string | null; + error: string | null; }; -export type RelevanceCacheEntry = { - decision: "yes" | "no"; - raw: string | null; - tokens: number | null; - reason: string | null; +export type FilterRelevantResult = { + /** + * `yes` files in input order, original contents + */ + files: Record; + /** + * one per input file, in input order + */ + decisions: RelevanceDecision[]; + model: string; + /** + * decisions whose call threw + */ + errors: RelevanceDecision[]; + /** + * sum of `tokens` on decisions that were not cache hits + */ + billedTokens: number; + billedPromptTokens: number; + billedCompletionTokens: number; }; /** - * A cache is any object with `get(key)` and `set(key, value)`. Either method - * may be sync or async. Keys are `${model}\\0${taskHash}\\0${contentHash}` - * (SHA-256 hex). Pass `cache: null` to disable caching. + * Record stored in a cache. Snake_case so a JSON file matches the Python cache. */ -export type RelevanceCache = { - get: (key: string) => RelevanceCacheEntry | undefined | Promise; - set: (key: string, value: RelevanceCacheEntry) => void | Promise; +export type RelevanceCacheRecord = { + decision: "yes" | "no"; + reason?: string; + raw?: string | null; + tokens?: number; + prompt_tokens?: number; + completion_tokens?: number; + skipped_reason?: "binary" | "oversize" | "truncated" | null; + error?: string | null; + cache_key?: string; }; diff --git a/src/filter.js b/src/filter.js index a88b17d..84ae6c3 100644 --- a/src/filter.js +++ b/src/filter.js @@ -1,98 +1,220 @@ /** * Opt-in relevance filter for large file maps. * - * For each candidate file, a cheap model sees the full contents plus the task - * and answers "is this file relevant to the task? yes/no". Only `yes` - * decisions should be passed to `generateDiff` (see `buildEnvironment`'s - * `decisions` option). `smartapply` still wants the full map so files the - * diff never mentions stay put. + * For each candidate file, a cheap model sees the contents plus the task and + * answers "is this file relevant to the task? yes/no". Only `yes` files are + * returned for `generateDiff`. `smartapply` still wants the full map so files + * the diff never mentions stay put. * - * Mirrors the Python package's `filter_relevant` (255BITS/patchling-py): + * Mirrors `filter_relevant` in 255BITS/patchling-py (`cursor/relevance-filter-0192`). + * Field names are camelCase. Ambiguous answers and per-file errors include the + * file. An error is attached only when the call throws. * - * filter_relevant(files, task, *, model=None, concurrency=8, cache=None, - * max_bytes=262144, on_decision=None, api_key=None, - * base_url=None) -> list[decision] - * - * decision: path, decision ("yes" | "no" | "skip"), raw, tokens, cached, - * skipped_reason ("binary" | "too-large" | None), error, reason - * - * JS uses camelCase (`maxBytes`, `onDecision`, `skippedReason`) like the rest - * of this package. Ambiguous answers and per-file errors include the file - * (`decision: "yes"`) and surface the problem on `error` instead of throwing. + * The disk cache loads `node:fs` on demand so this module stays browser-safe. */ -import { stripThinkTags } from './text.js'; +import { swallowReasoning } from './text.js'; import { callLlm as defaultCallLlm, resolveApiKey, resolveBaseUrl } from './llm.js'; import { getEnv } from './env.js'; /** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ -export const DEFAULT_FILTER_MODEL = 'xiaomi/mimo-v2.5-pro-ultraspeed'; - -/** Files larger than this (UTF-8 bytes) are skipped unless `maxBytes` is set. */ -export const DEFAULT_FILTER_MAX_BYTES = 262144; +export const DEFAULT_FILTER_MODEL = 'gemini-2.0-flash'; /** In-flight classifications per call when `concurrency` is omitted. */ export const DEFAULT_FILTER_CONCURRENCY = 8; -const FILTER_SYSTEM_PROMPT = `You decide if one file is relevant to a coding task. Reply with "yes" or "no" as the first word. You may add one short reason after it. Do not quote the file.`; +/** UTF-8 size cap. `0` or `null` disables it. */ +export const DEFAULT_FILTER_MAX_BYTES = 200_000; + +/** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ +export const DEFAULT_FILTER_MAX_TOKENS = 12_000; -const AMBIGUOUS = 'ambiguous answer'; +/** Completion cap for the yes/no call. */ +export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 256; + +const POLICIES = ['include', 'exclude', 'truncate']; + +const ANSWER_PREFIX = /^(?:answer|relevance|relevant|decision)\s*[:\-]\s*/i; +const FENCE_LINE = /^```[a-zA-Z0-9_-]*\s*$/; + +const SYSTEM_PROMPT = ( + 'You are a relevance filter for a code-editing task. ' + + 'Decide whether the given file must be read or edited to accomplish the task, ' + + 'or is necessary context for that edit. ' + + 'Reply with a single line that starts with exactly "yes" or exactly "no", ' + + 'followed by a short reason. ' + + 'Answer "yes" when the file is likely to need changes or is required context. ' + + 'Answer "no" when it is unrelated. Do not write a diff.' +); /** * @typedef {object} RelevanceDecision * @property {string} path - * @property {'yes' | 'no' | 'skip'} decision `yes` files go on to generateDiff. - * Ambiguous answers and errors are `yes` so a bad classifier cannot drop a - * file. `skip` files were not sent to the model. - * @property {string | null} raw model text, unchanged (think tags included) - * @property {number | null} tokens total tokens for the call; null when skipped or the call failed - * @property {boolean} cached true when this answer was read from the cache - * @property {'binary' | 'too-large' | null} skippedReason - * @property {string | null} error set for thrown calls and ambiguous answers - * @property {string | null} reason short reason after the yes/no, when the model gave one + * @property {'yes' | 'no'} decision `yes` files go on to generateDiff. + * Ambiguous answers and errors are `yes`. + * @property {string} reason short explanation; the cleaned model text when a call was made + * @property {string | null} raw model text before parsing, or null when no call was made + * @property {number} tokens total tokens for the call; 0 when no call was made + * @property {number} promptTokens + * @property {number} completionTokens + * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) + * @property {'binary' | 'oversize' | 'truncated' | null} skippedReason + * set when the file was not sent in full + * @property {string | null} error set when the call threw; the file is still included */ /** - * @typedef {object} RelevanceCacheEntry + * @typedef {object} FilterRelevantResult + * @property {Record} files `yes` files in input order, original contents + * @property {RelevanceDecision[]} decisions one per input file, in input order + * @property {string} model + * @property {RelevanceDecision[]} errors decisions whose call threw + * @property {number} billedTokens sum of `tokens` on decisions that were not cache hits + * @property {number} billedPromptTokens + * @property {number} billedCompletionTokens + */ + +/** + * Record stored in a cache. Snake_case so a JSON file matches the Python cache. + * @typedef {object} RelevanceCacheRecord * @property {'yes' | 'no'} decision - * @property {string | null} raw - * @property {number | null} tokens - * @property {string | null} reason + * @property {string} [reason] + * @property {string | null} [raw] + * @property {number} [tokens] + * @property {number} [prompt_tokens] + * @property {number} [completion_tokens] + * @property {'binary' | 'oversize' | 'truncated' | null} [skipped_reason] + * @property {string | null} [error] + * @property {string} [cache_key] */ /** - * A cache is any object with `get(key)` and `set(key, value)`. Either method - * may be sync or async. Keys are `${model}\\0${taskHash}\\0${contentHash}` - * (SHA-256 hex). Pass `cache: null` to disable caching. + * In-memory relevance cache with an optional on-disk JSON file. * - * @typedef {object} RelevanceCache - * @property {(key: string) => RelevanceCacheEntry | undefined | Promise} get - * @property {(key: string, value: RelevanceCacheEntry) => void | Promise} set + * Keys are `model`, SHA-256 of the task, SHA-256 of the original contents, and + * a variant tag (`full`, `binary-preview`, or `truncated::`), + * separated by newlines. A judgment on a truncated body is not reused as a + * full-content judgment. */ +export class RelevanceCache { + /** + * @param {string} [path] JSON file. Omit for memory only. + * Loaded on the first read. Corrupt files are ignored. + */ + constructor(path) { + this.path = path ? String(path) : null; + /** @type {Record} */ + this._entries = Object.create(null); + /** @type {Promise | null} */ + this._loadPromise = null; + } + + /** + * @param {string} model + * @param {string} task + * @param {string} content original file text + * @param {string} [variant] + * @returns {Promise} + */ + static async makeKey(model, task, content, variant = 'full') { + const taskHash = await sha256(task); + const contentHash = await sha256(content); + return `${model}\n${taskHash}\n${contentHash}\n${variant}`; + } + + /** + * @param {string} key + * @returns {Promise} + */ + async get(key) { + await this.load(); + const record = this._entries[key]; + return record ? { ...record } : undefined; + } -let sharedMemoryCache; + /** + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ + async put(key, record) { + await this.load(); + this._entries[key] = { ...record }; + } + + /** + * Alias of {@link put} so a `{ get, set }` cache and a `RelevanceCache` share a shape. + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ + async set(key, record) { + await this.put(key, record); + } + + /** Write `{ version: 1, entries }` atomically. No-op without a path. */ + async flush() { + if (!this.path) return; + await this.load(); + const fs = await import('node:fs/promises'); + const pathMod = await import('node:path'); + const dir = pathMod.dirname(pathMod.resolve(this.path)); + await fs.mkdir(dir, { recursive: true }); + const tmp = pathMod.join( + dir, + `.relevance-cache-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.tmp`, + ); + const payload = JSON.stringify({ version: 1, entries: this._entries }); + try { + await fs.writeFile(tmp, payload, 'utf8'); + await fs.rename(tmp, this.path); + } catch (err) { + await fs.unlink(tmp).catch(() => {}); + throw err; + } + } + + /** @returns {Promise} */ + async load() { + if (!this.path) return; + if (!this._loadPromise) this._loadPromise = this._load(); + await this._loadPromise; + } + + /** @returns {Promise} */ + async _load() { + let fs; + try { + fs = await import('node:fs/promises'); + } catch { + throw new Error('A disk-backed relevance cache requires Node (node:fs).'); + } + let text; + try { + text = await fs.readFile(this.path, 'utf8'); + } catch (err) { + if (err && /** @type {NodeJS.ErrnoException} */ (err).code === 'ENOENT') return; + return; + } + let payload; + try { + payload = JSON.parse(text); + } catch { + return; + } + const entries = payload && typeof payload === 'object' ? payload.entries : null; + if (entries && typeof entries === 'object' && !Array.isArray(entries)) { + this._entries = entries; + } + } +} /** - * Process-wide in-memory cache used when `filterRelevant` is called without - * a `cache` option. Pass the result as `cache` to share or inspect it. + * Fresh in-memory cache. Pass one instance as `cache` to share it across calls. * @returns {RelevanceCache} */ export function createMemoryCache() { - /** @type {Map} */ - const store = new Map(); - return { - get(key) { - return store.has(key) ? store.get(key) : undefined; - }, - set(key, value) { - store.set(key, value); - }, - }; -} - -function memoryCache() { - if (!sharedMemoryCache) sharedMemoryCache = createMemoryCache(); - return sharedMemoryCache; + return new RelevanceCache(); } /** @@ -108,305 +230,754 @@ async function sha256(text) { } /** - * @param {string} model - * @param {string} taskHash - * @param {string} contentHash + * Drop `` / `` blocks, including an unclosed tail. + * @param {string} text * @returns {string} */ -function cacheKey(model, taskHash, contentHash) { - return `${model}\0${taskHash}\0${contentHash}`; +function stripFilterThinkTags(text) { + if (!text) return ''; + let cleaned = text.replace(/]*>[\s\S]*?<\/think>/gi, ''); + cleaned = cleaned.replace(/]*>[\s\S]*?<\/thinking>/gi, ''); + cleaned = cleaned.replace(/]*>[\s\S]*$/gi, ''); + cleaned = cleaned.replace(/]*>[\s\S]*$/gi, ''); + cleaned = cleaned.replace(/<\/think>/gi, ''); + cleaned = cleaned.replace(/<\/thinking>/gi, ''); + return cleaned.trim(); +} + +/** + * Parse a yes/no relevance answer. + * + * Think tags and chain-of-thought reasoning blocks are removed before the + * answer is read, so a "no" inside a think tag cannot override a final "yes". + * A single consistent yes or no wins, including "Answer: yes" and a fenced + * "NO". Both words, or neither, is ambiguous — callers include the file. + * + * @param {unknown} raw + * @returns {{ decision: 'yes' | 'no' | null, cleaned: string }} + */ +export function parseRelevanceAnswer(raw) { + let text = stripFilterThinkTags(raw == null ? '' : String(raw)); + const [swallowed] = swallowReasoning(text); + text = swallowed.trim(); + const fenceLines = text.split(/\r\n|\r|\n/); + if ( + fenceLines.length >= 2 + && FENCE_LINE.test(fenceLines[0].trim()) + && fenceLines[fenceLines.length - 1].trim() === '```' + ) { + text = fenceLines.slice(1, -1).join('\n').trim(); + } + const cleaned = text.trim().replace(/^`+|`+$/g, '').trim(); + const contentLines = cleaned.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); + if (!contentLines.length) return { decision: null, cleaned }; + + let answerText = contentLines[0].replace(ANSWER_PREFIX, '').trim(); + if (contentLines.length > 1) { + answerText = `${answerText}\n${contentLines.slice(1).join('\n')}`; + } + const words = answerText.match(/\b(yes|no)\b/gi) || []; + const lowered = words.map((word) => word.toLowerCase()); + if (lowered.length && new Set(lowered).size === 1) { + return { decision: /** @type {'yes' | 'no'} */ (lowered[0]), cleaned }; + } + return { decision: null, cleaned }; +} + +/** + * @param {string} name + * @param {unknown} value + * @returns {'include' | 'exclude' | 'truncate'} + */ +function checkPolicy(name, value) { + if (!POLICIES.includes(/** @type {string} */ (value))) { + throw new Error(`${name} must be one of ${POLICIES.join(', ')}; got ${JSON.stringify(value)}`); + } + return /** @type {'include' | 'exclude' | 'truncate'} */ (value); } +/** + * @param {string | undefined} model + * @returns {string} + */ function resolveFilterModel(model) { return model || getEnv('GPTDIFF_FILTER_MODEL') || DEFAULT_FILTER_MODEL; } /** - * @param {unknown} content - * @returns {boolean} + * `null` and `false` disable the cache. Omit or pass `true` for a fresh + * in-memory cache. A string is a JSON file. A `RelevanceCache` is used as-is. + * An object with `get` and `put`/`set` is a pluggable cache. + * @param {unknown} cache + * @returns {RelevanceCache | { get: Function, put?: Function, set?: Function, load?: Function, flush?: Function } | null} */ -function looksBinary(content) { - if (typeof content !== 'string') return true; - const n = Math.min(content.length, 8192); - let controls = 0; - for (let i = 0; i < n; i++) { - const c = content.charCodeAt(i); - if (c === 0) return true; - // C0 controls other than tab, LF, and CR. A dense run is not text. - if (c < 32 && c !== 9 && c !== 10 && c !== 13) controls++; +function resolveCache(cache) { + if (cache === false || cache === null) return null; + if (cache === undefined || cache === true) return new RelevanceCache(); + if (cache instanceof RelevanceCache) return cache; + if (typeof cache === 'string') return new RelevanceCache(cache); + if ( + cache + && typeof cache === 'object' + && typeof /** @type {{ get?: unknown }} */ (cache).get === 'function' + && ( + typeof /** @type {{ put?: unknown }} */ (cache).put === 'function' + || typeof /** @type {{ set?: unknown }} */ (cache).set === 'function' + ) + ) { + return /** @type {{ get: Function, put?: Function, set?: Function }} */ (cache); } - return n > 0 && controls / n > 0.3; + throw new TypeError('cache must be a RelevanceCache, a filesystem path, null, or false'); } /** - * UTF-8 byte length, stopping once `maxBytes` is exceeded. - * @param {string} content + * Character-length stand-in for a tokenizer (Python's fallback when tiktoken + * is absent). Empty text is 0; anything else is at least 1. + * @param {string} text + * @returns {number} + */ +export function countTokens(text) { + if (!text) return 0; + return Math.max(1, Math.floor(String(text).length / 4)); +} + +/** + * @param {string} text + * @param {number | null} maxBytes + * @param {number | null} maxTokens + * @returns {string} + */ +function truncateText(text, maxBytes, maxTokens) { + let out = text; + if (maxTokens && countTokens(out) > maxTokens) { + out = out.slice(0, maxTokens * 4); + } + if (maxBytes) out = truncateUtf8(out, maxBytes); + return out; +} + +/** + * UTF-8 prefix. An incomplete trailing character is dropped, matching + * Python's `errors="ignore"` on a cut byte string. + * @param {string} text * @param {number} maxBytes - * @returns {boolean} + * @returns {string} */ -function exceedsMaxBytes(content, maxBytes) { - if (!Number.isFinite(maxBytes)) return false; - let bytes = 0; - for (let i = 0; i < content.length; i++) { - const c = content.charCodeAt(i); - if (c < 0x80) bytes += 1; - else if (c < 0x800) bytes += 2; - else if (c >= 0xd800 && c <= 0xdbff) { - bytes += 4; - i += 1; - } else bytes += 3; - if (bytes > maxBytes) return true; - } - return false; +function truncateUtf8(text, maxBytes) { + const bytes = new TextEncoder().encode(text); + if (bytes.length <= maxBytes) return text; + let end = maxBytes; + while (end > 0 && (bytes[end - 1] & 0xc0) === 0x80) end -= 1; + if (end > 0) { + const lead = bytes[end - 1]; + const need = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4; + if (end - 1 + need > maxBytes) end -= 1; + } + return new TextDecoder().decode(bytes.subarray(0, end)); } /** - * @param {unknown} response + * @param {string} text * @returns {number} */ -function totalTokens(response) { - const usage = response && /** @type {{ usage?: Record }} */ (response).usage; - if (!usage) return 0; - const prompt = usage.prompt_tokens ?? usage.input_tokens ?? 0; - const completion = usage.completion_tokens ?? usage.output_tokens ?? 0; - return usage.total_tokens ?? prompt + completion; +function utf8ByteLength(text) { + return new TextEncoder().encode(text).length; } /** - * Parse a yes/no classification. Think tags and "> Reasoning" preambles are - * removed first. The first word wins when it is yes or no. If both words - * appear, or neither does, the answer is ambiguous and the caller includes - * the file. - * - * @param {unknown} raw - * @returns {{ decision: 'yes' | 'no', reason: string | null, ambiguous: boolean }} + * True when `content` looks like a binary file (NUL or a dense run of + * non-text controls). Non-text values other than byte arrays are not binary; + * {@link filterRelevant} rejects them when it coerces the file. + * @param {unknown} content + * @returns {boolean} */ -export function parseRelevanceAnswer(raw) { - const stripped = stripThinkTags(raw == null ? '' : String(raw)).trim(); - const unfenced = stripped.replace(/^```[a-z0-9]*\s*/i, '').replace(/```\s*$/i, '').trim(); - const lines = unfenced.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); - const first = (lines[0] || '').replace(/^[*_>"'`]+/, '').replace(/[*_"`']+$/g, '').trim(); - const lead = first.match(/^(yes|no)\b[\s.:,\-—!]*(.*)$/i); - - if (lead) { - const decision = /** @type {'yes' | 'no'} */ (lead[1].toLowerCase()); - const sameLine = lead[2].trim(); - const reason = [sameLine, lines.slice(1).join(' ')].filter(Boolean).join(' ').trim() || null; - const firstHasYes = /\byes\b/i.test(first); - const firstHasNo = /\bno\b/i.test(first); - if (firstHasYes && firstHasNo) { - return { decision: 'yes', reason: unfenced || null, ambiguous: true }; - } - return { decision, reason, ambiguous: false }; +export function isBinaryContent(content) { + if (content instanceof Uint8Array) return binaryBytes(content); + if (typeof content !== 'string') return false; + return textLooksBinary(content.slice(0, 8192)); +} + +/** + * @param {Uint8Array} bytes + * @returns {boolean} + */ +function binaryBytes(bytes) { + const sample = bytes.subarray(0, 8192); + for (let i = 0; i < sample.length; i++) { + if (sample[i] === 0) return true; + } + try { + const text = new TextDecoder('utf-8', { fatal: true }).decode(sample); + return textLooksBinary(text); + } catch { + return true; } +} - const hasYes = /\byes\b/i.test(unfenced); - const hasNo = /\bno\b/i.test(unfenced); - if (hasYes && !hasNo) return { decision: 'yes', reason: unfenced || null, ambiguous: false }; - if (hasNo && !hasYes) return { decision: 'no', reason: unfenced || null, ambiguous: false }; - return { decision: 'yes', reason: unfenced || null, ambiguous: true }; +/** + * @param {string} sample + * @returns {boolean} + */ +function textLooksBinary(sample) { + if (!sample) return false; + if (sample.includes('\0')) return true; + let controls = 0; + for (let i = 0; i < sample.length; i++) { + const c = sample.charCodeAt(i); + if (c < 32 && c !== 9 && c !== 10 && c !== 13 && c !== 12) controls += 1; + } + return controls / sample.length > 0.3; } /** - * @param {Array<[string, string]>} items + * @param {unknown} content + * @returns {string} + */ +function coerceText(content) { + if (typeof content === 'string') return content; + if (content instanceof Uint8Array) { + return new TextDecoder('utf-8', { fatal: false }).decode(content); + } + const name = content === null ? 'null' : typeof content; + throw new TypeError(`file content must be str or bytes, got ${name}`); +} + +/** + * @param {unknown} value + * @param {number} fallback + * @returns {number | null} null disables the limit (`0` or `null`) + */ +function resolveLimit(value, fallback) { + const chosen = value === undefined ? fallback : value; + if (chosen === null || chosen === 0) return null; + const n = Number(chosen); + if (!Number.isFinite(n) || n <= 0) return null; + return n; +} + +/** + * @param {unknown} value + * @returns {string} + */ +/** + * Tag a limit the way Python's f-string does: omitted uses the default number, + * `null` is `None`, and `0` stays `0`. + * @param {unknown} value + * @param {number} fallback + * @returns {string} + */ +function rawLimitTag(value, fallback) { + const chosen = value === undefined ? fallback : value; + if (chosen === null || chosen === undefined) return 'None'; + return String(chosen); +} + +/** + * @param {object} fields + * @returns {RelevanceDecision} + */ +function decisionObject(fields) { + return { + path: fields.path, + decision: fields.decision, + reason: fields.reason ?? '', + raw: fields.raw ?? null, + tokens: fields.tokens ?? 0, + promptTokens: fields.promptTokens ?? 0, + completionTokens: fields.completionTokens ?? 0, + cached: !!fields.cached, + skippedReason: fields.skippedReason ?? null, + error: fields.error ?? null, + }; +} + +/** + * @param {string} path + * @param {RelevanceCacheRecord} record + * @param {boolean} cached + * @returns {RelevanceDecision} + */ +function decisionFromRecord(path, record, cached) { + const rec = /** @type {Record} */ (record); + return decisionObject({ + path, + decision: /** @type {'yes' | 'no'} */ (rec.decision), + reason: rec.reason || '', + raw: rec.raw ?? null, + tokens: Number(rec.tokens || 0), + promptTokens: Number(rec.prompt_tokens ?? rec.promptTokens ?? 0), + completionTokens: Number(rec.completion_tokens ?? rec.completionTokens ?? 0), + cached, + skippedReason: rec.skipped_reason ?? rec.skippedReason ?? null, + error: rec.error ?? null, + }); +} + +/** + * @param {string} path + * @param {'yes' | 'no'} decision + * @param {'binary' | 'oversize'} skippedReason + * @param {string} reason + * @returns {RelevanceDecision} + */ +function skipDecision(path, decision, skippedReason, reason) { + return decisionObject({ + path, + decision, + reason, + raw: null, + tokens: 0, + cached: false, + skippedReason, + error: null, + }); +} + +/** + * @param {string} task + * @param {string[]} paths + * @param {string} body + * @param {string | null} note + * @returns {string} + */ +function buildUserPrompt(task, paths, body, note) { + const label = paths.length === 1 ? 'File path' : 'File paths'; + const parts = [`Task:\n${task}`, `${label}:\n${paths.join('\n')}`]; + if (note) parts.push(note); + parts.push(`File contents:\n${body}`); + parts.push('Is this file relevant to the task? yes/no'); + return parts.join('\n\n'); +} + +/** + * @param {unknown} response + * @returns {{ raw: string, promptTokens: number, completionTokens: number, tokens: number }} + */ +function usageAndText(response) { + const message = /** @type {{ choices?: Array<{ message?: { content?: string | null } }>, usage?: Record }} */ (response); + const content = message?.choices?.[0]?.message?.content ?? ''; + const usage = message?.usage || {}; + const promptTokens = Number(usage.prompt_tokens ?? usage.input_tokens ?? 0); + const completionTokens = Number(usage.completion_tokens ?? usage.output_tokens ?? 0); + const tokens = Number(usage.total_tokens ?? (promptTokens + completionTokens)); + return { + raw: content || '', + promptTokens, + completionTokens, + tokens, + }; +} + +/** + * @param {unknown} err + * @returns {string} + */ +function errorMessage(err) { + if (err instanceof Error) return `${err.name}: ${err.message}`; + return String(err); +} + +/** + * @param {Array} items * @param {number} limit - * @param {(item: [string, string], index: number) => Promise} fn - * @returns {Promise} + * @param {(item: string) => Promise} fn + * @returns {Promise} */ async function mapLimit(items, limit, fn) { - /** @type {RelevanceDecision[]} */ - const results = new Array(items.length); - if (!items.length) return results; + if (!items.length) return; let cursor = 0; const workers = Math.min(limit, items.length); const run = async () => { while (cursor < items.length) { const index = cursor; cursor += 1; - results[index] = await fn(items[index], index); + await fn(items[index]); } }; await Promise.all(Array.from({ length: workers }, () => run())); - return results; } /** - * @param {object} args - * @param {string} args.path - * @param {unknown} args.content - * @param {string} args.task - * @param {string} args.taskHash - * @param {string} args.model - * @param {RelevanceCache | null} args.store - * @param {number} args.maxBytes - * @param {string | undefined} args.apiKey - * @param {string | undefined} args.baseUrl - * @param {Function} args.callLlm - * @param {number} args.maxTokens - * @param {number} args.temperature - * @returns {Promise} - */ -async function classifyFile(args) { - const { path, content, task, taskHash, model, store, maxBytes, apiKey, baseUrl, callLlm, maxTokens, temperature } = args; - - if (looksBinary(content)) { - return { - path, - decision: 'skip', - raw: null, - tokens: null, - cached: false, - skippedReason: 'binary', - error: null, - reason: null, - }; + * @param {{ get: Function, put?: Function, set?: Function }} store + * @param {string} key + * @returns {Promise} + */ +async function cacheGet(store, key) { + const hit = await store.get(key); + return hit || undefined; +} + +/** + * @param {{ put?: Function, set?: Function }} store + * @param {string} key + * @param {RelevanceCacheRecord} record + * @returns {Promise} + */ +async function cachePut(store, key, record) { + if (typeof store.put === 'function') await store.put(key, record); + else await store.set(key, record); +} + +/** + * Ask a cheap model which files are relevant to `task`. + * + * `files` is a `{ path: content }` map (string or byte array). The returned + * `files` map contains only the `yes` files, in input order, with their + * original contents — a truncated preview is what the model saw, not what is + * forwarded. `decisions` has one entry per input file. + * + * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that + * JSON file), `true` or omitted (a fresh in-memory cache for this call), or + * `null` / `false` (disabled). The cache key is the model, a hash of the task, + * a hash of the original file contents, and a truncation variant. + * + * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. + * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` + * (drop, no model call), or `truncate` (ask about a truncated or redacted + * preview). Pass `0` or `null` for `maxBytes` or `maxTokens` to disable that + * limit. Credentials resolve the same way as the rest of the library. + * + * @param {Record} files + * @param {string} task + * @param {object} [opts] + * @param {string} [opts.model] `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} + * @param {number} [opts.concurrency] max in-flight calls (default {@link DEFAULT_FILTER_CONCURRENCY}) + * @param {RelevanceCache | string | boolean | null | { get: Function, put?: Function, set?: Function }} [opts.cache] + * @param {number | null} [opts.maxBytes] UTF-8 cap; `0` or `null` disables; default {@link DEFAULT_FILTER_MAX_BYTES} + * @param {number | null} [opts.maxTokens] input-token cap; `0` or `null` disables; default {@link DEFAULT_FILTER_MAX_TOKENS} + * @param {'include' | 'exclude' | 'truncate'} [opts.onOversize] default `truncate` + * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` + * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) + * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) + * @param {string} [opts.apiKey] `GPTDIFF_LLM_API_KEY` when omitted + * @param {string} [opts.baseUrl] `GPTDIFF_LLM_BASE_URL` when omitted + * @param {Function} [opts.callLlm] injectable client (tests) + * @returns {Promise} + */ +export async function filterRelevant(files, task, opts = {}) { + if (files == null || typeof files !== 'object' || Array.isArray(files)) { + throw new TypeError('files must be an object of path -> content'); } + if (typeof task !== 'string' || task.trim() === '') { + throw new Error('task is required'); + } + const concurrency = Math.trunc(Number(opts.concurrency === undefined ? DEFAULT_FILTER_CONCURRENCY : opts.concurrency)); + if (!Number.isFinite(concurrency) || concurrency < 1) { + throw new Error('concurrency must be >= 1'); + } + const onOversize = checkPolicy('onOversize', opts.onOversize === undefined ? 'truncate' : opts.onOversize); + const onBinary = checkPolicy('onBinary', opts.onBinary === undefined ? 'exclude' : opts.onBinary); + const maxBytes = resolveLimit(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES); + const maxTokens = resolveLimit(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS); + const maxOutputTokens = opts.maxOutputTokens === undefined + ? DEFAULT_FILTER_MAX_OUTPUT_TOKENS + : opts.maxOutputTokens; - const text = /** @type {string} */ (content); - if (exceedsMaxBytes(text, maxBytes)) { - return { - path, - decision: 'skip', - raw: null, - tokens: null, - cached: false, - skippedReason: 'too-large', - error: null, - reason: null, - }; + const model = resolveFilterModel(opts.model); + const store = resolveCache(opts.cache); + const callLlm = opts.callLlm || defaultCallLlm; + const apiKey = opts.apiKey == null ? resolveApiKey(undefined) : opts.apiKey; + const baseUrl = resolveBaseUrl(opts.baseUrl); + const onDecision = opts.onDecision; + + /** @type {Map} */ + const byPath = new Map(); + /** @type {Map>} */ + const groups = new Map(); + /** @type {string[]} */ + const groupOrder = []; + const entries = Object.entries(files); + + /** + * @param {string} key + * @param {{ path: string, body: string, note: string | null, skippedReason: 'truncated' | null }} item + */ + function enqueue(key, item) { + if (!groups.has(key)) { + groups.set(key, []); + groupOrder.push(key); + } + groups.get(key).push(item); } - const contentHash = await sha256(text); - const key = cacheKey(model, taskHash, contentHash); - if (store) { - const hit = await store.get(key); - if (hit && (hit.decision === 'yes' || hit.decision === 'no')) { - return { - path, - decision: hit.decision, - raw: hit.raw ?? null, - tokens: hit.tokens ?? null, - cached: true, - skippedReason: null, - error: null, - reason: hit.reason ?? null, - }; + /** + * @param {RelevanceDecision} decision + * @returns {RelevanceDecision} + */ + function emit(decision) { + if (typeof onDecision === 'function') onDecision(decision); + return decision; + } + + for (const [path, content] of entries) { + const binary = isBinaryContent(content); + const text = coerceText(content); + if (binary) { + if (onBinary === 'truncate') { + const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); + const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; + const body = `[binary file, ${nbytes} bytes, contents omitted]`; + const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview'); + enqueue(key, { path, body, note, skippedReason: 'truncated' }); + } else { + const decision = onBinary === 'include' ? 'yes' : 'no'; + const verb = decision === 'yes' ? 'included' : 'excluded'; + const made = skipDecision(path, decision, 'binary', `binary file; ${verb} without a model call`); + byPath.set(path, made); + emit(made); + } + continue; + } + + const nBytes = utf8ByteLength(text); + const nTokens = countTokens(text); + const overBytes = !!maxBytes && nBytes > maxBytes; + const overTokens = !!maxTokens && nTokens > maxTokens; + if (overBytes || overTokens) { + if (onOversize === 'truncate') { + const body = truncateText(text, maxBytes, maxTokens); + const note = ( + 'Note: this file was truncated to fit the relevance check ' + + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` + ); + const variant = `truncated:${rawLimitTag(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS)}:${rawLimitTag(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES)}`; + const key = await RelevanceCache.makeKey(model, task, text, variant); + enqueue(key, { path, body, note, skippedReason: 'truncated' }); + } else { + const decision = onOversize === 'include' ? 'yes' : 'no'; + const verb = decision === 'yes' ? 'included' : 'excluded'; + const made = skipDecision( + path, + decision, + 'oversize', + `over size threshold (${nBytes} bytes, ${nTokens} tokens); ${verb} without a model call`, + ); + byPath.set(path, made); + emit(made); + } + continue; } + + const key = await RelevanceCache.makeKey(model, task, text, 'full'); + enqueue(key, { path, body: text, note: null, skippedReason: null }); } - const userContent = `Task:\n${task}\n\nFile: ${path}\n${text}\n\nis this file relevant to the task? yes/no`; - try { - const response = await callLlm({ - apiKey: resolveApiKey(apiKey), - baseUrl: resolveBaseUrl(baseUrl), - model, - messages: [ - { role: 'system', content: FILTER_SYSTEM_PROMPT }, - { role: 'user', content: userContent }, - ], - maxTokens, - temperature, - }); - const raw = response?.choices?.[0]?.message?.content ?? ''; + if (store && typeof store.load === 'function') await store.load(); + + /** + * @param {string} key + * @returns {Promise} + */ + async function cached(key) { + if (!store) return false; + const hit = await cacheGet(store, key); + return !!(hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error); + } + + let needsCall = false; + for (const key of groupOrder) { + if (!(await cached(key))) { + needsCall = true; + break; + } + } + if (needsCall && !apiKey) { + throw new Error( + 'GPTDIFF_LLM_API_KEY is required for the relevance filter. ' + + 'Set it in the environment or pass apiKey.', + ); + } + + /** + * @param {string} key + * @returns {Promise} + */ + async function runGroup(key) { + const items = groups.get(key); + if (store) { + const hit = await cacheGet(store, key); + if (hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error) { + for (const item of items) { + const decision = decisionFromRecord(item.path, { ...hit, cache_key: key }, true); + byPath.set(item.path, decision); + emit(decision); + } + return; + } + } + + const sample = items[0]; + let raw; + let promptTokens; + let completionTokens; + let totalTokens; + try { + const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); + let userContent = userPrompt; + if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; + const response = await callLlm({ + apiKey, + baseUrl, + model, + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: userContent }, + ], + maxTokens: maxOutputTokens, + temperature: 0, + budgetTokens: null, + }); + ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); + } catch (err) { + const message = errorMessage(err); + for (const item of items) { + const decision = decisionObject({ + path: item.path, + decision: 'yes', + reason: `error; included by default: ${message}`, + raw: null, + tokens: 0, + cached: false, + skippedReason: item.skippedReason, + error: message, + }); + byPath.set(item.path, decision); + emit(decision); + } + return; + } + const parsed = parseRelevanceAnswer(raw); - /** @type {RelevanceDecision} */ - const decision = { - path, - decision: parsed.decision, - raw: raw == null ? '' : String(raw), - tokens: totalTokens(response), - cached: false, - skippedReason: null, - error: parsed.ambiguous ? AMBIGUOUS : null, - reason: parsed.ambiguous ? (parsed.reason || null) : parsed.reason, + /** @type {'yes' | 'no'} */ + let decisionName; + /** @type {string} */ + let reason; + if (parsed.decision == null) { + decisionName = 'yes'; + reason = parsed.cleaned + ? `ambiguous answer; included by default: ${parsed.cleaned}` + : 'ambiguous answer; included by default'; + } else { + decisionName = parsed.decision; + reason = parsed.cleaned; + } + /** @type {RelevanceCacheRecord} */ + const record = { + decision: decisionName, + reason, + raw, + tokens: totalTokens, + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + skipped_reason: sample.skippedReason, + error: null, + cache_key: key, }; - // Keep failures and ambiguous answers out of the cache so a later call - // can try again. Definitive yes/no answers are stable for this content. - if (store && !parsed.ambiguous) { - await store.set(key, { - decision: parsed.decision, - raw: decision.raw, - tokens: decision.tokens, - reason: decision.reason, + if (store) await cachePut(store, key, record); + items.forEach((item, index) => { + const decision = decisionObject({ + path: item.path, + decision: decisionName, + reason, + raw, + tokens: totalTokens, + promptTokens, + completionTokens, + cached: index > 0, + skippedReason: item.skippedReason, + error: null, }); + byPath.set(item.path, decision); + emit(decision); + }); + } + + try { + await mapLimit(groupOrder, concurrency, runGroup); + } finally { + if (store && typeof store.flush === 'function') await store.flush(); + } + + /** @type {RelevanceDecision[]} */ + const decisions = entries.map(([path]) => byPath.get(path)); + /** @type {Record} */ + const included = {}; + for (const [path, content] of entries) { + if (byPath.get(path).decision === 'yes') { + included[path] = typeof content === 'string' ? content : coerceText(content); } - return decision; - } catch (err) { - const message = err && /** @type {Error} */ (err).message ? /** @type {Error} */ (err).message : String(err); - return { - path, - decision: 'yes', - raw: null, - tokens: null, - cached: false, - skippedReason: null, - error: message, - reason: null, - }; } + const errors = decisions.filter((decision) => decision.error); + let billedTokens = 0; + let billedPromptTokens = 0; + let billedCompletionTokens = 0; + for (const decision of decisions) { + if (decision.cached) continue; + billedTokens += decision.tokens; + billedPromptTokens += decision.promptTokens; + billedCompletionTokens += decision.completionTokens; + } + return { + files: included, + decisions, + model, + errors, + billedTokens, + billedPromptTokens, + billedCompletionTokens, + }; } /** - * Ask a cheap model which files are relevant to `task`. - * - * Files are classified with at most `concurrency` calls in flight. Binary - * files and files over `maxBytes` are not sent. The cache key is - * `(model, sha256(task), sha256(content))`. Omitting `cache` uses an - * in-memory cache shared by later calls; `cache: null` disables it. - * - * @param {Record} files - * @param {string} task - * @param {object} [opts] - * @param {string} [opts.model] cheap classifier; `GPTDIFF_FILTER_MODEL`, else {@link DEFAULT_FILTER_MODEL} - * @param {number} [opts.concurrency] max in-flight calls (default 8) - * @param {RelevanceCache | null} [opts.cache] pluggable cache; null disables; default is in-memory - * @param {number | null} [opts.maxBytes] UTF-8 size cap; null disables; default {@link DEFAULT_FILTER_MAX_BYTES} - * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) - * @param {string} [opts.apiKey] same resolution as the rest of the package (`GPTDIFF_LLM_API_KEY`) - * @param {string} [opts.baseUrl] same resolution as the rest of the package (`GPTDIFF_LLM_BASE_URL`) - * @param {Function} [opts.callLlm] injectable client (tests) - * @param {number} [opts.maxTokens] - * @param {number} [opts.temperature] - * @returns {Promise} one decision per file, in input order + * One-screen summary of a filter pass. Per-file lines when `verbose`. + * @param {FilterRelevantResult} result + * @param {boolean} [verbose] + * @returns {string} */ -export async function filterRelevant(files, task, opts = {}) { - const { - model, - concurrency = DEFAULT_FILTER_CONCURRENCY, - cache, - maxBytes = DEFAULT_FILTER_MAX_BYTES, - onDecision, - apiKey, - baseUrl, - callLlm = defaultCallLlm, - maxTokens = 128, - temperature = 0, - } = opts; - - const resolvedModel = resolveFilterModel(model); - const store = cache === undefined ? memoryCache() : cache; - const cap = maxBytes == null ? Infinity : maxBytes; - const limit = Math.max(1, Number.isFinite(concurrency) ? Math.floor(concurrency) : DEFAULT_FILTER_CONCURRENCY); - const taskText = task == null ? '' : String(task); - const taskHash = await sha256(taskText); - const entries = Object.entries(files || {}); - - return mapLimit(entries, limit, async ([path, content]) => { - const decision = await classifyFile({ - path, - content, - task: taskText, - taskHash, - model: resolvedModel, - store, - maxBytes: cap, - apiKey, - baseUrl, - callLlm, - maxTokens, - temperature, - }); - if (typeof onDecision === 'function') onDecision(decision); - return decision; - }); +export function formatFilterSummary(result, verbose = false) { + const total = result.decisions.length; + const kept = Object.keys(result.files).length; + const errorCount = result.errors.length; + let line = `Relevance filter (${result.model}): kept ${kept}/${total} files, ${result.billedTokens} tokens`; + if (errorCount) line += `, ${errorCount} error${errorCount === 1 ? '' : 's'}`; + const lines = [line]; + for (const decision of result.decisions) { + if (decision.error) { + lines.push(` error ${decision.path}: ${decision.error} (included by default)`); + } else if (verbose) { + const flags = []; + if (decision.cached) flags.push('cached'); + if (decision.skippedReason) flags.push(decision.skippedReason); + const suffix = flags.length ? ` (${flags.join(', ')})` : ''; + lines.push(` ${decision.decision.padEnd(3, ' ')} ${decision.path}${suffix}`); + } else if (decision.skippedReason && decision.decision === 'no') { + lines.push(` skip ${decision.path} (${decision.skippedReason})`); + } + } + return lines.join('\n'); +} + +/** + * Write the filter result as JSON so callers can inspect every decision. + * Decision fields stay camelCase. `files` is the list of kept paths. + * @param {FilterRelevantResult} result + * @param {string} path + * @returns {Promise} + */ +export async function dumpDecisions(result, path) { + const fs = await import('node:fs/promises'); + const pathMod = await import('node:path'); + const destination = String(path); + const directory = pathMod.dirname(pathMod.resolve(destination)); + if (directory) await fs.mkdir(directory, { recursive: true }); + const payload = { + model: result.model, + files: Object.keys(result.files), + decisions: result.decisions, + billedTokens: result.billedTokens, + billedPromptTokens: result.billedPromptTokens, + billedCompletionTokens: result.billedCompletionTokens, + errors: result.errors, + }; + await fs.writeFile(destination, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } diff --git a/src/gptdiff.d.ts b/src/gptdiff.d.ts index 2880528..00c3d4c 100644 --- a/src/gptdiff.d.ts +++ b/src/gptdiff.d.ts @@ -4,7 +4,7 @@ * Opt-in relevance filter: pass `{ decisions }` from `filterRelevant` and * only files whose decision is `"yes"` are included. An array of those * decisions is also accepted as the second argument. Omit it and every file - * is included, in insertion order. Skipped and `"no"` files stay out of the + * is included, in insertion order. Every other file stays out of the * string that `generateDiff` sees. * * @param {Record} filesDict diff --git a/src/gptdiff.js b/src/gptdiff.js index 4474061..373efb7 100644 --- a/src/gptdiff.js +++ b/src/gptdiff.js @@ -66,7 +66,7 @@ function decisionsFrom(opts) { * Opt-in relevance filter: pass `{ decisions }` from `filterRelevant` and * only files whose decision is `"yes"` are included. An array of those * decisions is also accepted as the second argument. Omit it and every file - * is included, in insertion order. Skipped and `"no"` files stay out of the + * is included, in insertion order. Every other file stays out of the * string that `generateDiff` sees. * * @param {Record} filesDict diff --git a/src/index.d.ts b/src/index.d.ts index f8234b0..51d6598 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -1,9 +1,8 @@ export * as oauth from "./oauth.js"; export type RelevanceDecision = import("./filter.js").RelevanceDecision; -export type RelevanceCache = import("./filter.js").RelevanceCache; -export type RelevanceCacheEntry = import("./filter.js").RelevanceCacheEntry; +export type FilterRelevantResult = import("./filter.js").FilterRelevantResult; export { generateDiff, smartapply, buildEnvironment, callLlmForDiff, callLlmForApply, callLlmForApplyWithThink } from "./gptdiff.js"; -export { filterRelevant, createMemoryCache, parseRelevanceAnswer, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_CONCURRENCY } from "./filter.js"; +export { filterRelevant, RelevanceCache, createMemoryCache, parseRelevanceAnswer, isBinaryContent, countTokens, formatFilterSummary, dumpDecisions, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_MAX_TOKENS, DEFAULT_FILTER_MAX_OUTPUT_TOKENS, DEFAULT_FILTER_CONCURRENCY } from "./filter.js"; export { parseDiffPerFile, applyDiff, applyPatchToFile } from "./applydiff.js"; export { colorCodeDiff, swallowReasoning, stripBadOutput, extractDiffBlocks, stripThinkTags } from "./text.js"; export { callLlm, resolveApiKey, resolveBaseUrl } from "./llm.js"; diff --git a/src/index.js b/src/index.js index ae45636..f01841c 100644 --- a/src/index.js +++ b/src/index.js @@ -8,17 +8,23 @@ export { generateDiff, smartapply, buildEnvironment, callLlmForDiff, callLlmForApply, callLlmForApplyWithThink } from './gptdiff.js'; export { filterRelevant, + RelevanceCache, createMemoryCache, parseRelevanceAnswer, + isBinaryContent, + countTokens, + formatFilterSummary, + dumpDecisions, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, + DEFAULT_FILTER_MAX_TOKENS, + DEFAULT_FILTER_MAX_OUTPUT_TOKENS, DEFAULT_FILTER_CONCURRENCY, } from './filter.js'; /** * @typedef {import('./filter.js').RelevanceDecision} RelevanceDecision - * @typedef {import('./filter.js').RelevanceCache} RelevanceCache - * @typedef {import('./filter.js').RelevanceCacheEntry} RelevanceCacheEntry + * @typedef {import('./filter.js').FilterRelevantResult} FilterRelevantResult */ export { parseDiffPerFile, applyDiff, applyPatchToFile } from './applydiff.js'; export { colorCodeDiff, swallowReasoning, stripBadOutput, extractDiffBlocks, stripThinkTags } from './text.js'; diff --git a/test/filter_relevant.test.js b/test/filter_relevant.test.js index 714250c..6a5279b 100644 --- a/test/filter_relevant.test.js +++ b/test/filter_relevant.test.js @@ -1,15 +1,33 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { filterRelevant, + RelevanceCache, createMemoryCache, parseRelevanceAnswer, + isBinaryContent, + countTokens, + formatFilterSummary, + dumpDecisions, buildEnvironment, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, + DEFAULT_FILTER_MAX_TOKENS, + DEFAULT_FILTER_MAX_OUTPUT_TOKENS, + DEFAULT_FILTER_CONCURRENCY, } from '../src/index.js'; import { setEnv } from '../src/env.js'; +function response(content, prompt = 11, completion = 3, total = prompt + completion) { + return { + choices: [{ message: { content } }], + usage: { prompt_tokens: prompt, completion_tokens: completion, total_tokens: total }, + }; +} + function llm(replies) { const calls = []; let inFlight = 0; @@ -19,303 +37,593 @@ function llm(replies) { maxInFlight = Math.max(maxInFlight, inFlight); const index = calls.length; calls.push(req); - await new Promise((r) => setTimeout(r, 15)); + const next = typeof replies === 'function' ? await replies(req, index) : replies[index]; inFlight -= 1; - const next = typeof replies === 'function' ? replies(req, index) : replies[index]; if (next instanceof Error) throw next; - const content = typeof next === 'string' || next == null ? (next ?? 'yes') : next.content; - const tokens = typeof next === 'object' && next && 'tokens' in next ? next.tokens : 11; - return { - choices: [{ message: { content } }], - usage: { prompt_tokens: 7, completion_tokens: Math.max(0, tokens - 7), total_tokens: tokens }, - }; + if (next && typeof next === 'object' && next.choices) return next; + return response(next == null ? 'yes' : String(next)); }; return { fn, calls, stats: () => ({ maxInFlight, count: calls.length }) }; } -function replyByPath(map) { - return (req) => { - const path = req.messages[1].content.match(/File: ([^\n]+)/)[1]; - if (!Object.prototype.hasOwnProperty.call(map, path)) throw new Error(`no reply for ${path}`); - return map[path]; - }; +function pathsOf(req) { + const user = req.messages[req.messages.length - 1].content; + const match = user.match(/File paths?:\n([\s\S]*?)\n\n/); + assert.ok(match, user); + return match[1].split('\n'); } -const QUESTION = 'is this file relevant to the task? yes/no'; +function bodyOf(req) { + const user = req.messages[req.messages.length - 1].content; + const marker = 'File contents:\n'; + const start = user.indexOf(marker); + assert.notEqual(start, -1); + return user.slice(start + marker.length).replace(/\n\nIs this file relevant to the task\? yes\/no$/, ''); +} -test('parseRelevanceAnswer strips think tags and accepts yes/no with a reason', () => { - assert.deepEqual(parseRelevanceAnswer('yes'), { decision: 'yes', reason: null, ambiguous: false }); - assert.deepEqual(parseRelevanceAnswer(' YES. '), { decision: 'yes', reason: null, ambiguous: false }); - assert.deepEqual(parseRelevanceAnswer('No'), { decision: 'no', reason: null, ambiguous: false }); - assert.deepEqual(parseRelevanceAnswer('no — arithmetic helpers'), { - decision: 'no', - reason: 'arithmetic helpers', - ambiguous: false, - }); - assert.deepEqual(parseRelevanceAnswer('the greeting lives here\n\nno'), { - decision: 'no', - reason: null, - ambiguous: false, - }); - assert.deepEqual(parseRelevanceAnswer('**yes**\ndefines greet'), { - decision: 'yes', - reason: 'defines greet', - ambiguous: false, - }); +function answerFor(req, unmatched = 'no — unrelated') { + const body = bodyOf(req); + if (body.includes('def keep') || body.includes('ALPHA_TOKEN')) return response('yes — needed for the task'); + if (body.includes('BINARY_PREVIEW')) return response('yes the path matches'); + return response(unmatched); +} + +const baseOpts = { model: 'fake-model', apiKey: 'test-key', cache: false }; + +let envLock = Promise.resolve(); +function withEnv(fn) { + const run = envLock.then(fn, fn); + envLock = run.then(() => {}, () => {}); + return run; +} + +test('parseRelevanceAnswer accepts yes/no variants and drops a fence', () => { + assert.equal(parseRelevanceAnswer('yes').decision, 'yes'); + assert.equal(parseRelevanceAnswer('Yes.').decision, 'yes'); + assert.equal(parseRelevanceAnswer('NO\nthis file is a changelog').decision, 'no'); + assert.equal(parseRelevanceAnswer('Answer: yes, the parser lives here').decision, 'yes'); + const fenced = parseRelevanceAnswer('```\nyes\n```'); + assert.equal(fenced.decision, 'yes'); + assert.ok(fenced.cleaned.includes('yes')); +}); + +test('parseRelevanceAnswer strips think tags before deciding', () => { + const yes = parseRelevanceAnswer('no, this file is irrelevant\nyes, the function to rename is here'); + assert.equal(yes.decision, 'yes'); + assert.equal(yes.cleaned.includes('irrelevant'), false); + + assert.equal(parseRelevanceAnswer('yes, definitely edit it\nno — tests only').decision, 'no'); + assert.equal(parseRelevanceAnswer('no, skip this file entirely').decision, null); + assert.equal(parseRelevanceAnswer('maybe\nyes').decision, 'yes'); +}); + +test('parseRelevanceAnswer is ambiguous when both words appear, or neither', () => { + assert.equal(parseRelevanceAnswer("I'm not sure about this one").decision, null); + assert.equal(parseRelevanceAnswer('yes for the helper, no for the tests').decision, null); + assert.equal(parseRelevanceAnswer('').decision, null); + assert.equal(parseRelevanceAnswer('nope').decision, null); assert.equal(parseRelevanceAnswer('The answer is no.').decision, 'no'); - const both = parseRelevanceAnswer('Yes and no'); - assert.equal(both.decision, 'yes'); - assert.equal(both.ambiguous, true); - const maybe = parseRelevanceAnswer('maybe later'); - assert.equal(maybe.decision, 'yes'); - assert.equal(maybe.ambiguous, true); - const empty = parseRelevanceAnswer('only thoughts'); - assert.equal(empty.decision, 'yes'); - assert.equal(empty.ambiguous, true); }); -test('filterRelevant sends full contents and the task, and returns per-file decisions', async () => { - const mock = llm(replyByPath({ - 'greet.py': 'yes — defines the greeting', - 'math_utils.py': 'no', - })); +test('filterRelevant keeps only yes files and reports camelCase decisions', async () => { + const mock = llm((req) => answerFor(req)); const files = { - 'greet.py': 'def greet():\n print("hello")\n', - 'math_utils.py': 'def add(a, b):\n return a + b\n', + 'keep.py': 'def keep():\n return 1\n', + 'drop.py': 'def drop():\n return 2\n', }; const seen = []; - const decisions = await filterRelevant(files, 'Say goodbye instead of hello', { - model: 'cheap-test', - apiKey: 'sk-test', + const result = await filterRelevant(files, 'Update keep()', { + ...baseOpts, baseUrl: 'https://example.test/v1', - cache: createMemoryCache(), callLlm: mock.fn, onDecision: (d) => seen.push(d.path), }); - assert.equal(decisions.length, 2); - assert.equal(decisions[0].path, 'greet.py'); - assert.equal(decisions[0].decision, 'yes'); - assert.equal(decisions[0].reason, 'defines the greeting'); - assert.equal(decisions[0].raw, 'yes — defines the greeting'); - assert.equal(decisions[0].tokens, 11); - assert.equal(decisions[0].cached, false); - assert.equal(decisions[0].skippedReason, null); - assert.equal(decisions[0].error, null); - assert.equal(decisions[1].decision, 'no'); - assert.deepEqual(seen.sort(), ['greet.py', 'math_utils.py']); + assert.deepEqual(Object.keys(result.files), ['keep.py']); + assert.equal(result.files['keep.py'], files['keep.py']); + assert.deepEqual(result.decisions.map((d) => d.path), ['keep.py', 'drop.py']); + assert.deepEqual(result.decisions.map((d) => d.decision), ['yes', 'no']); + assert.equal(result.model, 'fake-model'); + const yes = result.decisions[0]; + assert.ok(yes.raw.startsWith('yes')); + assert.ok(yes.reason.includes('needed')); + assert.equal(yes.tokens, 14); + assert.equal(yes.promptTokens, 11); + assert.equal(yes.completionTokens, 3); + assert.equal(yes.cached, false); + assert.equal(yes.skippedReason, null); + assert.equal(yes.error, null); + assert.equal('cacheKey' in yes, false); + assert.deepEqual(seen.sort(), ['drop.py', 'keep.py']); assert.equal(mock.calls.length, 2); - for (const [path, content] of Object.entries(files)) { - const call = mock.calls.find((c) => c.messages[1].content.includes(`File: ${path}\n`)); - assert.ok(call, path); - assert.equal(call.model, 'cheap-test'); - assert.equal(call.apiKey, 'sk-test'); + for (const call of mock.calls) { + assert.equal(call.model, 'fake-model'); + assert.equal(call.apiKey, 'test-key'); assert.equal(call.baseUrl, 'https://example.test/v1'); assert.equal(call.temperature, 0); - assert.match(call.messages[1].content, /Task:\nSay goodbye instead of hello/); - assert.ok(call.messages[1].content.includes(content)); - assert.ok(call.messages[1].content.includes(QUESTION)); + assert.equal(call.maxTokens, DEFAULT_FILTER_MAX_OUTPUT_TOKENS); + assert.match(call.messages[1].content, /Task:\nUpdate keep\(\)/); + assert.match(call.messages[1].content, /Is this file relevant to the task\? yes\/no/); } + assert.ok(bodyOf(mock.calls[0]).includes('def keep') || bodyOf(mock.calls[1]).includes('def keep')); }); -test('ambiguous answers and thrown errors include the file and surface the error', async () => { - const mock = llm(replyByPath({ - a: 'maybe', - b: new Error('connection reset'), - c: 'no', - })); - const decisions = await filterRelevant( - { a: 'aaa', b: 'bbb', c: 'ccc' }, - 'touch a', - { model: 'cheap-test', cache: null, callLlm: mock.fn }, +test('ambiguous answers and thrown errors include the file', async () => { + const mock = llm((req) => { + const paths = pathsOf(req); + if (paths.includes('ambiguous.py')) return response('no\nmaybe later'); + throw new Error('connection reset'); + }); + const result = await filterRelevant( + { 'ambiguous.py': 'x = 1\n', 'broken.py': 'y = 2\n' }, + 'Touch both', + { ...baseOpts, callLlm: mock.fn }, ); - assert.equal(decisions[0].decision, 'yes'); - assert.equal(decisions[0].error, 'ambiguous answer'); - assert.equal(decisions[0].raw, 'maybe'); - assert.equal(decisions[1].decision, 'yes'); - assert.equal(decisions[1].error, 'connection reset'); - assert.equal(decisions[1].raw, null); - assert.equal(decisions[1].tokens, null); - assert.equal(decisions[2].decision, 'no'); - assert.equal(decisions[2].error, null); + assert.deepEqual(Object.keys(result.files).sort(), ['ambiguous.py', 'broken.py']); + const by = Object.fromEntries(result.decisions.map((d) => [d.path, d])); + assert.equal(by['ambiguous.py'].decision, 'yes'); + assert.equal(by['ambiguous.py'].error, null); + assert.match(by['ambiguous.py'].reason, /ambiguous/); + assert.equal(by['broken.py'].decision, 'yes'); + assert.match(by['broken.py'].error, /connection reset/); + assert.match(by['broken.py'].reason, /connection reset/); + assert.equal(by['broken.py'].tokens, 0); + assert.deepEqual(result.errors, [by['broken.py']]); + const summary = formatFilterSummary(result); + assert.match(summary, /connection reset/); + assert.match(summary, /included by default/); }); -test('binary and huge files are skipped and never sent', async () => { - const mock = llm(['yes']); - const huge = 'x'.repeat(50); - const files = { - 'keep.py': 'print(1)\n', - 'logo.bin': 'PNG\0bytes', - 'notes.txt': huge, - 'odd.dat': '\u0001'.repeat(20), - }; - const decisions = await filterRelevant(files, 'print', { - model: 'cheap-test', - cache: null, - maxBytes: 40, - callLlm: mock.fn, +test('errors are not cached', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + if (n === 1) throw new Error('temporary'); + return response('no'); }); - const by = Object.fromEntries(decisions.map((d) => [d.path, d])); - assert.equal(by['keep.py'].decision, 'yes'); - assert.equal(by['logo.bin'].decision, 'skip'); - assert.equal(by['logo.bin'].skippedReason, 'binary'); - assert.equal(by['logo.bin'].tokens, null); - assert.equal(by['notes.txt'].decision, 'skip'); - assert.equal(by['notes.txt'].skippedReason, 'too-large'); - assert.equal(by['odd.dat'].skippedReason, 'binary'); - assert.equal(mock.stats().count, 1); - assert.ok(!mock.calls[0].messages[1].content.includes('PNG')); + const cache = new RelevanceCache(); + const files = { 'a.py': "print('a')\n" }; + const opts = { model: 'fake', cache, apiKey: 'test-key', callLlm: mock.fn }; + const first = await filterRelevant(files, 'edit a', opts); + const second = await filterRelevant(files, 'edit a', opts); + assert.equal(first.decisions[0].decision, 'yes'); + assert.ok(first.decisions[0].error); + assert.equal(second.decisions[0].decision, 'no'); + assert.equal(second.decisions[0].error, null); + assert.equal(n, 2); }); -test('maxBytes counts UTF-8 bytes and can be raised or disabled', async () => { - const mock = llm(['yes', 'yes', 'no']); - const accented = 'é'; // 1 UTF-16 unit, 2 UTF-8 bytes - const files = { a: accented, b: 'b' }; - const skipped = await filterRelevant(files, 't', { - model: 'cheap-test', - cache: null, - maxBytes: 1, - callLlm: mock.fn, +test('memory and disk cache round-trip', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return response('yes', 5, 1, 6); + }); + const dir = mkdtempSync(join(tmpdir(), 'patchling-filter-')); + const disk = join(dir, 'relevance-cache.json'); + const files = { 'a.py': "print('cached')\n" }; + const firstCache = new RelevanceCache(disk); + const first = await filterRelevant(files, 'ship it', { + model: 'fake', cache: firstCache, apiKey: 'test-key', callLlm: mock.fn, }); - assert.equal(skipped[0].skippedReason, 'too-large'); - assert.equal(skipped[1].decision, 'yes'); + assert.equal(first.decisions[0].cached, false); + assert.equal(n, 1); + const stored = JSON.parse(readFileSync(disk, 'utf8')); + assert.equal(stored.version, 1); + assert.equal(Object.keys(stored.entries).length, 1); + const [storedKey, storedValue] = Object.entries(stored.entries)[0]; + assert.equal(storedKey.split('\n').length, 4); + assert.equal(storedKey.split('\n')[3], 'full'); + assert.equal(storedValue.decision, 'yes'); + assert.equal(storedValue.prompt_tokens, 5); + assert.equal(storedValue.skipped_reason, null); - const sent = await filterRelevant({ a: accented }, 't', { - model: 'cheap-test', - cache: null, - maxBytes: 2, - callLlm: mock.fn, + const second = await filterRelevant(files, 'ship it', { + model: 'fake', cache: new RelevanceCache(disk), apiKey: 'test-key', callLlm: mock.fn, }); - assert.equal(sent[0].decision, 'yes'); - assert.equal(sent[0].skippedReason, null); + assert.equal(n, 1); + assert.equal(second.decisions[0].cached, true); + assert.equal(second.decisions[0].tokens, 6); + assert.equal(second.decisions[0].promptTokens, 5); + assert.equal(second.billedTokens, 0); + assert.ok(second.files['a.py']); - const open = await filterRelevant({ a: 'zzzz'.repeat(100) }, 't', { - model: 'cheap-test', - cache: null, - maxBytes: null, - callLlm: mock.fn, + await filterRelevant(files, 'other task', { model: 'fake', cache: firstCache, apiKey: 'k', callLlm: mock.fn }); + await filterRelevant(files, 'ship it', { model: 'other-model', cache: firstCache, apiKey: 'k', callLlm: mock.fn }); + await filterRelevant( + { 'a.py': "print('changed')\n" }, + 'ship it', + { model: 'fake', cache: firstCache, apiKey: 'k', callLlm: mock.fn }, + ); + assert.equal(n, 4); +}); + +test('a cache path string enables disk cache and false disables it', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return response('no'); }); - assert.equal(open[0].decision, 'no'); + const dir = mkdtempSync(join(tmpdir(), 'patchling-filter-')); + const disk = join(dir, 'nested', 'cache.json'); + const files = { 'a.py': 'v = 1\n' }; + await filterRelevant(files, 'task', { model: 'm', cache: disk, apiKey: 'k', callLlm: mock.fn }); + await filterRelevant(files, 'task', { model: 'm', cache: disk, apiKey: 'k', callLlm: mock.fn }); + assert.equal(n, 1); + + await filterRelevant(files, 'task', { model: 'm', cache: false, apiKey: 'k', callLlm: mock.fn }); + await filterRelevant(files, 'task', { model: 'm', cache: null, apiKey: 'k', callLlm: mock.fn }); + assert.equal(n, 3); }); -test('a file of exactly maxBytes is sent', async () => { - const mock = llm(['yes']); - const content = 'abcd'; - const decisions = await filterRelevant({ a: content }, 't', { - model: 'cheap-test', - cache: null, - maxBytes: 4, - callLlm: mock.fn, +test('a corrupt disk cache is ignored', async () => { + const dir = mkdtempSync(join(tmpdir(), 'patchling-filter-')); + const disk = join(dir, 'cache.json'); + writeFileSync(disk, '{not json'); + let n = 0; + const mock = llm(() => { + n += 1; + return response('yes'); + }); + const result = await filterRelevant( + { 'a.py': 'v = 1\n' }, + 'task', + { model: 'm', cache: new RelevanceCache(disk), apiKey: 'k', callLlm: mock.fn }, + ); + assert.equal(n, 1); + assert.equal(result.decisions[0].decision, 'yes'); + const rewritten = JSON.parse(readFileSync(disk, 'utf8')); + assert.equal(rewritten.version, 1); +}); + +test('duplicate contents share one request', async () => { + let n = 0; + const mock = llm((req) => { + n += 1; + const paths = pathsOf(req); + assert.ok(paths.includes('a.py') && paths.includes('b.py')); + return response('yes', 4, 1, 5); }); - assert.equal(decisions[0].decision, 'yes'); - assert.equal(mock.stats().count, 1); + const result = await filterRelevant( + { 'a.py': 'SAME', 'b.py': 'SAME' }, + 'task', + { ...baseOpts, model: 'm', callLlm: mock.fn }, + ); + assert.equal(n, 1); + assert.equal(result.decisions[0].cached, false); + assert.equal(result.decisions[1].cached, true); + assert.equal(result.billedTokens, 5); + assert.deepEqual(Object.keys(result.files).sort(), ['a.py', 'b.py']); + assert.equal(result.files['a.py'], 'SAME'); }); test('concurrency stays bounded and result order follows the input', async () => { - const names = ['a', 'b', 'c', 'd', 'e', 'f']; - const files = Object.fromEntries(names.map((n) => [n, `body ${n}`])); - const mock = llm(names.map(() => 'yes')); - const decisions = await filterRelevant(files, 'all', { - model: 'cheap-test', - concurrency: 2, - cache: null, + const names = Array.from({ length: 10 }, (_, i) => `file-${i}.py`); + const files = Object.fromEntries(names.map((name, i) => [name, `body ${i}\n`])); + const mock = llm(async (req) => { + const header = req.messages[1].content.split('File contents:')[0]; + if (header.includes('file-9.py')) await new Promise((r) => setTimeout(r, 40)); + else await new Promise((r) => setTimeout(r, 15)); + const relevant = header.includes('file-1.py') || header.includes('file-9.py'); + return response(relevant ? 'yes' : 'no', 1, 1, 2); + }); + const result = await filterRelevant(files, 'touch file-1 and file-9', { + model: 'm', + concurrency: 3, + cache: false, + apiKey: 'k', callLlm: mock.fn, }); - assert.deepEqual(decisions.map((d) => d.path), names); - assert.equal(mock.stats().maxInFlight, 2); - assert.equal(mock.stats().count, 6); + assert.ok(mock.stats().maxInFlight <= 3); + assert.ok(mock.stats().maxInFlight >= 2); + assert.deepEqual(result.decisions.map((d) => d.path), names); + assert.deepEqual(Object.keys(result.files).sort(), ['file-1.py', 'file-9.py']); +}); + +test('binary and oversize policies keep original contents on yes', async () => { + let n = 0; + const mock = llm((req) => { + n += 1; + const user = req.messages[1].content; + assert.equal(user.includes('SECRET_TAIL'), false); + if (user.includes('contents omitted')) { + assert.ok(user.includes('bin.dat')); + return response('no'); + } + return response('yes — prefix is enough'); + }); + const binary = 'hello\0world'; + const huge = `PREFIX ${'x'.repeat(80)} SECRET_TAIL`; + const limits = { maxBytes: 40, maxTokens: null }; + + const excluded = await filterRelevant( + { 'bin.dat': binary, 'big.py': huge }, + 'edit the prefix', + { model: 'm', cache: false, onBinary: 'exclude', onOversize: 'exclude', ...limits, callLlm: mock.fn }, + ); + assert.equal(n, 0); + assert.deepEqual(excluded.files, {}); + const excludedBy = Object.fromEntries(excluded.decisions.map((d) => [d.path, d])); + assert.equal(excludedBy['bin.dat'].decision, 'no'); + assert.equal(excludedBy['bin.dat'].skippedReason, 'binary'); + assert.equal(excludedBy['big.py'].decision, 'no'); + assert.equal(excludedBy['big.py'].skippedReason, 'oversize'); + assert.equal(excludedBy['big.py'].tokens, 0); + assert.equal(excludedBy['big.py'].reason.includes('over size threshold'), true); + + const included = await filterRelevant( + { 'bin.dat': binary, 'big.py': huge }, + 'edit the prefix', + { model: 'm', cache: false, onBinary: 'include', onOversize: 'include', ...limits, callLlm: mock.fn }, + ); + assert.equal(n, 0); + assert.deepEqual(Object.keys(included.files).sort(), ['big.py', 'bin.dat']); + assert.equal(included.files['big.py'], huge); + assert.equal(included.files['bin.dat'], binary); + assert.deepEqual(included.decisions.map((d) => d.decision), ['yes', 'yes']); + + n = 0; + const truncated = await filterRelevant( + { 'bin.dat': new Uint8Array([0, 1, ...Buffer.from('BINARY_PREVIEW')]), 'big.py': huge }, + 'edit the prefix', + { + model: 'm', + cache: false, + apiKey: 'k', + onBinary: 'truncate', + onOversize: 'truncate', + ...limits, + callLlm: mock.fn, + }, + ); + assert.equal(n, 2); + const truncatedBy = Object.fromEntries(truncated.decisions.map((d) => [d.path, d])); + assert.equal(truncatedBy['big.py'].decision, 'yes'); + assert.equal(truncatedBy['big.py'].skippedReason, 'truncated'); + assert.equal(truncated.files['big.py'], huge); + assert.equal(truncatedBy['bin.dat'].decision, 'no'); + assert.equal(truncatedBy['bin.dat'].skippedReason, 'truncated'); + assert.equal('bin.dat' in truncated.files, false); +}); + +test('a truncated judgment is not reused as a full read', async () => { + let n = 0; + const mock = llm((req) => { + n += 1; + const user = req.messages[1].content; + return response(user.includes('truncated') ? 'no' : 'yes'); + }); + const dir = mkdtempSync(join(tmpdir(), 'patchling-filter-')); + const cache = new RelevanceCache(join(dir, 'cache.json')); + const files = { 'big.py': `PREFIX ${'x'.repeat(50)}` }; + const first = await filterRelevant(files, 'task', { + model: 'm', cache, apiKey: 'k', maxBytes: 20, maxTokens: null, onOversize: 'truncate', callLlm: mock.fn, + }); + const second = await filterRelevant(files, 'task', { + model: 'm', cache, apiKey: 'k', maxBytes: 10_000, maxTokens: null, callLlm: mock.fn, + }); + assert.equal(first.decisions[0].decision, 'no'); + assert.equal(first.decisions[0].skippedReason, 'truncated'); + assert.equal(second.decisions[0].decision, 'yes'); + assert.equal(second.decisions[0].cached, false); + assert.equal(second.decisions[0].skippedReason, null); + assert.equal(n, 2); + const stored = JSON.parse(readFileSync(cache.path, 'utf8')); + const variants = Object.keys(stored.entries).map((key) => key.split('\n')[3]); + assert.ok(variants.includes('truncated:None:20')); + assert.ok(variants.includes('full')); }); -test('cache is keyed by model, task hash, and content hash', async () => { - const cache = createMemoryCache(); - const first = llm(['yes — greeting']); - const files = { 'greet.py': 'print("hello")\n' }; - const task = 'change the greeting'; - await filterRelevant(files, task, { model: 'cheap-a', cache, callLlm: first.fn }); - - const second = llm(['no']); - const again = await filterRelevant({ 'other.py': 'print("hello")\n' }, task, { - model: 'cheap-a', - cache, - callLlm: second.fn, +test('maxBytes counts UTF-8 bytes and a token cap truncates before the byte cap', async () => { + const seen = []; + const mock = llm((req) => { + seen.push(bodyOf(req)); + return response('yes'); }); - assert.equal(second.stats().count, 0); - assert.equal(again[0].cached, true); - assert.equal(again[0].path, 'other.py'); - assert.equal(again[0].decision, 'yes'); - assert.equal(again[0].reason, 'greeting'); - assert.equal(again[0].tokens, 11); - - const otherTask = llm(['no']); - const moved = await filterRelevant(files, 'rename the function', { - model: 'cheap-a', - cache, - callLlm: otherTask.fn, + const accented = 'é'; + const excluded = await filterRelevant({ a: accented, b: 'b' }, 't', { + ...baseOpts, + maxBytes: 1, + maxTokens: null, + onOversize: 'exclude', + callLlm: mock.fn, }); - assert.equal(moved[0].cached, false); - assert.equal(moved[0].decision, 'no'); - - const otherModel = llm(['no']); - const remodeled = await filterRelevant(files, task, { - model: 'cheap-b', - cache, - callLlm: otherModel.fn, + assert.equal(excluded.decisions[0].skippedReason, 'oversize'); + assert.equal(excluded.decisions[0].decision, 'no'); + assert.equal(excluded.decisions[1].decision, 'yes'); + assert.equal(excluded.decisions[1].skippedReason, null); + + const exact = await filterRelevant({ a: 'abcd' }, 't', { + ...baseOpts, maxBytes: 4, maxTokens: null, callLlm: mock.fn, }); - assert.equal(remodeled[0].cached, false); - assert.equal(otherModel.stats().count, 1); - - const stored = []; - const custom = { - async get(key) { - stored.push(['get', key]); - return undefined; - }, - async set(key, value) { - stored.push(['set', key, value.decision]); - }, - }; - await filterRelevant({ p: 'body' }, 'task', { model: 'm', cache: custom, callLlm: llm(['yes']).fn }); - const set = stored.find((row) => row[0] === 'set'); - assert.ok(set); - const parts = set[1].split('\0'); - assert.equal(parts.length, 3); - assert.equal(parts[0], 'm'); - assert.match(parts[1], /^[0-9a-f]{64}$/); - assert.match(parts[2], /^[0-9a-f]{64}$/); - assert.notEqual(parts[1], parts[2]); + assert.equal(exact.decisions[0].skippedReason, null); + assert.equal(exact.files.a, 'abcd'); + + const long = 'y'.repeat(40); + const truncated = await filterRelevant({ a: long }, 't', { + ...baseOpts, maxBytes: null, maxTokens: 4, callLlm: mock.fn, + }); + assert.equal(countTokens(long) > 4, true); + assert.equal(truncated.decisions[0].skippedReason, 'truncated'); + assert.equal(truncated.files.a, long); + assert.equal(seen.at(-1), long.slice(0, 16)); }); -test('ambiguous answers are not cached', async () => { - const cache = createMemoryCache(); - const first = llm(['maybe']); - await filterRelevant({ a: 'body' }, 'task', { model: 'm', cache, callLlm: first.fn }); - const second = llm(['no']); - const decisions = await filterRelevant({ a: 'body' }, 'task', { model: 'm', cache, callLlm: second.fn }); - assert.equal(second.stats().count, 1); - assert.equal(decisions[0].decision, 'no'); - assert.equal(decisions[0].cached, false); +test('zero disables both size limits', async () => { + const seen = []; + const mock = llm((req) => { + seen.push(req.messages[1].content); + return response('yes'); + }); + const body = 'y'.repeat(250_000); + const result = await filterRelevant({ 'big.py': body }, 'read the whole file', { + ...baseOpts, + maxBytes: 0, + maxTokens: 0, + onOversize: 'truncate', + callLlm: mock.fn, + }); + assert.equal(Object.keys(result.files).length, 1); + assert.equal(seen[0].includes('truncated to fit'), false); + assert.ok(seen[0].includes(body)); + assert.equal(result.files['big.py'], body); }); -test('default model and the shared memory cache', async () => { - const prev = process.env.GPTDIFF_FILTER_MODEL; - setEnv('GPTDIFF_FILTER_MODEL', 'from-env'); +test('missing api key throws only when a call is required', async () => withEnv(async () => { + const prev = process.env.GPTDIFF_LLM_API_KEY; + delete process.env.GPTDIFF_LLM_API_KEY; + setEnv('GPTDIFF_LLM_API_KEY', undefined); try { - const mock = llm(['yes']); - const task = `env-model-${Date.now()}`; - await filterRelevant({ a: `unique-${task}` }, task, { cache: undefined, callLlm: mock.fn }); - assert.equal(mock.calls[0].model, 'from-env'); - const again = llm(['no']); - const hit = await filterRelevant({ b: `unique-${task}` }, task, { callLlm: again.fn }); - assert.equal(again.stats().count, 0); - assert.equal(hit[0].cached, true); + const skipped = await filterRelevant({ 'bin.dat': '\0' }, 'task', { + model: 'm', + cache: false, + onBinary: 'exclude', + callLlm: () => { throw new Error('should not be called'); }, + }); + assert.deepEqual(skipped.files, {}); + + await assert.rejects( + () => filterRelevant({ 'a.py': 'x = 1\n' }, 'task', { + model: 'm', + cache: false, + callLlm: () => { throw new Error('should not be called'); }, + }), + /GPTDIFF_LLM_API_KEY/, + ); + } finally { + setEnv('GPTDIFF_LLM_API_KEY', undefined); + if (prev === undefined) delete process.env.GPTDIFF_LLM_API_KEY; + else process.env.GPTDIFF_LLM_API_KEY = prev; + } +})); + +test('cached files do not need an api key', async () => withEnv(async () => { + const prev = process.env.GPTDIFF_LLM_API_KEY; + delete process.env.GPTDIFF_LLM_API_KEY; + setEnv('GPTDIFF_LLM_API_KEY', undefined); + let n = 0; + const mock = llm(() => { + n += 1; + return response('no'); + }); + try { + const cache = createMemoryCache(); + assert.ok(cache instanceof RelevanceCache); + const files = { 'a.py': 'x = 1\n' }; + await filterRelevant(files, 'task', { model: 'm', cache, apiKey: 'k', callLlm: mock.fn }); + const result = await filterRelevant(files, 'task', { model: 'm', cache, callLlm: mock.fn }); + assert.equal(n, 1); + assert.equal(result.decisions[0].cached, true); + assert.equal(result.decisions[0].decision, 'no'); + } finally { + setEnv('GPTDIFF_LLM_API_KEY', undefined); + if (prev === undefined) delete process.env.GPTDIFF_LLM_API_KEY; + else process.env.GPTDIFF_LLM_API_KEY = prev; + } +})); + +test('invalid arguments throw', async () => { + await assert.rejects( + () => filterRelevant({ 'a.py': 'x' }, 'task', { concurrency: 0, cache: false, apiKey: 'k' }), + /concurrency/, + ); + await assert.rejects( + () => filterRelevant({ 'a.py': 'x' }, 'task', { onOversize: 'drop', cache: false, apiKey: 'k' }), + /onOversize/, + ); + await assert.rejects( + () => filterRelevant({ 'a.py': 'x' }, ' ', { cache: false, apiKey: 'k' }), + /task/, + ); + await assert.rejects( + () => filterRelevant([['a.py', 'x']], 'task', { cache: false, apiKey: 'k' }), + TypeError, + ); + await assert.rejects( + () => filterRelevant({ 'a.py': 'x' }, 'task', { cache: {}, apiKey: 'k' }), + TypeError, + ); + await assert.rejects( + () => filterRelevant({ 'a.py': 1 }, 'task', { cache: false, apiKey: 'k', callLlm: llm(['yes']).fn }), + TypeError, + ); +}); + +test('model defaults to GPTDIFF_FILTER_MODEL, else gemini-2.0-flash', async () => withEnv(async () => { + const prevFilter = process.env.GPTDIFF_FILTER_MODEL; + const prevBase = process.env.GPTDIFF_LLM_BASE_URL; + delete process.env.GPTDIFF_FILTER_MODEL; + setEnv('GPTDIFF_FILTER_MODEL', undefined); + setEnv('GPTDIFF_LLM_BASE_URL', 'https://example.test/v1/'); + const seen = []; + const mock = llm((req) => { + seen.push(req); + return response('yes'); + }); + try { + const fromEnv = await filterRelevant({ 'a.py': 'x = 1\n' }, 'task', { + cache: false, + apiKey: 'from-arg', + callLlm: mock.fn, + }); + setEnv('GPTDIFF_FILTER_MODEL', 'env-filter-model'); + const named = await filterRelevant({ 'a.py': 'x = 1\n' }, 'task', { + cache: false, + apiKey: 'from-arg', + callLlm: mock.fn, + }); + assert.equal(seen[0].model, DEFAULT_FILTER_MODEL); + assert.equal(fromEnv.model, 'gemini-2.0-flash'); + assert.equal(seen[0].baseUrl, 'https://example.test/v1/'); + assert.equal(seen[0].apiKey, 'from-arg'); + assert.ok(seen[0].messages[1].content.startsWith('You are a relevance filter')); + assert.equal(named.model, 'env-filter-model'); + assert.equal(seen[1].model, 'env-filter-model'); + assert.equal(seen[1].messages[1].content.startsWith('You are a relevance filter'), false); } finally { setEnv('GPTDIFF_FILTER_MODEL', undefined); - if (prev === undefined) delete process.env.GPTDIFF_FILTER_MODEL; - else process.env.GPTDIFF_FILTER_MODEL = prev; + setEnv('GPTDIFF_LLM_BASE_URL', undefined); + if (prevFilter === undefined) delete process.env.GPTDIFF_FILTER_MODEL; + else process.env.GPTDIFF_FILTER_MODEL = prevFilter; + if (prevBase === undefined) delete process.env.GPTDIFF_LLM_BASE_URL; + else process.env.GPTDIFF_LLM_BASE_URL = prevBase; } - assert.equal(typeof DEFAULT_FILTER_MODEL, 'string'); - assert.equal(DEFAULT_FILTER_MAX_BYTES, 262144); + assert.equal(DEFAULT_FILTER_MODEL, 'gemini-2.0-flash'); + assert.equal(DEFAULT_FILTER_MAX_BYTES, 200_000); + assert.equal(DEFAULT_FILTER_MAX_TOKENS, 12_000); + assert.equal(DEFAULT_FILTER_MAX_OUTPUT_TOKENS, 256); + assert.equal(DEFAULT_FILTER_CONCURRENCY, 8); + assert.equal(isBinaryContent('hello\0world'), true); + assert.equal(isBinaryContent('plain text'), false); + assert.equal(countTokens(''), 0); + assert.equal(countTokens('abcd'), 1); +})); + +test('omitting cache does not reuse a previous call', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return response('yes'); + }); + await filterRelevant({ a: 'body' }, 'task', { model: 'm', apiKey: 'k', callLlm: mock.fn }); + await filterRelevant({ a: 'body' }, 'task', { model: 'm', apiKey: 'k', callLlm: mock.fn }); + assert.equal(n, 2); +}); + +test('dumpDecisions writes camelCase decisions', async () => { + const mock = llm(() => response('yes — keep')); + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { ...baseOpts, callLlm: mock.fn }); + const dir = mkdtempSync(join(tmpdir(), 'patchling-filter-')); + const dest = join(dir, 'out', 'decisions.json'); + mkdirSync(join(dir, 'out'), { recursive: true }); + await dumpDecisions(result, dest); + const saved = JSON.parse(readFileSync(dest, 'utf8')); + assert.deepEqual(saved.files, ['a.py']); + assert.equal(saved.decisions[0].promptTokens, 11); + assert.equal(saved.decisions[0].skippedReason, null); + assert.equal('skipped_reason' in saved.decisions[0], false); + assert.equal(saved.billedTokens, 14); }); test('buildEnvironment includes only yes files when decisions are passed', () => { @@ -333,8 +641,8 @@ test('buildEnvironment includes only yes files when decisions are passed', () => const decisions = [ { path: 'greet.py', decision: 'yes' }, { path: 'main.py', decision: 'no' }, - { path: 'math_utils.py', decision: 'skip', skippedReason: 'binary' }, - { path: 'logo.bin', decision: 'skip', skippedReason: 'too-large' }, + { path: 'math_utils.py', decision: 'no', skippedReason: 'binary' }, + { path: 'logo.bin', decision: 'no', skippedReason: 'oversize' }, ]; const filtered = buildEnvironment(files, { decisions }); assert.match(filtered, /File: greet\.py/); @@ -345,7 +653,7 @@ test('buildEnvironment includes only yes files when decisions are passed', () => assert.equal(filtered, buildEnvironment(files, decisions)); const includedByError = buildEnvironment(files, { - decisions: [{ path: 'main.py', decision: 'yes', error: 'ambiguous answer' }], + decisions: [{ path: 'main.py', decision: 'yes', error: 'Error: connection reset' }], }); assert.match(includedByError, /File: main\.py/); assert.doesNotMatch(includedByError, /greet/); diff --git a/test/live-filter.test.js b/test/live-filter.test.js index 122738a..c1c9c19 100644 --- a/test/live-filter.test.js +++ b/test/live-filter.test.js @@ -31,15 +31,17 @@ test( 'math_utils.py': 'def add(a, b):\n return a + b\n', }; const task = 'Change the greeting from hello to goodbye. Do not edit unrelated helpers.'; - const decisions = await filterRelevant(files, task, { model: MODEL, apiKey }); - const by = Object.fromEntries(decisions.map((d) => [d.path, d])); + const result = await filterRelevant(files, task, { model: MODEL, apiKey }); + const by = Object.fromEntries(result.decisions.map((d) => [d.path, d])); assert.equal(by['greet.py'].decision, 'yes', JSON.stringify(by['greet.py'])); assert.equal(by['math_utils.py'].decision, 'no', JSON.stringify(by['math_utils.py'])); assert.equal(by['greet.py'].skippedReason, null); assert.ok(by['greet.py'].raw); + assert.equal(result.files['greet.py'], files['greet.py']); + assert.equal('math_utils.py' in result.files, false); - const env = buildEnvironment(files, { decisions }); + const env = buildEnvironment(files, { decisions: result.decisions }); assert.match(env, /File: greet\.py/); assert.doesNotMatch(env, /math_utils/); }, From 9e033e64156778a66f42e9469ed03dbad0bcdab1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 20:03:49 +0000 Subject: [PATCH 3/6] Keep reasoning models from caching an empty fail-open Raise the filter completion cap, send reasoning_effort low, and treat an empty answer as an error that is not cached. Retry timeouts and HTTP 5xx, count files that failed open, and skip lockfiles and generated paths. Default model is xiaomi/mimo-v2.6-flash. Co-authored-by: ntc-ai --- CHANGELOG.md | 4 +- README.md | 17 +- landing/index.html | 6 +- landing/lib/filter.js | 337 +++++++++++++++++++++++++++++------ landing/lib/llm.js | 4 + src/filter.d.ts | 57 +++++- src/filter.js | 337 +++++++++++++++++++++++++++++------ src/index.d.ts | 2 +- src/index.js | 4 + src/llm.d.ts | 5 +- src/llm.js | 4 + test/filter_relevant.test.js | 266 ++++++++++++++++++++++++++- 12 files changed, 908 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4421e92..e47bf40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### API -- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `gemini-2.0-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file. The cache key adds a truncation tag. Pass a path for an optional Node JSON cache (`{ version: 1, entries }`, corrupt files ignored). Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`). Ambiguous answers and thrown calls include the file; only a thrown call sets `error`. `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. +- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `xiaomi/mimo-v2.6-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file. The completion cap is 1024 and the call sends `reasoning_effort: low`, so a reasoning model does not spend the whole answer budget. An empty answer is an error and fails open. Timeouts and HTTP 5xx are retried with backoff. Lockfiles and generated files are skipped unless `ignore` is turned off or replaced. The cache key includes the truncation tag, the output-token limit, and the prompt version. Fallback and error decisions are not cached. `failedOpen` counts files included without a definitive answer. Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`, `ignored`). `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. - Renamed the package from `gptdiff-js` to `patchling` (npm name `patchling`; an intermediate publish used `gptdiff`). - `generateDiff` and `smartapply` stream tokens through `opts.onToken`. `generateDiff` reports token usage through `opts.onUsage`. - Ship JSDoc-generated `.d.ts` files, with `types` conditions on the package exports. @@ -27,7 +27,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### Site - `patchling.app` landing page: in-browser playground (canned demo and live NanoGPT), model picker, and Open Graph card. -- The playground can run `filterRelevant` before the diff: a toggle, a decision-model picker (default `gemini-2.0-flash`), and a per-file yes/no view (reasons, tokens, binary and oversize skips, truncated previews). Bundled sample projects exercise it without leaving the page. +- The playground can run `filterRelevant` before the diff: a toggle, a decision-model picker (default `xiaomi/mimo-v2.6-flash`), and a per-file yes/no view (reasons, tokens, binary, oversize, and ignored skips, truncated previews). Bundled sample projects exercise it without leaving the page. - README links that 404'd after the repo rename now point at patchling.app and the Python package. ### CI diff --git a/README.md b/README.md index 11c85b6..51f3056 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Configuration comes from environment variables (Node) or `setEnv(...)` overrides | `GPTDIFF_LLM_API_KEY` | NanoGPT API key (`sk-nano-…`) | — (required for live calls) | | `GPTDIFF_LLM_BASE_URL` | OpenAI-compatible base URL | `https://nano-gpt.com/api/v1/` | | `GPTDIFF_MODEL` | Model id | `xiaomi/mimo-v2.5-pro-ultraspeed` | -| `GPTDIFF_FILTER_MODEL` | Cheap model for `filterRelevant` | `gemini-2.0-flash` | +| `GPTDIFF_FILTER_MODEL` | Cheap model for `filterRelevant` | `xiaomi/mimo-v2.6-flash` | ```js import { setEnv } from 'patchling'; @@ -109,7 +109,7 @@ Opt-in pass for a large file map. Each candidate file is sent with the task to a import { filterRelevant, buildEnvironment, generateDiff, smartapply } from 'patchling'; const result = await filterRelevant(files, task, { - // model: 'gemini-2.0-flash', // GPTDIFF_FILTER_MODEL, else this default + // model: 'xiaomi/mimo-v2.6-flash', // GPTDIFF_FILTER_MODEL, else this default concurrency: 8, // max in-flight calls // cache: null, // omit for a fresh in-memory cache; a path string persists it // maxBytes: 200000, // 0 or null disables @@ -122,18 +122,21 @@ const diff = await generateDiff(buildEnvironment(result.files), task, { model }) const next = await smartapply(diff, files); ``` -- `opts.model`, `opts.apiKey`, `opts.baseUrl` — same key and base URL resolution as `generateDiff` (`GPTDIFF_LLM_API_KEY`, `GPTDIFF_LLM_BASE_URL`). The model falls back to `GPTDIFF_FILTER_MODEL`, then `gemini-2.0-flash`. +- `opts.model`, `opts.apiKey`, `opts.baseUrl` — same key and base URL resolution as `generateDiff` (`GPTDIFF_LLM_API_KEY`, `GPTDIFF_LLM_BASE_URL`). The model falls back to `GPTDIFF_FILTER_MODEL`, then `xiaomi/mimo-v2.6-flash`. - `opts.concurrency` — at most this many classifications in flight (default 8). -- `opts.cache` — a `RelevanceCache`, a `{ get, set }` object, or a filesystem path (Node writes `{ version: 1, entries }` atomically; a corrupt file is ignored). Omit it, or pass `true`, for a fresh in-memory cache used only by this call. `null` and `false` turn caching off. `createMemoryCache()` builds another memory cache you can share. The key is the model, SHA-256 of the task, SHA-256 of the original contents, and a variant tag (`full`, `binary-preview`, or `truncated::`), separated by newlines. Thrown errors are not stored. +- `opts.cache` — a `RelevanceCache`, a `{ get, set }` object, or a filesystem path (Node writes `{ version: 1, entries }` atomically; a corrupt file is ignored). Omit it, or pass `true`, for a fresh in-memory cache used only by this call. `null` and `false` turn caching off. `createMemoryCache()` builds another memory cache you can share. The key is the model, SHA-256 of the task, SHA-256 of the original contents, a variant tag (`full`, `binary-preview`, or `truncated::`), the output-token limit, and the prompt version, separated by newlines. Fallback and error decisions are not stored. - `opts.maxBytes` — UTF-8 size cap (default 200000). `opts.maxTokens` — input-token cap (default 12000; estimated as characters / 4). `0` or `null` disables either limit. - `opts.onOversize` and `opts.onBinary` — `include` (keep, no model call), `exclude` (drop, no model call), or `truncate` (ask about a preview). Defaults are `truncate` and `exclude`. A `yes` on a truncated preview still forwards the original text in `result.files`. +- `opts.ignore` — lockfiles and generated paths (`package-lock.json`, `dist/`, `*.min.js`, and the rest of `DEFAULT_IGNORE_PATTERNS`) are excluded with `skippedReason: "ignored"`. `false` classifies them. An array replaces the default patterns. +- `opts.maxOutputTokens` — completion cap (default 1024). The call also sends `reasoning_effort: "low"` so a reasoning model does not spend the whole budget before the yes/no. An empty answer is an error. +- `opts.retries` — extra attempts after a timeout or HTTP 5xx, including 503 (default 2), with exponential backoff. - `opts.onDecision` — called as each file finishes, in completion order. `result.decisions` stays in input order. - `opts.callLlm` — inject a mock client. Each decision is `{ path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error }`. -- `decision` is `"yes"` or `"no"`. Ambiguous text (after `` / `` blocks are stripped) and a thrown call become `"yes"`, so a bad answer cannot drop a file. `error` is set only when the call throws. -- `skippedReason` is `"binary"`, `"oversize"`, `"truncated"`, or `null`. +- `decision` is `"yes"` or `"no"`. Ambiguous text (after `` / `` blocks are stripped), an empty answer, and a thrown call become `"yes"`, so a bad answer cannot drop a file. `error` is set when the call throws or the answer is empty. Those fallbacks are not cached. `result.failedOpen` counts them, and `formatFilterSummary` includes that count. +- `skippedReason` is `"binary"`, `"oversize"`, `"truncated"`, `"ignored"`, or `null`. - `tokens`, `promptTokens`, and `completionTokens` are `0` when no model call was made. `reason` is a string. - `result.billedTokens` sums token counts for decisions that were not cache hits. Identical file bodies share one request; the later paths are `cached: true`. - `buildEnvironment(files, { decisions })` keeps only `"yes"` paths. Passing the decisions array itself does the same. With no second argument, every file is included. `buildEnvironment(result.files)` is the same set. @@ -151,7 +154,7 @@ Applies a diff to an in-memory file map with per-file, LLM-assisted conflict res Deterministic, no-LLM patch application (the strict "basic" applier). `changed` is `true` iff something was created, modified, or deleted; `files` is the updated map (equal to the input on failure — no partial writes). ### Other exports -`parseDiffPerFile`, `buildEnvironment`, `filterRelevant`, `RelevanceCache`, `createMemoryCache`, `parseRelevanceAnswer`, `isBinaryContent`, `countTokens`, `formatFilterSummary`, `dumpDecisions`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. +`parseDiffPerFile`, `buildEnvironment`, `filterRelevant`, `RelevanceCache`, `createMemoryCache`, `parseRelevanceAnswer`, `isBinaryContent`, `isIgnoredPath`, `countTokens`, `formatFilterSummary`, `dumpDecisions`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. ## Differences from the Python package diff --git a/landing/index.html b/landing/index.html index 638cb8f..04566c9 100644 --- a/landing/index.html +++ b/landing/index.html @@ -383,7 +383,7 @@

Give your app a “describe the change” box.

- Before the diff, a cheap model answers yes or no for each file. Binary files are excluded. Oversized files are truncated for the check; a yes still sends the original text. + Before the diff, a cheap model answers yes or no for each file. Binary files, lockfiles, and generated files are excluded. Oversized files are truncated for the check; a yes still sends the original text.
-

Each decision is { path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error }. decision is yes or no. skippedReason is binary, oversize, truncated, or null. The default model is gemini-2.0-flash (GPTDIFF_FILTER_MODEL). The cache key is the model, a hash of the task, a hash of the original contents, and a truncation tag. Python uses the same fields in snake_case (skipped_reason, prompt_tokens, max_bytes, max_tokens, on_oversize, on_binary).

+

Each decision is { path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error }. decision is yes or no. skippedReason is binary, oversize, truncated, ignored, or null. The default model is xiaomi/mimo-v2.6-flash (GPTDIFF_FILTER_MODEL). The cache key is the model, a hash of the task, a hash of the original contents, a truncation tag, the output-token limit, and the prompt version. Python uses the same fields in snake_case (skipped_reason, prompt_tokens, max_bytes, max_tokens, on_oversize, on_binary).

@@ -1559,7 +1559,7 @@

${esc(title)}

${cards}`; function renderDecision(d) { const row = document.createElement("div"); row.className = "drow"; - const skipped = d.skippedReason === "binary" || d.skippedReason === "oversize"; + const skipped = d.skippedReason === "binary" || d.skippedReason === "oversize" || d.skippedReason === "ignored"; const badge = document.createElement("span"); badge.className = "badge " + (skipped ? "skip" : d.decision === "yes" ? "yes" : "no"); badge.textContent = skipped ? "skipped" : d.decision; diff --git a/landing/lib/filter.js b/landing/lib/filter.js index 84ae6c3..6e1ea55 100644 --- a/landing/lib/filter.js +++ b/landing/lib/filter.js @@ -18,7 +18,7 @@ import { callLlm as defaultCallLlm, resolveApiKey, resolveBaseUrl } from './llm. import { getEnv } from './env.js'; /** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ -export const DEFAULT_FILTER_MODEL = 'gemini-2.0-flash'; +export const DEFAULT_FILTER_MODEL = 'xiaomi/mimo-v2.6-flash'; /** In-flight classifications per call when `concurrency` is omitted. */ export const DEFAULT_FILTER_CONCURRENCY = 8; @@ -29,8 +29,57 @@ export const DEFAULT_FILTER_MAX_BYTES = 200_000; /** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_TOKENS = 12_000; -/** Completion cap for the yes/no call. */ -export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 256; +/** + * Completion cap for the yes/no call. High enough that a reasoning model's + * scratchpad does not consume the whole answer. `reasoning_effort` is also + * sent as `low`. + */ +export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 1024; + +/** Sent on the filter call so reasoning models keep the yes/no inside the budget. */ +export const DEFAULT_FILTER_REASONING_EFFORT = 'low'; + +/** + * Bumped when the system prompt changes, and included in the cache key so an + * old judgment is not reused for a new prompt. + */ +export const FILTER_PROMPT_VERSION = 1; + +/** + * Lockfiles and generated paths skipped unless `ignore` is `false` or a custom list. + * Patterns are case-insensitive globs (`**` crosses directories, `*` does not). + */ +export const DEFAULT_IGNORE_PATTERNS = [ + '**/package-lock.json', + '**/yarn.lock', + '**/pnpm-lock.yaml', + '**/bun.lock', + '**/bun.lockb', + '**/npm-shrinkwrap.json', + '**/Cargo.lock', + '**/Gemfile.lock', + '**/poetry.lock', + '**/Pipfile.lock', + '**/composer.lock', + '**/go.sum', + '**/uv.lock', + '**/pdm.lock', + '**/flake.lock', + '**/pubspec.lock', + '**/packages.lock.json', + '**/Package.resolved', + '**/*.min.js', + '**/*.min.css', + '**/*.map', + '**/dist/**', + '**/build/**', + '**/.next/**', + '**/coverage/**', + '**/__pycache__/**', + '**/*.pyc', + '**/*.pyo', + '**/*.generated.*', +]; const POLICIES = ['include', 'exclude', 'truncate']; @@ -58,8 +107,8 @@ const SYSTEM_PROMPT = ( * @property {number} promptTokens * @property {number} completionTokens * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) - * @property {'binary' | 'oversize' | 'truncated' | null} skippedReason - * set when the file was not sent in full + * @property {'binary' | 'oversize' | 'truncated' | 'ignored' | null} skippedReason + * set when the file was not sent in full. `ignored` is a lockfile or generated path. * @property {string | null} error set when the call threw; the file is still included */ @@ -72,6 +121,8 @@ const SYSTEM_PROMPT = ( * @property {number} billedTokens sum of `tokens` on decisions that were not cache hits * @property {number} billedPromptTokens * @property {number} billedCompletionTokens + * @property {number} failedOpen files included because the call failed, the answer + * was empty, or the answer was ambiguous */ /** @@ -114,12 +165,21 @@ export class RelevanceCache { * @param {string} task * @param {string} content original file text * @param {string} [variant] + * @param {number} [maxOutputTokens] completion cap, so a larger budget is a different key + * @param {number} [promptVersion] {@link FILTER_PROMPT_VERSION} * @returns {Promise} */ - static async makeKey(model, task, content, variant = 'full') { + static async makeKey( + model, + task, + content, + variant = 'full', + maxOutputTokens = DEFAULT_FILTER_MAX_OUTPUT_TOKENS, + promptVersion = FILTER_PROMPT_VERSION, + ) { const taskHash = await sha256(task); const contentHash = await sha256(content); - return `${model}\n${taskHash}\n${contentHash}\n${variant}`; + return `${model}\n${taskHash}\n${contentHash}\n${variant}\n${maxOutputTokens}\n${promptVersion}`; } /** @@ -516,7 +576,7 @@ function decisionFromRecord(path, record, cached) { /** * @param {string} path * @param {'yes' | 'no'} decision - * @param {'binary' | 'oversize'} skippedReason + * @param {'binary' | 'oversize' | 'ignored'} skippedReason * @param {string} reason * @returns {RelevanceDecision} */ @@ -577,6 +637,120 @@ function errorMessage(err) { return String(err); } +/** + * Timeouts and HTTP 5xx (including 503). Other failures fail open immediately. + * @param {unknown} err + * @returns {boolean} + */ +function isTransientFilterError(err) { + const status = err && typeof err === 'object' + ? (/** @type {{ status?: number, statusCode?: number }} */ (err).status + ?? /** @type {{ statusCode?: number }} */ (err).statusCode) + : undefined; + if (typeof status === 'number' && status >= 500 && status <= 599) return true; + const code = err && typeof err === 'object' + ? (/** @type {{ code?: string, cause?: { code?: string } }} */ (err).code + || /** @type {{ cause?: { code?: string } }} */ (err).cause?.code) + : undefined; + if (typeof code === 'string' && /^(ETIMEDOUT|ECONNABORTED|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET)$/.test(code)) { + return true; + } + const name = err && typeof err === 'object' ? /** @type {{ name?: string }} */ (err).name : ''; + if (name === 'TimeoutError' || name === 'AbortError') return true; + const message = errorMessage(err); + if (/timeout|timed out/i.test(message)) return true; + if (/\b5\d\d\b/.test(message)) return true; + return false; +} + +/** + * @param {unknown} value + * @returns {number} + */ +function resolveRetries(value) { + if (value === undefined) return 2; + const n = Math.trunc(Number(value)); + if (!Number.isFinite(n) || n < 0) throw new Error('retries must be >= 0'); + return n; +} + +/** + * @param {string} pattern + * @returns {RegExp} + */ +function globToRegExp(pattern) { + let re = ''; + let i = 0; + const src = pattern.replace(/\\/g, '/').toLowerCase(); + while (i < src.length) { + if (src.startsWith('**/', i)) { + re += '(?:.*/)?'; + i += 3; + continue; + } + if (src.startsWith('**', i)) { + re += '.*'; + i += 2; + continue; + } + const ch = src[i]; + if (ch === '*') re += '[^/]*'; + else { + if (/[\\^$+?.()|{}[\]]/.test(ch)) re += '\\'; + re += ch; + } + i += 1; + } + return new RegExp(`^${re}$`); +} + +/** + * @param {string} filePath + * @param {string[]} patterns + * @returns {boolean} + */ +export function isIgnoredPath(filePath, patterns = DEFAULT_IGNORE_PATTERNS) { + const normalized = String(filePath).replace(/\\/g, '/').replace(/^\.\//, '').toLowerCase(); + return patterns.some((pattern) => globToRegExp(pattern).test(normalized)); +} + +/** + * @param {unknown} ignore + * @returns {string[]} + */ +function resolveIgnore(ignore) { + if (ignore === false || ignore === null) return []; + if (ignore === undefined || ignore === true) return DEFAULT_IGNORE_PATTERNS; + if (Array.isArray(ignore)) return ignore.map(String); + throw new TypeError('ignore must be true, false, or an array of patterns'); +} + +/** + * Included without a definitive yes/no: a thrown call, an empty answer, or an + * ambiguous answer. + * @param {RelevanceDecision} decision + * @returns {boolean} + */ +function isFailedOpen(decision) { + if (!decision) return false; + if (decision.error) return true; + return typeof decision.reason === 'string' + && decision.reason.startsWith('ambiguous answer; included by default'); +} + +/** + * A stored judgment is reusable only when it is a definitive yes or no. + * Fallbacks and errors are not, even if an older cache file still holds them. + * @param {RelevanceCacheRecord | undefined} hit + * @returns {boolean} + */ +function reusableHit(hit) { + if (!hit || (hit.decision !== 'yes' && hit.decision !== 'no') || hit.error) return false; + const reason = hit.reason || ''; + if (reason.startsWith('ambiguous answer') || reason.startsWith('error; included by default')) return false; + return true; +} + /** * @param {Array} items * @param {number} limit @@ -648,6 +822,12 @@ async function cachePut(store, key, record) { * @param {'include' | 'exclude' | 'truncate'} [opts.onOversize] default `truncate` * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) + * @param {string | null} [opts.reasoningEffort] default {@link DEFAULT_FILTER_REASONING_EFFORT}; `null` omits it + * @param {number} [opts.retries] extra attempts after a timeout or HTTP 5xx (default 2) + * @param {number} [opts.retryBaseMs] backoff base in ms (default 200, doubles, capped at 30s) + * @param {boolean | string[] | null} [opts.ignore] lockfile and generated-file patterns. + * `true` or omitted uses {@link DEFAULT_IGNORE_PATTERNS}. `false` or `null` classifies every path. + * An array replaces the default list. * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) * @param {string} [opts.apiKey] `GPTDIFF_LLM_API_KEY` when omitted * @param {string} [opts.baseUrl] `GPTDIFF_LLM_BASE_URL` when omitted @@ -672,6 +852,12 @@ export async function filterRelevant(files, task, opts = {}) { const maxOutputTokens = opts.maxOutputTokens === undefined ? DEFAULT_FILTER_MAX_OUTPUT_TOKENS : opts.maxOutputTokens; + const reasoningEffort = opts.reasoningEffort === undefined + ? DEFAULT_FILTER_REASONING_EFFORT + : opts.reasoningEffort; + const retries = resolveRetries(opts.retries); + const retryBaseMs = opts.retryBaseMs === undefined ? 200 : Math.max(0, Number(opts.retryBaseMs) || 0); + const ignorePatterns = resolveIgnore(opts.ignore); const model = resolveFilterModel(opts.model); const store = resolveCache(opts.cache); @@ -710,6 +896,18 @@ export async function filterRelevant(files, task, opts = {}) { } for (const [path, content] of entries) { + if (isIgnoredPath(path, ignorePatterns)) { + const made = skipDecision( + path, + 'no', + 'ignored', + 'lockfile or generated file; excluded without a model call', + ); + byPath.set(path, made); + emit(made); + continue; + } + const binary = isBinaryContent(content); const text = coerceText(content); if (binary) { @@ -717,7 +915,7 @@ export async function filterRelevant(files, task, opts = {}) { const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; const body = `[binary file, ${nbytes} bytes, contents omitted]`; - const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview'); + const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview', maxOutputTokens); enqueue(key, { path, body, note, skippedReason: 'truncated' }); } else { const decision = onBinary === 'include' ? 'yes' : 'no'; @@ -741,7 +939,7 @@ export async function filterRelevant(files, task, opts = {}) { + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` ); const variant = `truncated:${rawLimitTag(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS)}:${rawLimitTag(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES)}`; - const key = await RelevanceCache.makeKey(model, task, text, variant); + const key = await RelevanceCache.makeKey(model, task, text, variant, maxOutputTokens); enqueue(key, { path, body, note, skippedReason: 'truncated' }); } else { const decision = onOversize === 'include' ? 'yes' : 'no'; @@ -758,7 +956,7 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - const key = await RelevanceCache.makeKey(model, task, text, 'full'); + const key = await RelevanceCache.makeKey(model, task, text, 'full', maxOutputTokens); enqueue(key, { path, body: text, note: null, skippedReason: null }); } @@ -771,7 +969,7 @@ export async function filterRelevant(files, task, opts = {}) { async function cached(key) { if (!store) return false; const hit = await cacheGet(store, key); - return !!(hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error); + return reusableHit(hit); } let needsCall = false; @@ -796,7 +994,7 @@ export async function filterRelevant(files, task, opts = {}) { const items = groups.get(key); if (store) { const hit = await cacheGet(store, key); - if (hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error) { + if (reusableHit(hit)) { for (const item of items) { const decision = decisionFromRecord(item.path, { ...hit, cache_key: key }, true); byPath.set(item.path, decision); @@ -811,39 +1009,57 @@ export async function filterRelevant(files, task, opts = {}) { let promptTokens; let completionTokens; let totalTokens; - try { - const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); - let userContent = userPrompt; - if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; - const response = await callLlm({ - apiKey, - baseUrl, - model, - messages: [ - { role: 'system', content: SYSTEM_PROMPT }, - { role: 'user', content: userContent }, - ], - maxTokens: maxOutputTokens, - temperature: 0, - budgetTokens: null, - }); - ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); - } catch (err) { - const message = errorMessage(err); - for (const item of items) { + /** @type {unknown} */ + let lastError = null; + const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); + let userContent = userPrompt; + if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; + const attempts = retries + 1; + for (let attempt = 0; attempt < attempts; attempt++) { + try { + const response = await callLlm({ + apiKey, + baseUrl, + model, + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: userContent }, + ], + maxTokens: maxOutputTokens, + temperature: 0, + budgetTokens: null, + reasoningEffort: reasoningEffort || null, + maxRetries: 0, + }); + ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); + lastError = null; + break; + } catch (err) { + lastError = err; + const again = attempt + 1 < attempts && isTransientFilterError(err); + if (!again) break; + const wait = Math.min(retryBaseMs * 2 ** attempt, 30_000); + if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)); + } + } + if (lastError) { + const message = errorMessage(lastError); + items.forEach((item, index) => { const decision = decisionObject({ path: item.path, decision: 'yes', reason: `error; included by default: ${message}`, raw: null, tokens: 0, - cached: false, + promptTokens: 0, + completionTokens: 0, + cached: index > 0, skippedReason: item.skippedReason, error: message, }); byPath.set(item.path, decision); emit(decision); - } + }); return; } @@ -852,28 +1068,36 @@ export async function filterRelevant(files, task, opts = {}) { let decisionName; /** @type {string} */ let reason; - if (parsed.decision == null) { + /** @type {string | null} */ + let error = null; + let cacheable = false; + if (parsed.decision == null && !parsed.cleaned) { + decisionName = 'yes'; + error = 'empty answer'; + reason = `error; included by default: ${error}`; + } else if (parsed.decision == null) { decisionName = 'yes'; - reason = parsed.cleaned - ? `ambiguous answer; included by default: ${parsed.cleaned}` - : 'ambiguous answer; included by default'; + reason = `ambiguous answer; included by default: ${parsed.cleaned}`; } else { decisionName = parsed.decision; reason = parsed.cleaned; + cacheable = true; + } + if (store && cacheable) { + /** @type {RelevanceCacheRecord} */ + const record = { + decision: decisionName, + reason, + raw, + tokens: totalTokens, + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + skipped_reason: sample.skippedReason, + error: null, + cache_key: key, + }; + await cachePut(store, key, record); } - /** @type {RelevanceCacheRecord} */ - const record = { - decision: decisionName, - reason, - raw, - tokens: totalTokens, - prompt_tokens: promptTokens, - completion_tokens: completionTokens, - skipped_reason: sample.skippedReason, - error: null, - cache_key: key, - }; - if (store) await cachePut(store, key, record); items.forEach((item, index) => { const decision = decisionObject({ path: item.path, @@ -885,7 +1109,7 @@ export async function filterRelevant(files, task, opts = {}) { completionTokens, cached: index > 0, skippedReason: item.skippedReason, - error: null, + error, }); byPath.set(item.path, decision); emit(decision); @@ -908,6 +1132,7 @@ export async function filterRelevant(files, task, opts = {}) { } } const errors = decisions.filter((decision) => decision.error); + const failedOpen = decisions.filter(isFailedOpen).length; let billedTokens = 0; let billedPromptTokens = 0; let billedCompletionTokens = 0; @@ -925,6 +1150,7 @@ export async function filterRelevant(files, task, opts = {}) { billedTokens, billedPromptTokens, billedCompletionTokens, + failedOpen, }; } @@ -938,8 +1164,12 @@ export function formatFilterSummary(result, verbose = false) { const total = result.decisions.length; const kept = Object.keys(result.files).length; const errorCount = result.errors.length; + const failedOpen = Number.isFinite(result.failedOpen) + ? result.failedOpen + : result.decisions.filter(isFailedOpen).length; let line = `Relevance filter (${result.model}): kept ${kept}/${total} files, ${result.billedTokens} tokens`; if (errorCount) line += `, ${errorCount} error${errorCount === 1 ? '' : 's'}`; + if (failedOpen) line += `, ${failedOpen} failed open`; const lines = [line]; for (const decision of result.decisions) { if (decision.error) { @@ -977,6 +1207,7 @@ export async function dumpDecisions(result, path) { billedTokens: result.billedTokens, billedPromptTokens: result.billedPromptTokens, billedCompletionTokens: result.billedCompletionTokens, + failedOpen: result.failedOpen, errors: result.errors, }; await fs.writeFile(destination, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); diff --git a/landing/lib/llm.js b/landing/lib/llm.js index 6e6f80f..9631544 100644 --- a/landing/lib/llm.js +++ b/landing/lib/llm.js @@ -150,6 +150,8 @@ async function consumeSseStream(resp, onToken) { * @param {number} [opts.maxTokens] * @param {number} [opts.temperature] * @param {number} [opts.budgetTokens] Anthropic extended-thinking budget + * @param {string | null} [opts.reasoningEffort] OpenAI-compatible `reasoning_effort` + * (`low`, `medium`, `high`). Omitted when empty. * @param {(text: string) => void} [opts.onToken] when provided, the request is * made with `stream: true` and this is called once per content delta as * tokens arrive. The resolved value keeps the exact non-streaming shape @@ -168,6 +170,7 @@ export async function callLlm({ maxTokens = null, temperature = 1.0, budgetTokens = null, + reasoningEffort = null, onToken = null, fetchImpl, maxRetries = 3, @@ -235,6 +238,7 @@ export async function callLlm({ messages, temperature, ...(maxTokens ? { max_tokens: maxTokens } : {}), + ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), // Streaming: ask for SSE deltas plus a final usage chunk. ...(onToken ? { stream: true, stream_options: { include_usage: true } } : {}), }); diff --git a/src/filter.d.ts b/src/filter.d.ts index 5bfe9aa..bf30a7f 100644 --- a/src/filter.d.ts +++ b/src/filter.d.ts @@ -33,6 +33,12 @@ export function countTokens(text: string): number; * @returns {boolean} */ export function isBinaryContent(content: unknown): boolean; +/** + * @param {string} filePath + * @param {string[]} patterns + * @returns {boolean} + */ +export function isIgnoredPath(filePath: string, patterns?: string[]): boolean; /** * Ask a cheap model which files are relevant to `task`. * @@ -63,6 +69,12 @@ export function isBinaryContent(content: unknown): boolean; * @param {'include' | 'exclude' | 'truncate'} [opts.onOversize] default `truncate` * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) + * @param {string | null} [opts.reasoningEffort] default {@link DEFAULT_FILTER_REASONING_EFFORT}; `null` omits it + * @param {number} [opts.retries] extra attempts after a timeout or HTTP 5xx (default 2) + * @param {number} [opts.retryBaseMs] backoff base in ms (default 200, doubles, capped at 30s) + * @param {boolean | string[] | null} [opts.ignore] lockfile and generated-file patterns. + * `true` or omitted uses {@link DEFAULT_IGNORE_PATTERNS}. `false` or `null` classifies every path. + * An array replaces the default list. * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) * @param {string} [opts.apiKey] `GPTDIFF_LLM_API_KEY` when omitted * @param {string} [opts.baseUrl] `GPTDIFF_LLM_BASE_URL` when omitted @@ -82,6 +94,10 @@ export function filterRelevant(files: Record, task: onOversize?: "include" | "exclude" | "truncate"; onBinary?: "include" | "exclude" | "truncate"; maxOutputTokens?: number; + reasoningEffort?: string | null; + retries?: number; + retryBaseMs?: number; + ignore?: boolean | string[] | null; onDecision?: (decision: RelevanceDecision) => void; apiKey?: string; baseUrl?: string; @@ -103,15 +119,31 @@ export function formatFilterSummary(result: FilterRelevantResult, verbose?: bool */ export function dumpDecisions(result: FilterRelevantResult, path: string): Promise; /** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ -export const DEFAULT_FILTER_MODEL: "gemini-2.0-flash"; +export const DEFAULT_FILTER_MODEL: "xiaomi/mimo-v2.6-flash"; /** In-flight classifications per call when `concurrency` is omitted. */ export const DEFAULT_FILTER_CONCURRENCY: 8; /** UTF-8 size cap. `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_BYTES: 200000; /** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_TOKENS: 12000; -/** Completion cap for the yes/no call. */ -export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS: 256; +/** + * Completion cap for the yes/no call. High enough that a reasoning model's + * scratchpad does not consume the whole answer. `reasoning_effort` is also + * sent as `low`. + */ +export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS: 1024; +/** Sent on the filter call so reasoning models keep the yes/no inside the budget. */ +export const DEFAULT_FILTER_REASONING_EFFORT: "low"; +/** + * Bumped when the system prompt changes, and included in the cache key so an + * old judgment is not reused for a new prompt. + */ +export const FILTER_PROMPT_VERSION: 1; +/** + * Lockfiles and generated paths skipped unless `ignore` is `false` or a custom list. + * Patterns are case-insensitive globs (`**` crosses directories, `*` does not). + */ +export const DEFAULT_IGNORE_PATTERNS: string[]; /** * @typedef {object} RelevanceDecision * @property {string} path @@ -123,8 +155,8 @@ export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS: 256; * @property {number} promptTokens * @property {number} completionTokens * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) - * @property {'binary' | 'oversize' | 'truncated' | null} skippedReason - * set when the file was not sent in full + * @property {'binary' | 'oversize' | 'truncated' | 'ignored' | null} skippedReason + * set when the file was not sent in full. `ignored` is a lockfile or generated path. * @property {string | null} error set when the call threw; the file is still included */ /** @@ -136,6 +168,8 @@ export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS: 256; * @property {number} billedTokens sum of `tokens` on decisions that were not cache hits * @property {number} billedPromptTokens * @property {number} billedCompletionTokens + * @property {number} failedOpen files included because the call failed, the answer + * was empty, or the answer was ambiguous */ /** * Record stored in a cache. Snake_case so a JSON file matches the Python cache. @@ -164,9 +198,11 @@ export class RelevanceCache { * @param {string} task * @param {string} content original file text * @param {string} [variant] + * @param {number} [maxOutputTokens] completion cap, so a larger budget is a different key + * @param {number} [promptVersion] {@link FILTER_PROMPT_VERSION} * @returns {Promise} */ - static makeKey(model: string, task: string, content: string, variant?: string): Promise; + static makeKey(model: string, task: string, content: string, variant?: string, maxOutputTokens?: number, promptVersion?: number): Promise; /** * @param {string} [path] JSON file. Omit for memory only. * Loaded on the first read. Corrupt files are ignored. @@ -228,9 +264,9 @@ export type RelevanceDecision = { */ cached: boolean; /** - * set when the file was not sent in full + * set when the file was not sent in full. `ignored` is a lockfile or generated path. */ - skippedReason: "binary" | "oversize" | "truncated" | null; + skippedReason: "binary" | "oversize" | "truncated" | "ignored" | null; /** * set when the call threw; the file is still included */ @@ -256,6 +292,11 @@ export type FilterRelevantResult = { billedTokens: number; billedPromptTokens: number; billedCompletionTokens: number; + /** + * files included because the call failed, the answer + * was empty, or the answer was ambiguous + */ + failedOpen: number; }; /** * Record stored in a cache. Snake_case so a JSON file matches the Python cache. diff --git a/src/filter.js b/src/filter.js index 84ae6c3..6e1ea55 100644 --- a/src/filter.js +++ b/src/filter.js @@ -18,7 +18,7 @@ import { callLlm as defaultCallLlm, resolveApiKey, resolveBaseUrl } from './llm. import { getEnv } from './env.js'; /** Cheap default for yes/no classification. Override with GPTDIFF_FILTER_MODEL. */ -export const DEFAULT_FILTER_MODEL = 'gemini-2.0-flash'; +export const DEFAULT_FILTER_MODEL = 'xiaomi/mimo-v2.6-flash'; /** In-flight classifications per call when `concurrency` is omitted. */ export const DEFAULT_FILTER_CONCURRENCY = 8; @@ -29,8 +29,57 @@ export const DEFAULT_FILTER_MAX_BYTES = 200_000; /** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_TOKENS = 12_000; -/** Completion cap for the yes/no call. */ -export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 256; +/** + * Completion cap for the yes/no call. High enough that a reasoning model's + * scratchpad does not consume the whole answer. `reasoning_effort` is also + * sent as `low`. + */ +export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 1024; + +/** Sent on the filter call so reasoning models keep the yes/no inside the budget. */ +export const DEFAULT_FILTER_REASONING_EFFORT = 'low'; + +/** + * Bumped when the system prompt changes, and included in the cache key so an + * old judgment is not reused for a new prompt. + */ +export const FILTER_PROMPT_VERSION = 1; + +/** + * Lockfiles and generated paths skipped unless `ignore` is `false` or a custom list. + * Patterns are case-insensitive globs (`**` crosses directories, `*` does not). + */ +export const DEFAULT_IGNORE_PATTERNS = [ + '**/package-lock.json', + '**/yarn.lock', + '**/pnpm-lock.yaml', + '**/bun.lock', + '**/bun.lockb', + '**/npm-shrinkwrap.json', + '**/Cargo.lock', + '**/Gemfile.lock', + '**/poetry.lock', + '**/Pipfile.lock', + '**/composer.lock', + '**/go.sum', + '**/uv.lock', + '**/pdm.lock', + '**/flake.lock', + '**/pubspec.lock', + '**/packages.lock.json', + '**/Package.resolved', + '**/*.min.js', + '**/*.min.css', + '**/*.map', + '**/dist/**', + '**/build/**', + '**/.next/**', + '**/coverage/**', + '**/__pycache__/**', + '**/*.pyc', + '**/*.pyo', + '**/*.generated.*', +]; const POLICIES = ['include', 'exclude', 'truncate']; @@ -58,8 +107,8 @@ const SYSTEM_PROMPT = ( * @property {number} promptTokens * @property {number} completionTokens * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) - * @property {'binary' | 'oversize' | 'truncated' | null} skippedReason - * set when the file was not sent in full + * @property {'binary' | 'oversize' | 'truncated' | 'ignored' | null} skippedReason + * set when the file was not sent in full. `ignored` is a lockfile or generated path. * @property {string | null} error set when the call threw; the file is still included */ @@ -72,6 +121,8 @@ const SYSTEM_PROMPT = ( * @property {number} billedTokens sum of `tokens` on decisions that were not cache hits * @property {number} billedPromptTokens * @property {number} billedCompletionTokens + * @property {number} failedOpen files included because the call failed, the answer + * was empty, or the answer was ambiguous */ /** @@ -114,12 +165,21 @@ export class RelevanceCache { * @param {string} task * @param {string} content original file text * @param {string} [variant] + * @param {number} [maxOutputTokens] completion cap, so a larger budget is a different key + * @param {number} [promptVersion] {@link FILTER_PROMPT_VERSION} * @returns {Promise} */ - static async makeKey(model, task, content, variant = 'full') { + static async makeKey( + model, + task, + content, + variant = 'full', + maxOutputTokens = DEFAULT_FILTER_MAX_OUTPUT_TOKENS, + promptVersion = FILTER_PROMPT_VERSION, + ) { const taskHash = await sha256(task); const contentHash = await sha256(content); - return `${model}\n${taskHash}\n${contentHash}\n${variant}`; + return `${model}\n${taskHash}\n${contentHash}\n${variant}\n${maxOutputTokens}\n${promptVersion}`; } /** @@ -516,7 +576,7 @@ function decisionFromRecord(path, record, cached) { /** * @param {string} path * @param {'yes' | 'no'} decision - * @param {'binary' | 'oversize'} skippedReason + * @param {'binary' | 'oversize' | 'ignored'} skippedReason * @param {string} reason * @returns {RelevanceDecision} */ @@ -577,6 +637,120 @@ function errorMessage(err) { return String(err); } +/** + * Timeouts and HTTP 5xx (including 503). Other failures fail open immediately. + * @param {unknown} err + * @returns {boolean} + */ +function isTransientFilterError(err) { + const status = err && typeof err === 'object' + ? (/** @type {{ status?: number, statusCode?: number }} */ (err).status + ?? /** @type {{ statusCode?: number }} */ (err).statusCode) + : undefined; + if (typeof status === 'number' && status >= 500 && status <= 599) return true; + const code = err && typeof err === 'object' + ? (/** @type {{ code?: string, cause?: { code?: string } }} */ (err).code + || /** @type {{ cause?: { code?: string } }} */ (err).cause?.code) + : undefined; + if (typeof code === 'string' && /^(ETIMEDOUT|ECONNABORTED|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET)$/.test(code)) { + return true; + } + const name = err && typeof err === 'object' ? /** @type {{ name?: string }} */ (err).name : ''; + if (name === 'TimeoutError' || name === 'AbortError') return true; + const message = errorMessage(err); + if (/timeout|timed out/i.test(message)) return true; + if (/\b5\d\d\b/.test(message)) return true; + return false; +} + +/** + * @param {unknown} value + * @returns {number} + */ +function resolveRetries(value) { + if (value === undefined) return 2; + const n = Math.trunc(Number(value)); + if (!Number.isFinite(n) || n < 0) throw new Error('retries must be >= 0'); + return n; +} + +/** + * @param {string} pattern + * @returns {RegExp} + */ +function globToRegExp(pattern) { + let re = ''; + let i = 0; + const src = pattern.replace(/\\/g, '/').toLowerCase(); + while (i < src.length) { + if (src.startsWith('**/', i)) { + re += '(?:.*/)?'; + i += 3; + continue; + } + if (src.startsWith('**', i)) { + re += '.*'; + i += 2; + continue; + } + const ch = src[i]; + if (ch === '*') re += '[^/]*'; + else { + if (/[\\^$+?.()|{}[\]]/.test(ch)) re += '\\'; + re += ch; + } + i += 1; + } + return new RegExp(`^${re}$`); +} + +/** + * @param {string} filePath + * @param {string[]} patterns + * @returns {boolean} + */ +export function isIgnoredPath(filePath, patterns = DEFAULT_IGNORE_PATTERNS) { + const normalized = String(filePath).replace(/\\/g, '/').replace(/^\.\//, '').toLowerCase(); + return patterns.some((pattern) => globToRegExp(pattern).test(normalized)); +} + +/** + * @param {unknown} ignore + * @returns {string[]} + */ +function resolveIgnore(ignore) { + if (ignore === false || ignore === null) return []; + if (ignore === undefined || ignore === true) return DEFAULT_IGNORE_PATTERNS; + if (Array.isArray(ignore)) return ignore.map(String); + throw new TypeError('ignore must be true, false, or an array of patterns'); +} + +/** + * Included without a definitive yes/no: a thrown call, an empty answer, or an + * ambiguous answer. + * @param {RelevanceDecision} decision + * @returns {boolean} + */ +function isFailedOpen(decision) { + if (!decision) return false; + if (decision.error) return true; + return typeof decision.reason === 'string' + && decision.reason.startsWith('ambiguous answer; included by default'); +} + +/** + * A stored judgment is reusable only when it is a definitive yes or no. + * Fallbacks and errors are not, even if an older cache file still holds them. + * @param {RelevanceCacheRecord | undefined} hit + * @returns {boolean} + */ +function reusableHit(hit) { + if (!hit || (hit.decision !== 'yes' && hit.decision !== 'no') || hit.error) return false; + const reason = hit.reason || ''; + if (reason.startsWith('ambiguous answer') || reason.startsWith('error; included by default')) return false; + return true; +} + /** * @param {Array} items * @param {number} limit @@ -648,6 +822,12 @@ async function cachePut(store, key, record) { * @param {'include' | 'exclude' | 'truncate'} [opts.onOversize] default `truncate` * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) + * @param {string | null} [opts.reasoningEffort] default {@link DEFAULT_FILTER_REASONING_EFFORT}; `null` omits it + * @param {number} [opts.retries] extra attempts after a timeout or HTTP 5xx (default 2) + * @param {number} [opts.retryBaseMs] backoff base in ms (default 200, doubles, capped at 30s) + * @param {boolean | string[] | null} [opts.ignore] lockfile and generated-file patterns. + * `true` or omitted uses {@link DEFAULT_IGNORE_PATTERNS}. `false` or `null` classifies every path. + * An array replaces the default list. * @param {(decision: RelevanceDecision) => void} [opts.onDecision] called as each file finishes (completion order) * @param {string} [opts.apiKey] `GPTDIFF_LLM_API_KEY` when omitted * @param {string} [opts.baseUrl] `GPTDIFF_LLM_BASE_URL` when omitted @@ -672,6 +852,12 @@ export async function filterRelevant(files, task, opts = {}) { const maxOutputTokens = opts.maxOutputTokens === undefined ? DEFAULT_FILTER_MAX_OUTPUT_TOKENS : opts.maxOutputTokens; + const reasoningEffort = opts.reasoningEffort === undefined + ? DEFAULT_FILTER_REASONING_EFFORT + : opts.reasoningEffort; + const retries = resolveRetries(opts.retries); + const retryBaseMs = opts.retryBaseMs === undefined ? 200 : Math.max(0, Number(opts.retryBaseMs) || 0); + const ignorePatterns = resolveIgnore(opts.ignore); const model = resolveFilterModel(opts.model); const store = resolveCache(opts.cache); @@ -710,6 +896,18 @@ export async function filterRelevant(files, task, opts = {}) { } for (const [path, content] of entries) { + if (isIgnoredPath(path, ignorePatterns)) { + const made = skipDecision( + path, + 'no', + 'ignored', + 'lockfile or generated file; excluded without a model call', + ); + byPath.set(path, made); + emit(made); + continue; + } + const binary = isBinaryContent(content); const text = coerceText(content); if (binary) { @@ -717,7 +915,7 @@ export async function filterRelevant(files, task, opts = {}) { const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; const body = `[binary file, ${nbytes} bytes, contents omitted]`; - const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview'); + const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview', maxOutputTokens); enqueue(key, { path, body, note, skippedReason: 'truncated' }); } else { const decision = onBinary === 'include' ? 'yes' : 'no'; @@ -741,7 +939,7 @@ export async function filterRelevant(files, task, opts = {}) { + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` ); const variant = `truncated:${rawLimitTag(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS)}:${rawLimitTag(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES)}`; - const key = await RelevanceCache.makeKey(model, task, text, variant); + const key = await RelevanceCache.makeKey(model, task, text, variant, maxOutputTokens); enqueue(key, { path, body, note, skippedReason: 'truncated' }); } else { const decision = onOversize === 'include' ? 'yes' : 'no'; @@ -758,7 +956,7 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - const key = await RelevanceCache.makeKey(model, task, text, 'full'); + const key = await RelevanceCache.makeKey(model, task, text, 'full', maxOutputTokens); enqueue(key, { path, body: text, note: null, skippedReason: null }); } @@ -771,7 +969,7 @@ export async function filterRelevant(files, task, opts = {}) { async function cached(key) { if (!store) return false; const hit = await cacheGet(store, key); - return !!(hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error); + return reusableHit(hit); } let needsCall = false; @@ -796,7 +994,7 @@ export async function filterRelevant(files, task, opts = {}) { const items = groups.get(key); if (store) { const hit = await cacheGet(store, key); - if (hit && (hit.decision === 'yes' || hit.decision === 'no') && !hit.error) { + if (reusableHit(hit)) { for (const item of items) { const decision = decisionFromRecord(item.path, { ...hit, cache_key: key }, true); byPath.set(item.path, decision); @@ -811,39 +1009,57 @@ export async function filterRelevant(files, task, opts = {}) { let promptTokens; let completionTokens; let totalTokens; - try { - const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); - let userContent = userPrompt; - if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; - const response = await callLlm({ - apiKey, - baseUrl, - model, - messages: [ - { role: 'system', content: SYSTEM_PROMPT }, - { role: 'user', content: userContent }, - ], - maxTokens: maxOutputTokens, - temperature: 0, - budgetTokens: null, - }); - ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); - } catch (err) { - const message = errorMessage(err); - for (const item of items) { + /** @type {unknown} */ + let lastError = null; + const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); + let userContent = userPrompt; + if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; + const attempts = retries + 1; + for (let attempt = 0; attempt < attempts; attempt++) { + try { + const response = await callLlm({ + apiKey, + baseUrl, + model, + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: userContent }, + ], + maxTokens: maxOutputTokens, + temperature: 0, + budgetTokens: null, + reasoningEffort: reasoningEffort || null, + maxRetries: 0, + }); + ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); + lastError = null; + break; + } catch (err) { + lastError = err; + const again = attempt + 1 < attempts && isTransientFilterError(err); + if (!again) break; + const wait = Math.min(retryBaseMs * 2 ** attempt, 30_000); + if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)); + } + } + if (lastError) { + const message = errorMessage(lastError); + items.forEach((item, index) => { const decision = decisionObject({ path: item.path, decision: 'yes', reason: `error; included by default: ${message}`, raw: null, tokens: 0, - cached: false, + promptTokens: 0, + completionTokens: 0, + cached: index > 0, skippedReason: item.skippedReason, error: message, }); byPath.set(item.path, decision); emit(decision); - } + }); return; } @@ -852,28 +1068,36 @@ export async function filterRelevant(files, task, opts = {}) { let decisionName; /** @type {string} */ let reason; - if (parsed.decision == null) { + /** @type {string | null} */ + let error = null; + let cacheable = false; + if (parsed.decision == null && !parsed.cleaned) { + decisionName = 'yes'; + error = 'empty answer'; + reason = `error; included by default: ${error}`; + } else if (parsed.decision == null) { decisionName = 'yes'; - reason = parsed.cleaned - ? `ambiguous answer; included by default: ${parsed.cleaned}` - : 'ambiguous answer; included by default'; + reason = `ambiguous answer; included by default: ${parsed.cleaned}`; } else { decisionName = parsed.decision; reason = parsed.cleaned; + cacheable = true; + } + if (store && cacheable) { + /** @type {RelevanceCacheRecord} */ + const record = { + decision: decisionName, + reason, + raw, + tokens: totalTokens, + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + skipped_reason: sample.skippedReason, + error: null, + cache_key: key, + }; + await cachePut(store, key, record); } - /** @type {RelevanceCacheRecord} */ - const record = { - decision: decisionName, - reason, - raw, - tokens: totalTokens, - prompt_tokens: promptTokens, - completion_tokens: completionTokens, - skipped_reason: sample.skippedReason, - error: null, - cache_key: key, - }; - if (store) await cachePut(store, key, record); items.forEach((item, index) => { const decision = decisionObject({ path: item.path, @@ -885,7 +1109,7 @@ export async function filterRelevant(files, task, opts = {}) { completionTokens, cached: index > 0, skippedReason: item.skippedReason, - error: null, + error, }); byPath.set(item.path, decision); emit(decision); @@ -908,6 +1132,7 @@ export async function filterRelevant(files, task, opts = {}) { } } const errors = decisions.filter((decision) => decision.error); + const failedOpen = decisions.filter(isFailedOpen).length; let billedTokens = 0; let billedPromptTokens = 0; let billedCompletionTokens = 0; @@ -925,6 +1150,7 @@ export async function filterRelevant(files, task, opts = {}) { billedTokens, billedPromptTokens, billedCompletionTokens, + failedOpen, }; } @@ -938,8 +1164,12 @@ export function formatFilterSummary(result, verbose = false) { const total = result.decisions.length; const kept = Object.keys(result.files).length; const errorCount = result.errors.length; + const failedOpen = Number.isFinite(result.failedOpen) + ? result.failedOpen + : result.decisions.filter(isFailedOpen).length; let line = `Relevance filter (${result.model}): kept ${kept}/${total} files, ${result.billedTokens} tokens`; if (errorCount) line += `, ${errorCount} error${errorCount === 1 ? '' : 's'}`; + if (failedOpen) line += `, ${failedOpen} failed open`; const lines = [line]; for (const decision of result.decisions) { if (decision.error) { @@ -977,6 +1207,7 @@ export async function dumpDecisions(result, path) { billedTokens: result.billedTokens, billedPromptTokens: result.billedPromptTokens, billedCompletionTokens: result.billedCompletionTokens, + failedOpen: result.failedOpen, errors: result.errors, }; await fs.writeFile(destination, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); diff --git a/src/index.d.ts b/src/index.d.ts index 51d6598..ee46dee 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -2,7 +2,7 @@ export * as oauth from "./oauth.js"; export type RelevanceDecision = import("./filter.js").RelevanceDecision; export type FilterRelevantResult = import("./filter.js").FilterRelevantResult; export { generateDiff, smartapply, buildEnvironment, callLlmForDiff, callLlmForApply, callLlmForApplyWithThink } from "./gptdiff.js"; -export { filterRelevant, RelevanceCache, createMemoryCache, parseRelevanceAnswer, isBinaryContent, countTokens, formatFilterSummary, dumpDecisions, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_MAX_TOKENS, DEFAULT_FILTER_MAX_OUTPUT_TOKENS, DEFAULT_FILTER_CONCURRENCY } from "./filter.js"; +export { filterRelevant, RelevanceCache, createMemoryCache, parseRelevanceAnswer, isBinaryContent, isIgnoredPath, countTokens, formatFilterSummary, dumpDecisions, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_MAX_TOKENS, DEFAULT_FILTER_MAX_OUTPUT_TOKENS, DEFAULT_FILTER_REASONING_EFFORT, DEFAULT_FILTER_CONCURRENCY, FILTER_PROMPT_VERSION, DEFAULT_IGNORE_PATTERNS } from "./filter.js"; export { parseDiffPerFile, applyDiff, applyPatchToFile } from "./applydiff.js"; export { colorCodeDiff, swallowReasoning, stripBadOutput, extractDiffBlocks, stripThinkTags } from "./text.js"; export { callLlm, resolveApiKey, resolveBaseUrl } from "./llm.js"; diff --git a/src/index.js b/src/index.js index f01841c..5fb8766 100644 --- a/src/index.js +++ b/src/index.js @@ -12,6 +12,7 @@ export { createMemoryCache, parseRelevanceAnswer, isBinaryContent, + isIgnoredPath, countTokens, formatFilterSummary, dumpDecisions, @@ -19,7 +20,10 @@ export { DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_MAX_TOKENS, DEFAULT_FILTER_MAX_OUTPUT_TOKENS, + DEFAULT_FILTER_REASONING_EFFORT, DEFAULT_FILTER_CONCURRENCY, + FILTER_PROMPT_VERSION, + DEFAULT_IGNORE_PATTERNS, } from './filter.js'; /** diff --git a/src/llm.d.ts b/src/llm.d.ts index b752ed2..c45d0a2 100644 --- a/src/llm.d.ts +++ b/src/llm.d.ts @@ -9,6 +9,8 @@ * @param {number} [opts.maxTokens] * @param {number} [opts.temperature] * @param {number} [opts.budgetTokens] Anthropic extended-thinking budget + * @param {string | null} [opts.reasoningEffort] OpenAI-compatible `reasoning_effort` + * (`low`, `medium`, `high`). Omitted when empty. * @param {(text: string) => void} [opts.onToken] when provided, the request is * made with `stream: true` and this is called once per content delta as * tokens arrive. The resolved value keeps the exact non-streaming shape @@ -19,7 +21,7 @@ * @param {number} [opts.retryBaseMs] base backoff in ms (default 1500, exponential, capped 30s) * @returns {Promise<{choices: Array<{message: {content: string}}>, usage: object}>} */ -export function callLlm({ apiKey, baseUrl, model, messages, maxTokens, temperature, budgetTokens, onToken, fetchImpl, maxRetries, retryBaseMs, }: { +export function callLlm({ apiKey, baseUrl, model, messages, maxTokens, temperature, budgetTokens, reasoningEffort, onToken, fetchImpl, maxRetries, retryBaseMs, }: { apiKey?: string; baseUrl?: string; model: string; @@ -27,6 +29,7 @@ export function callLlm({ apiKey, baseUrl, model, messages, maxTokens, temperatu maxTokens?: number; temperature?: number; budgetTokens?: number; + reasoningEffort?: string | null; onToken?: (text: string) => void; fetchImpl?: typeof fetch; maxRetries?: number; diff --git a/src/llm.js b/src/llm.js index 6e6f80f..9631544 100644 --- a/src/llm.js +++ b/src/llm.js @@ -150,6 +150,8 @@ async function consumeSseStream(resp, onToken) { * @param {number} [opts.maxTokens] * @param {number} [opts.temperature] * @param {number} [opts.budgetTokens] Anthropic extended-thinking budget + * @param {string | null} [opts.reasoningEffort] OpenAI-compatible `reasoning_effort` + * (`low`, `medium`, `high`). Omitted when empty. * @param {(text: string) => void} [opts.onToken] when provided, the request is * made with `stream: true` and this is called once per content delta as * tokens arrive. The resolved value keeps the exact non-streaming shape @@ -168,6 +170,7 @@ export async function callLlm({ maxTokens = null, temperature = 1.0, budgetTokens = null, + reasoningEffort = null, onToken = null, fetchImpl, maxRetries = 3, @@ -235,6 +238,7 @@ export async function callLlm({ messages, temperature, ...(maxTokens ? { max_tokens: maxTokens } : {}), + ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), // Streaming: ask for SSE deltas plus a final usage chunk. ...(onToken ? { stream: true, stream_options: { include_usage: true } } : {}), }); diff --git a/test/filter_relevant.test.js b/test/filter_relevant.test.js index 6a5279b..5fc9c50 100644 --- a/test/filter_relevant.test.js +++ b/test/filter_relevant.test.js @@ -17,7 +17,12 @@ import { DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_MAX_TOKENS, DEFAULT_FILTER_MAX_OUTPUT_TOKENS, + DEFAULT_FILTER_REASONING_EFFORT, DEFAULT_FILTER_CONCURRENCY, + FILTER_PROMPT_VERSION, + DEFAULT_IGNORE_PATTERNS, + isIgnoredPath, + callLlm, } from '../src/index.js'; import { setEnv } from '../src/env.js'; @@ -143,6 +148,8 @@ test('filterRelevant keeps only yes files and reports camelCase decisions', asyn assert.equal(call.baseUrl, 'https://example.test/v1'); assert.equal(call.temperature, 0); assert.equal(call.maxTokens, DEFAULT_FILTER_MAX_OUTPUT_TOKENS); + assert.equal(call.reasoningEffort, 'low'); + assert.equal(call.maxRetries, 0); assert.match(call.messages[1].content, /Task:\nUpdate keep\(\)/); assert.match(call.messages[1].content, /Is this file relevant to the task\? yes\/no/); } @@ -170,9 +177,11 @@ test('ambiguous answers and thrown errors include the file', async () => { assert.match(by['broken.py'].reason, /connection reset/); assert.equal(by['broken.py'].tokens, 0); assert.deepEqual(result.errors, [by['broken.py']]); + assert.equal(result.failedOpen, 2); const summary = formatFilterSummary(result); assert.match(summary, /connection reset/); assert.match(summary, /included by default/); + assert.match(summary, /2 failed open/); }); test('errors are not cached', async () => { @@ -213,8 +222,10 @@ test('memory and disk cache round-trip', async () => { assert.equal(stored.version, 1); assert.equal(Object.keys(stored.entries).length, 1); const [storedKey, storedValue] = Object.entries(stored.entries)[0]; - assert.equal(storedKey.split('\n').length, 4); + assert.equal(storedKey.split('\n').length, 6); assert.equal(storedKey.split('\n')[3], 'full'); + assert.equal(storedKey.split('\n')[4], String(DEFAULT_FILTER_MAX_OUTPUT_TOKENS)); + assert.equal(storedKey.split('\n')[5], String(FILTER_PROMPT_VERSION)); assert.equal(storedValue.decision, 'yes'); assert.equal(storedValue.prompt_tokens, 5); assert.equal(storedValue.skipped_reason, null); @@ -548,9 +559,13 @@ test('invalid arguments throw', async () => { () => filterRelevant({ 'a.py': 1 }, 'task', { cache: false, apiKey: 'k', callLlm: llm(['yes']).fn }), TypeError, ); + await assert.rejects( + () => filterRelevant({ 'a.py': 'x' }, 'task', { cache: false, apiKey: 'k', ignore: 'nope' }), + TypeError, + ); }); -test('model defaults to GPTDIFF_FILTER_MODEL, else gemini-2.0-flash', async () => withEnv(async () => { +test('model defaults to GPTDIFF_FILTER_MODEL, else xiaomi/mimo-v2.6-flash', async () => withEnv(async () => { const prevFilter = process.env.GPTDIFF_FILTER_MODEL; const prevBase = process.env.GPTDIFF_LLM_BASE_URL; delete process.env.GPTDIFF_FILTER_MODEL; @@ -574,13 +589,22 @@ test('model defaults to GPTDIFF_FILTER_MODEL, else gemini-2.0-flash', async () = callLlm: mock.fn, }); assert.equal(seen[0].model, DEFAULT_FILTER_MODEL); - assert.equal(fromEnv.model, 'gemini-2.0-flash'); + assert.equal(fromEnv.model, 'xiaomi/mimo-v2.6-flash'); assert.equal(seen[0].baseUrl, 'https://example.test/v1/'); assert.equal(seen[0].apiKey, 'from-arg'); - assert.ok(seen[0].messages[1].content.startsWith('You are a relevance filter')); + assert.equal(seen[0].reasoningEffort, 'low'); + assert.ok(seen[0].messages[0].content.startsWith('You are a relevance filter')); + assert.ok(seen[0].messages[1].content.startsWith('Task:\n')); assert.equal(named.model, 'env-filter-model'); assert.equal(seen[1].model, 'env-filter-model'); - assert.equal(seen[1].messages[1].content.startsWith('You are a relevance filter'), false); + const gemini = llm(() => response('yes')); + await filterRelevant({ 'a.py': 'x = 1\n' }, 'task', { + model: 'gemini-2.0-flash', + cache: false, + apiKey: 'from-arg', + callLlm: gemini.fn, + }); + assert.ok(gemini.calls[0].messages[1].content.startsWith('You are a relevance filter')); } finally { setEnv('GPTDIFF_FILTER_MODEL', undefined); setEnv('GPTDIFF_LLM_BASE_URL', undefined); @@ -589,10 +613,12 @@ test('model defaults to GPTDIFF_FILTER_MODEL, else gemini-2.0-flash', async () = if (prevBase === undefined) delete process.env.GPTDIFF_LLM_BASE_URL; else process.env.GPTDIFF_LLM_BASE_URL = prevBase; } - assert.equal(DEFAULT_FILTER_MODEL, 'gemini-2.0-flash'); + assert.equal(DEFAULT_FILTER_MODEL, 'xiaomi/mimo-v2.6-flash'); assert.equal(DEFAULT_FILTER_MAX_BYTES, 200_000); assert.equal(DEFAULT_FILTER_MAX_TOKENS, 12_000); - assert.equal(DEFAULT_FILTER_MAX_OUTPUT_TOKENS, 256); + assert.equal(DEFAULT_FILTER_MAX_OUTPUT_TOKENS, 1024); + assert.equal(DEFAULT_FILTER_REASONING_EFFORT, 'low'); + assert.equal(FILTER_PROMPT_VERSION, 1); assert.equal(DEFAULT_FILTER_CONCURRENCY, 8); assert.equal(isBinaryContent('hello\0world'), true); assert.equal(isBinaryContent('plain text'), false); @@ -658,3 +684,229 @@ test('buildEnvironment includes only yes files when decisions are passed', () => assert.match(includedByError, /File: main\.py/); assert.doesNotMatch(includedByError, /greet/); }); + +function statusError(status) { + const err = new Error(`LLM request failed (${status}) from example: unavailable`); + err.status = status; + return err; +} + +test('an empty answer is an error, fails open, and is not cached', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + if (n === 1) return response('no, skip this file'); + return response('no'); + }); + const cache = new RelevanceCache(); + const files = { 'a.py': 'v = 1\n' }; + const first = await filterRelevant(files, 'task', { + model: 'm', cache, apiKey: 'k', callLlm: mock.fn, retryBaseMs: 0, + }); + assert.equal(first.decisions[0].decision, 'yes'); + assert.equal(first.decisions[0].error, 'empty answer'); + assert.equal(first.failedOpen, 1); + assert.equal('a.py' in first.files, true); + assert.match(formatFilterSummary(first), /1 failed open/); + const stored = await cache.get(await RelevanceCache.makeKey('m', 'task', 'v = 1\n', 'full')); + assert.equal(stored, undefined); + + const second = await filterRelevant(files, 'task', { + model: 'm', cache, apiKey: 'k', callLlm: mock.fn, retryBaseMs: 0, + }); + assert.equal(n, 2); + assert.equal(second.decisions[0].decision, 'no'); + assert.equal(second.decisions[0].error, null); + assert.equal(second.decisions[0].cached, false); + assert.equal(second.failedOpen, 0); +}); + +test('ambiguous answers fail open and are not cached', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return n === 1 ? response('maybe later') : response('no — unrelated'); + }); + const cache = createMemoryCache(); + const opts = { model: 'm', cache, apiKey: 'k', callLlm: mock.fn, retryBaseMs: 0 }; + const first = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', opts); + const second = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', opts); + assert.equal(n, 2); + assert.equal(first.decisions[0].decision, 'yes'); + assert.equal(first.decisions[0].error, null); + assert.match(first.decisions[0].reason, /ambiguous/); + assert.equal(first.failedOpen, 1); + assert.equal(second.decisions[0].decision, 'no'); + assert.equal(second.decisions[0].cached, false); +}); + +test('a stored fallback is not reused', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return response('no'); + }); + const cache = new RelevanceCache(); + const body = 'v = 1\n'; + const key = await RelevanceCache.makeKey('m', 'task', body, 'full'); + await cache.put(key, { + decision: 'yes', + reason: 'ambiguous answer; included by default: maybe', + error: null, + tokens: 4, + }); + const result = await filterRelevant({ 'a.py': body }, 'task', { + model: 'm', cache, apiKey: 'k', callLlm: mock.fn, + }); + assert.equal(n, 1); + assert.equal(result.decisions[0].decision, 'no'); + assert.equal(result.decisions[0].cached, false); +}); + +test('the output-token limit is part of the cache key', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return response('yes'); + }); + const cache = new RelevanceCache(); + const files = { 'a.py': 'v = 1\n' }; + const opts = { model: 'm', cache, apiKey: 'k', callLlm: mock.fn }; + await filterRelevant(files, 'task', { ...opts, maxOutputTokens: 128 }); + await filterRelevant(files, 'task', { ...opts, maxOutputTokens: 2048 }); + assert.equal(n, 2); + await filterRelevant(files, 'task', { ...opts, maxOutputTokens: 128 }); + assert.equal(n, 2); +}); + +test('transient 503 and timeouts are retried with backoff', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + if (n < 3) return statusError(503); + return response('no'); + }); + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { + ...baseOpts, callLlm: mock.fn, retryBaseMs: 0, + }); + assert.equal(n, 3); + assert.equal(result.decisions[0].decision, 'no'); + assert.equal(result.decisions[0].error, null); + assert.equal(result.failedOpen, 0); + + let timeouts = 0; + const timed = llm(() => { + timeouts += 1; + if (timeouts === 1) { + const err = new Error('request timed out'); + err.name = 'TimeoutError'; + return err; + } + return response('yes'); + }); + const afterTimeout = await filterRelevant({ 'a.py': 'v = 2\n' }, 'task', { + ...baseOpts, callLlm: timed.fn, retryBaseMs: 0, + }); + assert.equal(timeouts, 2); + assert.equal(afterTimeout.decisions[0].decision, 'yes'); + assert.equal(afterTimeout.decisions[0].error, null); +}); + +test('a non-transient error is not retried and still fails open', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return new Error('connection reset'); + }); + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { + ...baseOpts, callLlm: mock.fn, retries: 4, retryBaseMs: 0, + }); + assert.equal(n, 1); + assert.equal(result.decisions[0].decision, 'yes'); + assert.match(result.decisions[0].error, /connection reset/); + assert.equal(result.failedOpen, 1); + assert.equal(result.files['a.py'], 'v = 1\n'); +}); + +test('lockfiles and generated files are skipped unless ignore is configured', async () => { + assert.equal(isIgnoredPath('package-lock.json', DEFAULT_IGNORE_PATTERNS), true); + assert.equal(isIgnoredPath('app/yarn.lock', DEFAULT_IGNORE_PATTERNS), true); + assert.equal(isIgnoredPath('src/app.js', DEFAULT_IGNORE_PATTERNS), false); + assert.equal(isIgnoredPath('dist/app.js'), true); + assert.equal(isIgnoredPath('web/dist/main.min.js'), true); + assert.equal(isIgnoredPath('src/app.min.js'), true); + assert.equal(isIgnoredPath('keep.py'), false); + + const seen = []; + const mock = llm((req) => { + seen.push(...pathsOf(req)); + return response('yes'); + }); + const files = { + 'src/app.js': 'export const n = 1;\n', + 'package-lock.json': '{ "lock": true }\n', + 'dist/app.js': 'bundled\n', + 'src/app.min.js': 'min\n', + }; + const skipped = await filterRelevant(files, 'rename n', { ...baseOpts, callLlm: mock.fn }); + assert.deepEqual(seen, ['src/app.js']); + assert.deepEqual(Object.keys(skipped.files), ['src/app.js']); + const by = Object.fromEntries(skipped.decisions.map((d) => [d.path, d])); + assert.equal(by['package-lock.json'].decision, 'no'); + assert.equal(by['package-lock.json'].skippedReason, 'ignored'); + assert.equal(by['dist/app.js'].skippedReason, 'ignored'); + assert.equal(by['src/app.min.js'].skippedReason, 'ignored'); + assert.equal(by['src/app.js'].skippedReason, null); + + seen.length = 0; + const sent = await filterRelevant( + { 'package-lock.json': '{}\n', 'secret.txt': 'nope\n', 'src/app.js': 'ok\n' }, + 'task', + { ...baseOpts, ignore: ['**/secret.txt'], callLlm: mock.fn }, + ); + assert.ok(seen.includes('package-lock.json')); + assert.ok(seen.includes('src/app.js')); + assert.equal(seen.includes('secret.txt'), false); + assert.equal(sent.decisions.find((d) => d.path === 'secret.txt').skippedReason, 'ignored'); + + seen.length = 0; + await filterRelevant({ 'package-lock.json': '{}\n' }, 'task', { + ...baseOpts, ignore: false, callLlm: mock.fn, + }); + assert.deepEqual(seen, ['package-lock.json']); +}); + +test('callLlm sends reasoning_effort only when the filter asks for it', async () => { + /** @type {Record[]} */ + const bodies = []; + const fetchImpl = async (_url, init) => { + bodies.push(JSON.parse(init.body)); + return { + ok: true, + json: async () => ({ choices: [{ message: { content: 'yes' } }], usage: { total_tokens: 1 } }), + text: async () => '', + headers: { get: () => null }, + }; + }; + await callLlm({ + apiKey: 'k', + baseUrl: 'https://example.test/v1', + model: 'm', + messages: [{ role: 'user', content: 'hi' }], + maxTokens: 1024, + temperature: 0, + reasoningEffort: 'low', + fetchImpl, + }); + await callLlm({ + apiKey: 'k', + baseUrl: 'https://example.test/v1', + model: 'm', + messages: [{ role: 'user', content: 'hi' }], + temperature: 0, + fetchImpl, + }); + assert.equal(bodies[0].reasoning_effort, 'low'); + assert.equal(bodies[0].max_tokens, 1024); + assert.equal('reasoning_effort' in bodies[1], false); +}); From 8fe3b58946f3a1a609faabbccc102756188219bc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 21:35:10 +0000 Subject: [PATCH 4/6] Hash relevance cache keys without a global crypto Node 18's test runner does not expose crypto, so SHA-256 via the bare global threw and every filter test failed open. Use globalThis.crypto when it is present, and node:crypto's webcrypto export otherwise. Co-authored-by: ntc-ai --- CHANGELOG.md | 2 +- landing/lib/filter.js | 21 ++++++++++++++++++++- src/filter.js | 21 ++++++++++++++++++++- test/filter_relevant.test.js | 27 +++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47bf40..067e805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### API -- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `xiaomi/mimo-v2.6-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file. The completion cap is 1024 and the call sends `reasoning_effort: low`, so a reasoning model does not spend the whole answer budget. An empty answer is an error and fails open. Timeouts and HTTP 5xx are retried with backoff. Lockfiles and generated files are skipped unless `ignore` is turned off or replaced. The cache key includes the truncation tag, the output-token limit, and the prompt version. Fallback and error decisions are not cached. `failedOpen` counts files included without a definitive answer. Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`, `ignored`). `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. +- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `xiaomi/mimo-v2.6-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file. The completion cap is 1024 and the call sends `reasoning_effort: low`, so a reasoning model does not spend the whole answer budget. An empty answer is an error and fails open. Timeouts and HTTP 5xx are retried with backoff. Lockfiles and generated files are skipped unless `ignore` is turned off or replaced. The cache key includes the truncation tag, the output-token limit, and the prompt version. Cache-key hashing uses Web Crypto, and on Node 18 (where `crypto` is not a global) the `webcrypto` export of `node:crypto`. Fallback and error decisions are not cached. `failedOpen` counts files included without a definitive answer. Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`, `ignored`). `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. - Renamed the package from `gptdiff-js` to `patchling` (npm name `patchling`; an intermediate publish used `gptdiff`). - `generateDiff` and `smartapply` stream tokens through `opts.onToken`. `generateDiff` reports token usage through `opts.onUsage`. - Ship JSDoc-generated `.d.ts` files, with `types` conditions on the package exports. diff --git a/landing/lib/filter.js b/landing/lib/filter.js index 6e1ea55..2d0cad6 100644 --- a/landing/lib/filter.js +++ b/landing/lib/filter.js @@ -277,13 +277,32 @@ export function createMemoryCache() { return new RelevanceCache(); } +/** + * Browsers and Node 19+ expose Web Crypto as `globalThis.crypto`. Node 18 + * keeps the same API on `node:crypto` as `webcrypto`. That module is imported + * only when the global is missing, so this file still loads in the browser. + * @returns {Promise} + */ +async function getWebCrypto() { + const exposed = typeof globalThis !== 'undefined' ? globalThis.crypto : undefined; + if (exposed && exposed.subtle) return exposed; + try { + const mod = await import('node:crypto'); + if (mod.webcrypto && mod.webcrypto.subtle) return mod.webcrypto; + } catch { + // A browser without `globalThis.crypto`, or a host that cannot load node:crypto. + } + throw new Error('Web Crypto (crypto.subtle) is required to hash relevance cache keys.'); +} + /** * @param {string} text * @returns {Promise} hex SHA-256 */ async function sha256(text) { const data = new TextEncoder().encode(text); - const digest = await crypto.subtle.digest('SHA-256', data); + const webcrypto = await getWebCrypto(); + const digest = await webcrypto.subtle.digest('SHA-256', data); let hex = ''; for (const byte of new Uint8Array(digest)) hex += byte.toString(16).padStart(2, '0'); return hex; diff --git a/src/filter.js b/src/filter.js index 6e1ea55..2d0cad6 100644 --- a/src/filter.js +++ b/src/filter.js @@ -277,13 +277,32 @@ export function createMemoryCache() { return new RelevanceCache(); } +/** + * Browsers and Node 19+ expose Web Crypto as `globalThis.crypto`. Node 18 + * keeps the same API on `node:crypto` as `webcrypto`. That module is imported + * only when the global is missing, so this file still loads in the browser. + * @returns {Promise} + */ +async function getWebCrypto() { + const exposed = typeof globalThis !== 'undefined' ? globalThis.crypto : undefined; + if (exposed && exposed.subtle) return exposed; + try { + const mod = await import('node:crypto'); + if (mod.webcrypto && mod.webcrypto.subtle) return mod.webcrypto; + } catch { + // A browser without `globalThis.crypto`, or a host that cannot load node:crypto. + } + throw new Error('Web Crypto (crypto.subtle) is required to hash relevance cache keys.'); +} + /** * @param {string} text * @returns {Promise} hex SHA-256 */ async function sha256(text) { const data = new TextEncoder().encode(text); - const digest = await crypto.subtle.digest('SHA-256', data); + const webcrypto = await getWebCrypto(); + const digest = await webcrypto.subtle.digest('SHA-256', data); let hex = ''; for (const byte of new Uint8Array(digest)) hex += byte.toString(16).padStart(2, '0'); return hex; diff --git a/test/filter_relevant.test.js b/test/filter_relevant.test.js index 5fc9c50..060a85b 100644 --- a/test/filter_relevant.test.js +++ b/test/filter_relevant.test.js @@ -1,5 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -763,6 +764,32 @@ test('a stored fallback is not reused', async () => { assert.equal(result.decisions[0].cached, false); }); +test('cache keys hash when global crypto is missing (Node 18)', async () => { + const descriptor = Object.getOwnPropertyDescriptor(globalThis, 'crypto'); + Object.defineProperty(globalThis, 'crypto', { + configurable: true, + writable: true, + value: undefined, + }); + try { + const body = 'v = 1\n'; + const key = await RelevanceCache.makeKey('m', 'task', body); + const parts = key.split('\n'); + assert.equal(parts[0], 'm'); + assert.equal(parts[1], createHash('sha256').update('task').digest('hex')); + assert.equal(parts[2], createHash('sha256').update(body).digest('hex')); + const mock = llm(() => response('yes')); + const result = await filterRelevant({ 'a.py': body }, 'task', { + model: 'm', cache: false, apiKey: 'k', callLlm: mock.fn, + }); + assert.equal(result.decisions[0].decision, 'yes'); + assert.equal(result.decisions[0].error, null); + } finally { + if (descriptor) Object.defineProperty(globalThis, 'crypto', descriptor); + else delete globalThis.crypto; + } +}); + test('the output-token limit is part of the cache key', async () => { let n = 0; const mock = llm(() => { From 5311c068a0334068bad2fb56172b8edfcdfb665a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 22:05:23 +0000 Subject: [PATCH 5/6] Classify each path on its own and parse a leading yes or no Prompt version 3. The cache key hashes the path, the body that was sent, and the reasoning effort, and a malformed answer fails open without being stored. Truncation stays on a code point, and a short UTF-8 tail is not binary. Flask recall 1.0 and 0.8x token figures are unverified. Co-authored-by: ntc-ai --- CHANGELOG.md | 2 +- README.md | 20 +- landing/index.html | 2 +- landing/lib/filter.js | 524 +++++++++++++++++++--------- src/filter.d.ts | 78 +++-- src/filter.js | 524 +++++++++++++++++++--------- src/index.d.ts | 2 +- src/index.js | 1 + test/filter_relevant.test.js | 241 +++++++++++-- test/fixtures/parity/relevance.json | 149 ++++++++ test/relevance_parity.test.js | 161 +++++++++ 11 files changed, 1286 insertions(+), 418 deletions(-) create mode 100644 test/fixtures/parity/relevance.json create mode 100644 test/relevance_parity.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 067e805..32fe17d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### API -- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `xiaomi/mimo-v2.6-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file. The completion cap is 1024 and the call sends `reasoning_effort: low`, so a reasoning model does not spend the whole answer budget. An empty answer is an error and fails open. Timeouts and HTTP 5xx are retried with backoff. Lockfiles and generated files are skipped unless `ignore` is turned off or replaced. The cache key includes the truncation tag, the output-token limit, and the prompt version. Cache-key hashing uses Web Crypto, and on Node 18 (where `crypto` is not a global) the `webcrypto` export of `node:crypto`. Fallback and error decisions are not cached. `failedOpen` counts files included without a definitive answer. Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`, `ignored`). `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. +- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `xiaomi/mimo-v2.6-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000, characters/4) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file, and the cut stays on a code-point boundary. The completion cap is 1024 and the call sends `reasoning_effort: low`. A 4xx that names that parameter is retried once without it. An empty answer, and any answer that is not a leading standalone yes/no, fails open and is not cached. Timeouts, HTTP 5xx, and billing-reservation errors are retried with backoff. Lockfiles and generated files are skipped unless `ignore` is turned off or replaced (case-insensitive; the canonical list is `DEFAULT_IGNORE_PATTERNS`). Prompt version is 3. The cache key hashes prompt version, model, output-token limit, reasoning effort, task, path, and the body that was sent. Each path is classified on its own. A cache read or write error is logged and does not abort the run. Cache-key hashing uses Web Crypto, and on Node 18 (where `crypto` is not a global) the `webcrypto` export of `node:crypto`. `failedOpen` on each decision, and the `failedOpen` count on the result, include malformed answers. Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`, `ignored`). `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. Reported pallets/flask figures (recall 1.0, about 0.8× the tokens of a full-context read) are UNVERIFIED. - Renamed the package from `gptdiff-js` to `patchling` (npm name `patchling`; an intermediate publish used `gptdiff`). - `generateDiff` and `smartapply` stream tokens through `opts.onToken`. `generateDiff` reports token usage through `opts.onUsage`. - Ship JSDoc-generated `.d.ts` files, with `types` conditions on the package exports. diff --git a/README.md b/README.md index 51f3056..5738b49 100644 --- a/README.md +++ b/README.md @@ -124,24 +124,26 @@ const next = await smartapply(diff, files); - `opts.model`, `opts.apiKey`, `opts.baseUrl` — same key and base URL resolution as `generateDiff` (`GPTDIFF_LLM_API_KEY`, `GPTDIFF_LLM_BASE_URL`). The model falls back to `GPTDIFF_FILTER_MODEL`, then `xiaomi/mimo-v2.6-flash`. - `opts.concurrency` — at most this many classifications in flight (default 8). -- `opts.cache` — a `RelevanceCache`, a `{ get, set }` object, or a filesystem path (Node writes `{ version: 1, entries }` atomically; a corrupt file is ignored). Omit it, or pass `true`, for a fresh in-memory cache used only by this call. `null` and `false` turn caching off. `createMemoryCache()` builds another memory cache you can share. The key is the model, SHA-256 of the task, SHA-256 of the original contents, a variant tag (`full`, `binary-preview`, or `truncated::`), the output-token limit, and the prompt version, separated by newlines. Fallback and error decisions are not stored. +- `opts.cache` — a `RelevanceCache`, a `{ get, set }` object, or a filesystem path (Node writes `{ version: 1, entries }` atomically; a corrupt file is ignored). Omit it, or pass `true`, for a fresh in-memory cache used only by this call. `null` and `false` turn caching off. `createMemoryCache()` builds another memory cache you can share. The key is the SHA-256 of the length-prefixed classification inputs: prompt version (`3`), model, output-token limit, reasoning effort, task, file path, and the body that was sent (after truncation). Each path is its own decision, including paths with identical contents. A cache read or write error is logged and does not abort the run. Fallback and error decisions are not stored. - `opts.maxBytes` — UTF-8 size cap (default 200000). `opts.maxTokens` — input-token cap (default 12000; estimated as characters / 4). `0` or `null` disables either limit. - `opts.onOversize` and `opts.onBinary` — `include` (keep, no model call), `exclude` (drop, no model call), or `truncate` (ask about a preview). Defaults are `truncate` and `exclude`. A `yes` on a truncated preview still forwards the original text in `result.files`. -- `opts.ignore` — lockfiles and generated paths (`package-lock.json`, `dist/`, `*.min.js`, and the rest of `DEFAULT_IGNORE_PATTERNS`) are excluded with `skippedReason: "ignored"`. `false` classifies them. An array replaces the default patterns. -- `opts.maxOutputTokens` — completion cap (default 1024). The call also sends `reasoning_effort: "low"` so a reasoning model does not spend the whole budget before the yes/no. An empty answer is an error. -- `opts.retries` — extra attempts after a timeout or HTTP 5xx, including 503 (default 2), with exponential backoff. +- `opts.ignore` — lockfiles and generated paths are excluded with `skippedReason: "ignored"`. Matching is case-insensitive. `**` crosses directories and `*` does not. `false` classifies every path. An array replaces the default list. The canonical list is `DEFAULT_IGNORE_PATTERNS`: `**/package-lock.json`, `**/yarn.lock`, `**/pnpm-lock.yaml`, `**/bun.lock`, `**/bun.lockb`, `**/npm-shrinkwrap.json`, `**/Cargo.lock`, `**/Gemfile.lock`, `**/poetry.lock`, `**/Pipfile.lock`, `**/composer.lock`, `**/go.sum`, `**/uv.lock`, `**/pdm.lock`, `**/flake.lock`, `**/pubspec.lock`, `**/packages.lock.json`, `**/Package.resolved`, `**/*.min.js`, `**/*.min.mjs`, `**/*.min.css`, `**/*.map`, `**/dist/**`, `**/build/**`, `**/.next/**`, `**/coverage/**`, `**/__pycache__/**`, `**/*.pyc`, `**/*.pyo`, `**/*.generated.*`. The same list is in `test/fixtures/parity/relevance.json`. +- `opts.maxOutputTokens` — completion cap (default 1024). The call also sends `reasoning_effort: "low"` so a reasoning model does not spend the whole budget before the yes/no. A 4xx that names that parameter is retried once without it. An empty answer is an error. +- `opts.retries` — extra attempts after a timeout, an HTTP 5xx (including 503), or a billing-reservation error (default 2), with exponential backoff. - `opts.onDecision` — called as each file finishes, in completion order. `result.decisions` stays in input order. - `opts.callLlm` — inject a mock client. Each decision is `{ path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error }`. -- `decision` is `"yes"` or `"no"`. Ambiguous text (after `` / `` blocks are stripped), an empty answer, and a thrown call become `"yes"`, so a bad answer cannot drop a file. `error` is set when the call throws or the answer is empty. Those fallbacks are not cached. `result.failedOpen` counts them, and `formatFilterSummary` includes that count. +- `decision` is `"yes"` or `"no"`. After `` / `` blocks are stripped, the decision is a leading standalone `yes` or `no` (quotes, markdown bold, backticks, and an `Answer:` or `Decision:` prefix are wrappers). Any other wording, including a negated "no" later in the sentence, is malformed: the file is included, `failedOpen` is true, and the judgment is not cached. An empty answer and a thrown call do the same. `error` is set when the call throws or the answer is empty. `result.failedOpen` counts those files, and `formatFilterSummary` includes that count. - `skippedReason` is `"binary"`, `"oversize"`, `"truncated"`, `"ignored"`, or `null`. -- `tokens`, `promptTokens`, and `completionTokens` are `0` when no model call was made. `reason` is a string. -- `result.billedTokens` sums token counts for decisions that were not cache hits. Identical file bodies share one request; the later paths are `cached: true`. +- `tokens`, `promptTokens`, and `completionTokens` are `0` when no model call was made. `reason` is a string. `failedOpen` is true only when the file was included without a definitive yes/no. +- `result.billedTokens` sums token counts for decisions that were not cache hits. Identical contents at different paths are separate calls. - `buildEnvironment(files, { decisions })` keeps only `"yes"` paths. Passing the decisions array itself does the same. With no second argument, every file is included. `buildEnvironment(result.files)` is the same set. -Python (`filter_relevant` in [patchling-py](https://github.com/255BITS/patchling-py)) returns the same result. Its fields are snake_case: `max_bytes`, `max_tokens`, `on_oversize`, `on_binary`, `skipped_reason`, `prompt_tokens`, `completion_tokens`. +Python (`filter_relevant` in [patchling-py](https://github.com/255BITS/patchling-py)) returns the same result. Its fields are snake_case: `max_bytes`, `max_tokens`, `on_oversize`, `on_binary`, `skipped_reason`, `prompt_tokens`, `completion_tokens`, `failed_open`. Prompt version 3, the prompt text, the cache-key encoding, the ignore list, and the characters/4 token estimate are the shared contract in `test/fixtures/parity/relevance.json`. + +A pallets/flask labeling pass has been described as recall 1.0 for `xiaomi/mimo-v2.6-flash`, with the filter using about 0.8× the tokens of a full-context read. Those figures are **UNVERIFIED**. ### `smartapply(diffText, files, opts?) → Promise<{ path: content }>` Applies a diff to an in-memory file map with per-file, LLM-assisted conflict resolution (runs files concurrently). Handles creation, modification, and deletion; `…` and reasoning preambles are stripped automatically. Returns a **new** map; deleted files are omitted. @@ -154,7 +156,7 @@ Applies a diff to an in-memory file map with per-file, LLM-assisted conflict res Deterministic, no-LLM patch application (the strict "basic" applier). `changed` is `true` iff something was created, modified, or deleted; `files` is the updated map (equal to the input on failure — no partial writes). ### Other exports -`parseDiffPerFile`, `buildEnvironment`, `filterRelevant`, `RelevanceCache`, `createMemoryCache`, `parseRelevanceAnswer`, `isBinaryContent`, `isIgnoredPath`, `countTokens`, `formatFilterSummary`, `dumpDecisions`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. +`parseDiffPerFile`, `buildEnvironment`, `filterRelevant`, `RelevanceCache`, `createMemoryCache`, `parseRelevanceAnswer`, `isBinaryContent`, `isIgnoredPath`, `countTokens`, `formatFilterSummary`, `dumpDecisions`, `PROMPT_VERSION`, `colorCodeDiff`, `swallowReasoning`, `stripBadOutput`, `extractDiffBlocks`, `callLlm`, `resolveApiKey`, `resolveBaseUrl`, `getEnv`, `setEnv`, and the `oauth` namespace. ## Differences from the Python package diff --git a/landing/index.html b/landing/index.html index 04566c9..8a23cde 100644 --- a/landing/index.html +++ b/landing/index.html @@ -606,7 +606,7 @@

The same filter in JavaScript and Python.

updated = smartapply(diff, files) -

Each decision is { path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error }. decision is yes or no. skippedReason is binary, oversize, truncated, ignored, or null. The default model is xiaomi/mimo-v2.6-flash (GPTDIFF_FILTER_MODEL). The cache key is the model, a hash of the task, a hash of the original contents, a truncation tag, the output-token limit, and the prompt version. Python uses the same fields in snake_case (skipped_reason, prompt_tokens, max_bytes, max_tokens, on_oversize, on_binary).

+

Each decision is { path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error, failedOpen }. decision is yes or no. A leading standalone yes or no is required; other wording fails open and is not cached. skippedReason is binary, oversize, truncated, ignored, or null. The default model is xiaomi/mimo-v2.6-flash (GPTDIFF_FILTER_MODEL). The cache key hashes prompt version 3, the model, the output-token limit, the reasoning effort, the task, the file path, and the body that was sent. Python uses the same fields in snake_case (skipped_reason, prompt_tokens, failed_open, max_bytes, max_tokens, on_oversize, on_binary). Reported Flask recall 1.0 and 0.8× token figures are UNVERIFIED.

diff --git a/landing/lib/filter.js b/landing/lib/filter.js index 2d0cad6..373b62b 100644 --- a/landing/lib/filter.js +++ b/landing/lib/filter.js @@ -40,10 +40,12 @@ export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 1024; export const DEFAULT_FILTER_REASONING_EFFORT = 'low'; /** - * Bumped when the system prompt changes, and included in the cache key so an - * old judgment is not reused for a new prompt. + * Bumped when the prompt or the yes/no contract changes so an old judgment is + * not reused. The cache key stringifies this value. `PROMPT_VERSION` is the + * shared name; `FILTER_PROMPT_VERSION` is the same number. */ -export const FILTER_PROMPT_VERSION = 1; +export const PROMPT_VERSION = 3; +export const FILTER_PROMPT_VERSION = PROMPT_VERSION; /** * Lockfiles and generated paths skipped unless `ignore` is `false` or a custom list. @@ -69,6 +71,7 @@ export const DEFAULT_IGNORE_PATTERNS = [ '**/packages.lock.json', '**/Package.resolved', '**/*.min.js', + '**/*.min.mjs', '**/*.min.css', '**/*.map', '**/dist/**', @@ -83,7 +86,9 @@ export const DEFAULT_IGNORE_PATTERNS = [ const POLICIES = ['include', 'exclude', 'truncate']; -const ANSWER_PREFIX = /^(?:answer|relevance|relevant|decision)\s*[:\-]\s*/i; +const LEADING_WRAPPER = /^(?:answer|decision)\s*:\s*|^[\s"'`*_“”‘’]+/i; +const TRAILING_WRAPPER = /[\s"'`*_“”‘’]+$/; +const LEADING_YES_NO = /^(yes|no)\b/i; const FENCE_LINE = /^```[a-zA-Z0-9_-]*\s*$/; const SYSTEM_PROMPT = ( @@ -109,7 +114,11 @@ const SYSTEM_PROMPT = ( * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) * @property {'binary' | 'oversize' | 'truncated' | 'ignored' | null} skippedReason * set when the file was not sent in full. `ignored` is a lockfile or generated path. - * @property {string | null} error set when the call threw; the file is still included + * @property {string | null} error set when the call threw or the answer was empty; + * the file is still included + * @property {boolean} failedOpen true when the file was included without a + * definitive yes/no (empty, malformed, thrown, or a cache failure that + * blocked a decision) */ /** @@ -136,16 +145,17 @@ const SYSTEM_PROMPT = ( * @property {number} [completion_tokens] * @property {'binary' | 'oversize' | 'truncated' | null} [skipped_reason] * @property {string | null} [error] + * @property {boolean} [failed_open] * @property {string} [cache_key] */ /** * In-memory relevance cache with an optional on-disk JSON file. * - * Keys are `model`, SHA-256 of the task, SHA-256 of the original contents, and - * a variant tag (`full`, `binary-preview`, or `truncated::`), - * separated by newlines. A judgment on a truncated body is not reused as a - * full-content judgment. + * The key is the SHA-256 of the exact classification inputs, length-prefixed + * in this order: prompt version, model, output-token limit, reasoning effort, + * intent, file path, and the content actually sent (after truncation). Two + * paths never share a key, even when their contents match. */ export class RelevanceCache { /** @@ -161,25 +171,35 @@ export class RelevanceCache { } /** - * @param {string} model - * @param {string} task - * @param {string} content original file text - * @param {string} [variant] - * @param {number} [maxOutputTokens] completion cap, so a larger budget is a different key - * @param {number} [promptVersion] {@link FILTER_PROMPT_VERSION} + * SHA-256 hex of the length-prefixed classification inputs. + * Each field is its UTF-8 bytes, written as `byteLength`, a newline, the + * bytes, and a newline. Field order: prompt version, model, output-token + * limit, reasoning effort (`""` when omitted), intent, path, content. + * @param {object} fields + * @param {string} fields.model + * @param {string} fields.intent task text + * @param {string} fields.path + * @param {string} fields.content body sent to the model, after truncation + * @param {number} [fields.maxOutputTokens] + * @param {string | null} [fields.reasoningEffort] + * @param {number | string} [fields.promptVersion] * @returns {Promise} */ - static async makeKey( - model, - task, - content, - variant = 'full', - maxOutputTokens = DEFAULT_FILTER_MAX_OUTPUT_TOKENS, - promptVersion = FILTER_PROMPT_VERSION, - ) { - const taskHash = await sha256(task); - const contentHash = await sha256(content); - return `${model}\n${taskHash}\n${contentHash}\n${variant}\n${maxOutputTokens}\n${promptVersion}`; + static async makeKey(fields) { + const promptVersion = String( + fields.promptVersion === undefined ? FILTER_PROMPT_VERSION : fields.promptVersion, + ); + const model = String(fields.model); + const tokenLimit = String( + fields.maxOutputTokens === undefined ? DEFAULT_FILTER_MAX_OUTPUT_TOKENS : fields.maxOutputTokens, + ); + const effort = fields.reasoningEffort == null || fields.reasoningEffort === undefined + ? '' + : String(fields.reasoningEffort); + const intent = String(fields.intent ?? ''); + const path = String(fields.path); + const content = String(fields.content); + return sha256(lengthPrefixed([promptVersion, model, tokenLimit, effort, intent, path, content])); } /** @@ -281,14 +301,14 @@ export function createMemoryCache() { * Browsers and Node 19+ expose Web Crypto as `globalThis.crypto`. Node 18 * keeps the same API on `node:crypto` as `webcrypto`. That module is imported * only when the global is missing, so this file still loads in the browser. - * @returns {Promise} + * @returns {Promise<{ subtle: { digest: Function } }>} */ async function getWebCrypto() { const exposed = typeof globalThis !== 'undefined' ? globalThis.crypto : undefined; if (exposed && exposed.subtle) return exposed; try { const mod = await import('node:crypto'); - if (mod.webcrypto && mod.webcrypto.subtle) return mod.webcrypto; + if (mod.webcrypto && mod.webcrypto.subtle) return /** @type {{ subtle: { digest: Function } }} */ (mod.webcrypto); } catch { // A browser without `globalThis.crypto`, or a host that cannot load node:crypto. } @@ -296,13 +316,38 @@ async function getWebCrypto() { } /** - * @param {string} text + * Length-prefixed UTF-8 encoding shared with the Python filter. + * Each part is `decimalByteLength + "\\n" + utf8 + "\\n"`. + * @param {string[]} parts + * @returns {Uint8Array} + */ +function lengthPrefixed(parts) { + const encoder = new TextEncoder(); + const encoded = parts.map((part) => encoder.encode(part)); + let size = 0; + for (const bytes of encoded) size += String(bytes.length).length + 1 + bytes.length + 1; + const out = new Uint8Array(size); + let offset = 0; + for (const bytes of encoded) { + const header = encoder.encode(`${bytes.length}\n`); + out.set(header, offset); + offset += header.length; + out.set(bytes, offset); + offset += bytes.length; + out[offset] = 0x0a; + offset += 1; + } + return out; +} + +/** + * @param {string | Uint8Array} data * @returns {Promise} hex SHA-256 */ -async function sha256(text) { - const data = new TextEncoder().encode(text); +async function sha256(data) { + const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data; const webcrypto = await getWebCrypto(); - const digest = await webcrypto.subtle.digest('SHA-256', data); + const digest = await webcrypto.subtle.digest('SHA-256', bytes); let hex = ''; for (const byte of new Uint8Array(digest)) hex += byte.toString(16).padStart(2, '0'); return hex; @@ -324,13 +369,30 @@ function stripFilterThinkTags(text) { return cleaned.trim(); } +/** + * Strip a leading `Answer:` / `Decision:` prefix and surrounding quotes, + * markdown bold, and backticks. Repeats so `Answer: **yes**` unwraps fully. + * @param {string} text + * @returns {string} + */ +function unwrapAnswerWrappers(text) { + let value = text.trim(); + for (let i = 0; i < 8; i++) { + const next = value.replace(LEADING_WRAPPER, '').replace(TRAILING_WRAPPER, '').trim(); + if (next === value) break; + value = next; + } + return value; +} + /** * Parse a yes/no relevance answer. * - * Think tags and chain-of-thought reasoning blocks are removed before the - * answer is read, so a "no" inside a think tag cannot override a final "yes". - * A single consistent yes or no wins, including "Answer: yes" and a fenced - * "NO". Both words, or neither, is ambiguous — callers include the file. + * Think tags and chain-of-thought reasoning blocks are removed first. The + * decision is a leading standalone `yes` or `no` after whitespace, quotes, + * markdown bold or backticks, and an `Answer:` or `Decision:` prefix. A word + * later in the sentence does not count. Anything else is malformed (`null`); + * callers include the file and do not cache it. * * @param {unknown} raw * @returns {{ decision: 'yes' | 'no' | null, cleaned: string }} @@ -347,20 +409,12 @@ export function parseRelevanceAnswer(raw) { ) { text = fenceLines.slice(1, -1).join('\n').trim(); } - const cleaned = text.trim().replace(/^`+|`+$/g, '').trim(); - const contentLines = cleaned.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); - if (!contentLines.length) return { decision: null, cleaned }; - - let answerText = contentLines[0].replace(ANSWER_PREFIX, '').trim(); - if (contentLines.length > 1) { - answerText = `${answerText}\n${contentLines.slice(1).join('\n')}`; - } - const words = answerText.match(/\b(yes|no)\b/gi) || []; - const lowered = words.map((word) => word.toLowerCase()); - if (lowered.length && new Set(lowered).size === 1) { - return { decision: /** @type {'yes' | 'no'} */ (lowered[0]), cleaned }; - } - return { decision: null, cleaned }; + const cleaned = text.trim(); + const unwrapped = unwrapAnswerWrappers(cleaned); + if (!unwrapped) return { decision: null, cleaned }; + const match = LEADING_YES_NO.exec(unwrapped); + if (!match) return { decision: null, cleaned }; + return { decision: /** @type {'yes' | 'no'} */ (match[1].toLowerCase()), cleaned }; } /** @@ -436,23 +490,26 @@ function truncateText(text, maxBytes, maxTokens) { } /** - * UTF-8 prefix. An incomplete trailing character is dropped, matching - * Python's `errors="ignore"` on a cut byte string. + * UTF-8 prefix cut on a code-point boundary. Never emits U+FFFD: a character + * that does not fit entirely inside `maxBytes` is dropped. * @param {string} text * @param {number} maxBytes * @returns {string} */ function truncateUtf8(text, maxBytes) { const bytes = new TextEncoder().encode(text); - if (bytes.length <= maxBytes) return text; + if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text; let end = maxBytes; while (end > 0 && (bytes[end - 1] & 0xc0) === 0x80) end -= 1; if (end > 0) { - const lead = bytes[end - 1]; + const leadIndex = end - 1; + const lead = bytes[leadIndex]; const need = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4; - if (end - 1 + need > maxBytes) end -= 1; + const have = maxBytes - leadIndex; + if (have < need) end = leadIndex; + else end = leadIndex + need; } - return new TextDecoder().decode(bytes.subarray(0, end)); + return new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, end)); } /** @@ -473,7 +530,70 @@ function utf8ByteLength(text) { export function isBinaryContent(content) { if (content instanceof Uint8Array) return binaryBytes(content); if (typeof content !== 'string') return false; - return textLooksBinary(content.slice(0, 8192)); + return binaryBytes(utf8Prefix(content, 8192)); +} + +/** + * First `maxBytes` UTF-8 bytes of `text`, which may end on an incomplete + * code point when the next character does not fit. + * @param {string} text + * @param {number} maxBytes + * @returns {Uint8Array} + */ +function utf8Prefix(text, maxBytes) { + const encoder = new TextEncoder(); + /** @type {number[]} */ + const out = []; + let n = 0; + for (const ch of text) { + if (n >= maxBytes) break; + const bytes = encoder.encode(ch); + for (const byte of bytes) out.push(byte); + n += bytes.length; + } + return Uint8Array.from(out).subarray(0, maxBytes); +} + +/** + * @param {number} lead + * @returns {number} expected sequence length, or 0 when `lead` cannot start one + */ +function utf8SequenceLength(lead) { + if (lead <= 0x7f) return 1; + if (lead >= 0xc2 && lead <= 0xdf) return 2; + if (lead >= 0xe0 && lead <= 0xef) return 3; + if (lead >= 0xf0 && lead <= 0xf4) return 4; + return 0; +} + +/** + * Decode `sample` as UTF-8. An incomplete trailing sequence of at most 3 + * bytes is dropped. Genuinely invalid bytes return null. + * @param {Uint8Array} sample + * @returns {string | null} + */ +function decodeUtf8AllowingShortTail(sample) { + const fatal = () => new TextDecoder('utf-8', { fatal: true }); + try { + return fatal().decode(sample); + } catch { + // The sample is not well-formed. A cut at 8192 may end inside one character. + } + let index = sample.length - 1; + let continuations = 0; + while (continuations < 3 && index >= 0 && (sample[index] & 0xc0) === 0x80) { + continuations += 1; + index -= 1; + } + if (index < 0) return null; + const need = utf8SequenceLength(sample[index]); + const have = sample.length - index; + if (!need || have >= need || need - have > 3 || continuations !== have - 1) return null; + try { + return fatal().decode(sample.subarray(0, index)); + } catch { + return null; + } } /** @@ -485,12 +605,9 @@ function binaryBytes(bytes) { for (let i = 0; i < sample.length; i++) { if (sample[i] === 0) return true; } - try { - const text = new TextDecoder('utf-8', { fatal: true }).decode(sample); - return textLooksBinary(text); - } catch { - return true; - } + const text = decodeUtf8AllowingShortTail(sample); + if (text == null) return true; + return textLooksBinary(text); } /** @@ -534,23 +651,6 @@ function resolveLimit(value, fallback) { return n; } -/** - * @param {unknown} value - * @returns {string} - */ -/** - * Tag a limit the way Python's f-string does: omitted uses the default number, - * `null` is `None`, and `0` stays `0`. - * @param {unknown} value - * @param {number} fallback - * @returns {string} - */ -function rawLimitTag(value, fallback) { - const chosen = value === undefined ? fallback : value; - if (chosen === null || chosen === undefined) return 'None'; - return String(chosen); -} - /** * @param {object} fields * @returns {RelevanceDecision} @@ -567,6 +667,7 @@ function decisionObject(fields) { cached: !!fields.cached, skippedReason: fields.skippedReason ?? null, error: fields.error ?? null, + failedOpen: !!fields.failedOpen, }; } @@ -589,6 +690,7 @@ function decisionFromRecord(path, record, cached) { cached, skippedReason: rec.skipped_reason ?? rec.skippedReason ?? null, error: rec.error ?? null, + failedOpen: !!(rec.failed_open ?? rec.failedOpen), }); } @@ -614,14 +716,13 @@ function skipDecision(path, decision, skippedReason, reason) { /** * @param {string} task - * @param {string[]} paths + * @param {string} path * @param {string} body * @param {string | null} note * @returns {string} */ -function buildUserPrompt(task, paths, body, note) { - const label = paths.length === 1 ? 'File path' : 'File paths'; - const parts = [`Task:\n${task}`, `${label}:\n${paths.join('\n')}`]; +function buildUserPrompt(task, path, body, note) { + const parts = [`Task:\n${task}`, `File path:\n${path}`]; if (note) parts.push(note); parts.push(`File contents:\n${body}`); parts.push('Is this file relevant to the task? yes/no'); @@ -679,9 +780,37 @@ function isTransientFilterError(err) { const message = errorMessage(err); if (/timeout|timed out/i.test(message)) return true; if (/\b5\d\d\b/.test(message)) return true; + if (/billing reservation/i.test(message)) return true; return false; } +/** + * A 4xx that names `reasoning_effort`. The filter retries that call once + * without the parameter. + * @param {unknown} err + * @returns {boolean} + */ +function isReasoningEffortRejected(err) { + const message = errorMessage(err); + if (!/reasoning[_ ]effort/i.test(message)) return false; + const status = err && typeof err === 'object' + ? (/** @type {{ status?: number, statusCode?: number }} */ (err).status + ?? /** @type {{ statusCode?: number }} */ (err).statusCode) + : undefined; + if (typeof status === 'number') return status >= 400 && status < 500; + return /\b4\d\d\b/.test(message); +} + +/** + * @param {unknown} err + */ +function logCacheError(err) { + const message = errorMessage(err); + if (typeof console !== 'undefined' && typeof console.warn === 'function') { + console.warn(`relevance cache error: ${message}`); + } +} + /** * @param {unknown} value * @returns {number} @@ -752,9 +881,12 @@ function resolveIgnore(ignore) { */ function isFailedOpen(decision) { if (!decision) return false; + if (decision.failedOpen) return true; if (decision.error) return true; - return typeof decision.reason === 'string' - && decision.reason.startsWith('ambiguous answer; included by default'); + return typeof decision.reason === 'string' && ( + decision.reason.startsWith('ambiguous answer; included by default') + || decision.reason.startsWith('malformed answer; included by default') + ); } /** @@ -765,15 +897,21 @@ function isFailedOpen(decision) { */ function reusableHit(hit) { if (!hit || (hit.decision !== 'yes' && hit.decision !== 'no') || hit.error) return false; + if (hit.failed_open || /** @type {{ failedOpen?: boolean }} */ (hit).failedOpen) return false; const reason = hit.reason || ''; - if (reason.startsWith('ambiguous answer') || reason.startsWith('error; included by default')) return false; + if ( + reason.startsWith('ambiguous answer') + || reason.startsWith('malformed answer') + || reason.startsWith('error; included by default') + ) return false; return true; } /** - * @param {Array} items + * @template T + * @param {T[]} items * @param {number} limit - * @param {(item: string) => Promise} fn + * @param {(item: T) => Promise} fn * @returns {Promise} */ async function mapLimit(items, limit, fn) { @@ -821,8 +959,10 @@ async function cachePut(store, key, record) { * * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that * JSON file), `true` or omitted (a fresh in-memory cache for this call), or - * `null` / `false` (disabled). The cache key is the model, a hash of the task, - * a hash of the original file contents, and a truncation variant. + * `null` / `false` (disabled). The cache key hashes the prompt version, model, + * output-token limit, reasoning effort, task, path, and the body that was + * sent (after truncation). Each path is classified on its own, including + * paths whose contents match. * * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` @@ -842,7 +982,7 @@ async function cachePut(store, key, record) { * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) * @param {string | null} [opts.reasoningEffort] default {@link DEFAULT_FILTER_REASONING_EFFORT}; `null` omits it - * @param {number} [opts.retries] extra attempts after a timeout or HTTP 5xx (default 2) + * @param {number} [opts.retries] extra attempts after a timeout, HTTP 5xx, or a billing-reservation error (default 2). A 4xx that names `reasoning_effort` is retried once without that parameter and does not consume one of these attempts. * @param {number} [opts.retryBaseMs] backoff base in ms (default 200, doubles, capped at 30s) * @param {boolean | string[] | null} [opts.ignore] lockfile and generated-file patterns. * `true` or omitted uses {@link DEFAULT_IGNORE_PATTERNS}. `false` or `null` classifies every path. @@ -887,23 +1027,16 @@ export async function filterRelevant(files, task, opts = {}) { /** @type {Map} */ const byPath = new Map(); - /** @type {Map>} */ - const groups = new Map(); - /** @type {string[]} */ - const groupOrder = []; - const entries = Object.entries(files); - /** - * @param {string} key - * @param {{ path: string, body: string, note: string | null, skippedReason: 'truncated' | null }} item + * @typedef {object} PendingFile + * @property {string} path + * @property {string} body content sent to the model + * @property {string | null} note + * @property {'truncated' | null} skippedReason */ - function enqueue(key, item) { - if (!groups.has(key)) { - groups.set(key, []); - groupOrder.push(key); - } - groups.get(key).push(item); - } + /** @type {PendingFile[]} */ + const pending = []; + const entries = Object.entries(files); /** * @param {RelevanceDecision} decision @@ -934,8 +1067,7 @@ export async function filterRelevant(files, task, opts = {}) { const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; const body = `[binary file, ${nbytes} bytes, contents omitted]`; - const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview', maxOutputTokens); - enqueue(key, { path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, skippedReason: 'truncated' }); } else { const decision = onBinary === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -957,9 +1089,7 @@ export async function filterRelevant(files, task, opts = {}) { 'Note: this file was truncated to fit the relevance check ' + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` ); - const variant = `truncated:${rawLimitTag(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS)}:${rawLimitTag(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES)}`; - const key = await RelevanceCache.makeKey(model, task, text, variant, maxOutputTokens); - enqueue(key, { path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, skippedReason: 'truncated' }); } else { const decision = onOversize === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -975,25 +1105,54 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - const key = await RelevanceCache.makeKey(model, task, text, 'full', maxOutputTokens); - enqueue(key, { path, body: text, note: null, skippedReason: null }); + pending.push({ path, body: text, note: null, skippedReason: null }); } - if (store && typeof store.load === 'function') await store.load(); + if (store && typeof store.load === 'function') { + try { + await store.load(); + } catch (err) { + logCacheError(err); + } + } /** * @param {string} key - * @returns {Promise} + * @returns {Promise} */ - async function cached(key) { - if (!store) return false; - const hit = await cacheGet(store, key); - return reusableHit(hit); + async function readCache(key) { + if (!store) return undefined; + try { + return await cacheGet(store, key); + } catch (err) { + logCacheError(err); + return undefined; + } } + /** + * Key for a lookup before the call, using the configured reasoning effort. + * @param {PendingFile} item + * @param {string | null} effort + * @returns {Promise} + */ + function keyFor(item, effort) { + return RelevanceCache.makeKey({ + model, + intent: task, + path: item.path, + content: item.body, + maxOutputTokens, + reasoningEffort: effort, + promptVersion: FILTER_PROMPT_VERSION, + }); + } + + const configuredEffort = reasoningEffort || null; let needsCall = false; - for (const key of groupOrder) { - if (!(await cached(key))) { + for (const item of pending) { + const hit = await readCache(await keyFor(item, configuredEffort)); + if (!reusableHit(hit)) { needsCall = true; break; } @@ -1006,34 +1165,32 @@ export async function filterRelevant(files, task, opts = {}) { } /** - * @param {string} key + * @param {PendingFile} item * @returns {Promise} */ - async function runGroup(key) { - const items = groups.get(key); - if (store) { - const hit = await cacheGet(store, key); - if (reusableHit(hit)) { - for (const item of items) { - const decision = decisionFromRecord(item.path, { ...hit, cache_key: key }, true); - byPath.set(item.path, decision); - emit(decision); - } - return; - } + async function runOne(item) { + const lookupKey = await keyFor(item, configuredEffort); + const hit = await readCache(lookupKey); + if (reusableHit(hit)) { + const decision = decisionFromRecord(item.path, { ...hit, cache_key: lookupKey }, true); + byPath.set(item.path, decision); + emit(decision); + return; } - const sample = items[0]; let raw; let promptTokens; let completionTokens; let totalTokens; /** @type {unknown} */ let lastError = null; - const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); + /** @type {string | null} */ + let effortUsed = configuredEffort; + const userPrompt = buildUserPrompt(task, item.path, item.body, item.note); let userContent = userPrompt; if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; const attempts = retries + 1; + let strippedEffort = false; for (let attempt = 0; attempt < attempts; attempt++) { try { const response = await callLlm({ @@ -1047,7 +1204,7 @@ export async function filterRelevant(files, task, opts = {}) { maxTokens: maxOutputTokens, temperature: 0, budgetTokens: null, - reasoningEffort: reasoningEffort || null, + reasoningEffort: effortUsed, maxRetries: 0, }); ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); @@ -1055,6 +1212,12 @@ export async function filterRelevant(files, task, opts = {}) { break; } catch (err) { lastError = err; + if (!strippedEffort && effortUsed && isReasoningEffortRejected(err)) { + effortUsed = null; + strippedEffort = true; + attempt -= 1; + continue; + } const again = attempt + 1 < attempts && isTransientFilterError(err); if (!again) break; const wait = Math.min(retryBaseMs * 2 ** attempt, 30_000); @@ -1063,22 +1226,21 @@ export async function filterRelevant(files, task, opts = {}) { } if (lastError) { const message = errorMessage(lastError); - items.forEach((item, index) => { - const decision = decisionObject({ - path: item.path, - decision: 'yes', - reason: `error; included by default: ${message}`, - raw: null, - tokens: 0, - promptTokens: 0, - completionTokens: 0, - cached: index > 0, - skippedReason: item.skippedReason, - error: message, - }); - byPath.set(item.path, decision); - emit(decision); + const decision = decisionObject({ + path: item.path, + decision: 'yes', + reason: `error; included by default: ${message}`, + raw: null, + tokens: 0, + promptTokens: 0, + completionTokens: 0, + cached: false, + skippedReason: item.skippedReason, + error: message, + failedOpen: true, }); + byPath.set(item.path, decision); + emit(decision); return; } @@ -1090,18 +1252,22 @@ export async function filterRelevant(files, task, opts = {}) { /** @type {string | null} */ let error = null; let cacheable = false; + let failedOpen = false; if (parsed.decision == null && !parsed.cleaned) { decisionName = 'yes'; error = 'empty answer'; reason = `error; included by default: ${error}`; + failedOpen = true; } else if (parsed.decision == null) { decisionName = 'yes'; - reason = `ambiguous answer; included by default: ${parsed.cleaned}`; + reason = `malformed answer; included by default: ${parsed.cleaned}`; + failedOpen = true; } else { decisionName = parsed.decision; reason = parsed.cleaned; cacheable = true; } + const storeKey = await keyFor(item, effortUsed); if (store && cacheable) { /** @type {RelevanceCacheRecord} */ const record = { @@ -1111,34 +1277,44 @@ export async function filterRelevant(files, task, opts = {}) { tokens: totalTokens, prompt_tokens: promptTokens, completion_tokens: completionTokens, - skipped_reason: sample.skippedReason, + skipped_reason: item.skippedReason, error: null, - cache_key: key, + cache_key: storeKey, + failed_open: false, }; - await cachePut(store, key, record); + try { + await cachePut(store, storeKey, record); + } catch (err) { + logCacheError(err); + } } - items.forEach((item, index) => { - const decision = decisionObject({ - path: item.path, - decision: decisionName, - reason, - raw, - tokens: totalTokens, - promptTokens, - completionTokens, - cached: index > 0, - skippedReason: item.skippedReason, - error, - }); - byPath.set(item.path, decision); - emit(decision); + const decision = decisionObject({ + path: item.path, + decision: decisionName, + reason, + raw, + tokens: totalTokens, + promptTokens, + completionTokens, + cached: false, + skippedReason: item.skippedReason, + error, + failedOpen, }); + byPath.set(item.path, decision); + emit(decision); } try { - await mapLimit(groupOrder, concurrency, runGroup); + await mapLimit(pending, concurrency, runOne); } finally { - if (store && typeof store.flush === 'function') await store.flush(); + if (store && typeof store.flush === 'function') { + try { + await store.flush(); + } catch (err) { + logCacheError(err); + } + } } /** @type {RelevanceDecision[]} */ diff --git a/src/filter.d.ts b/src/filter.d.ts index bf30a7f..bfcd9b1 100644 --- a/src/filter.d.ts +++ b/src/filter.d.ts @@ -6,10 +6,11 @@ export function createMemoryCache(): RelevanceCache; /** * Parse a yes/no relevance answer. * - * Think tags and chain-of-thought reasoning blocks are removed before the - * answer is read, so a "no" inside a think tag cannot override a final "yes". - * A single consistent yes or no wins, including "Answer: yes" and a fenced - * "NO". Both words, or neither, is ambiguous — callers include the file. + * Think tags and chain-of-thought reasoning blocks are removed first. The + * decision is a leading standalone `yes` or `no` after whitespace, quotes, + * markdown bold or backticks, and an `Answer:` or `Decision:` prefix. A word + * later in the sentence does not count. Anything else is malformed (`null`); + * callers include the file and do not cache it. * * @param {unknown} raw * @returns {{ decision: 'yes' | 'no' | null, cleaned: string }} @@ -49,8 +50,10 @@ export function isIgnoredPath(filePath: string, patterns?: string[]): boolean; * * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that * JSON file), `true` or omitted (a fresh in-memory cache for this call), or - * `null` / `false` (disabled). The cache key is the model, a hash of the task, - * a hash of the original file contents, and a truncation variant. + * `null` / `false` (disabled). The cache key hashes the prompt version, model, + * output-token limit, reasoning effort, task, path, and the body that was + * sent (after truncation). Each path is classified on its own, including + * paths whose contents match. * * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` @@ -70,7 +73,7 @@ export function isIgnoredPath(filePath: string, patterns?: string[]): boolean; * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) * @param {string | null} [opts.reasoningEffort] default {@link DEFAULT_FILTER_REASONING_EFFORT}; `null` omits it - * @param {number} [opts.retries] extra attempts after a timeout or HTTP 5xx (default 2) + * @param {number} [opts.retries] extra attempts after a timeout, HTTP 5xx, or a billing-reservation error (default 2). A 4xx that names `reasoning_effort` is retried once without that parameter and does not consume one of these attempts. * @param {number} [opts.retryBaseMs] backoff base in ms (default 200, doubles, capped at 30s) * @param {boolean | string[] | null} [opts.ignore] lockfile and generated-file patterns. * `true` or omitted uses {@link DEFAULT_IGNORE_PATTERNS}. `false` or `null` classifies every path. @@ -135,10 +138,12 @@ export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS: 1024; /** Sent on the filter call so reasoning models keep the yes/no inside the budget. */ export const DEFAULT_FILTER_REASONING_EFFORT: "low"; /** - * Bumped when the system prompt changes, and included in the cache key so an - * old judgment is not reused for a new prompt. + * Bumped when the prompt or the yes/no contract changes so an old judgment is + * not reused. The cache key stringifies this value. `PROMPT_VERSION` is the + * shared name; `FILTER_PROMPT_VERSION` is the same number. */ -export const FILTER_PROMPT_VERSION: 1; +export const PROMPT_VERSION: 3; +export const FILTER_PROMPT_VERSION: 3; /** * Lockfiles and generated paths skipped unless `ignore` is `false` or a custom list. * Patterns are case-insensitive globs (`**` crosses directories, `*` does not). @@ -157,7 +162,11 @@ export const DEFAULT_IGNORE_PATTERNS: string[]; * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) * @property {'binary' | 'oversize' | 'truncated' | 'ignored' | null} skippedReason * set when the file was not sent in full. `ignored` is a lockfile or generated path. - * @property {string | null} error set when the call threw; the file is still included + * @property {string | null} error set when the call threw or the answer was empty; + * the file is still included + * @property {boolean} failedOpen true when the file was included without a + * definitive yes/no (empty, malformed, thrown, or a cache failure that + * blocked a decision) */ /** * @typedef {object} FilterRelevantResult @@ -182,27 +191,42 @@ export const DEFAULT_IGNORE_PATTERNS: string[]; * @property {number} [completion_tokens] * @property {'binary' | 'oversize' | 'truncated' | null} [skipped_reason] * @property {string | null} [error] + * @property {boolean} [failed_open] * @property {string} [cache_key] */ /** * In-memory relevance cache with an optional on-disk JSON file. * - * Keys are `model`, SHA-256 of the task, SHA-256 of the original contents, and - * a variant tag (`full`, `binary-preview`, or `truncated::`), - * separated by newlines. A judgment on a truncated body is not reused as a - * full-content judgment. + * The key is the SHA-256 of the exact classification inputs, length-prefixed + * in this order: prompt version, model, output-token limit, reasoning effort, + * intent, file path, and the content actually sent (after truncation). Two + * paths never share a key, even when their contents match. */ export class RelevanceCache { /** - * @param {string} model - * @param {string} task - * @param {string} content original file text - * @param {string} [variant] - * @param {number} [maxOutputTokens] completion cap, so a larger budget is a different key - * @param {number} [promptVersion] {@link FILTER_PROMPT_VERSION} + * SHA-256 hex of the length-prefixed classification inputs. + * Each field is its UTF-8 bytes, written as `byteLength`, a newline, the + * bytes, and a newline. Field order: prompt version, model, output-token + * limit, reasoning effort (`""` when omitted), intent, path, content. + * @param {object} fields + * @param {string} fields.model + * @param {string} fields.intent task text + * @param {string} fields.path + * @param {string} fields.content body sent to the model, after truncation + * @param {number} [fields.maxOutputTokens] + * @param {string | null} [fields.reasoningEffort] + * @param {number | string} [fields.promptVersion] * @returns {Promise} */ - static makeKey(model: string, task: string, content: string, variant?: string, maxOutputTokens?: number, promptVersion?: number): Promise; + static makeKey(fields: { + model: string; + intent: string; + path: string; + content: string; + maxOutputTokens?: number; + reasoningEffort?: string | null; + promptVersion?: number | string; + }): Promise; /** * @param {string} [path] JSON file. Omit for memory only. * Loaded on the first read. Corrupt files are ignored. @@ -268,9 +292,16 @@ export type RelevanceDecision = { */ skippedReason: "binary" | "oversize" | "truncated" | "ignored" | null; /** - * set when the call threw; the file is still included + * set when the call threw or the answer was empty; + * the file is still included */ error: string | null; + /** + * true when the file was included without a + * definitive yes/no (empty, malformed, thrown, or a cache failure that + * blocked a decision) + */ + failedOpen: boolean; }; export type FilterRelevantResult = { /** @@ -310,5 +341,6 @@ export type RelevanceCacheRecord = { completion_tokens?: number; skipped_reason?: "binary" | "oversize" | "truncated" | null; error?: string | null; + failed_open?: boolean; cache_key?: string; }; diff --git a/src/filter.js b/src/filter.js index 2d0cad6..373b62b 100644 --- a/src/filter.js +++ b/src/filter.js @@ -40,10 +40,12 @@ export const DEFAULT_FILTER_MAX_OUTPUT_TOKENS = 1024; export const DEFAULT_FILTER_REASONING_EFFORT = 'low'; /** - * Bumped when the system prompt changes, and included in the cache key so an - * old judgment is not reused for a new prompt. + * Bumped when the prompt or the yes/no contract changes so an old judgment is + * not reused. The cache key stringifies this value. `PROMPT_VERSION` is the + * shared name; `FILTER_PROMPT_VERSION` is the same number. */ -export const FILTER_PROMPT_VERSION = 1; +export const PROMPT_VERSION = 3; +export const FILTER_PROMPT_VERSION = PROMPT_VERSION; /** * Lockfiles and generated paths skipped unless `ignore` is `false` or a custom list. @@ -69,6 +71,7 @@ export const DEFAULT_IGNORE_PATTERNS = [ '**/packages.lock.json', '**/Package.resolved', '**/*.min.js', + '**/*.min.mjs', '**/*.min.css', '**/*.map', '**/dist/**', @@ -83,7 +86,9 @@ export const DEFAULT_IGNORE_PATTERNS = [ const POLICIES = ['include', 'exclude', 'truncate']; -const ANSWER_PREFIX = /^(?:answer|relevance|relevant|decision)\s*[:\-]\s*/i; +const LEADING_WRAPPER = /^(?:answer|decision)\s*:\s*|^[\s"'`*_“”‘’]+/i; +const TRAILING_WRAPPER = /[\s"'`*_“”‘’]+$/; +const LEADING_YES_NO = /^(yes|no)\b/i; const FENCE_LINE = /^```[a-zA-Z0-9_-]*\s*$/; const SYSTEM_PROMPT = ( @@ -109,7 +114,11 @@ const SYSTEM_PROMPT = ( * @property {boolean} cached true when this judgment was reused (a cache hit, or a later path in a shared request) * @property {'binary' | 'oversize' | 'truncated' | 'ignored' | null} skippedReason * set when the file was not sent in full. `ignored` is a lockfile or generated path. - * @property {string | null} error set when the call threw; the file is still included + * @property {string | null} error set when the call threw or the answer was empty; + * the file is still included + * @property {boolean} failedOpen true when the file was included without a + * definitive yes/no (empty, malformed, thrown, or a cache failure that + * blocked a decision) */ /** @@ -136,16 +145,17 @@ const SYSTEM_PROMPT = ( * @property {number} [completion_tokens] * @property {'binary' | 'oversize' | 'truncated' | null} [skipped_reason] * @property {string | null} [error] + * @property {boolean} [failed_open] * @property {string} [cache_key] */ /** * In-memory relevance cache with an optional on-disk JSON file. * - * Keys are `model`, SHA-256 of the task, SHA-256 of the original contents, and - * a variant tag (`full`, `binary-preview`, or `truncated::`), - * separated by newlines. A judgment on a truncated body is not reused as a - * full-content judgment. + * The key is the SHA-256 of the exact classification inputs, length-prefixed + * in this order: prompt version, model, output-token limit, reasoning effort, + * intent, file path, and the content actually sent (after truncation). Two + * paths never share a key, even when their contents match. */ export class RelevanceCache { /** @@ -161,25 +171,35 @@ export class RelevanceCache { } /** - * @param {string} model - * @param {string} task - * @param {string} content original file text - * @param {string} [variant] - * @param {number} [maxOutputTokens] completion cap, so a larger budget is a different key - * @param {number} [promptVersion] {@link FILTER_PROMPT_VERSION} + * SHA-256 hex of the length-prefixed classification inputs. + * Each field is its UTF-8 bytes, written as `byteLength`, a newline, the + * bytes, and a newline. Field order: prompt version, model, output-token + * limit, reasoning effort (`""` when omitted), intent, path, content. + * @param {object} fields + * @param {string} fields.model + * @param {string} fields.intent task text + * @param {string} fields.path + * @param {string} fields.content body sent to the model, after truncation + * @param {number} [fields.maxOutputTokens] + * @param {string | null} [fields.reasoningEffort] + * @param {number | string} [fields.promptVersion] * @returns {Promise} */ - static async makeKey( - model, - task, - content, - variant = 'full', - maxOutputTokens = DEFAULT_FILTER_MAX_OUTPUT_TOKENS, - promptVersion = FILTER_PROMPT_VERSION, - ) { - const taskHash = await sha256(task); - const contentHash = await sha256(content); - return `${model}\n${taskHash}\n${contentHash}\n${variant}\n${maxOutputTokens}\n${promptVersion}`; + static async makeKey(fields) { + const promptVersion = String( + fields.promptVersion === undefined ? FILTER_PROMPT_VERSION : fields.promptVersion, + ); + const model = String(fields.model); + const tokenLimit = String( + fields.maxOutputTokens === undefined ? DEFAULT_FILTER_MAX_OUTPUT_TOKENS : fields.maxOutputTokens, + ); + const effort = fields.reasoningEffort == null || fields.reasoningEffort === undefined + ? '' + : String(fields.reasoningEffort); + const intent = String(fields.intent ?? ''); + const path = String(fields.path); + const content = String(fields.content); + return sha256(lengthPrefixed([promptVersion, model, tokenLimit, effort, intent, path, content])); } /** @@ -281,14 +301,14 @@ export function createMemoryCache() { * Browsers and Node 19+ expose Web Crypto as `globalThis.crypto`. Node 18 * keeps the same API on `node:crypto` as `webcrypto`. That module is imported * only when the global is missing, so this file still loads in the browser. - * @returns {Promise} + * @returns {Promise<{ subtle: { digest: Function } }>} */ async function getWebCrypto() { const exposed = typeof globalThis !== 'undefined' ? globalThis.crypto : undefined; if (exposed && exposed.subtle) return exposed; try { const mod = await import('node:crypto'); - if (mod.webcrypto && mod.webcrypto.subtle) return mod.webcrypto; + if (mod.webcrypto && mod.webcrypto.subtle) return /** @type {{ subtle: { digest: Function } }} */ (mod.webcrypto); } catch { // A browser without `globalThis.crypto`, or a host that cannot load node:crypto. } @@ -296,13 +316,38 @@ async function getWebCrypto() { } /** - * @param {string} text + * Length-prefixed UTF-8 encoding shared with the Python filter. + * Each part is `decimalByteLength + "\\n" + utf8 + "\\n"`. + * @param {string[]} parts + * @returns {Uint8Array} + */ +function lengthPrefixed(parts) { + const encoder = new TextEncoder(); + const encoded = parts.map((part) => encoder.encode(part)); + let size = 0; + for (const bytes of encoded) size += String(bytes.length).length + 1 + bytes.length + 1; + const out = new Uint8Array(size); + let offset = 0; + for (const bytes of encoded) { + const header = encoder.encode(`${bytes.length}\n`); + out.set(header, offset); + offset += header.length; + out.set(bytes, offset); + offset += bytes.length; + out[offset] = 0x0a; + offset += 1; + } + return out; +} + +/** + * @param {string | Uint8Array} data * @returns {Promise} hex SHA-256 */ -async function sha256(text) { - const data = new TextEncoder().encode(text); +async function sha256(data) { + const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data; const webcrypto = await getWebCrypto(); - const digest = await webcrypto.subtle.digest('SHA-256', data); + const digest = await webcrypto.subtle.digest('SHA-256', bytes); let hex = ''; for (const byte of new Uint8Array(digest)) hex += byte.toString(16).padStart(2, '0'); return hex; @@ -324,13 +369,30 @@ function stripFilterThinkTags(text) { return cleaned.trim(); } +/** + * Strip a leading `Answer:` / `Decision:` prefix and surrounding quotes, + * markdown bold, and backticks. Repeats so `Answer: **yes**` unwraps fully. + * @param {string} text + * @returns {string} + */ +function unwrapAnswerWrappers(text) { + let value = text.trim(); + for (let i = 0; i < 8; i++) { + const next = value.replace(LEADING_WRAPPER, '').replace(TRAILING_WRAPPER, '').trim(); + if (next === value) break; + value = next; + } + return value; +} + /** * Parse a yes/no relevance answer. * - * Think tags and chain-of-thought reasoning blocks are removed before the - * answer is read, so a "no" inside a think tag cannot override a final "yes". - * A single consistent yes or no wins, including "Answer: yes" and a fenced - * "NO". Both words, or neither, is ambiguous — callers include the file. + * Think tags and chain-of-thought reasoning blocks are removed first. The + * decision is a leading standalone `yes` or `no` after whitespace, quotes, + * markdown bold or backticks, and an `Answer:` or `Decision:` prefix. A word + * later in the sentence does not count. Anything else is malformed (`null`); + * callers include the file and do not cache it. * * @param {unknown} raw * @returns {{ decision: 'yes' | 'no' | null, cleaned: string }} @@ -347,20 +409,12 @@ export function parseRelevanceAnswer(raw) { ) { text = fenceLines.slice(1, -1).join('\n').trim(); } - const cleaned = text.trim().replace(/^`+|`+$/g, '').trim(); - const contentLines = cleaned.split(/\r\n|\r|\n/).map((line) => line.trim()).filter(Boolean); - if (!contentLines.length) return { decision: null, cleaned }; - - let answerText = contentLines[0].replace(ANSWER_PREFIX, '').trim(); - if (contentLines.length > 1) { - answerText = `${answerText}\n${contentLines.slice(1).join('\n')}`; - } - const words = answerText.match(/\b(yes|no)\b/gi) || []; - const lowered = words.map((word) => word.toLowerCase()); - if (lowered.length && new Set(lowered).size === 1) { - return { decision: /** @type {'yes' | 'no'} */ (lowered[0]), cleaned }; - } - return { decision: null, cleaned }; + const cleaned = text.trim(); + const unwrapped = unwrapAnswerWrappers(cleaned); + if (!unwrapped) return { decision: null, cleaned }; + const match = LEADING_YES_NO.exec(unwrapped); + if (!match) return { decision: null, cleaned }; + return { decision: /** @type {'yes' | 'no'} */ (match[1].toLowerCase()), cleaned }; } /** @@ -436,23 +490,26 @@ function truncateText(text, maxBytes, maxTokens) { } /** - * UTF-8 prefix. An incomplete trailing character is dropped, matching - * Python's `errors="ignore"` on a cut byte string. + * UTF-8 prefix cut on a code-point boundary. Never emits U+FFFD: a character + * that does not fit entirely inside `maxBytes` is dropped. * @param {string} text * @param {number} maxBytes * @returns {string} */ function truncateUtf8(text, maxBytes) { const bytes = new TextEncoder().encode(text); - if (bytes.length <= maxBytes) return text; + if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text; let end = maxBytes; while (end > 0 && (bytes[end - 1] & 0xc0) === 0x80) end -= 1; if (end > 0) { - const lead = bytes[end - 1]; + const leadIndex = end - 1; + const lead = bytes[leadIndex]; const need = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4; - if (end - 1 + need > maxBytes) end -= 1; + const have = maxBytes - leadIndex; + if (have < need) end = leadIndex; + else end = leadIndex + need; } - return new TextDecoder().decode(bytes.subarray(0, end)); + return new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, end)); } /** @@ -473,7 +530,70 @@ function utf8ByteLength(text) { export function isBinaryContent(content) { if (content instanceof Uint8Array) return binaryBytes(content); if (typeof content !== 'string') return false; - return textLooksBinary(content.slice(0, 8192)); + return binaryBytes(utf8Prefix(content, 8192)); +} + +/** + * First `maxBytes` UTF-8 bytes of `text`, which may end on an incomplete + * code point when the next character does not fit. + * @param {string} text + * @param {number} maxBytes + * @returns {Uint8Array} + */ +function utf8Prefix(text, maxBytes) { + const encoder = new TextEncoder(); + /** @type {number[]} */ + const out = []; + let n = 0; + for (const ch of text) { + if (n >= maxBytes) break; + const bytes = encoder.encode(ch); + for (const byte of bytes) out.push(byte); + n += bytes.length; + } + return Uint8Array.from(out).subarray(0, maxBytes); +} + +/** + * @param {number} lead + * @returns {number} expected sequence length, or 0 when `lead` cannot start one + */ +function utf8SequenceLength(lead) { + if (lead <= 0x7f) return 1; + if (lead >= 0xc2 && lead <= 0xdf) return 2; + if (lead >= 0xe0 && lead <= 0xef) return 3; + if (lead >= 0xf0 && lead <= 0xf4) return 4; + return 0; +} + +/** + * Decode `sample` as UTF-8. An incomplete trailing sequence of at most 3 + * bytes is dropped. Genuinely invalid bytes return null. + * @param {Uint8Array} sample + * @returns {string | null} + */ +function decodeUtf8AllowingShortTail(sample) { + const fatal = () => new TextDecoder('utf-8', { fatal: true }); + try { + return fatal().decode(sample); + } catch { + // The sample is not well-formed. A cut at 8192 may end inside one character. + } + let index = sample.length - 1; + let continuations = 0; + while (continuations < 3 && index >= 0 && (sample[index] & 0xc0) === 0x80) { + continuations += 1; + index -= 1; + } + if (index < 0) return null; + const need = utf8SequenceLength(sample[index]); + const have = sample.length - index; + if (!need || have >= need || need - have > 3 || continuations !== have - 1) return null; + try { + return fatal().decode(sample.subarray(0, index)); + } catch { + return null; + } } /** @@ -485,12 +605,9 @@ function binaryBytes(bytes) { for (let i = 0; i < sample.length; i++) { if (sample[i] === 0) return true; } - try { - const text = new TextDecoder('utf-8', { fatal: true }).decode(sample); - return textLooksBinary(text); - } catch { - return true; - } + const text = decodeUtf8AllowingShortTail(sample); + if (text == null) return true; + return textLooksBinary(text); } /** @@ -534,23 +651,6 @@ function resolveLimit(value, fallback) { return n; } -/** - * @param {unknown} value - * @returns {string} - */ -/** - * Tag a limit the way Python's f-string does: omitted uses the default number, - * `null` is `None`, and `0` stays `0`. - * @param {unknown} value - * @param {number} fallback - * @returns {string} - */ -function rawLimitTag(value, fallback) { - const chosen = value === undefined ? fallback : value; - if (chosen === null || chosen === undefined) return 'None'; - return String(chosen); -} - /** * @param {object} fields * @returns {RelevanceDecision} @@ -567,6 +667,7 @@ function decisionObject(fields) { cached: !!fields.cached, skippedReason: fields.skippedReason ?? null, error: fields.error ?? null, + failedOpen: !!fields.failedOpen, }; } @@ -589,6 +690,7 @@ function decisionFromRecord(path, record, cached) { cached, skippedReason: rec.skipped_reason ?? rec.skippedReason ?? null, error: rec.error ?? null, + failedOpen: !!(rec.failed_open ?? rec.failedOpen), }); } @@ -614,14 +716,13 @@ function skipDecision(path, decision, skippedReason, reason) { /** * @param {string} task - * @param {string[]} paths + * @param {string} path * @param {string} body * @param {string | null} note * @returns {string} */ -function buildUserPrompt(task, paths, body, note) { - const label = paths.length === 1 ? 'File path' : 'File paths'; - const parts = [`Task:\n${task}`, `${label}:\n${paths.join('\n')}`]; +function buildUserPrompt(task, path, body, note) { + const parts = [`Task:\n${task}`, `File path:\n${path}`]; if (note) parts.push(note); parts.push(`File contents:\n${body}`); parts.push('Is this file relevant to the task? yes/no'); @@ -679,9 +780,37 @@ function isTransientFilterError(err) { const message = errorMessage(err); if (/timeout|timed out/i.test(message)) return true; if (/\b5\d\d\b/.test(message)) return true; + if (/billing reservation/i.test(message)) return true; return false; } +/** + * A 4xx that names `reasoning_effort`. The filter retries that call once + * without the parameter. + * @param {unknown} err + * @returns {boolean} + */ +function isReasoningEffortRejected(err) { + const message = errorMessage(err); + if (!/reasoning[_ ]effort/i.test(message)) return false; + const status = err && typeof err === 'object' + ? (/** @type {{ status?: number, statusCode?: number }} */ (err).status + ?? /** @type {{ statusCode?: number }} */ (err).statusCode) + : undefined; + if (typeof status === 'number') return status >= 400 && status < 500; + return /\b4\d\d\b/.test(message); +} + +/** + * @param {unknown} err + */ +function logCacheError(err) { + const message = errorMessage(err); + if (typeof console !== 'undefined' && typeof console.warn === 'function') { + console.warn(`relevance cache error: ${message}`); + } +} + /** * @param {unknown} value * @returns {number} @@ -752,9 +881,12 @@ function resolveIgnore(ignore) { */ function isFailedOpen(decision) { if (!decision) return false; + if (decision.failedOpen) return true; if (decision.error) return true; - return typeof decision.reason === 'string' - && decision.reason.startsWith('ambiguous answer; included by default'); + return typeof decision.reason === 'string' && ( + decision.reason.startsWith('ambiguous answer; included by default') + || decision.reason.startsWith('malformed answer; included by default') + ); } /** @@ -765,15 +897,21 @@ function isFailedOpen(decision) { */ function reusableHit(hit) { if (!hit || (hit.decision !== 'yes' && hit.decision !== 'no') || hit.error) return false; + if (hit.failed_open || /** @type {{ failedOpen?: boolean }} */ (hit).failedOpen) return false; const reason = hit.reason || ''; - if (reason.startsWith('ambiguous answer') || reason.startsWith('error; included by default')) return false; + if ( + reason.startsWith('ambiguous answer') + || reason.startsWith('malformed answer') + || reason.startsWith('error; included by default') + ) return false; return true; } /** - * @param {Array} items + * @template T + * @param {T[]} items * @param {number} limit - * @param {(item: string) => Promise} fn + * @param {(item: T) => Promise} fn * @returns {Promise} */ async function mapLimit(items, limit, fn) { @@ -821,8 +959,10 @@ async function cachePut(store, key, record) { * * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that * JSON file), `true` or omitted (a fresh in-memory cache for this call), or - * `null` / `false` (disabled). The cache key is the model, a hash of the task, - * a hash of the original file contents, and a truncation variant. + * `null` / `false` (disabled). The cache key hashes the prompt version, model, + * output-token limit, reasoning effort, task, path, and the body that was + * sent (after truncation). Each path is classified on its own, including + * paths whose contents match. * * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` @@ -842,7 +982,7 @@ async function cachePut(store, key, record) { * @param {'include' | 'exclude' | 'truncate'} [opts.onBinary] default `exclude` * @param {number} [opts.maxOutputTokens] completion cap (default {@link DEFAULT_FILTER_MAX_OUTPUT_TOKENS}) * @param {string | null} [opts.reasoningEffort] default {@link DEFAULT_FILTER_REASONING_EFFORT}; `null` omits it - * @param {number} [opts.retries] extra attempts after a timeout or HTTP 5xx (default 2) + * @param {number} [opts.retries] extra attempts after a timeout, HTTP 5xx, or a billing-reservation error (default 2). A 4xx that names `reasoning_effort` is retried once without that parameter and does not consume one of these attempts. * @param {number} [opts.retryBaseMs] backoff base in ms (default 200, doubles, capped at 30s) * @param {boolean | string[] | null} [opts.ignore] lockfile and generated-file patterns. * `true` or omitted uses {@link DEFAULT_IGNORE_PATTERNS}. `false` or `null` classifies every path. @@ -887,23 +1027,16 @@ export async function filterRelevant(files, task, opts = {}) { /** @type {Map} */ const byPath = new Map(); - /** @type {Map>} */ - const groups = new Map(); - /** @type {string[]} */ - const groupOrder = []; - const entries = Object.entries(files); - /** - * @param {string} key - * @param {{ path: string, body: string, note: string | null, skippedReason: 'truncated' | null }} item + * @typedef {object} PendingFile + * @property {string} path + * @property {string} body content sent to the model + * @property {string | null} note + * @property {'truncated' | null} skippedReason */ - function enqueue(key, item) { - if (!groups.has(key)) { - groups.set(key, []); - groupOrder.push(key); - } - groups.get(key).push(item); - } + /** @type {PendingFile[]} */ + const pending = []; + const entries = Object.entries(files); /** * @param {RelevanceDecision} decision @@ -934,8 +1067,7 @@ export async function filterRelevant(files, task, opts = {}) { const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; const body = `[binary file, ${nbytes} bytes, contents omitted]`; - const key = await RelevanceCache.makeKey(model, task, text, 'binary-preview', maxOutputTokens); - enqueue(key, { path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, skippedReason: 'truncated' }); } else { const decision = onBinary === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -957,9 +1089,7 @@ export async function filterRelevant(files, task, opts = {}) { 'Note: this file was truncated to fit the relevance check ' + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` ); - const variant = `truncated:${rawLimitTag(opts.maxTokens, DEFAULT_FILTER_MAX_TOKENS)}:${rawLimitTag(opts.maxBytes, DEFAULT_FILTER_MAX_BYTES)}`; - const key = await RelevanceCache.makeKey(model, task, text, variant, maxOutputTokens); - enqueue(key, { path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, skippedReason: 'truncated' }); } else { const decision = onOversize === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -975,25 +1105,54 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - const key = await RelevanceCache.makeKey(model, task, text, 'full', maxOutputTokens); - enqueue(key, { path, body: text, note: null, skippedReason: null }); + pending.push({ path, body: text, note: null, skippedReason: null }); } - if (store && typeof store.load === 'function') await store.load(); + if (store && typeof store.load === 'function') { + try { + await store.load(); + } catch (err) { + logCacheError(err); + } + } /** * @param {string} key - * @returns {Promise} + * @returns {Promise} */ - async function cached(key) { - if (!store) return false; - const hit = await cacheGet(store, key); - return reusableHit(hit); + async function readCache(key) { + if (!store) return undefined; + try { + return await cacheGet(store, key); + } catch (err) { + logCacheError(err); + return undefined; + } } + /** + * Key for a lookup before the call, using the configured reasoning effort. + * @param {PendingFile} item + * @param {string | null} effort + * @returns {Promise} + */ + function keyFor(item, effort) { + return RelevanceCache.makeKey({ + model, + intent: task, + path: item.path, + content: item.body, + maxOutputTokens, + reasoningEffort: effort, + promptVersion: FILTER_PROMPT_VERSION, + }); + } + + const configuredEffort = reasoningEffort || null; let needsCall = false; - for (const key of groupOrder) { - if (!(await cached(key))) { + for (const item of pending) { + const hit = await readCache(await keyFor(item, configuredEffort)); + if (!reusableHit(hit)) { needsCall = true; break; } @@ -1006,34 +1165,32 @@ export async function filterRelevant(files, task, opts = {}) { } /** - * @param {string} key + * @param {PendingFile} item * @returns {Promise} */ - async function runGroup(key) { - const items = groups.get(key); - if (store) { - const hit = await cacheGet(store, key); - if (reusableHit(hit)) { - for (const item of items) { - const decision = decisionFromRecord(item.path, { ...hit, cache_key: key }, true); - byPath.set(item.path, decision); - emit(decision); - } - return; - } + async function runOne(item) { + const lookupKey = await keyFor(item, configuredEffort); + const hit = await readCache(lookupKey); + if (reusableHit(hit)) { + const decision = decisionFromRecord(item.path, { ...hit, cache_key: lookupKey }, true); + byPath.set(item.path, decision); + emit(decision); + return; } - const sample = items[0]; let raw; let promptTokens; let completionTokens; let totalTokens; /** @type {unknown} */ let lastError = null; - const userPrompt = buildUserPrompt(task, items.map((item) => item.path), sample.body, sample.note); + /** @type {string | null} */ + let effortUsed = configuredEffort; + const userPrompt = buildUserPrompt(task, item.path, item.body, item.note); let userContent = userPrompt; if (model && model.includes('gemini')) userContent = `${SYSTEM_PROMPT}\n${userPrompt}`; const attempts = retries + 1; + let strippedEffort = false; for (let attempt = 0; attempt < attempts; attempt++) { try { const response = await callLlm({ @@ -1047,7 +1204,7 @@ export async function filterRelevant(files, task, opts = {}) { maxTokens: maxOutputTokens, temperature: 0, budgetTokens: null, - reasoningEffort: reasoningEffort || null, + reasoningEffort: effortUsed, maxRetries: 0, }); ({ raw, promptTokens, completionTokens, tokens: totalTokens } = usageAndText(response)); @@ -1055,6 +1212,12 @@ export async function filterRelevant(files, task, opts = {}) { break; } catch (err) { lastError = err; + if (!strippedEffort && effortUsed && isReasoningEffortRejected(err)) { + effortUsed = null; + strippedEffort = true; + attempt -= 1; + continue; + } const again = attempt + 1 < attempts && isTransientFilterError(err); if (!again) break; const wait = Math.min(retryBaseMs * 2 ** attempt, 30_000); @@ -1063,22 +1226,21 @@ export async function filterRelevant(files, task, opts = {}) { } if (lastError) { const message = errorMessage(lastError); - items.forEach((item, index) => { - const decision = decisionObject({ - path: item.path, - decision: 'yes', - reason: `error; included by default: ${message}`, - raw: null, - tokens: 0, - promptTokens: 0, - completionTokens: 0, - cached: index > 0, - skippedReason: item.skippedReason, - error: message, - }); - byPath.set(item.path, decision); - emit(decision); + const decision = decisionObject({ + path: item.path, + decision: 'yes', + reason: `error; included by default: ${message}`, + raw: null, + tokens: 0, + promptTokens: 0, + completionTokens: 0, + cached: false, + skippedReason: item.skippedReason, + error: message, + failedOpen: true, }); + byPath.set(item.path, decision); + emit(decision); return; } @@ -1090,18 +1252,22 @@ export async function filterRelevant(files, task, opts = {}) { /** @type {string | null} */ let error = null; let cacheable = false; + let failedOpen = false; if (parsed.decision == null && !parsed.cleaned) { decisionName = 'yes'; error = 'empty answer'; reason = `error; included by default: ${error}`; + failedOpen = true; } else if (parsed.decision == null) { decisionName = 'yes'; - reason = `ambiguous answer; included by default: ${parsed.cleaned}`; + reason = `malformed answer; included by default: ${parsed.cleaned}`; + failedOpen = true; } else { decisionName = parsed.decision; reason = parsed.cleaned; cacheable = true; } + const storeKey = await keyFor(item, effortUsed); if (store && cacheable) { /** @type {RelevanceCacheRecord} */ const record = { @@ -1111,34 +1277,44 @@ export async function filterRelevant(files, task, opts = {}) { tokens: totalTokens, prompt_tokens: promptTokens, completion_tokens: completionTokens, - skipped_reason: sample.skippedReason, + skipped_reason: item.skippedReason, error: null, - cache_key: key, + cache_key: storeKey, + failed_open: false, }; - await cachePut(store, key, record); + try { + await cachePut(store, storeKey, record); + } catch (err) { + logCacheError(err); + } } - items.forEach((item, index) => { - const decision = decisionObject({ - path: item.path, - decision: decisionName, - reason, - raw, - tokens: totalTokens, - promptTokens, - completionTokens, - cached: index > 0, - skippedReason: item.skippedReason, - error, - }); - byPath.set(item.path, decision); - emit(decision); + const decision = decisionObject({ + path: item.path, + decision: decisionName, + reason, + raw, + tokens: totalTokens, + promptTokens, + completionTokens, + cached: false, + skippedReason: item.skippedReason, + error, + failedOpen, }); + byPath.set(item.path, decision); + emit(decision); } try { - await mapLimit(groupOrder, concurrency, runGroup); + await mapLimit(pending, concurrency, runOne); } finally { - if (store && typeof store.flush === 'function') await store.flush(); + if (store && typeof store.flush === 'function') { + try { + await store.flush(); + } catch (err) { + logCacheError(err); + } + } } /** @type {RelevanceDecision[]} */ diff --git a/src/index.d.ts b/src/index.d.ts index ee46dee..6a71f24 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -2,7 +2,7 @@ export * as oauth from "./oauth.js"; export type RelevanceDecision = import("./filter.js").RelevanceDecision; export type FilterRelevantResult = import("./filter.js").FilterRelevantResult; export { generateDiff, smartapply, buildEnvironment, callLlmForDiff, callLlmForApply, callLlmForApplyWithThink } from "./gptdiff.js"; -export { filterRelevant, RelevanceCache, createMemoryCache, parseRelevanceAnswer, isBinaryContent, isIgnoredPath, countTokens, formatFilterSummary, dumpDecisions, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_MAX_TOKENS, DEFAULT_FILTER_MAX_OUTPUT_TOKENS, DEFAULT_FILTER_REASONING_EFFORT, DEFAULT_FILTER_CONCURRENCY, FILTER_PROMPT_VERSION, DEFAULT_IGNORE_PATTERNS } from "./filter.js"; +export { filterRelevant, RelevanceCache, createMemoryCache, parseRelevanceAnswer, isBinaryContent, isIgnoredPath, countTokens, formatFilterSummary, dumpDecisions, DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MAX_BYTES, DEFAULT_FILTER_MAX_TOKENS, DEFAULT_FILTER_MAX_OUTPUT_TOKENS, DEFAULT_FILTER_REASONING_EFFORT, DEFAULT_FILTER_CONCURRENCY, FILTER_PROMPT_VERSION, PROMPT_VERSION, DEFAULT_IGNORE_PATTERNS } from "./filter.js"; export { parseDiffPerFile, applyDiff, applyPatchToFile } from "./applydiff.js"; export { colorCodeDiff, swallowReasoning, stripBadOutput, extractDiffBlocks, stripThinkTags } from "./text.js"; export { callLlm, resolveApiKey, resolveBaseUrl } from "./llm.js"; diff --git a/src/index.js b/src/index.js index 5fb8766..f9a4c32 100644 --- a/src/index.js +++ b/src/index.js @@ -23,6 +23,7 @@ export { DEFAULT_FILTER_REASONING_EFFORT, DEFAULT_FILTER_CONCURRENCY, FILTER_PROMPT_VERSION, + PROMPT_VERSION, DEFAULT_IGNORE_PATTERNS, } from './filter.js'; diff --git a/test/filter_relevant.test.js b/test/filter_relevant.test.js index 060a85b..995a152 100644 --- a/test/filter_relevant.test.js +++ b/test/filter_relevant.test.js @@ -1,6 +1,5 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { createHash } from 'node:crypto'; import { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -21,6 +20,7 @@ import { DEFAULT_FILTER_REASONING_EFFORT, DEFAULT_FILTER_CONCURRENCY, FILTER_PROMPT_VERSION, + PROMPT_VERSION, DEFAULT_IGNORE_PATTERNS, isIgnoredPath, callLlm, @@ -103,12 +103,33 @@ test('parseRelevanceAnswer strips think tags before deciding', () => { assert.equal(parseRelevanceAnswer('maybe\nyes').decision, 'yes'); }); -test('parseRelevanceAnswer is ambiguous when both words appear, or neither', () => { +test('parseRelevanceAnswer is malformed without a leading standalone yes or no', () => { assert.equal(parseRelevanceAnswer("I'm not sure about this one").decision, null); - assert.equal(parseRelevanceAnswer('yes for the helper, no for the tests').decision, null); assert.equal(parseRelevanceAnswer('').decision, null); assert.equal(parseRelevanceAnswer('nope').decision, null); - assert.equal(parseRelevanceAnswer('The answer is no.').decision, 'no'); + assert.equal(parseRelevanceAnswer('The answer is no.').decision, null); + assert.equal(parseRelevanceAnswer('yes for the helper, no for the tests').decision, 'yes'); +}); + +test('parseRelevanceAnswer: quoted yes/no is a decision', () => { + assert.equal(parseRelevanceAnswer('"yes"').decision, 'yes'); + assert.equal(parseRelevanceAnswer("'no'").decision, 'no'); + assert.equal(parseRelevanceAnswer('**yes**').decision, 'yes'); + assert.equal(parseRelevanceAnswer('`no`').decision, 'no'); + assert.equal(parseRelevanceAnswer('Answer: **yes**').decision, 'yes'); + assert.equal(parseRelevanceAnswer('Decision: "no"').decision, 'no'); +}); + +test('parseRelevanceAnswer: negated "There is no doubt this file is required" is malformed', () => { + const parsed = parseRelevanceAnswer('There is no doubt this file is required'); + assert.equal(parsed.decision, null); + assert.match(parsed.cleaned, /no doubt/); +}); + +test('parseRelevanceAnswer: explanatory wording is malformed', () => { + assert.equal(parseRelevanceAnswer('The file defines helpers used by the task').decision, null); + assert.equal(parseRelevanceAnswer('I would say no because it is unrelated').decision, null); + assert.equal(parseRelevanceAnswer('This file is not relevant to the task').decision, null); }); test('filterRelevant keeps only yes files and reports camelCase decisions', async () => { @@ -172,8 +193,10 @@ test('ambiguous answers and thrown errors include the file', async () => { const by = Object.fromEntries(result.decisions.map((d) => [d.path, d])); assert.equal(by['ambiguous.py'].decision, 'yes'); assert.equal(by['ambiguous.py'].error, null); - assert.match(by['ambiguous.py'].reason, /ambiguous/); + assert.equal(by['ambiguous.py'].failedOpen, true); + assert.match(by['ambiguous.py'].reason, /malformed/); assert.equal(by['broken.py'].decision, 'yes'); + assert.equal(by['broken.py'].failedOpen, true); assert.match(by['broken.py'].error, /connection reset/); assert.match(by['broken.py'].reason, /connection reset/); assert.equal(by['broken.py'].tokens, 0); @@ -223,10 +246,17 @@ test('memory and disk cache round-trip', async () => { assert.equal(stored.version, 1); assert.equal(Object.keys(stored.entries).length, 1); const [storedKey, storedValue] = Object.entries(stored.entries)[0]; - assert.equal(storedKey.split('\n').length, 6); - assert.equal(storedKey.split('\n')[3], 'full'); - assert.equal(storedKey.split('\n')[4], String(DEFAULT_FILTER_MAX_OUTPUT_TOKENS)); - assert.equal(storedKey.split('\n')[5], String(FILTER_PROMPT_VERSION)); + assert.match(storedKey, /^[0-9a-f]{64}$/); + const expectedKey = await RelevanceCache.makeKey({ + model: 'fake', + intent: 'ship it', + path: 'a.py', + content: "print('cached')\n", + maxOutputTokens: DEFAULT_FILTER_MAX_OUTPUT_TOKENS, + reasoningEffort: 'low', + promptVersion: FILTER_PROMPT_VERSION, + }); + assert.equal(storedKey, expectedKey); assert.equal(storedValue.decision, 'yes'); assert.equal(storedValue.prompt_tokens, 5); assert.equal(storedValue.skipped_reason, null); @@ -289,25 +319,50 @@ test('a corrupt disk cache is ignored', async () => { assert.equal(rewritten.version, 1); }); -test('duplicate contents share one request', async () => { +test('same-batch: empty unrelated.py "no" does not exclude empty src/target.py', async () => { + const mock = llm((req) => { + const paths = pathsOf(req); + assert.equal(paths.length, 1); + return response(paths[0] === 'unrelated.py' ? 'no' : 'yes'); + }); + const result = await filterRelevant( + { 'unrelated.py': '', 'src/target.py': '' }, + 'edit the target', + { ...baseOpts, cache: false, callLlm: mock.fn }, + ); + assert.equal(mock.calls.length, 2); + const by = Object.fromEntries(result.decisions.map((d) => [d.path, d])); + assert.equal(by['unrelated.py'].decision, 'no'); + assert.equal(by['unrelated.py'].failedOpen, false); + assert.equal(by['src/target.py'].decision, 'yes'); + assert.equal(by['src/target.py'].cached, false); + assert.deepEqual(Object.keys(result.files), ['src/target.py']); + assert.equal(result.files['src/target.py'], ''); +}); + +test('cross-call: cached no for empty unrelated.py does not exclude empty src/target.py', async () => { + const cache = new RelevanceCache(); let n = 0; const mock = llm((req) => { n += 1; const paths = pathsOf(req); - assert.ok(paths.includes('a.py') && paths.includes('b.py')); - return response('yes', 4, 1, 5); + return response(paths[0] === 'unrelated.py' ? 'no' : 'yes'); }); - const result = await filterRelevant( - { 'a.py': 'SAME', 'b.py': 'SAME' }, - 'task', - { ...baseOpts, model: 'm', callLlm: mock.fn }, + const opts = { model: 'm', cache, apiKey: 'k', callLlm: mock.fn }; + const first = await filterRelevant({ 'unrelated.py': '' }, 'edit the target', opts); + assert.equal(first.decisions[0].decision, 'no'); + const second = await filterRelevant( + { 'unrelated.py': '', 'src/target.py': '' }, + 'edit the target', + opts, ); - assert.equal(n, 1); - assert.equal(result.decisions[0].cached, false); - assert.equal(result.decisions[1].cached, true); - assert.equal(result.billedTokens, 5); - assert.deepEqual(Object.keys(result.files).sort(), ['a.py', 'b.py']); - assert.equal(result.files['a.py'], 'SAME'); + assert.equal(n, 2); + const by = Object.fromEntries(second.decisions.map((d) => [d.path, d])); + assert.equal(by['unrelated.py'].decision, 'no'); + assert.equal(by['unrelated.py'].cached, true); + assert.equal(by['src/target.py'].decision, 'yes'); + assert.equal(by['src/target.py'].cached, false); + assert.equal('src/target.py' in second.files, true); }); test('concurrency stays bounded and result order follows the input', async () => { @@ -422,9 +477,7 @@ test('a truncated judgment is not reused as a full read', async () => { assert.equal(second.decisions[0].skippedReason, null); assert.equal(n, 2); const stored = JSON.parse(readFileSync(cache.path, 'utf8')); - const variants = Object.keys(stored.entries).map((key) => key.split('\n')[3]); - assert.ok(variants.includes('truncated:None:20')); - assert.ok(variants.includes('full')); + assert.equal(Object.keys(stored.entries).length, 2); }); test('maxBytes counts UTF-8 bytes and a token cap truncates before the byte cap', async () => { @@ -619,7 +672,8 @@ test('model defaults to GPTDIFF_FILTER_MODEL, else xiaomi/mimo-v2.6-flash', asyn assert.equal(DEFAULT_FILTER_MAX_TOKENS, 12_000); assert.equal(DEFAULT_FILTER_MAX_OUTPUT_TOKENS, 1024); assert.equal(DEFAULT_FILTER_REASONING_EFFORT, 'low'); - assert.equal(FILTER_PROMPT_VERSION, 1); + assert.equal(FILTER_PROMPT_VERSION, 3); + assert.equal(PROMPT_VERSION, 3); assert.equal(DEFAULT_FILTER_CONCURRENCY, 8); assert.equal(isBinaryContent('hello\0world'), true); assert.equal(isBinaryContent('plain text'), false); @@ -709,7 +763,9 @@ test('an empty answer is an error, fails open, and is not cached', async () => { assert.equal(first.failedOpen, 1); assert.equal('a.py' in first.files, true); assert.match(formatFilterSummary(first), /1 failed open/); - const stored = await cache.get(await RelevanceCache.makeKey('m', 'task', 'v = 1\n', 'full')); + const stored = await cache.get(await RelevanceCache.makeKey({ + model: 'm', intent: 'task', path: 'a.py', content: 'v = 1\n', + })); assert.equal(stored, undefined); const second = await filterRelevant(files, 'task', { @@ -735,7 +791,8 @@ test('ambiguous answers fail open and are not cached', async () => { assert.equal(n, 2); assert.equal(first.decisions[0].decision, 'yes'); assert.equal(first.decisions[0].error, null); - assert.match(first.decisions[0].reason, /ambiguous/); + assert.match(first.decisions[0].reason, /malformed/); + assert.equal(first.decisions[0].failedOpen, true); assert.equal(first.failedOpen, 1); assert.equal(second.decisions[0].decision, 'no'); assert.equal(second.decisions[0].cached, false); @@ -749,10 +806,12 @@ test('a stored fallback is not reused', async () => { }); const cache = new RelevanceCache(); const body = 'v = 1\n'; - const key = await RelevanceCache.makeKey('m', 'task', body, 'full'); + const key = await RelevanceCache.makeKey({ + model: 'm', intent: 'task', path: 'a.py', content: body, + }); await cache.put(key, { decision: 'yes', - reason: 'ambiguous answer; included by default: maybe', + reason: 'malformed answer; included by default: maybe', error: null, tokens: 4, }); @@ -773,11 +832,14 @@ test('cache keys hash when global crypto is missing (Node 18)', async () => { }); try { const body = 'v = 1\n'; - const key = await RelevanceCache.makeKey('m', 'task', body); - const parts = key.split('\n'); - assert.equal(parts[0], 'm'); - assert.equal(parts[1], createHash('sha256').update('task').digest('hex')); - assert.equal(parts[2], createHash('sha256').update(body).digest('hex')); + const key = await RelevanceCache.makeKey({ + model: 'm', intent: 'task', path: 'a.py', content: body, + }); + assert.match(key, /^[0-9a-f]{64}$/); + const again = await RelevanceCache.makeKey({ + model: 'm', intent: 'task', path: 'other.py', content: body, + }); + assert.notEqual(key, again); const mock = llm(() => response('yes')); const result = await filterRelevant({ 'a.py': body }, 'task', { model: 'm', cache: false, apiKey: 'k', callLlm: mock.fn, @@ -855,6 +917,113 @@ test('a non-transient error is not retried and still fails open', async () => { assert.equal(result.files['a.py'], 'v = 1\n'); }); +test('truncateUtf8 cuts éX at 2 bytes without U+FFFD', async () => { + const seen = []; + const mock = llm((req) => { + seen.push(bodyOf(req)); + return response('yes'); + }); + const cases = [ + ['éX', 2, 'é'], + ['€X', 2, ''], + ['😀X', 3, ''], + ['a€b', 4, 'a€'], + ]; + for (const [text, maxBytes, expected] of cases) { + await filterRelevant({ 'a.py': text }, 'task', { + ...baseOpts, maxBytes, maxTokens: null, onOversize: 'truncate', callLlm: mock.fn, + }); + assert.equal(seen.at(-1), expected); + assert.equal(seen.at(-1).includes('\uFFFD'), false); + } +}); + +test('isBinaryContent tolerates a multibyte character cut at byte 8192', () => { + function sample(pad, tail) { + const bytes = new Uint8Array(pad + tail.length); + bytes.fill(0x61, 0, pad); + bytes.set(tail, pad); + return bytes; + } + assert.equal(isBinaryContent(sample(8191, [0xc3])), false); + assert.equal(isBinaryContent(sample(8190, [0xe2, 0x82])), false); + assert.equal(isBinaryContent(sample(8189, [0xf0, 0x9f, 0x98])), false); + assert.equal(isBinaryContent(sample(8191, [0xff])), true); + assert.equal(isBinaryContent(Uint8Array.from([0x68, 0x69, 0x00])), true); + const valid = new Uint8Array(8192); + valid.fill(0x61); + assert.equal(isBinaryContent(valid), false); +}); + +test('a 4xx that names reasoning_effort is retried once without it', async () => { + const efforts = []; + const wrapped = async (req) => { + efforts.push(req.reasoningEffort ?? null); + if (efforts.length === 1) { + const err = new Error('LLM request failed (400) from example: invalid reasoning_effort'); + err.status = 400; + throw err; + } + return response('no'); + }; + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { + ...baseOpts, callLlm: wrapped, retryBaseMs: 0, retries: 0, + }); + assert.deepEqual(efforts, ['low', null]); + assert.equal(result.decisions[0].decision, 'no'); + assert.equal(result.decisions[0].failedOpen, false); + assert.equal(result.failedOpen, 0); +}); + +test('billing reservation errors are retried then fail open', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return new Error('billing reservation failed'); + }); + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { + ...baseOpts, callLlm: mock.fn, retries: 2, retryBaseMs: 0, + }); + assert.equal(n, 3); + assert.equal(result.decisions[0].decision, 'yes'); + assert.equal(result.decisions[0].failedOpen, true); + assert.equal(result.failedOpen, 1); +}); + +test('cache read and write errors are logged and do not abort the run', async () => { + const warnings = []; + const original = console.warn; + console.warn = (...args) => warnings.push(args.join(' ')); + try { + let puts = 0; + const cache = { + async get() { + throw new Error('disk read failed'); + }, + async put() { + puts += 1; + throw new Error('disk write failed'); + }, + async flush() { + throw new Error('disk flush failed'); + }, + }; + const mock = llm(() => response('no')); + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { + model: 'm', cache, apiKey: 'k', callLlm: mock.fn, + }); + assert.equal(result.decisions[0].decision, 'no'); + assert.equal(result.decisions[0].failedOpen, false); + assert.equal(puts, 1); + assert.match(warnings.join('\n'), /disk read failed/); + assert.match(warnings.join('\n'), /disk write failed/); + assert.match(warnings.join('\n'), /disk flush failed/); + assert.equal('a.py' in result.files, false); + } finally { + console.warn = original; + } +}); + test('lockfiles and generated files are skipped unless ignore is configured', async () => { assert.equal(isIgnoredPath('package-lock.json', DEFAULT_IGNORE_PATTERNS), true); assert.equal(isIgnoredPath('app/yarn.lock', DEFAULT_IGNORE_PATTERNS), true); @@ -862,6 +1031,8 @@ test('lockfiles and generated files are skipped unless ignore is configured', as assert.equal(isIgnoredPath('dist/app.js'), true); assert.equal(isIgnoredPath('web/dist/main.min.js'), true); assert.equal(isIgnoredPath('src/app.min.js'), true); + assert.equal(isIgnoredPath('src/app.min.mjs'), true); + assert.equal(isIgnoredPath('Src/Package-Lock.json'), true); assert.equal(isIgnoredPath('keep.py'), false); const seen = []; diff --git a/test/fixtures/parity/relevance.json b/test/fixtures/parity/relevance.json new file mode 100644 index 0000000..b6c894d --- /dev/null +++ b/test/fixtures/parity/relevance.json @@ -0,0 +1,149 @@ +{ + "promptVersion": 3, + "maxOutputTokens": 1024, + "reasoningEffort": "low", + "tokenEstimate": { + "rule": "characters/4", + "empty": 0, + "minimumNonEmpty": 1, + "examples": [ + { "text": "", "tokens": 0 }, + { "text": "abcd", "tokens": 1 }, + { "text": "abcde", "tokens": 1 }, + { "text": "abcdefgh", "tokens": 2 } + ] + }, + "systemPrompt": "You are a relevance filter for a code-editing task. Decide whether the given file must be read or edited to accomplish the task, or is necessary context for that edit. Reply with a single line that starts with exactly \"yes\" or exactly \"no\", followed by a short reason. Answer \"yes\" when the file is likely to need changes or is required context. Answer \"no\" when it is unrelated. Do not write a diff.", + "userPrompt": { + "withoutNote": "Task:\n{intent}\n\nFile path:\n{path}\n\nFile contents:\n{content}\n\nIs this file relevant to the task? yes/no", + "withNote": "Task:\n{intent}\n\nFile path:\n{path}\n\n{note}\n\nFile contents:\n{content}\n\nIs this file relevant to the task? yes/no" + }, + "cacheKey": { + "algorithm": "sha256-hex of length-prefixed utf-8 fields", + "fields": ["promptVersion", "model", "tokenLimit", "effort", "intent", "path", "content"], + "encoding": "For each field, write the decimal UTF-8 byte length, a newline, the UTF-8 bytes, and a newline. promptVersion and tokenLimit are decimal strings. effort is the reasoning-effort string, or empty when the parameter is omitted. content is the body sent to the model after truncation, not the original file. Different paths never share a key.", + "vectors": [ + { + "promptVersion": "3", + "model": "m", + "tokenLimit": "1024", + "effort": "low", + "intent": "edit the target", + "path": "src/target.py", + "content": "", + "sha256": "beef6d830371f183c68a46946633626a112d45fd6461ecddec666fc1d129af4a" + }, + { + "promptVersion": "3", + "model": "m", + "tokenLimit": "1024", + "effort": "low", + "intent": "edit the target", + "path": "unrelated.py", + "content": "", + "sha256": "7f10396aa96a1ac4e70209d7f8415efe532c3629a2c7f6ad19fb3f5237138046" + }, + { + "promptVersion": "3", + "model": "m", + "tokenLimit": "128", + "effort": "low", + "intent": "task", + "path": "a.py", + "content": "v = 1\n", + "sha256": "6dcec468a48967b4d50a063a3435f7e481c8db85e7cd4cb87828a7a80e3f3285" + }, + { + "promptVersion": "3", + "model": "m", + "tokenLimit": "1024", + "effort": "", + "intent": "task", + "path": "a.py", + "content": "é", + "sha256": "6f0021af86e723832cfb176d7fc140208b0d2c5d39b31b9f5a74003668972fdf" + } + ] + }, + "parser": [ + { "raw": "yes", "decision": "yes" }, + { "raw": "Yes.", "decision": "yes" }, + { "raw": "NO\nthis file is a changelog", "decision": "no" }, + { "raw": "Answer: yes, the parser lives here", "decision": "yes" }, + { "raw": "Decision: no", "decision": "no" }, + { "raw": "\"yes\"", "decision": "yes" }, + { "raw": "'no'", "decision": "no" }, + { "raw": "**yes**", "decision": "yes" }, + { "raw": "`no`", "decision": "no" }, + { "raw": "Answer: **yes**", "decision": "yes" }, + { "raw": "no, this file is irrelevant\nyes, the function to rename is here", "decision": "yes" }, + { "raw": "There is no doubt this file is required", "decision": null }, + { "raw": "The answer is no.", "decision": null }, + { "raw": "The file defines helpers used by the task", "decision": null }, + { "raw": "I would say no because it is unrelated", "decision": null }, + { "raw": "nope", "decision": null }, + { "raw": "", "decision": null }, + { "raw": "yes for the helper, no for the tests", "decision": "yes" } + ], + "malformedIsFailedOpen": true, + "truncation": [ + { "text": "éX", "maxBytes": 2, "expected": "é" }, + { "text": "€X", "maxBytes": 2, "expected": "" }, + { "text": "😀X", "maxBytes": 3, "expected": "" }, + { "text": "a€b", "maxBytes": 4, "expected": "a€" } + ], + "binary": { + "sampleBytes": 8192, + "rule": "Decode the first 8192 bytes as UTF-8. Tolerate only an incomplete trailing sequence of at most 3 bytes. A NUL or any other invalid byte is binary.", + "cases": [ + { "name": "2-byte cut", "pad": 8191, "tail": [195], "binary": false }, + { "name": "3-byte cut", "pad": 8190, "tail": [226, 130], "binary": false }, + { "name": "4-byte cut", "pad": 8189, "tail": [240, 159, 152], "binary": false }, + { "name": "invalid lead", "pad": 8191, "tail": [255], "binary": true }, + { "name": "nul", "pad": 0, "tail": [104, 105, 0], "binary": true } + ] + }, + "ignore": { + "caseSensitive": false, + "glob": "** crosses directories, * does not. Matching is on the normalized relative path.", + "patterns": [ + "**/package-lock.json", + "**/yarn.lock", + "**/pnpm-lock.yaml", + "**/bun.lock", + "**/bun.lockb", + "**/npm-shrinkwrap.json", + "**/Cargo.lock", + "**/Gemfile.lock", + "**/poetry.lock", + "**/Pipfile.lock", + "**/composer.lock", + "**/go.sum", + "**/uv.lock", + "**/pdm.lock", + "**/flake.lock", + "**/pubspec.lock", + "**/packages.lock.json", + "**/Package.resolved", + "**/*.min.js", + "**/*.min.mjs", + "**/*.min.css", + "**/*.map", + "**/dist/**", + "**/build/**", + "**/.next/**", + "**/coverage/**", + "**/__pycache__/**", + "**/*.pyc", + "**/*.pyo", + "**/*.generated.*" + ], + "examples": [ + { "path": "package-lock.json", "ignored": true }, + { "path": "Src/Package-Lock.json", "ignored": true }, + { "path": "web/dist/app.js", "ignored": true }, + { "path": "src/app.min.mjs", "ignored": true }, + { "path": "src/app.js", "ignored": false } + ] + } +} diff --git a/test/relevance_parity.test.js b/test/relevance_parity.test.js new file mode 100644 index 0000000..05b3dc8 --- /dev/null +++ b/test/relevance_parity.test.js @@ -0,0 +1,161 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { + filterRelevant, + RelevanceCache, + parseRelevanceAnswer, + isBinaryContent, + isIgnoredPath, + countTokens, + DEFAULT_IGNORE_PATTERNS, + FILTER_PROMPT_VERSION, + PROMPT_VERSION, + DEFAULT_FILTER_MAX_OUTPUT_TOKENS, + DEFAULT_FILTER_REASONING_EFFORT, +} from '../src/index.js'; + +const fixture = JSON.parse(readFileSync( + fileURLToPath(new URL('./fixtures/parity/relevance.json', import.meta.url)), + 'utf8', +)); + +function lengthPrefixed(parts) { + const chunks = []; + for (const part of parts) { + const bytes = Buffer.from(part, 'utf8'); + chunks.push(Buffer.from(`${bytes.length}\n`, 'utf8')); + chunks.push(bytes); + chunks.push(Buffer.from('\n', 'utf8')); + } + return Buffer.concat(chunks); +} + +function oracleKey(vector) { + const parts = [ + String(vector.promptVersion), + String(vector.model), + String(vector.tokenLimit), + vector.effort == null ? '' : String(vector.effort), + String(vector.intent), + String(vector.path), + String(vector.content), + ]; + return createHash('sha256').update(lengthPrefixed(parts)).digest('hex'); +} + +function fill(template, fields) { + return template.replace(/\{(\w+)\}/g, (_, name) => fields[name] ?? ''); +} + +test('parity fixture: prompt version, token estimate, and ignore list', () => { + assert.equal(PROMPT_VERSION, fixture.promptVersion); + assert.equal(FILTER_PROMPT_VERSION, fixture.promptVersion); + assert.equal(DEFAULT_FILTER_MAX_OUTPUT_TOKENS, fixture.maxOutputTokens); + assert.equal(DEFAULT_FILTER_REASONING_EFFORT, fixture.reasoningEffort); + for (const example of fixture.tokenEstimate.examples) { + assert.equal(countTokens(example.text), example.tokens); + } + assert.deepEqual(DEFAULT_IGNORE_PATTERNS, fixture.ignore.patterns); + for (const example of fixture.ignore.examples) { + assert.equal(isIgnoredPath(example.path), example.ignored); + } +}); + +test('parity fixture: parser, including quoted, negated, and explanatory answers', () => { + for (const row of fixture.parser) { + assert.equal(parseRelevanceAnswer(row.raw).decision, row.decision, row.raw); + } + assert.equal(fixture.malformedIsFailedOpen, true); +}); + +test('parity fixture: cache key hashes path, effort, and truncated content', async () => { + for (const vector of fixture.cacheKey.vectors) { + assert.equal(oracleKey(vector), vector.sha256, vector.path); + const key = await RelevanceCache.makeKey({ + model: vector.model, + intent: vector.intent, + path: vector.path, + content: vector.content, + maxOutputTokens: Number(vector.tokenLimit), + reasoningEffort: vector.effort === '' ? null : vector.effort, + promptVersion: Number(vector.promptVersion), + }); + assert.equal(key, vector.sha256); + } + assert.notEqual(fixture.cacheKey.vectors[0].sha256, fixture.cacheKey.vectors[1].sha256); +}); + +test('parity fixture: prompt text is path, intent, and content', async () => { + let user = ''; + let system = ''; + await filterRelevant({ 'src/target.py': '' }, 'edit the target', { + model: 'm', + cache: false, + apiKey: 'k', + callLlm: async (req) => { + system = req.messages[0].content; + user = req.messages[1].content; + return { choices: [{ message: { content: 'yes' } }], usage: {} }; + }, + }); + assert.equal(system, fixture.systemPrompt); + assert.equal(user, fill(fixture.userPrompt.withoutNote, { + intent: 'edit the target', + path: 'src/target.py', + content: '', + })); +}); + +test('parity fixture: truncation never emits U+FFFD', async () => { + for (const row of fixture.truncation) { + let body = null; + await filterRelevant({ 'a.py': row.text }, 'task', { + model: 'm', + cache: false, + apiKey: 'k', + maxBytes: row.maxBytes, + maxTokens: null, + onOversize: 'truncate', + callLlm: async (req) => { + body = req.messages[1].content.split('File contents:\n')[1] + .replace(/\n\nIs this file relevant to the task\? yes\/no$/, ''); + return { choices: [{ message: { content: 'no' } }], usage: {} }; + }, + }); + assert.equal(body, row.expected, row.text); + assert.equal(String(body).includes('\uFFFD'), false); + } +}); + +test('parity fixture: binary detection tolerates only a short trailing sequence', () => { + for (const row of fixture.binary.cases) { + const bytes = new Uint8Array(row.pad + row.tail.length); + bytes.fill(0x61, 0, row.pad); + bytes.set(row.tail, row.pad); + assert.equal(isBinaryContent(bytes), row.binary, row.name); + } +}); + +test('parity fixture: a negated no fails open and is not cached', async () => { + let n = 0; + const cache = new RelevanceCache(); + const callLlm = async () => { + n += 1; + return { + choices: [{ message: { content: 'There is no doubt this file is required' } }], + usage: { prompt_tokens: 2, completion_tokens: 2, total_tokens: 4 }, + }; + }; + const opts = { model: 'm', cache, apiKey: 'k', callLlm }; + const first = await filterRelevant({ 'src/target.py': 'x = 1\n' }, 'edit the target', opts); + const second = await filterRelevant({ 'src/target.py': 'x = 1\n' }, 'edit the target', opts); + assert.equal(n, 2); + assert.equal(first.decisions[0].decision, 'yes'); + assert.equal(first.decisions[0].failedOpen, true); + assert.equal(first.failedOpen, 1); + assert.equal(second.decisions[0].cached, false); + assert.equal(fixture.malformedIsFailedOpen, true); +}); From 2c121ba76ef63b99ef25e6548f1894d75c5e44bf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 22:39:44 +0000 Subject: [PATCH 6/6] Match the relevance cache key and token count to parity contract v2 Hash length-prefixed fields with a colon, including mode, note, and the effort that was actually sent. Count Unicode code points, decode valid UTF-8 bytes, and fail open on a bad response envelope. The playground keeps going when no file is relevant, ignores a stale run, and shows classifier tokens. Co-authored-by: ntc-ai --- CHANGELOG.md | 2 +- README.md | 6 +- landing/index.html | 90 +++++++++++--- landing/lib/filter.js | 169 +++++++++++++++++++------- src/filter.d.ts | 48 +++++--- src/filter.js | 169 +++++++++++++++++++------- test/filter_relevant.test.js | 180 ++++++++++++++++++++++++++++ test/fixtures/parity/relevance.json | 121 ++++++++++++++----- test/playground_run.test.js | 91 ++++++++++++++ test/relevance_parity.test.js | 127 ++++++++++++++++---- 10 files changed, 826 insertions(+), 177 deletions(-) create mode 100644 test/playground_run.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fe17d..dc326ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ Not tagged and not published. `package.json` is `0.2.1`; the only release tag an ### API -- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `xiaomi/mimo-v2.6-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000, characters/4) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file, and the cut stays on a code-point boundary. The completion cap is 1024 and the call sends `reasoning_effort: low`. A 4xx that names that parameter is retried once without it. An empty answer, and any answer that is not a leading standalone yes/no, fails open and is not cached. Timeouts, HTTP 5xx, and billing-reservation errors are retried with backoff. Lockfiles and generated files are skipped unless `ignore` is turned off or replaced (case-insensitive; the canonical list is `DEFAULT_IGNORE_PATTERNS`). Prompt version is 3. The cache key hashes prompt version, model, output-token limit, reasoning effort, task, path, and the body that was sent. Each path is classified on its own. A cache read or write error is logged and does not abort the run. Cache-key hashing uses Web Crypto, and on Node 18 (where `crypto` is not a global) the `webcrypto` export of `node:crypto`. `failedOpen` on each decision, and the `failedOpen` count on the result, include malformed answers. Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`, `ignored`). `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. Reported pallets/flask figures (recall 1.0, about 0.8× the tokens of a full-context read) are UNVERIFIED. +- `filterRelevant(files, task, opts)` asks a cheap model, per file, whether that file is relevant to the task. The default model is `xiaomi/mimo-v2.6-flash` (`GPTDIFF_FILTER_MODEL`), with concurrency 8. `onOversize` and `onBinary` are `include`, `exclude`, or `truncate` (defaults `truncate` and `exclude`). `maxBytes` (200000) and `maxTokens` (12000, `ceil(code points / 4)`) cap what is sent; `0` disables either limit. A truncated `yes` still forwards the original file, and the cut stays on a code-point boundary. The completion cap is 1024 and the call sends `reasoning_effort: low`. A 4xx that names that parameter is retried once without it. An empty answer, and any answer that is not a leading standalone yes/no, fails open and is not cached. Timeouts, HTTP 5xx, and billing-reservation errors are retried with backoff. Lockfiles and generated files are skipped unless `ignore` is turned off or replaced (case-insensitive; the canonical list is `DEFAULT_IGNORE_PATTERNS`). Prompt version is 3. The cache key hashes prompt version, model, output-token limit, the reasoning effort actually sent, task, path, mode (`full`, `truncated`, or `binary`), the preview note, and the body that was sent. Each field is length-prefixed as `byteLength:utf8\n`. A lookup tries the configured effort, then the empty effort used when that parameter was rejected. Each path is classified on its own. A cache read or write error is logged and does not abort the run. Cache-key hashing uses Web Crypto, and on Node 18 (where `crypto` is not a global) the `webcrypto` export of `node:crypto`. `failedOpen` on each decision, and the `failedOpen` count on the result, include malformed answers. Decisions use camelCase: `promptTokens`, `completionTokens`, `skippedReason` (`binary`, `oversize`, `truncated`, `ignored`). `buildEnvironment(files, { decisions })` keeps only the `yes` files for `generateDiff`. Reported pallets/flask figures (recall 1.0, about 0.8× the tokens of a full-context read) are UNVERIFIED. - Renamed the package from `gptdiff-js` to `patchling` (npm name `patchling`; an intermediate publish used `gptdiff`). - `generateDiff` and `smartapply` stream tokens through `opts.onToken`. `generateDiff` reports token usage through `opts.onUsage`. - Ship JSDoc-generated `.d.ts` files, with `types` conditions on the package exports. diff --git a/README.md b/README.md index 5738b49..03ca6c5 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,8 @@ const next = await smartapply(diff, files); - `opts.model`, `opts.apiKey`, `opts.baseUrl` — same key and base URL resolution as `generateDiff` (`GPTDIFF_LLM_API_KEY`, `GPTDIFF_LLM_BASE_URL`). The model falls back to `GPTDIFF_FILTER_MODEL`, then `xiaomi/mimo-v2.6-flash`. - `opts.concurrency` — at most this many classifications in flight (default 8). -- `opts.cache` — a `RelevanceCache`, a `{ get, set }` object, or a filesystem path (Node writes `{ version: 1, entries }` atomically; a corrupt file is ignored). Omit it, or pass `true`, for a fresh in-memory cache used only by this call. `null` and `false` turn caching off. `createMemoryCache()` builds another memory cache you can share. The key is the SHA-256 of the length-prefixed classification inputs: prompt version (`3`), model, output-token limit, reasoning effort, task, file path, and the body that was sent (after truncation). Each path is its own decision, including paths with identical contents. A cache read or write error is logged and does not abort the run. Fallback and error decisions are not stored. -- `opts.maxBytes` — UTF-8 size cap (default 200000). `opts.maxTokens` — input-token cap (default 12000; estimated as characters / 4). `0` or `null` disables either limit. +- `opts.cache` — a `RelevanceCache`, a `{ get, set }` object, or a filesystem path (Node writes `{ version: 1, entries }` atomically; a corrupt file is ignored). Omit it, or pass `true`, for a fresh in-memory cache used only by this call. `null` and `false` turn caching off. `createMemoryCache()` builds another memory cache you can share. The key is the SHA-256 of the length-prefixed classification inputs. Each field is written as its UTF-8 byte length, a colon, the bytes, and a newline, in this order: prompt version (`3`), model, output-token limit, the reasoning effort actually sent (`""` when that parameter was omitted or rejected), task, file path, mode (`full`, `truncated`, or `binary`), the preview note (or `""`), and the body that was sent. A later lookup tries the configured effort first, then the empty effort. A full file and a preview truncated to the same bytes do not share a key. Each path is its own decision, including paths with identical contents. A cache read failure is a miss. A cache write failure is logged and the run continues. Fallback and error decisions are not stored. +- `opts.maxBytes` — UTF-8 size cap (default 200000). `opts.maxTokens` — input-token cap (default 12000; `ceil(code points / 4)`, so a surrogate pair is one code point). `0` or `null` disables either limit. Token truncation keeps the first `maxTokens * 4` code points. Byte truncation stays on a UTF-8 code-point boundary. - `opts.onOversize` and `opts.onBinary` — `include` (keep, no model call), `exclude` (drop, no model call), or `truncate` (ask about a preview). Defaults are `truncate` and `exclude`. A `yes` on a truncated preview still forwards the original text in `result.files`. - `opts.ignore` — lockfiles and generated paths are excluded with `skippedReason: "ignored"`. Matching is case-insensitive. `**` crosses directories and `*` does not. `false` classifies every path. An array replaces the default list. The canonical list is `DEFAULT_IGNORE_PATTERNS`: `**/package-lock.json`, `**/yarn.lock`, `**/pnpm-lock.yaml`, `**/bun.lock`, `**/bun.lockb`, `**/npm-shrinkwrap.json`, `**/Cargo.lock`, `**/Gemfile.lock`, `**/poetry.lock`, `**/Pipfile.lock`, `**/composer.lock`, `**/go.sum`, `**/uv.lock`, `**/pdm.lock`, `**/flake.lock`, `**/pubspec.lock`, `**/packages.lock.json`, `**/Package.resolved`, `**/*.min.js`, `**/*.min.mjs`, `**/*.min.css`, `**/*.map`, `**/dist/**`, `**/build/**`, `**/.next/**`, `**/coverage/**`, `**/__pycache__/**`, `**/*.pyc`, `**/*.pyo`, `**/*.generated.*`. The same list is in `test/fixtures/parity/relevance.json`. - `opts.maxOutputTokens` — completion cap (default 1024). The call also sends `reasoning_effort: "low"` so a reasoning model does not spend the whole budget before the yes/no. A 4xx that names that parameter is retried once without it. An empty answer is an error. @@ -141,7 +141,7 @@ Each decision is `{ path, decision, reason, raw, tokens, promptTokens, completio - `result.billedTokens` sums token counts for decisions that were not cache hits. Identical contents at different paths are separate calls. - `buildEnvironment(files, { decisions })` keeps only `"yes"` paths. Passing the decisions array itself does the same. With no second argument, every file is included. `buildEnvironment(result.files)` is the same set. -Python (`filter_relevant` in [patchling-py](https://github.com/255BITS/patchling-py)) returns the same result. Its fields are snake_case: `max_bytes`, `max_tokens`, `on_oversize`, `on_binary`, `skipped_reason`, `prompt_tokens`, `completion_tokens`, `failed_open`. Prompt version 3, the prompt text, the cache-key encoding, the ignore list, and the characters/4 token estimate are the shared contract in `test/fixtures/parity/relevance.json`. +Python (`filter_relevant` in [patchling-py](https://github.com/255BITS/patchling-py)) returns the same result. Its fields are snake_case: `max_bytes`, `max_tokens`, `on_oversize`, `on_binary`, `skipped_reason`, `prompt_tokens`, `completion_tokens`, `failed_open`. Prompt version 3, the prompt text, the cache-key encoding, the ignore list, and the code-point token estimate are the shared contract in `test/fixtures/parity/relevance.json`. Valid UTF-8 bytes are decoded to text. Invalid bytes take the binary path. A response that is not a chat-completion envelope fails that file open and is not cached. A pallets/flask labeling pass has been described as recall 1.0 for `xiaomi/mimo-v2.6-flash`, with the filter using about 0.8× the tokens of a full-context read. Those figures are **UNVERIFIED**. diff --git a/landing/index.html b/landing/index.html index 8a23cde..58f5673 100644 --- a/landing/index.html +++ b/landing/index.html @@ -606,7 +606,7 @@

The same filter in JavaScript and Python.

updated = smartapply(diff, files) -

Each decision is { path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error, failedOpen }. decision is yes or no. A leading standalone yes or no is required; other wording fails open and is not cached. skippedReason is binary, oversize, truncated, ignored, or null. The default model is xiaomi/mimo-v2.6-flash (GPTDIFF_FILTER_MODEL). The cache key hashes prompt version 3, the model, the output-token limit, the reasoning effort, the task, the file path, and the body that was sent. Python uses the same fields in snake_case (skipped_reason, prompt_tokens, failed_open, max_bytes, max_tokens, on_oversize, on_binary). Reported Flask recall 1.0 and 0.8× token figures are UNVERIFIED.

+

Each decision is { path, decision, reason, raw, tokens, promptTokens, completionTokens, cached, skippedReason, error, failedOpen }. decision is yes or no. A leading standalone yes or no is required; other wording fails open and is not cached. skippedReason is binary, oversize, truncated, ignored, or null. The default model is xiaomi/mimo-v2.6-flash (GPTDIFF_FILTER_MODEL). The cache key hashes prompt version 3, the model, the output-token limit, the reasoning effort actually sent, the task, the file path, the mode (`full`, `truncated`, or `binary`), the preview note, and the body that was sent (`byteLength:utf8` for each field). Token estimates use `ceil(code points / 4)`. Python uses the same fields in snake_case (skipped_reason, prompt_tokens, failed_open, max_bytes, max_tokens, on_oversize, on_binary). Reported Flask recall 1.0 and 0.8× token figures are UNVERIFIED.

@@ -900,7 +900,7 @@

${esc(title)}

${cards}`; "main.py": 'from greet import greet\n\ndef main():\n greet("World")\n', "math_utils.py": 'def add(a, b):\n return a + b\n', "README.md": 'A tiny greeting program. The words live in greet.py.\n', - "assets/logo.bin": "\u0000PNG logo", + "assets/logo.bin": "\u0000PNG\u0000logo\u0000", "vendor/HISTORY.txt": "Internal release notes. Not part of the greeting.\n".repeat(12), }, decisions: { @@ -1352,10 +1352,56 @@

${esc(title)}

${cards}`; $("results").replaceChildren(p); } +/* PLAYGROUND_HELPERS:START */ +let activeRun = 0; +function beginRun() { + activeRun += 1; + return activeRun; +} +function isCurrentRun(runId) { + return runId === activeRun; +} +function classifierTokens(decisions) { + if (!decisions) return 0; + let total = 0; + for (const decision of decisions) total += Number(decision && decision.tokens) || 0; + return total; +} +function classifierCost(decisions, price) { + const list = Array.isArray(decisions) ? decisions : []; + let tokens = 0; + let prompt = 0; + let completion = 0; + let priced = true; + for (const decision of list) { + const total = Number(decision && decision.tokens) || 0; + tokens += total; + const hasSplit = !!(decision && (decision.promptTokens != null || decision.completionTokens != null)); + if (total > 0 && !hasSplit) priced = false; + if (hasSplit) { + prompt += Number(decision.promptTokens) || 0; + completion += Number(decision.completionTokens) || 0; + } + } + let dollars = null; + if (priced && price && price.prompt != null && price.completion != null) { + dollars = (prompt * Number(price.prompt) + completion * Number(price.completion)) / 1000000; + } + return { tokens, dollars }; +} +function formatClassifierCost(decisions, price) { + const cost = classifierCost(decisions, price); + const noun = cost.tokens === 1 ? "token" : "tokens"; + let text = "classifier " + cost.tokens + " " + noun; + if (cost.dollars != null) text += " · $" + cost.dollars.toFixed(4); + return text; +} +/* PLAYGROUND_HELPERS:END */ + // The punchline, told only after it happened: which files git apply would // have bounced, and what smartapply did about it. Built from the actual run // outcome, so it's just as true in real mode as in the canned demo. -function renderVerdict(perFile, inputFiles, updated, llmResolved) { +function renderVerdict(perFile, inputFiles, updated, llmResolved, decisions) { const v = $("verdict"); v.innerHTML = ""; const repaired = [...llmResolved].filter((p) => p in updated); @@ -1385,6 +1431,8 @@

${esc(title)}

${cards}`; line("ok", "✓ smartapply — " + okBits.join(" · ")); } + if (decisions) line("ok", formatClassifierCost(decisions, priceOf(FILTER_MODEL))); + if (mode === "demo" && SCENARIOS[scenarioKey].reveal) { const why = document.createElement("p"); why.className = "vwhy"; @@ -1403,7 +1451,6 @@

${esc(title)}

${cards}`; /* ---------- the run ---------- */ async function run() { - if (running) return; const scn = SCENARIOS[scenarioKey]; // Demo mode replays a canned response, so it must run against the canned @@ -1422,6 +1469,7 @@

${esc(title)}

${cards}`; const goal = $("goal").value.trim(); if (!goal) return; + const runId = beginRun(); const inputFiles = { ...files }; running = true; @@ -1439,6 +1487,7 @@

${esc(title)}

${cards}`; const diffPane = $("diffpane"); const onDiffToken = (t) => { + if (!isCurrentRun(runId)) return; diffPane.textContent += t; diffPane.scrollTop = diffPane.scrollHeight; }; @@ -1449,15 +1498,11 @@

${esc(title)}

${cards}`; if (filtering) { $("decisionwrap").style.display = "block"; $("filtersrc").textContent = mode === "demo" ? "· pre-recorded decisions" : "· live, cheap model"; - decisions = await collectDecisions(inputFiles, goal); - if (!decisions.some((d) => d.decision === "yes")) { - $("phase").textContent = "nothing relevant — diff skipped"; - document.querySelector("#status .spin").style.display = "none"; - resultMsg("No file was marked relevant, so generateDiff did not run."); - return; - } + decisions = await collectDecisions(inputFiles, goal, runId); + if (!isCurrentRun(runId)) return; } + if (!isCurrentRun(runId)) return; $("phase").textContent = "generating diff…"; $("diffsrc").textContent = mode === "demo" ? "· pre-recorded reply, real engine" : "· live from NanoGPT"; $("diffwrap").style.display = "block"; @@ -1466,10 +1511,12 @@

${esc(title)}

${cards}`; : { apiKey: key, model: MODEL, onToken: onDiffToken }; const diff = await generateDiff(buildEnvironment(filesForDiff(inputFiles, decisions)), goal, genOpts); + if (!isCurrentRun(runId)) return; renderDiff(diff); const perFile = parseDiffPerFile(diff); if (!diff.trim() || !perFile.length) { + if (!isCurrentRun(runId)) return; resultMsg("⚠ The model returned no applicable diff. Try rephrasing the goal."); $("phase").textContent = "no diff"; document.querySelector("#status .spin").style.display = "none"; @@ -1490,6 +1537,7 @@

${esc(title)}

${cards}`; // means the deterministic patcher handled them. const llmResolved = new Set(); const onApplyToken = (t, path) => { + if (!isCurrentRun(runId)) return; llmResolved.add(path); const card = cards.get(path); if (!card) return; @@ -1505,6 +1553,7 @@

${esc(title)}

${cards}`; : { apiKey: key, model: APPLY_MODEL, onToken: onApplyToken }; const updated = await smartapply(diff, inputFiles, applyOpts); + if (!isCurrentRun(runId)) return; for (const [path, card] of cards) { const pre = card.querySelector("pre"); @@ -1531,18 +1580,24 @@

${esc(title)}

${cards}`; renderEditors(); renderPreview(); - renderVerdict(perFile, inputFiles, updated, llmResolved); + renderVerdict(perFile, inputFiles, updated, llmResolved, decisions); - $("phase").textContent = "done ✓ — the preview and files above are the new map"; + const cost = decisions ? formatClassifierCost(decisions, priceOf(FILTER_MODEL)) : ""; + $("phase").textContent = cost + ? "done ✓ — " + cost + " — the preview and files above are the new map" + : "done ✓ — the preview and files above are the new map"; document.querySelector("#status .spin").style.display = "none"; if (mode === "demo") $("upsell").style.display = "block"; } catch (e) { + if (!isCurrentRun(runId)) return; $("phase").textContent = "error"; document.querySelector("#status .spin").style.display = "none"; resultMsg("Error: " + e.message); } finally { - running = false; - renderRun(); + if (isCurrentRun(runId)) { + running = false; + renderRun(); + } } } @@ -1579,11 +1634,12 @@

${esc(title)}

${cards}`; $("decisionlist").appendChild(row); } -async function collectDecisions(inputFiles, goal) { +async function collectDecisions(inputFiles, goal, runId) { if (mode === "demo") { const canned = SCENARIOS[scenarioKey].decisions || {}; const out = []; for (const path of Object.keys(inputFiles)) { + if (!isCurrentRun(runId)) return out; const src = canned[path] || { decision: "yes", reason: "included — this demo has no recorded decision for the file", @@ -1615,7 +1671,7 @@

${esc(title)}

${cards}`; maxTokens: scn.maxTokens, onOversize: scn.onOversize, onBinary: scn.onBinary, - onDecision: renderDecision, + onDecision: (d) => { if (isCurrentRun(runId)) renderDecision(d); }, }); return result.decisions; } diff --git a/landing/lib/filter.js b/landing/lib/filter.js index 373b62b..8419811 100644 --- a/landing/lib/filter.js +++ b/landing/lib/filter.js @@ -26,7 +26,7 @@ export const DEFAULT_FILTER_CONCURRENCY = 8; /** UTF-8 size cap. `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_BYTES = 200_000; -/** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ +/** Input-token cap (`ceil(code points / 4)`). `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_TOKENS = 12_000; /** @@ -88,7 +88,7 @@ const POLICIES = ['include', 'exclude', 'truncate']; const LEADING_WRAPPER = /^(?:answer|decision)\s*:\s*|^[\s"'`*_“”‘’]+/i; const TRAILING_WRAPPER = /[\s"'`*_“”‘’]+$/; -const LEADING_YES_NO = /^(yes|no)\b/i; +const YES_NO_TOKEN = /^(yes|no)(?=$|\s|[.,!:;)])/i; const FENCE_LINE = /^```[a-zA-Z0-9_-]*\s*$/; const SYSTEM_PROMPT = ( @@ -153,9 +153,11 @@ const SYSTEM_PROMPT = ( * In-memory relevance cache with an optional on-disk JSON file. * * The key is the SHA-256 of the exact classification inputs, length-prefixed - * in this order: prompt version, model, output-token limit, reasoning effort, - * intent, file path, and the content actually sent (after truncation). Two - * paths never share a key, even when their contents match. + * in this order: prompt version, model, output-token limit, reasoning effort + * actually sent, task, path, mode (`full`, `truncated`, or `binary`), the + * preview note (or `""`), and the content actually sent. Two paths never + * share a key, even when their contents match. A full file and a preview + * truncated to the same bytes do not share a key either. */ export class RelevanceCache { /** @@ -172,16 +174,21 @@ export class RelevanceCache { /** * SHA-256 hex of the length-prefixed classification inputs. - * Each field is its UTF-8 bytes, written as `byteLength`, a newline, the + * Each field is its UTF-8 bytes, written as `byteLength`, a colon, the * bytes, and a newline. Field order: prompt version, model, output-token - * limit, reasoning effort (`""` when omitted), intent, path, content. + * limit, reasoning effort actually sent (`""` when omitted or rejected), + * task, path, mode (`full`, `truncated`, or `binary`), note (or `""`), + * content. * @param {object} fields * @param {string} fields.model - * @param {string} fields.intent task text + * @param {string} [fields.intent] task text + * @param {string} [fields.task] task text when `intent` is omitted * @param {string} fields.path * @param {string} fields.content body sent to the model, after truncation + * @param {'full' | 'truncated' | 'binary'} [fields.mode] default `full` + * @param {string} [fields.note] exact preview note, or empty * @param {number} [fields.maxOutputTokens] - * @param {string | null} [fields.reasoningEffort] + * @param {string | null} [fields.reasoningEffort] effort actually sent * @param {number | string} [fields.promptVersion] * @returns {Promise} */ @@ -196,10 +203,14 @@ export class RelevanceCache { const effort = fields.reasoningEffort == null || fields.reasoningEffort === undefined ? '' : String(fields.reasoningEffort); - const intent = String(fields.intent ?? ''); + const task = String(fields.task ?? fields.intent ?? ''); const path = String(fields.path); + const mode = fields.mode == null || fields.mode === undefined ? 'full' : String(fields.mode); + const note = fields.note == null || fields.note === undefined ? '' : String(fields.note); const content = String(fields.content); - return sha256(lengthPrefixed([promptVersion, model, tokenLimit, effort, intent, path, content])); + return sha256(lengthPrefixed([ + promptVersion, model, tokenLimit, effort, task, path, mode, note, content, + ])); } /** @@ -317,7 +328,7 @@ async function getWebCrypto() { /** * Length-prefixed UTF-8 encoding shared with the Python filter. - * Each part is `decimalByteLength + "\\n" + utf8 + "\\n"`. + * Each part is `decimalByteLength + ":" + utf8 + "\\n"`. * @param {string[]} parts * @returns {Uint8Array} */ @@ -329,7 +340,7 @@ function lengthPrefixed(parts) { const out = new Uint8Array(size); let offset = 0; for (const bytes of encoded) { - const header = encoder.encode(`${bytes.length}\n`); + const header = encoder.encode(`${bytes.length}:`); out.set(header, offset); offset += header.length; out.set(bytes, offset); @@ -388,10 +399,11 @@ function unwrapAnswerWrappers(text) { /** * Parse a yes/no relevance answer. * - * Think tags and chain-of-thought reasoning blocks are removed first. The - * decision is a leading standalone `yes` or `no` after whitespace, quotes, - * markdown bold or backticks, and an `Answer:` or `Decision:` prefix. A word - * later in the sentence does not count. Anything else is malformed (`null`); + * Think tags, one fenced block, and chain-of-thought reasoning are removed + * first. Whitespace, quotes, markdown bold or backticks, and one `Answer:` + * or `Decision:` prefix are wrappers. The first token must then be exactly + * `yes` or `no`, and the next character must be the end of the string, + * whitespace, or one of `. , ! : ; )`. Anything else is malformed (`null`); * callers include the file and do not cache it. * * @param {unknown} raw @@ -412,7 +424,7 @@ export function parseRelevanceAnswer(raw) { const cleaned = text.trim(); const unwrapped = unwrapAnswerWrappers(cleaned); if (!unwrapped) return { decision: null, cleaned }; - const match = LEADING_YES_NO.exec(unwrapped); + const match = YES_NO_TOKEN.exec(unwrapped); if (!match) return { decision: null, cleaned }; return { decision: /** @type {'yes' | 'no'} */ (match[1].toLowerCase()), cleaned }; } @@ -464,14 +476,27 @@ function resolveCache(cache) { } /** - * Character-length stand-in for a tokenizer (Python's fallback when tiktoken - * is absent). Empty text is 0; anything else is at least 1. + * Code-point stand-in for a tokenizer. Empty text is 0. Otherwise + * `ceil(codePoints / 4)`. A surrogate pair counts as one code point. * @param {string} text * @returns {number} */ export function countTokens(text) { - if (!text) return 0; - return Math.max(1, Math.floor(String(text).length / 4)); + if (text == null || text === '') return 0; + const codePoints = Array.from(String(text)).length; + if (!codePoints) return 0; + return Math.ceil(codePoints / 4); +} + +/** + * First `count` Unicode code points. Never splits a surrogate pair. + * @param {string} text + * @param {number} count + * @returns {string} + */ +function takeCodePoints(text, count) { + if (count <= 0) return ''; + return Array.from(text).slice(0, count).join(''); } /** @@ -483,7 +508,7 @@ export function countTokens(text) { function truncateText(text, maxBytes, maxTokens) { let out = text; if (maxTokens && countTokens(out) > maxTokens) { - out = out.slice(0, maxTokens * 4); + out = takeCodePoints(out, maxTokens * 4); } if (maxBytes) out = truncateUtf8(out, maxBytes); return out; @@ -607,6 +632,7 @@ function binaryBytes(bytes) { } const text = decodeUtf8AllowingShortTail(sample); if (text == null) return true; + if (sample.length > 0 && text === '') return true; return textLooksBinary(text); } @@ -632,6 +658,8 @@ function textLooksBinary(sample) { function coerceText(content) { if (typeof content === 'string') return content; if (content instanceof Uint8Array) { + const text = decodeUtf8AllowingShortTail(content); + if (text != null) return text; return new TextDecoder('utf-8', { fatal: false }).decode(content); } const name = content === null ? 'null' : typeof content; @@ -729,19 +757,41 @@ function buildUserPrompt(task, path, body, note) { return parts.join('\n\n'); } +/** + * A response that is not a chat-completion envelope. The caller fails that + * file open and does not retry it. + * @param {string} detail + * @returns {Error} + */ +function envelopeError(detail) { + const err = new Error(detail); + /** @type {{ envelope?: boolean }} */ (err).envelope = true; + return err; +} + /** * @param {unknown} response * @returns {{ raw: string, promptTokens: number, completionTokens: number, tokens: number }} */ function usageAndText(response) { - const message = /** @type {{ choices?: Array<{ message?: { content?: string | null } }>, usage?: Record }} */ (response); - const content = message?.choices?.[0]?.message?.content ?? ''; - const usage = message?.usage || {}; + if (!response || typeof response !== 'object' || Array.isArray(response)) { + throw envelopeError('relevance response was not JSON'); + } + const body = /** @type {{ choices?: unknown, usage?: Record }} */ (response); + if (!Array.isArray(body.choices) || body.choices.length === 0) { + throw envelopeError('relevance response envelope was incomplete'); + } + const message = /** @type {{ message?: { content?: unknown } }} */ (body.choices[0]).message; + if (!message || message.content == null) { + throw envelopeError('relevance response envelope was incomplete'); + } + const content = typeof message.content === 'string' ? message.content : String(message.content); + const usage = body.usage || {}; const promptTokens = Number(usage.prompt_tokens ?? usage.input_tokens ?? 0); const completionTokens = Number(usage.completion_tokens ?? usage.output_tokens ?? 0); const tokens = Number(usage.total_tokens ?? (promptTokens + completionTokens)); return { - raw: content || '', + raw: content, promptTokens, completionTokens, tokens, @@ -960,9 +1010,10 @@ async function cachePut(store, key, record) { * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that * JSON file), `true` or omitted (a fresh in-memory cache for this call), or * `null` / `false` (disabled). The cache key hashes the prompt version, model, - * output-token limit, reasoning effort, task, path, and the body that was - * sent (after truncation). Each path is classified on its own, including - * paths whose contents match. + * output-token limit, the reasoning effort actually sent, task, path, mode, + * preview note, and the body that was sent. A cache read failure is a miss. + * A cache write failure is logged and the run continues. Each path is + * classified on its own, including paths whose contents match. * * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` @@ -1031,7 +1082,8 @@ export async function filterRelevant(files, task, opts = {}) { * @typedef {object} PendingFile * @property {string} path * @property {string} body content sent to the model - * @property {string | null} note + * @property {string} note exact preview note, or "" + * @property {'full' | 'truncated' | 'binary'} mode * @property {'truncated' | null} skippedReason */ /** @type {PendingFile[]} */ @@ -1060,14 +1112,21 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - const binary = isBinaryContent(content); - const text = coerceText(content); + let binary = isBinaryContent(content); + /** @type {string} */ + let text = ''; + if (!binary) { + text = coerceText(content); + if (content instanceof Uint8Array && content.length > 0 && text === '') binary = true; + } if (binary) { if (onBinary === 'truncate') { - const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); + const nbytes = content instanceof Uint8Array + ? content.length + : utf8ByteLength(typeof content === 'string' ? content : text); const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; const body = `[binary file, ${nbytes} bytes, contents omitted]`; - pending.push({ path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, mode: 'binary', skippedReason: 'truncated' }); } else { const decision = onBinary === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -1089,7 +1148,7 @@ export async function filterRelevant(files, task, opts = {}) { 'Note: this file was truncated to fit the relevance check ' + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` ); - pending.push({ path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, mode: 'truncated', skippedReason: 'truncated' }); } else { const decision = onOversize === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -1105,7 +1164,7 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - pending.push({ path, body: text, note: null, skippedReason: null }); + pending.push({ path, body: text, note: '', mode: 'full', skippedReason: null }); } if (store && typeof store.load === 'function') { @@ -1130,6 +1189,8 @@ export async function filterRelevant(files, task, opts = {}) { } } + const configuredEffort = reasoningEffort || null; + /** * Key for a lookup before the call, using the configured reasoning effort. * @param {PendingFile} item @@ -1142,17 +1203,35 @@ export async function filterRelevant(files, task, opts = {}) { intent: task, path: item.path, content: item.body, + mode: item.mode, + note: item.note || '', maxOutputTokens, reasoningEffort: effort, promptVersion: FILTER_PROMPT_VERSION, }); } - const configuredEffort = reasoningEffort || null; + /** + * The configured effort is tried first. A judgment stored after the + * provider rejected that parameter lives under the empty effort. + * @param {PendingFile} item + * @returns {Promise<{ hit: RelevanceCacheRecord, key: string } | null>} + */ + async function loadReusable(item) { + const primaryKey = await keyFor(item, configuredEffort); + const primary = await readCache(primaryKey); + if (primary && reusableHit(primary)) return { hit: primary, key: primaryKey }; + if (configuredEffort) { + const fallbackKey = await keyFor(item, null); + const fallback = await readCache(fallbackKey); + if (fallback && reusableHit(fallback)) return { hit: fallback, key: fallbackKey }; + } + return null; + } + let needsCall = false; for (const item of pending) { - const hit = await readCache(await keyFor(item, configuredEffort)); - if (!reusableHit(hit)) { + if (!(await loadReusable(item))) { needsCall = true; break; } @@ -1169,10 +1248,9 @@ export async function filterRelevant(files, task, opts = {}) { * @returns {Promise} */ async function runOne(item) { - const lookupKey = await keyFor(item, configuredEffort); - const hit = await readCache(lookupKey); - if (reusableHit(hit)) { - const decision = decisionFromRecord(item.path, { ...hit, cache_key: lookupKey }, true); + const cached = await loadReusable(item); + if (cached) { + const decision = decisionFromRecord(item.path, { ...cached.hit, cache_key: cached.key }, true); byPath.set(item.path, decision); emit(decision); return; @@ -1212,6 +1290,9 @@ export async function filterRelevant(files, task, opts = {}) { break; } catch (err) { lastError = err; + if (err && typeof err === 'object' && /** @type {{ envelope?: boolean }} */ (err).envelope) { + break; + } if (!strippedEffort && effortUsed && isReasoningEffortRejected(err)) { effortUsed = null; strippedEffort = true; diff --git a/src/filter.d.ts b/src/filter.d.ts index bfcd9b1..758c31a 100644 --- a/src/filter.d.ts +++ b/src/filter.d.ts @@ -6,10 +6,11 @@ export function createMemoryCache(): RelevanceCache; /** * Parse a yes/no relevance answer. * - * Think tags and chain-of-thought reasoning blocks are removed first. The - * decision is a leading standalone `yes` or `no` after whitespace, quotes, - * markdown bold or backticks, and an `Answer:` or `Decision:` prefix. A word - * later in the sentence does not count. Anything else is malformed (`null`); + * Think tags, one fenced block, and chain-of-thought reasoning are removed + * first. Whitespace, quotes, markdown bold or backticks, and one `Answer:` + * or `Decision:` prefix are wrappers. The first token must then be exactly + * `yes` or `no`, and the next character must be the end of the string, + * whitespace, or one of `. , ! : ; )`. Anything else is malformed (`null`); * callers include the file and do not cache it. * * @param {unknown} raw @@ -20,8 +21,8 @@ export function parseRelevanceAnswer(raw: unknown): { cleaned: string; }; /** - * Character-length stand-in for a tokenizer (Python's fallback when tiktoken - * is absent). Empty text is 0; anything else is at least 1. + * Code-point stand-in for a tokenizer. Empty text is 0. Otherwise + * `ceil(codePoints / 4)`. A surrogate pair counts as one code point. * @param {string} text * @returns {number} */ @@ -51,9 +52,10 @@ export function isIgnoredPath(filePath: string, patterns?: string[]): boolean; * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that * JSON file), `true` or omitted (a fresh in-memory cache for this call), or * `null` / `false` (disabled). The cache key hashes the prompt version, model, - * output-token limit, reasoning effort, task, path, and the body that was - * sent (after truncation). Each path is classified on its own, including - * paths whose contents match. + * output-token limit, the reasoning effort actually sent, task, path, mode, + * preview note, and the body that was sent. A cache read failure is a miss. + * A cache write failure is logged and the run continues. Each path is + * classified on its own, including paths whose contents match. * * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` @@ -127,7 +129,7 @@ export const DEFAULT_FILTER_MODEL: "xiaomi/mimo-v2.6-flash"; export const DEFAULT_FILTER_CONCURRENCY: 8; /** UTF-8 size cap. `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_BYTES: 200000; -/** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ +/** Input-token cap (`ceil(code points / 4)`). `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_TOKENS: 12000; /** * Completion cap for the yes/no call. High enough that a reasoning model's @@ -198,31 +200,41 @@ export const DEFAULT_IGNORE_PATTERNS: string[]; * In-memory relevance cache with an optional on-disk JSON file. * * The key is the SHA-256 of the exact classification inputs, length-prefixed - * in this order: prompt version, model, output-token limit, reasoning effort, - * intent, file path, and the content actually sent (after truncation). Two - * paths never share a key, even when their contents match. + * in this order: prompt version, model, output-token limit, reasoning effort + * actually sent, task, path, mode (`full`, `truncated`, or `binary`), the + * preview note (or `""`), and the content actually sent. Two paths never + * share a key, even when their contents match. A full file and a preview + * truncated to the same bytes do not share a key either. */ export class RelevanceCache { /** * SHA-256 hex of the length-prefixed classification inputs. - * Each field is its UTF-8 bytes, written as `byteLength`, a newline, the + * Each field is its UTF-8 bytes, written as `byteLength`, a colon, the * bytes, and a newline. Field order: prompt version, model, output-token - * limit, reasoning effort (`""` when omitted), intent, path, content. + * limit, reasoning effort actually sent (`""` when omitted or rejected), + * task, path, mode (`full`, `truncated`, or `binary`), note (or `""`), + * content. * @param {object} fields * @param {string} fields.model - * @param {string} fields.intent task text + * @param {string} [fields.intent] task text + * @param {string} [fields.task] task text when `intent` is omitted * @param {string} fields.path * @param {string} fields.content body sent to the model, after truncation + * @param {'full' | 'truncated' | 'binary'} [fields.mode] default `full` + * @param {string} [fields.note] exact preview note, or empty * @param {number} [fields.maxOutputTokens] - * @param {string | null} [fields.reasoningEffort] + * @param {string | null} [fields.reasoningEffort] effort actually sent * @param {number | string} [fields.promptVersion] * @returns {Promise} */ static makeKey(fields: { model: string; - intent: string; + intent?: string; + task?: string; path: string; content: string; + mode?: "full" | "truncated" | "binary"; + note?: string; maxOutputTokens?: number; reasoningEffort?: string | null; promptVersion?: number | string; diff --git a/src/filter.js b/src/filter.js index 373b62b..8419811 100644 --- a/src/filter.js +++ b/src/filter.js @@ -26,7 +26,7 @@ export const DEFAULT_FILTER_CONCURRENCY = 8; /** UTF-8 size cap. `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_BYTES = 200_000; -/** Input-token cap (characters / 4 when no tokenizer is bundled). `0` or `null` disables it. */ +/** Input-token cap (`ceil(code points / 4)`). `0` or `null` disables it. */ export const DEFAULT_FILTER_MAX_TOKENS = 12_000; /** @@ -88,7 +88,7 @@ const POLICIES = ['include', 'exclude', 'truncate']; const LEADING_WRAPPER = /^(?:answer|decision)\s*:\s*|^[\s"'`*_“”‘’]+/i; const TRAILING_WRAPPER = /[\s"'`*_“”‘’]+$/; -const LEADING_YES_NO = /^(yes|no)\b/i; +const YES_NO_TOKEN = /^(yes|no)(?=$|\s|[.,!:;)])/i; const FENCE_LINE = /^```[a-zA-Z0-9_-]*\s*$/; const SYSTEM_PROMPT = ( @@ -153,9 +153,11 @@ const SYSTEM_PROMPT = ( * In-memory relevance cache with an optional on-disk JSON file. * * The key is the SHA-256 of the exact classification inputs, length-prefixed - * in this order: prompt version, model, output-token limit, reasoning effort, - * intent, file path, and the content actually sent (after truncation). Two - * paths never share a key, even when their contents match. + * in this order: prompt version, model, output-token limit, reasoning effort + * actually sent, task, path, mode (`full`, `truncated`, or `binary`), the + * preview note (or `""`), and the content actually sent. Two paths never + * share a key, even when their contents match. A full file and a preview + * truncated to the same bytes do not share a key either. */ export class RelevanceCache { /** @@ -172,16 +174,21 @@ export class RelevanceCache { /** * SHA-256 hex of the length-prefixed classification inputs. - * Each field is its UTF-8 bytes, written as `byteLength`, a newline, the + * Each field is its UTF-8 bytes, written as `byteLength`, a colon, the * bytes, and a newline. Field order: prompt version, model, output-token - * limit, reasoning effort (`""` when omitted), intent, path, content. + * limit, reasoning effort actually sent (`""` when omitted or rejected), + * task, path, mode (`full`, `truncated`, or `binary`), note (or `""`), + * content. * @param {object} fields * @param {string} fields.model - * @param {string} fields.intent task text + * @param {string} [fields.intent] task text + * @param {string} [fields.task] task text when `intent` is omitted * @param {string} fields.path * @param {string} fields.content body sent to the model, after truncation + * @param {'full' | 'truncated' | 'binary'} [fields.mode] default `full` + * @param {string} [fields.note] exact preview note, or empty * @param {number} [fields.maxOutputTokens] - * @param {string | null} [fields.reasoningEffort] + * @param {string | null} [fields.reasoningEffort] effort actually sent * @param {number | string} [fields.promptVersion] * @returns {Promise} */ @@ -196,10 +203,14 @@ export class RelevanceCache { const effort = fields.reasoningEffort == null || fields.reasoningEffort === undefined ? '' : String(fields.reasoningEffort); - const intent = String(fields.intent ?? ''); + const task = String(fields.task ?? fields.intent ?? ''); const path = String(fields.path); + const mode = fields.mode == null || fields.mode === undefined ? 'full' : String(fields.mode); + const note = fields.note == null || fields.note === undefined ? '' : String(fields.note); const content = String(fields.content); - return sha256(lengthPrefixed([promptVersion, model, tokenLimit, effort, intent, path, content])); + return sha256(lengthPrefixed([ + promptVersion, model, tokenLimit, effort, task, path, mode, note, content, + ])); } /** @@ -317,7 +328,7 @@ async function getWebCrypto() { /** * Length-prefixed UTF-8 encoding shared with the Python filter. - * Each part is `decimalByteLength + "\\n" + utf8 + "\\n"`. + * Each part is `decimalByteLength + ":" + utf8 + "\\n"`. * @param {string[]} parts * @returns {Uint8Array} */ @@ -329,7 +340,7 @@ function lengthPrefixed(parts) { const out = new Uint8Array(size); let offset = 0; for (const bytes of encoded) { - const header = encoder.encode(`${bytes.length}\n`); + const header = encoder.encode(`${bytes.length}:`); out.set(header, offset); offset += header.length; out.set(bytes, offset); @@ -388,10 +399,11 @@ function unwrapAnswerWrappers(text) { /** * Parse a yes/no relevance answer. * - * Think tags and chain-of-thought reasoning blocks are removed first. The - * decision is a leading standalone `yes` or `no` after whitespace, quotes, - * markdown bold or backticks, and an `Answer:` or `Decision:` prefix. A word - * later in the sentence does not count. Anything else is malformed (`null`); + * Think tags, one fenced block, and chain-of-thought reasoning are removed + * first. Whitespace, quotes, markdown bold or backticks, and one `Answer:` + * or `Decision:` prefix are wrappers. The first token must then be exactly + * `yes` or `no`, and the next character must be the end of the string, + * whitespace, or one of `. , ! : ; )`. Anything else is malformed (`null`); * callers include the file and do not cache it. * * @param {unknown} raw @@ -412,7 +424,7 @@ export function parseRelevanceAnswer(raw) { const cleaned = text.trim(); const unwrapped = unwrapAnswerWrappers(cleaned); if (!unwrapped) return { decision: null, cleaned }; - const match = LEADING_YES_NO.exec(unwrapped); + const match = YES_NO_TOKEN.exec(unwrapped); if (!match) return { decision: null, cleaned }; return { decision: /** @type {'yes' | 'no'} */ (match[1].toLowerCase()), cleaned }; } @@ -464,14 +476,27 @@ function resolveCache(cache) { } /** - * Character-length stand-in for a tokenizer (Python's fallback when tiktoken - * is absent). Empty text is 0; anything else is at least 1. + * Code-point stand-in for a tokenizer. Empty text is 0. Otherwise + * `ceil(codePoints / 4)`. A surrogate pair counts as one code point. * @param {string} text * @returns {number} */ export function countTokens(text) { - if (!text) return 0; - return Math.max(1, Math.floor(String(text).length / 4)); + if (text == null || text === '') return 0; + const codePoints = Array.from(String(text)).length; + if (!codePoints) return 0; + return Math.ceil(codePoints / 4); +} + +/** + * First `count` Unicode code points. Never splits a surrogate pair. + * @param {string} text + * @param {number} count + * @returns {string} + */ +function takeCodePoints(text, count) { + if (count <= 0) return ''; + return Array.from(text).slice(0, count).join(''); } /** @@ -483,7 +508,7 @@ export function countTokens(text) { function truncateText(text, maxBytes, maxTokens) { let out = text; if (maxTokens && countTokens(out) > maxTokens) { - out = out.slice(0, maxTokens * 4); + out = takeCodePoints(out, maxTokens * 4); } if (maxBytes) out = truncateUtf8(out, maxBytes); return out; @@ -607,6 +632,7 @@ function binaryBytes(bytes) { } const text = decodeUtf8AllowingShortTail(sample); if (text == null) return true; + if (sample.length > 0 && text === '') return true; return textLooksBinary(text); } @@ -632,6 +658,8 @@ function textLooksBinary(sample) { function coerceText(content) { if (typeof content === 'string') return content; if (content instanceof Uint8Array) { + const text = decodeUtf8AllowingShortTail(content); + if (text != null) return text; return new TextDecoder('utf-8', { fatal: false }).decode(content); } const name = content === null ? 'null' : typeof content; @@ -729,19 +757,41 @@ function buildUserPrompt(task, path, body, note) { return parts.join('\n\n'); } +/** + * A response that is not a chat-completion envelope. The caller fails that + * file open and does not retry it. + * @param {string} detail + * @returns {Error} + */ +function envelopeError(detail) { + const err = new Error(detail); + /** @type {{ envelope?: boolean }} */ (err).envelope = true; + return err; +} + /** * @param {unknown} response * @returns {{ raw: string, promptTokens: number, completionTokens: number, tokens: number }} */ function usageAndText(response) { - const message = /** @type {{ choices?: Array<{ message?: { content?: string | null } }>, usage?: Record }} */ (response); - const content = message?.choices?.[0]?.message?.content ?? ''; - const usage = message?.usage || {}; + if (!response || typeof response !== 'object' || Array.isArray(response)) { + throw envelopeError('relevance response was not JSON'); + } + const body = /** @type {{ choices?: unknown, usage?: Record }} */ (response); + if (!Array.isArray(body.choices) || body.choices.length === 0) { + throw envelopeError('relevance response envelope was incomplete'); + } + const message = /** @type {{ message?: { content?: unknown } }} */ (body.choices[0]).message; + if (!message || message.content == null) { + throw envelopeError('relevance response envelope was incomplete'); + } + const content = typeof message.content === 'string' ? message.content : String(message.content); + const usage = body.usage || {}; const promptTokens = Number(usage.prompt_tokens ?? usage.input_tokens ?? 0); const completionTokens = Number(usage.completion_tokens ?? usage.output_tokens ?? 0); const tokens = Number(usage.total_tokens ?? (promptTokens + completionTokens)); return { - raw: content || '', + raw: content, promptTokens, completionTokens, tokens, @@ -960,9 +1010,10 @@ async function cachePut(store, key, record) { * `cache` is a {@link RelevanceCache}, a filesystem path (memory plus that * JSON file), `true` or omitted (a fresh in-memory cache for this call), or * `null` / `false` (disabled). The cache key hashes the prompt version, model, - * output-token limit, reasoning effort, task, path, and the body that was - * sent (after truncation). Each path is classified on its own, including - * paths whose contents match. + * output-token limit, the reasoning effort actually sent, task, path, mode, + * preview note, and the body that was sent. A cache read failure is a miss. + * A cache write failure is logged and the run continues. Each path is + * classified on its own, including paths whose contents match. * * Binary files and files over `maxBytes` / `maxTokens` are not sent in full. * `onBinary` and `onOversize` are `include` (keep, no model call), `exclude` @@ -1031,7 +1082,8 @@ export async function filterRelevant(files, task, opts = {}) { * @typedef {object} PendingFile * @property {string} path * @property {string} body content sent to the model - * @property {string | null} note + * @property {string} note exact preview note, or "" + * @property {'full' | 'truncated' | 'binary'} mode * @property {'truncated' | null} skippedReason */ /** @type {PendingFile[]} */ @@ -1060,14 +1112,21 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - const binary = isBinaryContent(content); - const text = coerceText(content); + let binary = isBinaryContent(content); + /** @type {string} */ + let text = ''; + if (!binary) { + text = coerceText(content); + if (content instanceof Uint8Array && content.length > 0 && text === '') binary = true; + } if (binary) { if (onBinary === 'truncate') { - const nbytes = content instanceof Uint8Array ? content.length : utf8ByteLength(text); + const nbytes = content instanceof Uint8Array + ? content.length + : utf8ByteLength(typeof content === 'string' ? content : text); const note = 'Note: this file is binary. Its bytes are not shown; judge relevance from the path and this note.'; const body = `[binary file, ${nbytes} bytes, contents omitted]`; - pending.push({ path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, mode: 'binary', skippedReason: 'truncated' }); } else { const decision = onBinary === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -1089,7 +1148,7 @@ export async function filterRelevant(files, task, opts = {}) { 'Note: this file was truncated to fit the relevance check ' + `(${nBytes} bytes, ${nTokens} tokens). Judge from the portion shown.` ); - pending.push({ path, body, note, skippedReason: 'truncated' }); + pending.push({ path, body, note, mode: 'truncated', skippedReason: 'truncated' }); } else { const decision = onOversize === 'include' ? 'yes' : 'no'; const verb = decision === 'yes' ? 'included' : 'excluded'; @@ -1105,7 +1164,7 @@ export async function filterRelevant(files, task, opts = {}) { continue; } - pending.push({ path, body: text, note: null, skippedReason: null }); + pending.push({ path, body: text, note: '', mode: 'full', skippedReason: null }); } if (store && typeof store.load === 'function') { @@ -1130,6 +1189,8 @@ export async function filterRelevant(files, task, opts = {}) { } } + const configuredEffort = reasoningEffort || null; + /** * Key for a lookup before the call, using the configured reasoning effort. * @param {PendingFile} item @@ -1142,17 +1203,35 @@ export async function filterRelevant(files, task, opts = {}) { intent: task, path: item.path, content: item.body, + mode: item.mode, + note: item.note || '', maxOutputTokens, reasoningEffort: effort, promptVersion: FILTER_PROMPT_VERSION, }); } - const configuredEffort = reasoningEffort || null; + /** + * The configured effort is tried first. A judgment stored after the + * provider rejected that parameter lives under the empty effort. + * @param {PendingFile} item + * @returns {Promise<{ hit: RelevanceCacheRecord, key: string } | null>} + */ + async function loadReusable(item) { + const primaryKey = await keyFor(item, configuredEffort); + const primary = await readCache(primaryKey); + if (primary && reusableHit(primary)) return { hit: primary, key: primaryKey }; + if (configuredEffort) { + const fallbackKey = await keyFor(item, null); + const fallback = await readCache(fallbackKey); + if (fallback && reusableHit(fallback)) return { hit: fallback, key: fallbackKey }; + } + return null; + } + let needsCall = false; for (const item of pending) { - const hit = await readCache(await keyFor(item, configuredEffort)); - if (!reusableHit(hit)) { + if (!(await loadReusable(item))) { needsCall = true; break; } @@ -1169,10 +1248,9 @@ export async function filterRelevant(files, task, opts = {}) { * @returns {Promise} */ async function runOne(item) { - const lookupKey = await keyFor(item, configuredEffort); - const hit = await readCache(lookupKey); - if (reusableHit(hit)) { - const decision = decisionFromRecord(item.path, { ...hit, cache_key: lookupKey }, true); + const cached = await loadReusable(item); + if (cached) { + const decision = decisionFromRecord(item.path, { ...cached.hit, cache_key: cached.key }, true); byPath.set(item.path, decision); emit(decision); return; @@ -1212,6 +1290,9 @@ export async function filterRelevant(files, task, opts = {}) { break; } catch (err) { lastError = err; + if (err && typeof err === 'object' && /** @type {{ envelope?: boolean }} */ (err).envelope) { + break; + } if (!strippedEffort && effortUsed && isReasoningEffortRejected(err)) { effortUsed = null; strippedEffort = true; diff --git a/test/filter_relevant.test.js b/test/filter_relevant.test.js index 995a152..1bd1f3f 100644 --- a/test/filter_relevant.test.js +++ b/test/filter_relevant.test.js @@ -109,6 +109,11 @@ test('parseRelevanceAnswer is malformed without a leading standalone yes or no', assert.equal(parseRelevanceAnswer('nope').decision, null); assert.equal(parseRelevanceAnswer('The answer is no.').decision, null); assert.equal(parseRelevanceAnswer('yes for the helper, no for the tests').decision, 'yes'); + assert.equal(parseRelevanceAnswer('yes/no').decision, null); + assert.equal(parseRelevanceAnswer('yes-no').decision, null); + assert.equal(parseRelevanceAnswer('yes?').decision, null); + assert.equal(parseRelevanceAnswer('yes!').decision, 'yes'); + assert.equal(parseRelevanceAnswer('no)').decision, 'no'); }); test('parseRelevanceAnswer: quoted yes/no is a decision', () => { @@ -1108,3 +1113,178 @@ test('callLlm sends reasoning_effort only when the filter asks for it', async () assert.equal(bodies[0].max_tokens, 1024); assert.equal('reasoning_effort' in bodies[1], false); }); + +test('a full file and a preview truncated to the same bytes do not share a cache key', async () => { + let n = 0; + const mock = llm(() => { + n += 1; + return response('yes'); + }); + const cache = new RelevanceCache(); + const opts = { model: 'm', cache, apiKey: 'k', callLlm: mock.fn, maxTokens: null }; + await filterRelevant({ 'a.txt': 'abc' }, 'edit', opts); + await filterRelevant({ 'a.txt': 'abcXXXX' }, 'edit', { + ...opts, maxBytes: 3, onOversize: 'truncate', + }); + assert.equal(n, 2); + const full = await RelevanceCache.makeKey({ + model: 'm', intent: 'edit', path: 'a.txt', content: 'abc', mode: 'full', note: '', + }); + const truncated = await RelevanceCache.makeKey({ + model: 'm', intent: 'edit', path: 'a.txt', content: 'abc', mode: 'truncated', note: 'x', + }); + assert.notEqual(full, truncated); +}); + +test('a rejected reasoning_effort is cached under the empty effort', async () => { + const efforts = []; + let n = 0; + const wrapped = async (req) => { + n += 1; + efforts.push(req.reasoningEffort ?? null); + if (efforts.length === 1) { + const err = new Error('LLM request failed (400) from example: invalid reasoning_effort'); + err.status = 400; + throw err; + } + return response('no'); + }; + const cache = new RelevanceCache(); + const opts = { model: 'm', cache, apiKey: 'k', callLlm: wrapped, retryBaseMs: 0, retries: 0 }; + const first = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', opts); + assert.deepEqual(efforts, ['low', null]); + assert.equal(first.decisions[0].decision, 'no'); + assert.equal(first.decisions[0].cached, false); + const lowKey = await RelevanceCache.makeKey({ + model: 'm', intent: 'task', path: 'a.py', content: 'v = 1\n', + reasoningEffort: 'low', mode: 'full', note: '', + }); + const emptyKey = await RelevanceCache.makeKey({ + model: 'm', intent: 'task', path: 'a.py', content: 'v = 1\n', + reasoningEffort: null, mode: 'full', note: '', + }); + assert.equal(await cache.get(lowKey), undefined); + assert.ok(await cache.get(emptyKey)); + const second = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', opts); + assert.equal(second.decisions[0].cached, true); + assert.equal(second.decisions[0].decision, 'no'); + assert.equal(n, 2); +}); + +test('a TypeError is not retried and does not strip reasoning_effort', async () => { + const efforts = []; + const wrapped = async (req) => { + efforts.push(req.reasoningEffort ?? null); + throw new TypeError('choices is not iterable'); + }; + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { + ...baseOpts, callLlm: wrapped, retries: 2, retryBaseMs: 0, + }); + assert.deepEqual(efforts, ['low']); + assert.equal(result.decisions[0].decision, 'yes'); + assert.equal(result.decisions[0].failedOpen, true); + assert.equal(result.failedOpen, 1); +}); + +test('reasoning_effort is not retried when it was not sent', async () => { + let n = 0; + const wrapped = async () => { + n += 1; + const err = new Error('LLM request failed (400) from example: invalid reasoning_effort'); + err.status = 400; + throw err; + }; + const result = await filterRelevant({ 'a.py': 'v = 1\n' }, 'task', { + ...baseOpts, reasoningEffort: null, callLlm: wrapped, retries: 2, retryBaseMs: 0, + }); + assert.equal(n, 1); + assert.equal(result.decisions[0].failedOpen, true); +}); + +test('a bad response envelope fails open, stays uncached, and does not abort other files', async () => { + let n = 0; + const wrapped = async (req) => { + n += 1; + const path = pathsOf(req)[0]; + if (path === 'bad.py') return { usage: { total_tokens: 3 } }; + if (path === 'empty-choices.py') return { choices: [] }; + if (path === 'no-message.py') return { choices: [{}] }; + if (path === 'no-content.py') return { choices: [{ message: {} }] }; + if (path === 'json.py') throw new SyntaxError('Unexpected token'); + return response('no'); + }; + const cache = new RelevanceCache(); + const files = { + 'bad.py': 'a\n', + 'empty-choices.py': 'b\n', + 'no-message.py': 'c\n', + 'no-content.py': 'd\n', + 'json.py': 'e\n', + 'ok.py': 'f\n', + }; + const result = await filterRelevant(files, 'task', { + ...baseOpts, cache, callLlm: wrapped, retries: 2, retryBaseMs: 0, + }); + for (const path of ['bad.py', 'empty-choices.py', 'no-message.py', 'no-content.py', 'json.py']) { + const decision = result.decisions.find((item) => item.path === path); + assert.equal(decision.decision, 'yes', path); + assert.equal(decision.failedOpen, true, path); + assert.equal(decision.cached, false, path); + assert.equal(path in result.files, true, path); + } + assert.equal(result.decisions.find((item) => item.path === 'ok.py').decision, 'no'); + assert.equal(n, 6); + const again = await filterRelevant({ 'bad.py': 'a\n' }, 'task', { + ...baseOpts, cache, callLlm: wrapped, retries: 2, retryBaseMs: 0, + }); + assert.equal(again.decisions[0].cached, false); + assert.equal(n, 7); +}); + +test('valid UTF-8 bytes are decoded and a changed buffer is a separate call', async () => { + const seen = []; + let n = 0; + const mock = llm((req) => { + n += 1; + seen.push(bodyOf(req)); + return response('yes'); + }); + const cache = new RelevanceCache(); + const opts = { ...baseOpts, cache, callLlm: mock.fn }; + const first = await filterRelevant({ 'a.py': new TextEncoder().encode('hello') }, 'task', opts); + assert.equal(first.files['a.py'], 'hello'); + assert.equal(first.decisions[0].skippedReason, null); + assert.equal(seen[0], 'hello'); + const second = await filterRelevant({ 'a.py': new TextEncoder().encode('hello!') }, 'task', opts); + assert.equal(second.files['a.py'], 'hello!'); + assert.equal(n, 2); + const invalid = await filterRelevant({ 'a.py': Uint8Array.from([0xc3]) }, 'task', opts); + assert.equal(invalid.decisions[0].skippedReason, 'binary'); + assert.equal(invalid.decisions[0].decision, 'no'); + assert.equal(n, 2); + assert.equal('a.py' in invalid.files, false); +}); + +test('code-point truncation keeps a whole emoji, CJK run, and combining mark', async () => { + const seen = []; + const mock = llm((req) => { + seen.push(bodyOf(req)); + return response('no'); + }); + const cases = [ + ['abc😀def', 'abc😀'], + ['中文测试啊', '中文测试'], + ['e\u0301XYZ', 'e\u0301XY'], + ]; + for (const [text, expected] of cases) { + await filterRelevant({ 'a.py': text }, 'task', { + ...baseOpts, maxBytes: null, maxTokens: 1, callLlm: mock.fn, + }); + assert.equal(seen.at(-1), expected); + assert.equal(seen.at(-1).includes('\uFFFD'), false); + } + assert.equal(countTokens('abcde'), 2); + assert.equal(countTokens('abc😀def'), 2); + assert.equal(countTokens('中'), 1); + assert.equal(countTokens('e\u0301'), 1); +}); diff --git a/test/fixtures/parity/relevance.json b/test/fixtures/parity/relevance.json index b6c894d..d44d609 100644 --- a/test/fixtures/parity/relevance.json +++ b/test/fixtures/parity/relevance.json @@ -3,65 +3,112 @@ "maxOutputTokens": 1024, "reasoningEffort": "low", "tokenEstimate": { - "rule": "characters/4", + "rule": "ceil(code_points/4)", "empty": 0, - "minimumNonEmpty": 1, "examples": [ { "text": "", "tokens": 0 }, { "text": "abcd", "tokens": 1 }, - { "text": "abcde", "tokens": 1 }, - { "text": "abcdefgh", "tokens": 2 } + { "text": "abcde", "tokens": 2 }, + { "text": "abcdefgh", "tokens": 2 }, + { "text": "abc😀def", "tokens": 2 }, + { "text": "中", "tokens": 1 }, + { "text": "中文测试", "tokens": 1 }, + { "text": "中文测试啊", "tokens": 2 }, + { "text": "e\u0301", "tokens": 1 }, + { "text": "é", "tokens": 1 }, + { "text": "€", "tokens": 1 }, + { "text": "😀", "tokens": 1 } ] }, + "tokenTruncation": [ + { "text": "abc😀def", "maxTokens": 1, "expected": "abc😀" }, + { "text": "中文测试啊", "maxTokens": 1, "expected": "中文测试" }, + { "text": "e\u0301XYZ", "maxTokens": 1, "expected": "e\u0301XY" }, + { "text": "é", "maxTokens": 1, "expected": "é" }, + { "text": "€", "maxTokens": 1, "expected": "€" }, + { "text": "😀", "maxTokens": 1, "expected": "😀" } + ], "systemPrompt": "You are a relevance filter for a code-editing task. Decide whether the given file must be read or edited to accomplish the task, or is necessary context for that edit. Reply with a single line that starts with exactly \"yes\" or exactly \"no\", followed by a short reason. Answer \"yes\" when the file is likely to need changes or is required context. Answer \"no\" when it is unrelated. Do not write a diff.", "userPrompt": { - "withoutNote": "Task:\n{intent}\n\nFile path:\n{path}\n\nFile contents:\n{content}\n\nIs this file relevant to the task? yes/no", - "withNote": "Task:\n{intent}\n\nFile path:\n{path}\n\n{note}\n\nFile contents:\n{content}\n\nIs this file relevant to the task? yes/no" + "withoutNote": "Task:\n{task}\n\nFile path:\n{path}\n\nFile contents:\n{content}\n\nIs this file relevant to the task? yes/no", + "withNote": "Task:\n{task}\n\nFile path:\n{path}\n\n{note}\n\nFile contents:\n{content}\n\nIs this file relevant to the task? yes/no" }, "cacheKey": { "algorithm": "sha256-hex of length-prefixed utf-8 fields", - "fields": ["promptVersion", "model", "tokenLimit", "effort", "intent", "path", "content"], - "encoding": "For each field, write the decimal UTF-8 byte length, a newline, the UTF-8 bytes, and a newline. promptVersion and tokenLimit are decimal strings. effort is the reasoning-effort string, or empty when the parameter is omitted. content is the body sent to the model after truncation, not the original file. Different paths never share a key.", + "fields": ["prompt_version", "model", "max_output_tokens", "effort_sent", "task", "path", "mode", "note", "content"], + "encoding": "For each field, write the decimal UTF-8 byte length, a colon, the UTF-8 bytes, and a newline. prompt_version and max_output_tokens are decimal strings. effort_sent is the reasoning-effort string actually sent, or empty when the parameter was omitted or rejected. mode is full, truncated, or binary. note is the exact preview note, or empty. content is the body sent to the model after truncation, not the original file. A full file and a preview whose bytes match do not share a key. Different paths never share a key.", "vectors": [ { - "promptVersion": "3", + "prompt_version": "3", "model": "m", - "tokenLimit": "1024", - "effort": "low", - "intent": "edit the target", + "max_output_tokens": "1024", + "effort_sent": "low", + "task": "edit the target", "path": "src/target.py", + "mode": "full", + "note": "", "content": "", - "sha256": "beef6d830371f183c68a46946633626a112d45fd6461ecddec666fc1d129af4a" + "sha256": "410e63b003a0df8e18269d1b77297ec8bb901be6578c56261bb6159606d4b208" }, { - "promptVersion": "3", + "prompt_version": "3", "model": "m", - "tokenLimit": "1024", - "effort": "low", - "intent": "edit the target", + "max_output_tokens": "1024", + "effort_sent": "low", + "task": "edit the target", "path": "unrelated.py", + "mode": "full", + "note": "", "content": "", - "sha256": "7f10396aa96a1ac4e70209d7f8415efe532c3629a2c7f6ad19fb3f5237138046" + "sha256": "1beb2b108f5a538cb6188ba61ac10393b2d946ad8855f8ddb789736d95dcfd08" }, { - "promptVersion": "3", + "prompt_version": "3", "model": "m", - "tokenLimit": "128", - "effort": "low", - "intent": "task", + "max_output_tokens": "128", + "effort_sent": "low", + "task": "task", "path": "a.py", + "mode": "full", + "note": "", "content": "v = 1\n", - "sha256": "6dcec468a48967b4d50a063a3435f7e481c8db85e7cd4cb87828a7a80e3f3285" + "sha256": "4c635fbcf4c2316dd7e876dbc7623593b1fdf37c151eb4bc617d501f66ca53d5" }, { - "promptVersion": "3", + "prompt_version": "3", "model": "m", - "tokenLimit": "1024", - "effort": "", - "intent": "task", + "max_output_tokens": "1024", + "effort_sent": "", + "task": "task", "path": "a.py", + "mode": "full", + "note": "", "content": "é", - "sha256": "6f0021af86e723832cfb176d7fc140208b0d2c5d39b31b9f5a74003668972fdf" + "sha256": "b04f8ba70487cf354258d98902bed43b7159e526760315d1d0102d7b80362cb9" + }, + { + "prompt_version": "3", + "model": "m", + "max_output_tokens": "1024", + "effort_sent": "low", + "task": "edit", + "path": "a.txt", + "mode": "full", + "note": "", + "content": "abc", + "sha256": "98f5da60c1d2001abfcb8bb8b9e7f2fa61b6636e7a91efa0bde86f8a6b858347" + }, + { + "prompt_version": "3", + "model": "m", + "max_output_tokens": "1024", + "effort_sent": "low", + "task": "edit", + "path": "a.txt", + "mode": "truncated", + "note": "Note: truncated", + "content": "abc", + "sha256": "accdc713f73c3b5e334ed3d0cd502e18569b921a5a549dadc92c53623878b824" } ] }, @@ -76,12 +123,18 @@ { "raw": "**yes**", "decision": "yes" }, { "raw": "`no`", "decision": "no" }, { "raw": "Answer: **yes**", "decision": "yes" }, + { "raw": "yes!", "decision": "yes" }, + { "raw": "no)", "decision": "no" }, { "raw": "no, this file is irrelevant\nyes, the function to rename is here", "decision": "yes" }, + { "raw": "```json\nyes\n```", "decision": "yes" }, { "raw": "There is no doubt this file is required", "decision": null }, { "raw": "The answer is no.", "decision": null }, { "raw": "The file defines helpers used by the task", "decision": null }, { "raw": "I would say no because it is unrelated", "decision": null }, { "raw": "nope", "decision": null }, + { "raw": "yes/no", "decision": null }, + { "raw": "yes-no", "decision": null }, + { "raw": "yes?", "decision": null }, { "raw": "", "decision": null }, { "raw": "yes for the helper, no for the tests", "decision": "yes" } ], @@ -92,19 +145,26 @@ { "text": "😀X", "maxBytes": 3, "expected": "" }, { "text": "a€b", "maxBytes": 4, "expected": "a€" } ], + "bytes": { + "rule": "Valid UTF-8 bytes decode to text and a non-empty buffer is never decoded as empty. Invalid UTF-8 takes the binary path. Different bytes at the same path do not share a key.", + "validUtf8": "café", + "changed": ["alpha", "beta"] + }, "binary": { "sampleBytes": 8192, - "rule": "Decode the first 8192 bytes as UTF-8. Tolerate only an incomplete trailing sequence of at most 3 bytes. A NUL or any other invalid byte is binary.", + "rule": "Decode the first 8192 bytes as UTF-8. Tolerate only an incomplete trailing sequence of at most 3 bytes. A NUL or any other invalid byte is binary. A non-empty buffer that decodes to empty text is binary.", "cases": [ { "name": "2-byte cut", "pad": 8191, "tail": [195], "binary": false }, { "name": "3-byte cut", "pad": 8190, "tail": [226, 130], "binary": false }, { "name": "4-byte cut", "pad": 8189, "tail": [240, 159, 152], "binary": false }, { "name": "invalid lead", "pad": 8191, "tail": [255], "binary": true }, - { "name": "nul", "pad": 0, "tail": [104, 105, 0], "binary": true } + { "name": "nul", "pad": 0, "tail": [104, 105, 0], "binary": true }, + { "name": "incomplete only", "pad": 0, "tail": [195], "binary": true } ] }, "ignore": { "caseSensitive": false, + "label": "ignored", "glob": "** crosses directories, * does not. Matching is on the normalized relative path.", "patterns": [ "**/package-lock.json", @@ -142,6 +202,7 @@ { "path": "package-lock.json", "ignored": true }, { "path": "Src/Package-Lock.json", "ignored": true }, { "path": "web/dist/app.js", "ignored": true }, + { "path": "src/app.min.js", "ignored": true }, { "path": "src/app.min.mjs", "ignored": true }, { "path": "src/app.js", "ignored": false } ] diff --git a/test/playground_run.test.js b/test/playground_run.test.js new file mode 100644 index 0000000..39bc794 --- /dev/null +++ b/test/playground_run.test.js @@ -0,0 +1,91 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { generateDiff, smartapply, buildEnvironment } from '../src/index.js'; + +const html = readFileSync(new URL('../landing/index.html', import.meta.url), 'utf8'); + +function helpers() { + const start = html.indexOf('/* PLAYGROUND_HELPERS:START */'); + const end = html.indexOf('/* PLAYGROUND_HELPERS:END */'); + assert.ok(start >= 0 && end > start); + const block = html.slice(start, end); + return new Function(`${block}\nreturn { beginRun, isCurrentRun, classifierTokens, classifierCost, formatClassifierCost };`)(); +} + +test('a newer playground run supersedes an older one', () => { + const api = helpers(); + const first = api.beginRun(); + const second = api.beginRun(); + assert.equal(api.isCurrentRun(first), false); + assert.equal(api.isCurrentRun(second), true); + assert.equal(api.isCurrentRun(first), false); +}); + +test('classifier cost is the sum of decision tokens, priced when the split is known', () => { + const api = helpers(); + const canned = [ + { tokens: 48 }, + { tokens: 41 }, + { tokens: 0 }, + ]; + assert.equal(api.classifierTokens(canned), 89); + assert.equal(api.formatClassifierCost(canned, null), 'classifier 89 tokens'); + const live = [ + { tokens: 14, promptTokens: 11, completionTokens: 3 }, + { tokens: 6, promptTokens: 5, completionTokens: 1 }, + ]; + const cost = api.classifierCost(live, { prompt: 0.1, completion: 0.4 }); + assert.equal(cost.tokens, 20); + assert.equal(cost.dollars, (16 * 0.1 + 4 * 0.4) / 1000000); + assert.match(api.formatClassifierCost(live, { prompt: 0.1, completion: 0.4 }), /classifier 20 tokens · \$/); +}); + +test('the playground does not skip an empty context and applies onto the original map', () => { + assert.equal(html.includes('No file was marked relevant, so generateDiff did not run.'), false); + assert.match(html, /generateDiff\(buildEnvironment\(filesForDiff\(inputFiles, decisions\)\)/); + assert.match(html, /smartapply\(diff, inputFiles, applyOpts\)/); + assert.match(html, /if \(!isCurrentRun\(runId\)\) return;/); + const logo = html.match(/"assets\/logo\.bin": "([^"]+)"/); + assert.ok(logo); + assert.ok(logo[1].split('\\u0000').length >= 3, logo[1]); +}); + +test('an empty relevance context still applies a new file onto the full original map', async () => { + const original = { + 'keep.txt': 'untouched\n', + 'other.txt': 'also\n', + }; + let sawKeep = false; + const diff = await generateDiff(buildEnvironment({}), 'add a note', { + callLlm: async (req) => { + const blob = req.messages.map((message) => message.content).join('\n'); + if (blob.includes('keep.txt')) sawKeep = true; + return { + choices: [{ + message: { + content: [ + '```diff', + 'diff --git a/note.txt b/note.txt', + 'new file mode 100644', + '--- /dev/null', + '+++ b/note.txt', + '@@ -0,0 +1 @@', + '+hello', + '```', + ].join('\n'), + }, + }], + }; + }, + }); + assert.equal(sawKeep, false); + const updated = await smartapply(diff, original, { + callLlmForApply: async () => { + throw new Error('the new file should apply without a model'); + }, + }); + assert.equal(updated['keep.txt'], 'untouched\n'); + assert.equal(updated['other.txt'], 'also\n'); + assert.match(updated['note.txt'], /^hello\n?$/); +}); diff --git a/test/relevance_parity.test.js b/test/relevance_parity.test.js index 05b3dc8..a5df863 100644 --- a/test/relevance_parity.test.js +++ b/test/relevance_parity.test.js @@ -17,16 +17,17 @@ import { DEFAULT_FILTER_REASONING_EFFORT, } from '../src/index.js'; -const fixture = JSON.parse(readFileSync( - fileURLToPath(new URL('./fixtures/parity/relevance.json', import.meta.url)), - 'utf8', -)); +const fixturePath = fileURLToPath(new URL('./fixtures/parity/relevance.json', import.meta.url)); +const fixtureBytes = readFileSync(fixturePath); +const fixture = JSON.parse(fixtureBytes.toString('utf8')); + +const FIXTURE_SHA256 = '90b7299375518fbc6cb9bfeacb536f46f95c17ed72f8f66d063592bfdaec4665'; function lengthPrefixed(parts) { const chunks = []; for (const part of parts) { const bytes = Buffer.from(part, 'utf8'); - chunks.push(Buffer.from(`${bytes.length}\n`, 'utf8')); + chunks.push(Buffer.from(`${bytes.length}:`, 'utf8')); chunks.push(bytes); chunks.push(Buffer.from('\n', 'utf8')); } @@ -35,12 +36,14 @@ function lengthPrefixed(parts) { function oracleKey(vector) { const parts = [ - String(vector.promptVersion), + String(vector.prompt_version), String(vector.model), - String(vector.tokenLimit), - vector.effort == null ? '' : String(vector.effort), - String(vector.intent), + String(vector.max_output_tokens), + vector.effort_sent == null ? '' : String(vector.effort_sent), + String(vector.task), String(vector.path), + String(vector.mode), + vector.note == null ? '' : String(vector.note), String(vector.content), ]; return createHash('sha256').update(lengthPrefixed(parts)).digest('hex'); @@ -50,17 +53,25 @@ function fill(template, fields) { return template.replace(/\{(\w+)\}/g, (_, name) => fields[name] ?? ''); } +test('parity fixture bytes match the recorded sha256', () => { + assert.equal(createHash('sha256').update(fixtureBytes).digest('hex'), FIXTURE_SHA256); +}); + test('parity fixture: prompt version, token estimate, and ignore list', () => { assert.equal(PROMPT_VERSION, fixture.promptVersion); assert.equal(FILTER_PROMPT_VERSION, fixture.promptVersion); assert.equal(DEFAULT_FILTER_MAX_OUTPUT_TOKENS, fixture.maxOutputTokens); assert.equal(DEFAULT_FILTER_REASONING_EFFORT, fixture.reasoningEffort); + assert.equal(fixture.tokenEstimate.rule, 'ceil(code_points/4)'); for (const example of fixture.tokenEstimate.examples) { - assert.equal(countTokens(example.text), example.tokens); + assert.equal(countTokens(example.text), example.tokens, JSON.stringify(example.text)); } + assert.equal(fixture.ignore.label, 'ignored'); assert.deepEqual(DEFAULT_IGNORE_PATTERNS, fixture.ignore.patterns); + assert.ok(fixture.ignore.patterns.includes('**/*.min.js')); + assert.ok(fixture.ignore.patterns.includes('**/*.min.mjs')); for (const example of fixture.ignore.examples) { - assert.equal(isIgnoredPath(example.path), example.ignored); + assert.equal(isIgnoredPath(example.path), example.ignored, example.path); } }); @@ -71,24 +82,29 @@ test('parity fixture: parser, including quoted, negated, and explanatory answers assert.equal(fixture.malformedIsFailedOpen, true); }); -test('parity fixture: cache key hashes path, effort, and truncated content', async () => { +test('parity fixture: cache key hashes mode, note, effort, and content', async () => { for (const vector of fixture.cacheKey.vectors) { - assert.equal(oracleKey(vector), vector.sha256, vector.path); + assert.equal(oracleKey(vector), vector.sha256, vector.path + ' ' + vector.mode); const key = await RelevanceCache.makeKey({ model: vector.model, - intent: vector.intent, + task: vector.task, path: vector.path, content: vector.content, - maxOutputTokens: Number(vector.tokenLimit), - reasoningEffort: vector.effort === '' ? null : vector.effort, - promptVersion: Number(vector.promptVersion), + mode: vector.mode, + note: vector.note, + maxOutputTokens: Number(vector.max_output_tokens), + reasoningEffort: vector.effort_sent === '' ? null : vector.effort_sent, + promptVersion: Number(vector.prompt_version), }); assert.equal(key, vector.sha256); } + const full = fixture.cacheKey.vectors.find((vector) => vector.mode === 'full' && vector.content === 'abc'); + const truncated = fixture.cacheKey.vectors.find((vector) => vector.mode === 'truncated' && vector.content === 'abc'); + assert.notEqual(full.sha256, truncated.sha256); assert.notEqual(fixture.cacheKey.vectors[0].sha256, fixture.cacheKey.vectors[1].sha256); }); -test('parity fixture: prompt text is path, intent, and content', async () => { +test('parity fixture: prompt text is task, path, and content', async () => { let user = ''; let system = ''; await filterRelevant({ 'src/target.py': '' }, 'edit the target', { @@ -103,13 +119,13 @@ test('parity fixture: prompt text is path, intent, and content', async () => { }); assert.equal(system, fixture.systemPrompt); assert.equal(user, fill(fixture.userPrompt.withoutNote, { - intent: 'edit the target', + task: 'edit the target', path: 'src/target.py', content: '', })); }); -test('parity fixture: truncation never emits U+FFFD', async () => { +test('parity fixture: truncation never emits U+FFFD and keeps code points', async () => { for (const row of fixture.truncation) { let body = null; await filterRelevant({ 'a.py': row.text }, 'task', { @@ -128,6 +144,24 @@ test('parity fixture: truncation never emits U+FFFD', async () => { assert.equal(body, row.expected, row.text); assert.equal(String(body).includes('\uFFFD'), false); } + for (const row of fixture.tokenTruncation) { + let body = null; + await filterRelevant({ 'a.py': row.text }, 'task', { + model: 'm', + cache: false, + apiKey: 'k', + maxBytes: null, + maxTokens: row.maxTokens, + onOversize: 'truncate', + callLlm: async (req) => { + body = req.messages[1].content.split('File contents:\n')[1] + .replace(/\n\nIs this file relevant to the task\? yes\/no$/, ''); + return { choices: [{ message: { content: 'no' } }], usage: {} }; + }, + }); + assert.equal(body, row.expected, row.text); + assert.equal(String(body).includes('\uFFFD'), false); + } }); test('parity fixture: binary detection tolerates only a short trailing sequence', () => { @@ -139,6 +173,59 @@ test('parity fixture: binary detection tolerates only a short trailing sequence' } }); +test('parity fixture: valid UTF-8 bytes stay text and changed bytes miss the cache', async () => { + const seen = []; + let n = 0; + const cache = new RelevanceCache(); + const callLlm = async (req) => { + n += 1; + seen.push(req.messages[1].content); + return { choices: [{ message: { content: 'yes' } }], usage: {} }; + }; + const opts = { model: 'm', cache, apiKey: 'k', callLlm, maxBytes: null, maxTokens: null }; + const encoded = new TextEncoder().encode(fixture.bytes.validUtf8); + const first = await filterRelevant({ 'a.py': encoded }, 'task', opts); + assert.equal(first.files['a.py'], fixture.bytes.validUtf8); + assert.notEqual(first.files['a.py'], ''); + assert.equal(first.decisions[0].skippedReason, null); + assert.match(seen[0], new RegExp(fixture.bytes.validUtf8)); + + const [left, right] = fixture.bytes.changed; + await filterRelevant({ 'a.py': new TextEncoder().encode(left) }, 'task', opts); + await filterRelevant({ 'a.py': new TextEncoder().encode(right) }, 'task', opts); + assert.equal(n, 3); + + const invalid = await filterRelevant( + { 'a.py': Uint8Array.from([0xff]) }, + 'task', + opts, + ); + assert.equal(invalid.decisions[0].skippedReason, 'binary'); + assert.equal(invalid.decisions[0].decision, 'no'); + assert.equal(n, 3); +}); + +test('parity fixture: an ignored minified module is labeled ignored', async () => { + let n = 0; + const result = await filterRelevant( + { 'src/app.min.mjs': 'var a=1;\n', 'src/app.js': 'export const a = 1;\n' }, + 'task', + { + model: 'm', + cache: false, + apiKey: 'k', + callLlm: async () => { + n += 1; + return { choices: [{ message: { content: 'yes' } }], usage: {} }; + }, + }, + ); + const skipped = result.decisions.find((decision) => decision.path === 'src/app.min.mjs'); + assert.equal(skipped.decision, 'no'); + assert.equal(skipped.skippedReason, fixture.ignore.label); + assert.equal(n, 1); +}); + test('parity fixture: a negated no fails open and is not cached', async () => { let n = 0; const cache = new RelevanceCache();