diff --git a/CHANGELOG.md b/CHANGELOG.md index 52730cb..235ac0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,49 @@ All notable changes to Code Context Control (C3) are documented here. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.120.0] - 2026-09-06 + +### Added — every file map is measured (c3_compress remediation, C0) + +Measured on this repository's telemetry (3,932 tool calls, Jul–Sep 2026): +`c3_compress` was called 132 times, 130 of them `mode='map'`, and its rows +never said which backend built the map, which parser extracted the +symbols, whether the record was already fresh, or whether the map led to +a targeted read. This release adds the instrument before the renderer +changes (2.121.0) so the change can be graded. + +- `file_memory` records carry `parser` (`tree_sitter | regex | generic`); + a pre-2.120.0 record without it is re-extracted on its next update. +- `c3_compress` and `c3_read` rows carry a flat `detail`: requested mode, + backend (`file_memory`, `compressor`, `large_fast_path`, `batch`, + `source`), parser, `cache_hit`, section count, symbols requested, and + why a read served a map (`map_only`, `symbols_not_found`) or how much + source it served (`ranges`, `lines_served`, `file_lines`). +- `aggregate_tool_telemetry` folds them into `map_by_backend` (calls, + tokens, cache hits, ratio p50/p95 per backend/parser) and + `map_read_chain` (maps followed by a read of the same file within five + calls, reads preceded by a map) — the before/after instrument for the + remediation. The chain works on `target` alone, so rows written since + 2.111.0 count. +- `c3 map-eval`: a hand-annotated gold harness for the file map + (`services/bench/map_eval.py`, `tests/map_eval/`), mirroring + `c3 shell-eval`: 16 fixtures across Python, TypeScript, JavaScript, Go, + Rust, Markdown, YAML, JSON, CSS, HTML, R (regex fallback), Java + (generic), a malformed file, a minified bundle and a 250 KB generated + file. Each case scores tokens, symbol recall/precision, signature + completeness, range accuracy and determinism against the annotation — + not against tree-sitter, which would be circular. Baseline committed in + `tests/map_eval/baseline_fixture.json` (docs/map-eval.md). + +### Measured on the baseline (what 2.121.0 fixes) + +Signature completeness 0.20, symbol recall 0.72, 14% of map characters are +emoji and padding; multi-line Python signatures print with newlines; YAML +files that start with a comment render no keys; HTML ids, CSS media +queries, Go const blocks and receivers, Rust impl methods and nested-class +methods are missing; a 20 KB minified line becomes 549 symbols and an +11 MB record. + ## [2.119.0] - 2026-09-05 ### Added — native Codex integration: hooks, thread-bound handoffs, additive installs diff --git a/cli/c3.py b/cli/c3.py index d2afb3a..c7fac5f 100644 --- a/cli/c3.py +++ b/cli/c3.py @@ -92,7 +92,7 @@ # Config CONFIG_DIR = ".c3" CONFIG_FILE = ".c3/config.json" -__version__ = "2.119.0" +__version__ = "2.120.0" def _compress_file_cli(compressor, path, mode="smart", **kw): @@ -1570,6 +1570,42 @@ def cmd_shell_eval(args): sys.exit(1) +def cmd_map_eval(args): + """Run the file-map gold suite and compare it to its baseline. + + Renders the hand-annotated fixtures through the renderer the MCP tool + uses and grades symbol recall, signatures, ranges and determinism. Exit + status 1 when the baseline is violated (a must_pass case failed, an + aggregate fell under its floor or rose over its ceiling, a complete map + grew). See docs/map-eval.md. + """ + from services.bench import map_eval as me + + try: + report = me.run_suite(args.suite, baseline=args.baseline) + except FileNotFoundError as e: + raise RuntimeError(str(e)) + + if args.update_baseline: + path = args.baseline or me.BUNDLED_BASELINES.get(report.suite) + if not path: + raise RuntimeError( + f"suite {report.suite!r} has no bundled baseline; pass --baseline PATH") + floors = json.loads(args.floors) if args.floors else None + ceilings = json.loads(args.ceilings) if args.ceilings else None + me.write_baseline(report, path, floors=floors, ceilings=ceilings) + report.baseline_violations, report.baseline_warnings = me.compare_to_baseline( + report, me.load_baseline(path)) + print(f"baseline written: {path}") + + if args.json: + print(json.dumps(report.to_dict(), indent=2)) + else: + print(report.render()) + if report.baseline_violations: + sys.exit(1) + + def _benchmark_extract_preview(full_path: Path, compressor: CodeCompressor, pattern: str = "", max_lines: int = 50) -> str: """Approximate c3_filter behavior for local benchmarking without MCP startup.""" import re as _re @@ -9261,6 +9297,7 @@ def main(): "stats": cmd_stats, "search-eval": cmd_search_eval, "shell-eval": cmd_shell_eval, + "map-eval": cmd_map_eval, "benchmark": cmd_benchmark, "session-benchmark": cmd_session_benchmark, "benchmark-e2e": cmd_benchmark_e2e, diff --git a/cli/commands/parser.py b/cli/commands/parser.py index aa6f33a..3aeaf51 100644 --- a/cli/commands/parser.py +++ b/cli/commands/parser.py @@ -76,6 +76,19 @@ def build_parser(version: str, parse_cli_ide_arg): help='With --update-baseline: JSON object of metric ceilings, e.g. \'{"bytes_p95": 18432}\'') p_shell_eval.add_argument("--json", action="store_true", help="Emit the full report as JSON") + p_map_eval = subparsers.add_parser( + "map-eval", help="Run the file-map gold suite (docs/map-eval.md)") + p_map_eval.add_argument("--suite", default="fixture", + help="'fixture' (hand-annotated sources, CI gate) or a .jsonl path") + p_map_eval.add_argument("--baseline", default=None, help="Baseline JSON to compare against / write") + p_map_eval.add_argument("--update-baseline", action="store_true", + help="Write aggregates + per-case status to the baseline (floors/ceilings are kept)") + p_map_eval.add_argument("--floors", default=None, + help='With --update-baseline: JSON object of metric floors, e.g. \'{"symbol_recall_mean": 0.9}\'') + p_map_eval.add_argument("--ceilings", default=None, + help='With --update-baseline: JSON object of metric ceilings, e.g. \'{"tokens_p95": 4000}\'') + p_map_eval.add_argument("--json", action="store_true", help="Emit the full report as JSON") + p_compress = subparsers.add_parser("compress", help="Compress a file") p_compress.add_argument("file", help="File to compress") p_compress.add_argument("--mode", choices=["map", "dense_map", "smart", "diff"], default="smart") diff --git a/cli/tools/compress.py b/cli/tools/compress.py index c803df9..252fbcd 100644 --- a/cli/tools/compress.py +++ b/cli/tools/compress.py @@ -14,6 +14,48 @@ from services import access_guard +def map_detail(svc, rel: str, requested_mode: str, *, cache_hit: bool, + symbols: int = 0, fallback: str = "") -> dict: + """Telemetry `detail` for a map served from file_memory (C0, 2.120.0). + + backend names the code path, parser the extractor that built the record + (tree_sitter | regex | generic, "" for a pre-2.120.0 record), cache_hit + whether the record was already fresh. `symbols` is how many the caller + asked for and `fallback` why a read served a map instead of source. + Flat so telemetry.aggregate_tool_telemetry can fold it without a schema. + """ + record = None + try: + record = svc.file_memory.get(rel) + except Exception: + record = None + if not isinstance(record, dict): + record = None + detail = { + "requested_mode": requested_mode, + "backend": "file_memory", + "parser": str((record or {}).get("parser") or ""), + "cache_hit": bool(cache_hit), + "sections": len((record or {}).get("sections") or []), + } + if symbols: + detail["symbols"] = int(symbols) + if fallback: + detail["fallback"] = fallback + return detail + + +def compressor_detail(res: dict, requested_mode: str) -> dict: + """Telemetry `detail` for a CodeCompressor result (smart/diff/bug_scan…).""" + backend = "large_fast_path" if res.get("fast_path") else "compressor" + return { + "requested_mode": requested_mode, + "backend": backend, + "actual_mode": str(res.get("mode") or ""), + "cache_hit": bool(res.get("cache_hit")), + } + + def _run_memory_mcp_cli(args: list, cwd: str, timeout: int = 30) -> tuple: """Run codebase-memory-mcp CLI and return (success, output_or_error).""" binary = shutil.which("codebase-memory-mcp") @@ -150,6 +192,8 @@ def _compress_single(file_path: str, mode: str, svc, finalize, maybe_facts) -> s return ("[file_map:error] not found. To create a new file use " "c3_edit(file_path=..., old_string='', new_string=).") rel = str(full.resolve().relative_to(Path(svc.project_path).resolve())).replace("\\", "/") + # C0 measurement: was the record already fresh before this call? + cache_hit = not svc.file_memory.needs_update(rel) queued = svc.file_memory.drain_queue() completed = [] failed = [] @@ -183,7 +227,8 @@ def _compress_single(file_path: str, mode: str, svc, finalize, maybe_facts) -> s finalize, svc, "c3_compress", {"file_path": file_path, "mode": mode}, res, summary, raw_tokens=raw_tokens, optimized_tokens=map_tokens or None, - response_tokens=map_tokens) + response_tokens=map_tokens, + detail=map_detail(svc, rel, mode, cache_hit=cache_hit)) try: res = svc.compressor.compress_file(str(full), mode) @@ -197,7 +242,8 @@ def _compress_single(file_path: str, mode: str, svc, finalize, maybe_facts) -> s finalize, svc, "c3_compress", {"file_path": file_path}, resp + maybe_facts(svc, Path(file_path).name), mode, raw_tokens=res.get('original_tokens'), - optimized_tokens=res.get('compressed_tokens')) + optimized_tokens=res.get('compressed_tokens'), + detail=compressor_detail(res, mode)) def _compress_batch(paths: list, mode: str, svc, finalize, maybe_facts) -> str: @@ -288,4 +334,6 @@ def _do_one(fp): {"file_path": ",".join(paths), "mode": mode, "batch": True}, body, f"batch {total_ok}/{len(paths)}", raw_tokens=total_raw if measured else None, - optimized_tokens=total_opt if measured else None) + optimized_tokens=total_opt if measured else None, + detail={"requested_mode": mode, "backend": "batch", + "files": len(paths), "ok": total_ok}) diff --git a/cli/tools/read.py b/cli/tools/read.py index 58893f5..8c79ba4 100644 --- a/cli/tools/read.py +++ b/cli/tools/read.py @@ -8,6 +8,7 @@ from cli.tools import _grants from cli.tools._helpers import finalize_with_tokens, maybe_related_facts +from cli.tools.compress import map_detail from core import count_tokens from services import access_guard @@ -301,15 +302,19 @@ def _mark(secs): ranges.extend(main_ranges) if not ranges and symbols: + map_cached = not svc.file_memory.needs_update(rel_path) file_map = svc.file_memory.get_or_build_map(rel_path) resp = f"[read:{file_path}] symbols not found: {symbols}. Showing file map:\n{file_map}" map_tok = count_tokens(file_map) return finalize_with_tokens( finalize, svc, "c3_read", {"file": file_path, "symbols": symbols}, resp, f"{full_file_tokens()}->{map_tok}tok", - raw_tokens=full_file_tokens(), optimized_tokens=map_tok) + raw_tokens=full_file_tokens(), optimized_tokens=map_tok, + detail=map_detail(svc, rel_path, "read", cache_hit=map_cached, + symbols=len(symbols), fallback="symbols_not_found")) if not ranges: + map_cached = not svc.file_memory.needs_update(rel_path) file_map = svc.file_memory.get_or_build_map(rel_path) resp = (file_map + "\n[map only — pass lines=[start,end] or symbols=[...] for exact source]" @@ -319,7 +324,9 @@ def _mark(secs): finalize, svc, "c3_read", {"file": file_path}, resp, f"{full_file_tokens()}->{map_tok}tok", raw_tokens=full_file_tokens(), optimized_tokens=map_tok, - response_tokens=map_tok) + response_tokens=map_tok, + detail=map_detail(svc, rel_path, "read", cache_hit=map_cached, + fallback="map_only")) else: # Sort and merge overlapping ranges ranges.sort() @@ -359,7 +366,16 @@ def _mark(secs): resp = final_content + maybe_related_facts(svc, rel_path, top_k=3, context="read") tokens = count_tokens(resp) summary = f"{full_file_tokens()}->{tokens}tok" if tokens < full_file_tokens() else f"{tokens}tok" + # C0 measurement: what the read asked for and how much source it took. + read_detail = { + "backend": "source", + "symbols": len(symbols) if symbols else 0, + "by_lines": bool(lines), + "ranges": len(ranges), + "lines_served": sum(e - s_ + 1 for s_, e in ranges), + "file_lines": len(content_lines), + } return finalize_with_tokens( finalize, svc, "c3_read", {"file": file_path, "symbols": symbols}, resp, summary, raw_tokens=full_file_tokens(), optimized_tokens=tokens, - response_tokens=tokens) + response_tokens=tokens, detail=read_detail) diff --git a/docs/map-eval.md b/docs/map-eval.md new file mode 100644 index 0000000..a6b7692 --- /dev/null +++ b/docs/map-eval.md @@ -0,0 +1,277 @@ +# File map evaluation — `c3 map-eval` + +A gold harness for the file map C3 hands the agent (`c3_compress mode=map`, +bare `c3_read`, the `c3_search` prefetch). It exists so the compress +remediation phases (C1 canonical renderer, C2 mode retirement, C3 fold into +`c3_read`, C4 large files) can be argued with numbers against fixtures whose +answers were written by hand, and so a map that silently loses a symbol has +somewhere to fail. + +The truth is the annotation, never the parser. Every fixture under +`tests/map_eval/fixtures/` carries, in `tests/map_eval/fixture_suite.jsonl`, +the list of symbols a correct map must show, with the kind, the qualified +name, the parameters and return type as written in the source, and the +1-based inclusive line range — each verified with `cat -n`. Grading +tree-sitter against tree-sitter would be circular; grading it against a +human reading of the file is not. + +## What it measures + +Each case is copied (or generated) into a temporary project directory, a +`FileMemoryStore` is built on it, and the map is rendered through the same +callable the MCP tool uses (see § The renderer under test). The map is +parsed with both grammars below and compared to the annotation. + +Per case: + +| Metric | Meaning | +|---|---| +| `raw_tokens`, `map_tokens`, `ratio` | cl100k tokens of the source and of the map; `map / raw` | +| `symbol_recall` | share of expected symbols present in the map | +| `symbol_precision` | share of rendered symbols that were expected (info: fields, members and other unannotated lines lower it) | +| `signature_completeness` | share of expected symbols with `params` whose rendered params equal the expected ones after whitespace normalization AND, when `ret` is expected, whose return type equals it | +| `range_accuracy` | share of matched symbols with the exact `line_start` and `line_end` | +| `determinism` | the map rendered twice from a cold store (record dropped, new `FileMemoryStore`) is byte-identical | +| `render_ms` | wall time of the first (cold) render, parse included | +| `chrome_share` | share of map characters that are emoji / variation selectors or runs of 2+ spaces after a non-space (column padding) — info | +| `record_bytes` | size of the persisted `.c3/file_memory` record — info | +| `must_not_contain_hits` | forbidden names (inner functions, fenced headings, nested keys) that came out as symbols | + +Aggregates: pass rate per gate, `symbol_recall_mean` (all and must_pass +only), `symbol_precision_mean`, `signature_completeness_mean`, +`range_accuracy_mean`, `determinism_rate`, `chrome_share_mean`, `ratio` +mean/p50/p95, map `tokens` p50/p95/total, `render_ms` p50/p95, the grammar +each map was parsed with, `must_pass_failed`, `xfail_passing`, per-phase +counts. + +Matching. Under the canonical grammar a symbol matches on `(kind, name)` +with the qualified name (`Worker.run`, `Registry.Entry.call`). Under the +legacy grammar it matches on the bare name only (`run`), as a multiset so two +classes sharing a method name need two rendered `run`s; the legacy type word +is mapped onto the canonical kind (function→F, method→M, class→C, +constant→K, variable→V, interface→IF, type→T, enum→E, struct→S, trait→TR, +impl→IM, heading→H, section→SEC, property→P) so the baseline is fair. Within +a name, the candidate with the exact range pairs first, then the same start +line, then whatever is left. Import lines (`I …`) are counted, never matched. + +## Grammar + +### Canonical (phase C1, `services/file_map.py: render_map`) + +``` +# (L ) +I imports # when > 6 imports +I [L-L] # each, when <= 6 +K [L-L] # constant (Python UPPER_CASE, JS/TS/Go/Rust const) +V [L-L] # other module-level binding +C () [L-L] # bases only when present in source + M .() -> [L-L] + P . [L-L] # @property / TS getter +F () -> [L-L] +F async () [L-L] # async prefix before the name (M async … for methods) +IF [L-L] # ts interface +T [L-L] # ts/rs type alias +E [L-L] +S [L-L] # go/rs struct +TR [L-L] # rs trait +IM [L-L] # rs impl block; methods nest as M . +H [L-L] # markdown / html headings +SEC [L-L] # yaml/json top-level keys, css selectors, html #ids +``` + +Every symbol line matches the spec regex, which the harness uses verbatim: + +``` +^(?P *)(?PI|K|V|C|M|P|F|IF|T|E|S|TR|IM|H|SEC) (?Pasync )?(?P.+?)(?:\((?P.*)\))?(?: -> (?P.+?))? \[L(?P\d+)-L(?P\d+)\]$ +``` + +Conventions the annotations follow (and C1 must, to pass): + +- `params` is the parameter list as written, whitespace-normalized to single + spaces, multi-line joined, never truncated, a trailing comma dropped; + `self`/`cls` kept. `ret` only when the source has a return annotation. + Go receivers fold into the qualified name (`M Server.Start(ctx + context.Context) -> error`); the receiver is not a parameter. +- Ranges are the symbol's full extent: decorators and Rust attributes + excluded, body included, one-line symbols `[L7-L7]`. A markdown or html + heading spans its section — to the line before the next heading of the + same or a higher level, else end of file — so a heading range is the read + target for "that section". +- Nested classes qualify through every level (`C Registry.Entry`, `M + Registry.Entry.call`). Rust methods nest under the implemented type, for + inherent and trait impls alike (`IM Point` twice, `M Point.fmt`). +- `K` is a declared constant: a `const` keyword in JS/TS/Go/Rust or an + UPPER_CASE assignment in Python/R. Everything else module-level is `V`. +- Not symbols: inner functions, class fields (unannotated, precision only), + interface/enum members, nested yaml/json keys, css declarations, html + elements without an id, `module.exports`, anything inside a code fence. + Parens inside a heading, selector or key (`H Usage (advanced)`) are part + of the name, not params. + +### Legacy (today, `services/file_memory.py: _format_map`) + +``` +# ( lines, ) +[] + + imports statements (collapsed) # when > 6 imports + - [ ][async ] [()] + + - [ ][async ]() # child method + - # other child +``` + +The legacy parser joins the physical continuation lines a multi-line +signature leaves behind (the params are copied with their newlines), reads +the type from the type word or, where the label has none, from the emoji, +strips the decorations that are formatting rather than identity (`h2: ` on +headings, ` (tag)` on html ids, `impl ` on impl names), qualifies child +lines with their parent, and never sees a return type — today's map has +none, so `signature_completeness` is 0 wherever `ret` is expected. + +## Gates and phases + +| Gate | Meaning | +|---|---| +| `must_pass` | a failure fails CI. Always on: `symbol_recall == 1.0`, no `must_not_contain` hit, deterministic. Plus every check the suite line switches on. | +| `xfail` | known broken; `phase` names the campaign phase that fixes it (`C1` renderer, `C2` mode retirement, `C3` fold, `C4` large files). A pass is reported as *XFAIL PASSING* so the case can be promoted. | +| `info` | measured, never gates (Java has no parser today; the case is the gold a future one must hit). | + +A must_pass case must pass on the renderer at the commit that sets it. +The 2026-09-06 baseline against the legacy renderer: 4 must_pass +(`py_malformed`, `js_basic`, `json_basic`, `py_large_generated`), 10 +xfail(C1), 1 xfail(C4) (`js_minified`), 1 info (`java_generic`). + +Baseline floors and ceilings (`tests/map_eval/baseline_fixture.json`, +hand-set): floors `pass_rate_must_pass 1.0`, `symbol_recall_mean_must_pass +1.0`, `symbol_recall_mean 0.70`, `determinism_rate 1.0`, +`range_accuracy_mean 0.95`; ceilings `tokens_p95 10000`, `ratio_p95 1.10`, +`chrome_share_mean 0.15` (render time is reported, not gated — CI runners +are not a timing bench). Two per-case comparisons +ride along: a case whose `symbol_recall` fell below its baseline value +(violation for must_pass, warning otherwise — the campaign only moves recall +up), and the campaign's token gate — *never more tokens than today's map for +the same file* — applied where the baseline map was complete (recall 1.0): +`map_tokens` above the baseline is a violation for must_pass and a warning +otherwise. A map that missed symbols is no bar to clear, so `yaml_basic` +(10 tokens, nothing rendered) sets no ceiling for C1. + +## Checks vocabulary + +| Check | Meaning | +|---|---| +| `signature_completeness: ` | gate the case when its measured completeness is below the rate (1.0 = every annotated signature exact) | +| `range_accuracy: ` | gate the case when the share of exact ranges is below the rate | +| `max_map_tokens: ` | gate the case when the map exceeds `n` cl100k tokens (`js_minified` uses the 600-token prefetch cap) | + +Both rate metrics are measured for every case; they gate only where the +suite line names them. Cases that fail them on the legacy renderer are +`xfail(C1)` rather than switched off, so C1 is graded by the same line. + +## Case format + +```json +{"id": "py_basic", "file": "py_basic.py", "gate": "xfail", "phase": "C1", + "expected": [{"kind": "M", "name": "Worker.run", "params": "self, job: dict", "ret": "bool", + "line_start": 34, "line_end": 36}, + {"kind": "F", "name": "gather_all", "async": true, "params": "urls: list", "ret": "list", + "line_start": 60, "line_end": 63}], + "must_not_contain": ["wrap"], + "checks": {"signature_completeness": 1.0, "range_accuracy": 1.0}, + "tags": ["python", "ast"], "why": "..."} +``` + +`file` names a fixture under `tests/map_eval/fixtures/`; `generator` (+ +`params`) names a seeded generator in `tests/map_eval/generators.py` whose +`Generated` carries its own `expected` and `must_not_contain`, computed from +the layout it produced (a 20 KB single line and a 250 KB / 400-function +module are not checked in as literals). `params: ""` on an expected symbol +means "no parameters, and I want that checked". + +## Adding a fixture + +1. Write the source under `tests/map_eval/fixtures/.`, 20–80 + lines, exercising one thing the map must get right. Keep every text-mode + `open()` / `read_text()` in a Python fixture on `encoding="utf-8"` — + `tests/test_windows_reliability.py` scans fixtures too. The directory is + excluded from ruff (one fixture is deliberately malformed). +2. `cat -n` the file and write the suite line by hand: every symbol's kind, + qualified name, params/ret as written, exact range; the names that must + not become symbols; the checks to gate on. +3. Run `c3 map-eval` (from a source checkout: `python -m cli.c3 map-eval`). + If the case passes today, `gate: must_pass`; if the renderer misses it, + `gate: xfail` with the phase that will fix it and a one-line finding in + `why`. Do not weaken the annotation to make the renderer pass. +4. `c3 map-eval --update-baseline` so the baseline knows the case + (`test_baseline_covers_every_case` fails until it does). Floors and + ceilings are kept from the file; pass `--floors` / `--ceilings` JSON to + move them, deliberately. + +## Workflow + +``` +c3 map-eval # fixture suite, compare to the bundled baseline, exit 1 on violation +c3 map-eval --json # full report as JSON (per-case missing / extra / misses lists) +c3 map-eval --suite path/to.jsonl --baseline path/to.json +c3 map-eval --update-baseline --floors '{"symbol_recall_mean": 0.9}' --ceilings '{"tokens_p95": 4000}' +python -m pytest tests/test_map_eval.py -q # the CI gate: same suite, same baseline +``` + +When a phase lands: run the suite, read the *XFAIL PASSING* lines, flip +those cases to `must_pass`, refresh the baseline, raise the floors to what +was measured. An xfail that stays failing after its phase is a finding +against that phase, not a reason to relax the annotation. + +## The renderer under test + +`services.bench.map_eval.resolve_renderer(store)` returns the callable and +its name; the report header prints the name (`renderer=`) so a baseline can +never be mistaken for one produced by a different code path. + +- Today: `lambda rel: store.get_or_build_map(rel)` on a + `FileMemoryStore(work_dir)` — the exact call behind `c3_compress mode=map` + and bare `c3_read` (`cli/tools/compress.py`, `cli/tools/read.py`). +- From C1: when `services.file_map.render_map` can be imported it is + preferred, rendered from `store.update(rel)`'s record, and reported as + `services.file_map.render_map`. The suite, annotations and baseline do not + change; only the grammar the parser detects does. + +Determinism is probed by dropping the record and rendering again from a +fresh store, so a renderer that is stable only through its in-memory cache +is caught. + +## Findings from the 2026-09-06 baseline (legacy renderer) + +Concrete for C1, each with the fixture that pins it: + +- No return types anywhere; `@property` and TS getters render as methods + (`py_basic`, `ts_basic`); class bases are not shown (`ts_basic`). +- Multi-line Python signatures are printed with their newlines, breaking the + one-symbol-per-line shape (`py_basic` `render`). +- Methods of a nested class are dropped — only one level of children is + rendered (`py_few_imports`). +- Go: no `const` block, the struct is typed `type`, the receiver is shown as + the parameter list (`go_basic`). +- Rust: no `const`, `impl impl Point`, impl methods flat as `function`, no + return types (`rs_basic`). +- Markdown: the last headings end one line past EOF — the parser splits on + `\n` and counts the trailing empty element (`md_basic`). +- YAML: `_walk_yaml` breaks after the first stream child, so a file that + starts with a comment renders no keys at all; nested mapping ranges end + one line late (`yaml_basic`). +- CSS: a selector spanning two lines is cut to the first line's columns + (`.card .t`); a media query renders as `@media @media` (`css_basic`). +- HTML: `id` attributes are never found (tree-sitter wraps the value in + `quoted_attribute_value`), so no `#id` sections (`html_basic`). +- R (regex path): symbols are named by their raw source line + (`safe_mean <- function(x, na.rm = TRUE) {`), local assignments inside + functions become top-level constants, ranges are off by one + (`r_regex_fallback`). +- Java: no parser, one `(full file)` line (`java_generic`, info). + +For C4: a 20 KB single-line bundle renders 549 `variable vN` lines (7,133 +tokens for a 7,117-token file) and persists the whole line as every +symbol's `signature` — an 11.3 MB record for a 20 KB file (`js_minified`). +The 250 KB / 400-function module is parsed completely today (recall 1.0, +ranges exact, 41 ms) and is `must_pass` so C4's bounds may never drop a +symbol (`py_large_generated`). diff --git a/pyproject.toml b/pyproject.toml index 194eac3..c14c5de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "code-context-control" -version = "2.119.0" +version = "2.120.0" description = "Local MCP code-intelligence for AI coding tools: surgical search/read/edit, agent-config version history, path-level access + masking guards, and a multi-project hub." readme = "README.md" requires-python = ">=3.10" @@ -163,6 +163,8 @@ extend-exclude = [ "commercial", "tui/build.bat", "tui/build.sh", + # map-eval fixtures are parser input, one of them deliberately malformed. + "tests/map_eval/fixtures", ] [tool.ruff.lint] diff --git a/services/bench/map_eval.py b/services/bench/map_eval.py new file mode 100644 index 0000000..cc50fc1 --- /dev/null +++ b/services/bench/map_eval.py @@ -0,0 +1,999 @@ +"""Gold evaluation harness for C3's file map (``c3 map-eval``). + +Renders hand-annotated source fixtures (``tests/map_eval/fixtures``, suite +``tests/map_eval/fixture_suite.jsonl``) through the SAME renderer the MCP +tool uses — today ``FileMemoryStore.get_or_build_map`` (c3_compress +mode=map and bare c3_read), from phase C1 ``services.file_map.render_map`` +when it exists — and grades the map against the annotation: did every +symbol the author listed come out (recall), with the right parameters and +return type (signature completeness), on the right lines (range +accuracy), and nothing that must stay hidden (inner functions, fenced +headings). The annotation is the truth; tree-sitter output is not. + +Two grammars are parsed so a baseline taken against today's renderer and a +run against C1's can be graded from the same fixtures: + +* canonical (C1, docs/map-eval.md § Grammar): ``K Qualified.name(params) + -> ret [La-Lb]`` — matched on (kind, name); +* legacy (today, ``file_memory._format_map``): `` a-b type + name(params)`` — matched on the bare name, with the legacy type words + mapped onto the canonical kinds so the baseline is fair. + +Metrics per case: raw/map tokens and their ratio, symbol recall and +precision, signature completeness, range accuracy, determinism (render +twice from a cold store, byte-equal), render time, chrome share (emoji + +padding, info), persisted record size (info). Aggregates: means and +p50/p95, pass rates per gate, ``must_pass_failed``, ``xfail_passing``. A +checked-in baseline carries FLOORS and CEILINGS; ``compare_to_baseline`` +turns them into violations (exit 1) and warnings, exactly like +``services.bench.shell_eval``. + +Gates per case: ``must_pass`` (a failure fails CI), ``xfail`` (known +broken; ``phase`` names the campaign phase — C1 renderer, C2 mode +retirement, C3 fold, C4 large files — that fixes it; a pass is reported so +the case can be promoted), ``info`` (measured, never gates). A must_pass +case fails when recall < 1.0, a forbidden name is a symbol, the render is +not deterministic, or a check the suite line switches on +(``signature_completeness``, ``range_accuracy``, ``max_map_tokens``) is +missed. +""" + +from __future__ import annotations + +import importlib.util +import json +import re +import shutil +import statistics +import sys +import tempfile +import time +from collections import Counter, defaultdict +from dataclasses import asdict, dataclass, field +from pathlib import Path + +KINDS = ("C", "M", "F", "K", "V", "IF", "T", "E", "S", "TR", "IM", "H", "SEC", "P") +QUALIFIED_KINDS = ("M", "P", "C", "IM") # kinds whose name may carry a `Parent.` prefix +GATES = ("must_pass", "xfail", "info") +PHASES = ("C1", "C2", "C3", "C4") +CHECK_KEYS = ("signature_completeness", "range_accuracy", "max_map_tokens") + +SUITE_DIR = Path(__file__).resolve().parents[2] / "tests" / "map_eval" +FIXTURES_DIR = SUITE_DIR / "fixtures" +GENERATORS_PATH = SUITE_DIR / "generators.py" +BUNDLED_SUITES = {"fixture": SUITE_DIR / "fixture_suite.jsonl"} +BUNDLED_BASELINES = {"fixture": SUITE_DIR / "baseline_fixture.json"} + +# Legacy section types (file_memory / parser) onto canonical kinds. +LEGACY_TYPE_TO_KIND = { + "function": "F", "method": "M", "class": "C", "constant": "K", "variable": "V", + "interface": "IF", "type": "T", "enum": "E", "struct": "S", "trait": "TR", "impl": "IM", + "heading": "H", "section": "SEC", "property": "P", +} +# The legacy renderer's icon table (file_memory._format_map), inverted, so a +# label that carries no type word (headings, sections, child methods) can +# still be typed. Variation selectors are stripped before lookup. +LEGACY_ICON_TO_TYPE = { + "\U0001F3D7": "class", "\u2728": "function", "\u2699": "method", "\U0001F4E6": "import", + "\U0001F48E": "constant", "\U0001F4C4": "variable", "\U0001F9E9": "interface", + "\U0001F3F7": "type", "\U0001F522": "enum", "\U0001F4AC": "comment", "\U0001F527": "property", + "\U0001F3A8": "decorator", "\U0001F516": "heading", "\U0001F4CD": "section", + "\U0001F9F1": "struct", "\U0001F4DC": "trait", "\U0001F6E0": "impl", +} +LEGACY_TYPE_WORDS = set(LEGACY_TYPE_TO_KIND) | {"import", "comment", "decorator", "content"} +LEGACY_ACCESS_WORDS = ("public", "private", "protected") + +# Canonical symbol line (docs/map-eval.md § Grammar). ``I`` lines are import +# summaries, parsed but never matched as symbols. +CANON_LINE_RE = re.compile( + r"^(?P *)(?PI|K|V|C|M|P|F|IF|T|E|S|TR|IM|H|SEC) (?Pasync )?" + r"(?P.+?)(?:\((?P.*)\))?(?: -> (?P.+?))? \[L(?P\d+)-L(?P\d+)\]$") +CANON_HEADER_RE = re.compile(r"^# (?P.+?) \((?P\d+)L (?P\S*)\)$") +CANON_IMPORTS_RE = re.compile(r"^ *I (?P\d+) imports$") + +LEGACY_HEADER_RE = re.compile(r"^# (?P.+?) \((?P\d+) lines, (?P[^)]*)\)$") +LEGACY_IMPORTS_RE = re.compile(r"^ imports\s+(?P\d+) statements \(collapsed\)$") +LEGACY_TOP_RE = re.compile(r"^ (?P\d+)-(?P\d+)\s*(?P\d+)-(?P\d+)\s*(?P +
+
+

Installing

+

Run the installer.

+
+
+

Usage

+

Call c3 map-eval from the project root.

+
+
+
+

Footer text that must not become a symbol.

+
+ + diff --git a/tests/map_eval/fixtures/java_generic.java b/tests/map_eval/fixtures/java_generic.java new file mode 100644 index 0000000..a76b493 --- /dev/null +++ b/tests/map_eval/fixtures/java_generic.java @@ -0,0 +1,28 @@ +package eval; + +import java.util.ArrayList; +import java.util.List; + +/** A greeter: no tree-sitter grammar and no regex patterns, so the generic path runs. */ +public class Greeter { + private static final String DEFAULT_NAME = "world"; + private final List greeted = new ArrayList<>(); + + public Greeter() { + } + + public String greet(String name) { + String who = name == null ? DEFAULT_NAME : name; + greeted.add(who); + return "Hello, " + who + "!"; + } + + public int count() { + return greeted.size(); + } + + public static void main(String[] args) { + Greeter g = new Greeter(); + System.out.println(g.greet(args.length > 0 ? args[0] : null)); + } +} diff --git a/tests/map_eval/fixtures/js_basic.js b/tests/map_eval/fixtures/js_basic.js new file mode 100644 index 0000000..4c30902 --- /dev/null +++ b/tests/map_eval/fixtures/js_basic.js @@ -0,0 +1,39 @@ +const fs = require("fs"); +const path = require("path"); + +const MAX_DEPTH = 8; + +function readConfig(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +const joinAll = (...parts) => path.join(...parts); + +const walk = async (dir, depth = 0) => { + if (depth > MAX_DEPTH) return []; + return fs.readdirSync(dir).map((name) => joinAll(dir, name)); +}; + +class Cache { + constructor(ttl) { + this.ttl = ttl; + this.store = new Map(); + } + + get(key) { + return this.store.get(key); + } + + set(key, value) { + this.store.set(key, value); + return this; + } +} + +function main() { + const cache = new Cache(60); + cache.set("config", readConfig("config.json")); + return cache; +} + +module.exports = { readConfig, walk, Cache, main }; diff --git a/tests/map_eval/fixtures/json_basic.json b/tests/map_eval/fixtures/json_basic.json new file mode 100644 index 0000000..c01d016 --- /dev/null +++ b/tests/map_eval/fixtures/json_basic.json @@ -0,0 +1,21 @@ +{ + "name": "map-eval", + "version": "1.0.0", + "private": true, + "scripts": { + "test": "pytest -q tests/test_map_eval.py", + "lint": "ruff check services/bench/map_eval.py" + }, + "dependencies": { + "tree-sitter": "^0.21", + "tiktoken": "^0.7" + }, + "keywords": [ + "map", + "eval", + "c3" + ], + "engines": { + "python": ">=3.10" + } +} diff --git a/tests/map_eval/fixtures/md_basic.md b/tests/map_eval/fixtures/md_basic.md new file mode 100644 index 0000000..963c10f --- /dev/null +++ b/tests/map_eval/fixtures/md_basic.md @@ -0,0 +1,29 @@ +# Map Eval Guide + +A short guide used as a markdown fixture. Three heading levels, one fence. + +## Installing + +Run the installer from a source checkout: + +```bash +# heading-looking comment inside a fence +c3 init +``` + +### Requirements + +Python 3.10 or newer and the tree-sitter wheels. + +## Usage + +Call `c3 map-eval` from the project root. + +### Flags + +- `--json` emits the report as JSON +- `--update-baseline` rewrites the baseline + +## Troubleshooting + +Nothing yet. Open an issue with the rendered map attached. diff --git a/tests/map_eval/fixtures/py_basic.py b/tests/map_eval/fixtures/py_basic.py new file mode 100644 index 0000000..d5306fa --- /dev/null +++ b/tests/map_eval/fixtures/py_basic.py @@ -0,0 +1,72 @@ +"""Basic Python fixture for the map-eval harness (hand-annotated in fixture_suite.jsonl).""" +import asyncio +import json +import os +import re +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +MAX_RETRIES = 3 +DEFAULT_TIMEOUT = 30.0 + + +def retry(times: int): + """Decorator factory; ``wrap`` is an inner function and must not be a symbol.""" + def wrap(fn): + return fn + return wrap + + +class Worker: + """A worker that runs jobs.""" + + def __init__(self, name: str, timeout: float = DEFAULT_TIMEOUT): + self.name = name + self.timeout = timeout + + @property + def label(self) -> str: + return f"worker:{self.name}" + + def run(self, job: dict) -> bool: + """Run one job.""" + return bool(job) + + +class Scheduler: + """Schedules workers.""" + + def __init__(self, workers: list): + self.workers = workers + + def run(self, jobs: list) -> int: + """Run every job on the first worker.""" + return sum(1 for j in jobs if self.workers[0].run(j)) + + @staticmethod + def build(names: list) -> "Scheduler": + return Scheduler([Worker(n) for n in names]) + + +@retry(times=MAX_RETRIES) +def fetch(url: str, timeout: float = DEFAULT_TIMEOUT) -> Optional[str]: + """Fetch a URL (decorated).""" + return url if timeout > 0 else None + + +async def gather_all(urls: list) -> list: + """Fetch many URLs concurrently.""" + await asyncio.sleep(0) + return [fetch(u) for u in urls] + + +def render( + template: str, + context: dict, + strict: bool = False, +) -> str: + """Multi-line signature with a return annotation.""" + return template.format(**context) if strict else template diff --git a/tests/map_eval/fixtures/py_few_imports.py b/tests/map_eval/fixtures/py_few_imports.py new file mode 100644 index 0000000..bbf513b --- /dev/null +++ b/tests/map_eval/fixtures/py_few_imports.py @@ -0,0 +1,50 @@ +import os +import sys +from pathlib import Path + +CACHE_DIR = Path(os.environ.get("CACHE_DIR", "/tmp/cache")) + + +def build_index(root: Path) -> dict: + """Walk root and index files; ``visit`` is local and must not be a symbol.""" + index = {} + + def visit(path: Path) -> None: + if path.is_file(): + index[path.name] = path.stat().st_size + else: + for child in path.iterdir(): + visit(child) + + visit(root) + return index + + +class Registry: + """Holds handlers keyed by name.""" + + class Entry: + """A single registration (nested class).""" + + def __init__(self, name: str, handler): + self.name = name + self.handler = handler + + def call(self, *args): + return self.handler(*args) + + def __init__(self): + self.entries = {} + + def register(self, name: str, handler) -> "Registry.Entry": + entry = Registry.Entry(name, handler) + self.entries[name] = entry + return entry + + def lookup(self, name: str): + return self.entries.get(name) + + +def main(argv: list = sys.argv) -> int: + print(build_index(CACHE_DIR), argv) + return 0 diff --git a/tests/map_eval/fixtures/py_malformed.py b/tests/map_eval/fixtures/py_malformed.py new file mode 100644 index 0000000..aa39d11 --- /dev/null +++ b/tests/map_eval/fixtures/py_malformed.py @@ -0,0 +1,37 @@ +"""Malformed module: an unclosed paren in ``broken`` splits the file in two. + +Symbols before and after the error must still be listed. +""" +import os + +LIMIT = 10 + + +class Before: + """Defined before the syntax error.""" + + def ok(self, x: int) -> int: + return x + LIMIT + + def twice(self, x: int) -> int: + return self.ok(x) * 2 + + +def broken(a, b): + total = compute(a, (b + 1 + return total + + +def after(y: int) -> int: + """Still a symbol after the syntax error.""" + return y * 2 + + +class After: + """Defined after the syntax error.""" + + def fine(self) -> None: + pass + + def also_fine(self, path: str) -> bool: + return os.path.exists(path) diff --git a/tests/map_eval/fixtures/r_regex_fallback.r b/tests/map_eval/fixtures/r_regex_fallback.r new file mode 100644 index 0000000..c890ad0 --- /dev/null +++ b/tests/map_eval/fixtures/r_regex_fallback.r @@ -0,0 +1,28 @@ +library(stats) +library(utils) + +DEFAULT_ALPHA <- 0.05 + +# Compute the mean of a numeric vector, ignoring NA. +safe_mean <- function(x, na.rm = TRUE) { + if (length(x) == 0) { + return(NA_real_) + } + mean(x, na.rm = na.rm) +} + +# Standardize a vector. +zscore <- function(x) { + (x - safe_mean(x)) / sd(x, na.rm = TRUE) +} + +# Run a t-test and report whether it is significant. +p_value <- function(a, b, alpha = DEFAULT_ALPHA) { + result <- t.test(a, b) + result$p.value < alpha +} + +main <- function() { + sample <- rnorm(10) + print(zscore(sample)) +} diff --git a/tests/map_eval/fixtures/rs_basic.rs b/tests/map_eval/fixtures/rs_basic.rs new file mode 100644 index 0000000..2beb165 --- /dev/null +++ b/tests/map_eval/fixtures/rs_basic.rs @@ -0,0 +1,49 @@ +use std::collections::HashMap; +use std::fmt; + +pub const MAX_ITEMS: usize = 128; + +#[derive(Debug, Clone)] +pub struct Point { + pub x: f64, + pub y: f64, +} + +pub enum Shape { + Circle(Point, f64), + Rect(Point, Point), +} + +pub trait Area { + fn area(&self) -> f64; + fn label(&self) -> String; +} + +impl Point { + pub fn new(x: f64, y: f64) -> Self { + Point { x, y } + } + + pub fn dist(&self, other: &Point) -> f64 { + ((self.x - other.x).powi(2) + (self.y - other.y).powi(2)).sqrt() + } +} + +impl fmt::Display for Point { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "({}, {})", self.x, self.y) + } +} + +pub fn centroid(points: &[Point]) -> Option { + if points.is_empty() { + return None; + } + let n = points.len() as f64; + let (sx, sy) = points.iter().fold((0.0, 0.0), |(ax, ay), p| (ax + p.x, ay + p.y)); + Some(Point::new(sx / n, sy / n)) +} + +pub fn index_by_label(points: &[Point]) -> HashMap { + points.iter().map(|p| (format!("{}", p), p.clone())).collect() +} diff --git a/tests/map_eval/fixtures/ts_basic.ts b/tests/map_eval/fixtures/ts_basic.ts new file mode 100644 index 0000000..471d821 --- /dev/null +++ b/tests/map_eval/fixtures/ts_basic.ts @@ -0,0 +1,51 @@ +import { EventEmitter } from "events"; +import type { Logger } from "./logger"; + +export interface Job { + id: string; + payload: Record; + retries?: number; +} + +export type JobHandler = (job: Job) => Promise; + +export enum JobState { + Pending = "pending", + Running = "running", + Done = "done", +} + +export const DEFAULT_CONCURRENCY = 4; + +export class Queue extends EventEmitter { + private jobs: Job[] = []; + + constructor(private readonly log: Logger) { + super(); + } + + get size(): number { + return this.jobs.length; + } + + push(job: Job): void { + this.jobs.push(job); + this.emit("push", job); + } + + async drain(handler: JobHandler): Promise { + let count = 0; + while (this.jobs.length) { + await handler(this.jobs.shift() as Job); + count += 1; + } + this.log.info(`drained ${count}`); + return count; + } +} + +export const makeQueue = (log: Logger): Queue => new Queue(log); + +export default function createJob(id: string, payload: Record): Job { + return { id, payload, retries: 0 }; +} diff --git a/tests/map_eval/fixtures/yaml_basic.yaml b/tests/map_eval/fixtures/yaml_basic.yaml new file mode 100644 index 0000000..e09018e --- /dev/null +++ b/tests/map_eval/fixtures/yaml_basic.yaml @@ -0,0 +1,20 @@ +# Fixture config for the map-eval harness: five top-level keys. +name: map-eval +version: 1 + +# Paths are project-relative. +paths: + fixtures: tests/map_eval/fixtures + baseline: tests/map_eval/baseline_fixture.json + +# Gate vocabulary, in order of severity. +gates: + - must_pass + - xfail + - info + +floors: + pass_rate_must_pass: 1.0 + symbol_recall_mean: 0.9 + # nested comment inside a mapping + determinism_rate: 1.0 diff --git a/tests/map_eval/generators.py b/tests/map_eval/generators.py new file mode 100644 index 0000000..74de665 --- /dev/null +++ b/tests/map_eval/generators.py @@ -0,0 +1,137 @@ +"""Deterministic fixture generators for the ``c3 map-eval`` suite. + +Two fixtures are too large to check in as literals — a 20 KB single-line +minified bundle and a 250 KB Python module with 400 functions — so they are +built here from a seed, the way ``tests/shell_eval/generators.py`` builds +its streams. Because the generator lays the file out itself, it also knows +the gold: every planted symbol's name, parameters, return annotation and +exact line range. That gold is computed from the layout, never from a +parser, so it cannot be circular. + +Determinism rules: all randomness goes through ``random.Random(seed)``; the +seed is ``params.seed`` or a CRC32 of the case id (never ``hash()``). +""" + +from __future__ import annotations + +import random +import zlib +from dataclasses import dataclass, field + + +@dataclass +class Generated: + filename: str + content: str + expected: list[dict] = field(default_factory=list) + must_not_contain: list[str] = field(default_factory=list) + + +def seed_for(case_id: str, params: dict | None = None) -> int: + params = params or {} + if "seed" in params: + return int(params["seed"]) + return zlib.crc32(case_id.encode("utf-8")) & 0xFFFFFFFF + + +# ── js_minified ───────────────────────────────────────────────────────────── + + +def js_minified(params: dict, rng: random.Random) -> Generated: + """One line of at least ``min_bytes`` (default 20 KiB): ``named`` planted + top-level function declarations (the gold) buried in ``var`` soup of + anonymous function expressions, the shape webpack/terser output takes. + + Gold: the planted functions, all on line 1. The filler ``var`` bindings + are deliberately NOT gold — a map that lists hundreds of one-letter + bindings on a single line is noise, which is what ``max_map_tokens`` + in the suite line measures. + """ + named = int(params.get("named", 6)) + min_bytes = int(params.get("min_bytes", 20 * 1024)) + pieces: list[str] = [] + expected: list[dict] = [] + forbidden: list[str] = [] + + def filler(i: int) -> str: + k = rng.randint(2, 97) + name = f"v{i}" + forbidden.append(name) + return f"var {name}=function(t){{return t*{k}+{i}}};" + + slots = max(named, 1) + per_slot = 6 + i = 0 + for n in range(slots): + for _ in range(per_slot): + pieces.append(filler(i)) + i += 1 + fn = f"c3Plant{n}" + pieces.append(f"function {fn}(a,b){{return a+b+{n}}}") + expected.append({"kind": "F", "name": fn, "params": "a,b", "line_start": 1, "line_end": 1}) + while sum(len(p) for p in pieces) < min_bytes: + pieces.append(filler(i)) + i += 1 + content = "".join(pieces) + "\n" + return Generated("js_minified.js", content, expected, forbidden[:5]) + + +# ── py_large_generated ────────────────────────────────────────────────────── + + +def py_large_generated(params: dict, rng: random.Random) -> Generated: + """A Python module of ``functions`` top-level functions (default 400) + padded to at least ``min_bytes`` (default 250 KiB). Every function has + the same signature shape ``fn_NNNN(a: int, b: int) -> int`` and a + docstring; the body length varies with the seed so ranges are not + arithmetic. Gold: every function with its exact range.""" + n_funcs = int(params.get("functions", 400)) + min_bytes = int(params.get("min_bytes", 250 * 1024)) + per_func_target = max(1, min_bytes // n_funcs) + + lines: list[str] = ['"""Generated large module for the map-eval harness."""', "", "SCALE = 3", ""] + expected: list[dict] = [{"kind": "K", "name": "SCALE", "line_start": 3, "line_end": 3}] + forbidden: list[str] = [] + + for n in range(n_funcs): + name = f"fn_{n:04d}" + start = len(lines) + 1 + body = [ + f"def {name}(a: int, b: int) -> int:", + f' """Generated function {n}; the local helper must not be a symbol."""', + " total = a + b * SCALE", + ] + # Pad with distinct statements until the function is long enough. + j = 0 + while sum(len(x) + 1 for x in body) < per_func_target: + k = rng.randint(1, 999) + body.append(f" total = (total * {k} + {j}) % 1_000_003 # step {j}") + j += 1 + if n % 50 == 0: + helper = f"inner_{n:04d}" + forbidden.append(helper) + body.append(f" def {helper}(x: int) -> int:") + body.append(" return x + total") + body.append(f" total = {helper}(total)") + body.append(" return total") + lines.extend(body) + end = len(lines) + expected.append({"kind": "F", "name": name, "params": "a: int, b: int", "ret": "int", + "line_start": start, "line_end": end}) + lines.extend(["", ""]) + content = "\n".join(lines).rstrip("\n") + "\n" + return Generated("py_large_generated.py", content, expected, forbidden) + + +GENERATORS = { + "js_minified": js_minified, + "py_large_generated": py_large_generated, +} + + +def generate(name: str, params: dict | None = None, *, case_id: str = "") -> Generated: + if name not in GENERATORS: + raise KeyError(f"unknown generator {name!r}; known: {sorted(GENERATORS)}") + params = dict(params or {}) + rng = random.Random(seed_for(case_id or name, params)) + return GENERATORS[name](params, rng) diff --git a/tests/test_map_eval.py b/tests/test_map_eval.py new file mode 100644 index 0000000..16dca01 --- /dev/null +++ b/tests/test_map_eval.py @@ -0,0 +1,475 @@ +"""File-map gold gate (``c3 map-eval``). + +Renders the hand-annotated fixture suite (tests/map_eval/fixture_suite.jsonl) +through the renderer the MCP tool uses and enforces the checked-in baseline: + +* every ``must_pass`` case passes (recall 1.0, nothing forbidden, deterministic, + its switched-on checks met), +* every aggregate stays at or above its floor and at or below its ceiling, +* no complete map (baseline recall 1.0) grew in tokens, +* the baseline knows every case in the suite (add a case -> refresh it), +* an ``xfail`` that unexpectedly passes is reported, not failed. + +Both map grammars (canonical C1 and today's legacy) are parser-tested here so +the same fixtures grade either renderer. See docs/map-eval.md. +""" + +import json +import re +import sys +import types +import warnings +from pathlib import Path + +import pytest + +from services.bench import map_eval as me + +SUITE = me.BUNDLED_SUITES["fixture"] +BASELINE = me.BUNDLED_BASELINES["fixture"] + +# The spec's minimum case list. A rename here is a rename in the suite. +REQUIRED_CASES = { + "py_basic", "py_few_imports", "py_malformed", "ts_basic", "js_basic", "go_basic", "rs_basic", + "md_basic", "yaml_basic", "json_basic", "css_basic", "html_basic", "r_regex_fallback", + "java_generic", "js_minified", "py_large_generated", +} + +CANONICAL_SAMPLE = """# services/x.py (72L python) +I 9 imports +K MAX_RETRIES [L12-L12] +V settings [L14-L14] +C Worker [L23-L36] + M Worker.__init__(self, name: str, timeout: float = DEFAULT_TIMEOUT) [L26-L28] + P Worker.label [L31-L32] + M Worker.run(self, job: dict) -> bool [L34-L36] +F async gather_all(urls: list) -> list [L60-L63] +F render(template: str, context: dict, strict: bool = False) -> str [L66-L72] +F main() [L80-L82] +H Usage (advanced) [L18-L26] +SEC .card .title, .card h2 [L17-L20] +C Queue(EventEmitter) [L20-L45] + M async Queue.drain(handler: JobHandler) -> Promise [L36-L44] +M Server.Start(ctx context.Context) -> error [L26-L33] +IM Point [L22-L30] + M Point.new(x: f64, y: f64) -> Self [L23-L25] +TR Area [L17-L20] +… 3 more symbols +""" + +LEGACY_SAMPLE = """# py_basic.py (72 lines, python) + + imports 9 statements (collapsed) + 12-12 \U0001F48E constant MAX_RETRIES + 16-20 ✨ function retry(times: int) + Decorator factory; ``wrap`` is an inner function. + 23-36 \U0001F3D7️ class Worker + A worker that runs jobs. + 26-28 ⚙️ __init__(self, name: str, timeout: float = DEFAULT_TIMEOUT) + 31-32 ⚙️ label(self) + 34-36 ⚙️ run(self, job: dict) + 40-41 \U0001F3D7️ class Entry + 42-42 \U0001F527 private property jobs + 43-44 ⚙️ private async drain(handler: JobHandler) + 60-63 ✨ async function gather_all(urls: list) + 66-72 ✨ function render(template: str, + context: dict, + strict: bool = False,) + Multi-line signature with a return annotation. + 74-76 ✨ function long_one(aaaaaaaaaa: int, bbbbbbbbbb: int, cccccccccc: int, ddddd...) + 1-30 \U0001F516 h1: Map Eval Guide + 8-14 \U0001F4CD #top (header) + 17-20 \U0001F4CD .card .t + 22-30 \U0001F6E0️ impl impl Point + 26-33 ⚙️ method Start(s *Server) + 15-18 \U0001F9F1 struct Point + 1-28 ⚙️ content (full file) +""" + + +@pytest.fixture(scope="module") +def report(tmp_path_factory): + work = tmp_path_factory.mktemp("map_eval") + return me.run_suite("fixture", work_dir=work, baseline=BASELINE) + + +def _raw_cases(): + with open(SUITE, encoding="utf-8") as fh: + objs = [json.loads(line) for line in fh if line.strip()] + return objs[0], objs[1:] + + +class TestSuiteShape: + def test_header_and_size(self): + header, cases = me.load_suite(SUITE) + assert header.get("fixture") is True + assert set(header.get("kinds", [])) == set(me.KINDS) + assert len(cases) >= 16 + assert REQUIRED_CASES <= {c.id for c in cases} + + def test_every_fixture_exists_and_is_hand_sized(self): + _, cases = me.load_suite(SUITE) + for c in cases: + if not c.file: + continue + p = me.FIXTURES_DIR / c.file + assert p.exists(), f"{c.id}: fixture {p} missing" + n = len(p.read_text(encoding="utf-8").splitlines()) + assert 20 <= n <= 80, f"{c.id}: {n} lines, fixtures are 20-80 lines" + for e in c.expected: + assert e.b <= n, f"{c.id}: {e.name} ends at L{e.b} past EOF ({n})" + + def test_every_generator_exists(self): + _, cases = me.load_suite(SUITE) + gens = me.load_generators() + missing = sorted({c.generator for c in cases if c.generator} - set(gens.GENERATORS)) + assert not missing, f"cases name unknown generators: {missing}" + + def test_gates_are_declared_and_xfails_name_a_phase(self): + _, cases = me.load_suite(SUITE) + assert all(c.gate in me.GATES for c in cases) + bad = [c.id for c in cases if c.gate == "xfail" and c.phase not in me.PHASES] + assert not bad, f"xfail cases must name a phase: {bad}" + assert any(c.gate == "must_pass" for c in cases) + assert any(c.gate == "xfail" and c.phase == "C1" for c in cases) + assert any(c.gate == "xfail" and c.phase == "C4" for c in cases) + + def test_checks_use_known_vocabulary(self): + _, raw = _raw_cases() + for d in raw: + unknown = set(d.get("checks") or {}) - set(me.CHECK_KEYS) + assert not unknown, f"{d['id']}: unknown checks {sorted(unknown)}" + with pytest.raises(ValueError): + me.MapCase.from_dict({"id": "x", "file": "a.py", "expected": [], "gate": "xfail"}) # no phase + with pytest.raises(ValueError): + me.MapCase.from_dict({"id": "x", "file": "a.py", "expected": [], + "checks": {"budget": 1}}) + with pytest.raises(ValueError): + me.MapCase.from_dict({"id": "x", "file": "a.py"}) # fixture without annotation + with pytest.raises(ValueError): + me.MapCase.from_dict({"id": "x", "file": "a.py", "generator": "g", "expected": []}) + with pytest.raises(ValueError): + me.ExpectedSymbol.from_dict({"kind": "FN", "name": "f", "line_start": 1, "line_end": 1}, "x") + with pytest.raises(ValueError): + me.ExpectedSymbol.from_dict({"kind": "F", "name": "f", "line_start": 5, "line_end": 4}, "x") + + def test_expected_names_are_qualified_for_methods(self): + _, cases = me.load_suite(SUITE) + for c in cases: + for e in c.expected: + if e.kind in ("M", "P"): + assert "." in e.name, f"{c.id}: {e.kind} {e.name} must be Class.member" + + def test_baseline_covers_every_case(self): + _, cases = me.load_suite(SUITE) + baseline = me.load_baseline(BASELINE) + missing = sorted({c.id for c in cases} - set(baseline["per_case"])) + extra = sorted(set(baseline["per_case"]) - {c.id for c in cases}) + assert not missing and not extra, ( + f"baseline out of date (missing={missing}, extra={extra}); run " + "`c3 map-eval --update-baseline`") + + +class TestGenerators: + def test_deterministic_across_calls(self): + gens = me.load_generators() + for name in gens.GENERATORS: + a = gens.generate(name, case_id=name) + b = gens.generate(name, case_id=name) + assert (a.content, a.expected, a.must_not_contain) == (b.content, b.expected, b.must_not_contain), name + assert gens.seed_for("a") != gens.seed_for("b") + assert gens.seed_for("a", {"seed": 7}) == 7 + + def test_js_minified_is_one_long_line_with_planted_functions(self): + gens = me.load_generators() + g = gens.generate("js_minified", {"named": 6, "min_bytes": 20480}, case_id="x") + assert g.content.count("\n") == 1 and len(g.content) >= 20480 + assert len(g.expected) == 6 + for e in g.expected: + assert f"function {e['name']}(a,b)" in g.content + assert (e["line_start"], e["line_end"]) == (1, 1) + assert g.must_not_contain and all(f"var {v}=" in g.content for v in g.must_not_contain) + + def test_py_large_gold_matches_its_own_layout(self): + gens = me.load_generators() + g = gens.generate("py_large_generated", {"functions": 400, "min_bytes": 256000}, case_id="x") + assert len(g.content.encode("utf-8")) >= 256000 + lines = g.content.splitlines() + funcs = [e for e in g.expected if e["kind"] == "F"] + assert len(funcs) == 400 + for e in funcs: + assert lines[e["line_start"] - 1].startswith(f"def {e['name']}(a: int, b: int) -> int:") + assert lines[e["line_end"] - 1] == " return total" + assert g.must_not_contain and all(f"def {h}(" in g.content for h in g.must_not_contain) + + +class TestCanonicalParser: + def test_parses_every_kind_and_shape(self): + parsed = me.parse_canonical(CANONICAL_SAMPLE) + assert parsed is not None and parsed.grammar == "canonical" + assert parsed.imports == 9 + assert parsed.header == {"path": "services/x.py", "lines": 72, "lang": "python"} + by = {(s.kind, s.name): s for s in parsed.symbols} + assert ("K", "MAX_RETRIES") in by and by[("K", "MAX_RETRIES")].a == 12 + assert ("V", "settings") in by + init = by[("M", "Worker.__init__")] + assert init.params == "self, name: str, timeout: float = DEFAULT_TIMEOUT" and init.ret is None + assert init.bare == "__init__" and (init.a, init.b) == (26, 28) + run = by[("M", "Worker.run")] + assert run.params == "self, job: dict" and run.ret == "bool" + assert by[("P", "Worker.label")].params is None + ga = by[("F", "gather_all")] + assert ga.is_async and ga.ret == "list" + # whitespace inside params is normalized + assert by[("F", "render")].params == "template: str, context: dict, strict: bool = False" + assert by[("F", "main")].params == "" + # parens in a heading are part of the name, not params + assert ("H", "Usage (advanced)") in by and by[("H", "Usage (advanced)")].params is None + assert ("SEC", ".card .title, .card h2") in by + assert by[("C", "Queue")].params == "EventEmitter" + drain = by[("M", "Queue.drain")] + assert drain.is_async and drain.ret == "Promise" + assert by[("M", "Server.Start")].ret == "error" + assert ("IM", "Point") in by and by[("M", "Point.new")].ret == "Self" + assert ("TR", "Area") in by + # the `… k more symbols` tail and the I line are not symbols + assert not any(s.name.startswith("…") for s in parsed.symbols) + assert all(s.kind != "I" for s in parsed.symbols) + + def test_regex_is_the_spec_regex(self): + line = " M Worker.run(self, job: dict) -> bool [L34-L36]" + m = me.CANON_LINE_RE.match(line) + assert m and m["kind"] == "M" and m["name"] == "Worker.run" + assert m["params"] == "self, job: dict" and m["ret"] == "bool" + assert (m["a"], m["b"]) == ("34", "36") + assert me.CANON_LINE_RE.match("F async fetch(url) [L1-L1]")["async"] == "async " + assert not me.CANON_LINE_RE.match(" 12-12 constant MAX_RETRIES") + + def test_parse_map_picks_canonical(self): + parsed = me.parse_map(CANONICAL_SAMPLE) + assert parsed.grammar == "canonical" and len(parsed.symbols) == 17 + + +class TestLegacyParser: + def test_parses_todays_shape(self): + parsed = me.parse_legacy(LEGACY_SAMPLE) + assert parsed is not None and parsed.grammar == "legacy" + assert parsed.imports == 9 + assert parsed.header["lines"] == 72 + by = {} + for s in parsed.symbols: + by.setdefault(s.name, s) + assert by["MAX_RETRIES"].kind == "K" + assert by["retry"].kind == "F" and by["retry"].params == "times: int" + assert by["Worker"].kind == "C" and (by["Worker"].a, by["Worker"].b) == (23, 36) + # children are qualified with the parent and mapped to M / C / P + init = by["Worker.__init__"] + assert init.kind == "M" and init.bare == "__init__" and init.legacy_type == "method" + assert init.params == "self, name: str, timeout: float = DEFAULT_TIMEOUT" + assert by["Worker.Entry"].kind == "C" + assert by["Worker.jobs"].kind == "P" + drain = by["Worker.drain"] + assert drain.is_async and drain.params == "handler: JobHandler" + assert by["gather_all"].is_async and by["gather_all"].kind == "F" + # a multi-line signature is joined back into one symbol + assert by["render"].params == "template: str, context: dict, strict: bool = False" + assert (by["render"].a, by["render"].b) == (66, 72) + # the 60-char truncation is kept verbatim (it must fail completeness) + assert by["long_one"].params.endswith("...") + # headings / html sections / impl lose their legacy decoration + assert by["Map Eval Guide"].kind == "H" + assert by["#top"].kind == "SEC" + assert by[".card .t"].kind == "SEC" + assert by["Point"].kind in ("IM", "S") + kinds = {(s.kind, s.name) for s in parsed.symbols} + assert ("IM", "Point") in kinds and ("S", "Point") in kinds + go = by["Start"] + assert go.kind == "M" and go.params == "s *Server" + # docstring lines, the (full file) content line and I lines are not symbols + assert not any("Decorator factory" in s.name for s in parsed.symbols) + assert not any("full file" in s.name for s in parsed.symbols) + assert all(s.ret is None for s in parsed.symbols) + + def test_parse_map_picks_legacy_by_header(self): + parsed = me.parse_map(LEGACY_SAMPLE) + assert parsed.grammar == "legacy" + assert me.parse_map("[file_map] Could not build map for x — file not found.").grammar == "none" + + +class TestMetrics: + def _exp(self, kind, name, a, b, **kw): + return me.ExpectedSymbol.from_dict({"kind": kind, "name": name, "line_start": a, "line_end": b, **kw}, "t") + + def test_normalize_ws(self): + assert me.normalize_ws(" a: int ,\n b : str , ") == "a: int, b : str" + assert me.normalize_ws("template: str,\n context: dict,") == "template: str, context: dict" + assert me.normalize_ws("( a , b )") == "(a, b)" + assert me.normalize_ws(None) is None + + def test_multiset_match_pairs_duplicates_by_range(self): + expected = [self._exp("M", "Worker.run", 34, 36), self._exp("M", "Scheduler.run", 45, 47), + self._exp("M", "Registry.Entry.__init__", 29, 31), self._exp("M", "Registry.__init__", 36, 37)] + rendered = [me.Symbol("M", "Scheduler.run", 45, 47), me.Symbol("M", "Worker.run", 34, 36), + me.Symbol("M", "Registry.__init__", 36, 37)] + pairs, missing, extra = me.match_symbols(expected, rendered, legacy=True) + assert {(e.name, s.a) for e, s in pairs} == {("Worker.run", 34), ("Scheduler.run", 45), ("Registry.__init__", 36)} + assert [e.name for e in missing] == ["Registry.Entry.__init__"] + assert extra == [] + # canonical matching is on (kind, name): a P rendered as M is a miss + pairs, missing, _ = me.match_symbols([self._exp("P", "Worker.label", 31, 32)], + [me.Symbol("M", "Worker.label", 31, 32)], legacy=False) + assert not pairs and len(missing) == 1 + + def test_grade_metrics_and_gates(self): + case = me.MapCase(id="t", file="x.py", gate="must_pass", + checks={"signature_completeness": 1.0, "range_accuracy": 1.0, "max_map_tokens": 50}) + expected = [self._exp("F", "fetch", 55, 57, params="url: str", ret="str"), + self._exp("F", "render", 66, 72, params="a, b"), + self._exp("K", "MAX", 1, 1)] + rendered = [me.Symbol("F", "fetch", 55, 57, params="url: str", ret="str"), + me.Symbol("F", "render", 66, 70, params="a, b, c"), + me.Symbol("F", "wrap", 18, 19)] + parsed = me.ParsedMap("canonical", rendered) + result = me.CaseResult(id="t", file="x.py", gate="must_pass", phase=None, status="error") + result.map_tokens = 80 + result.determinism = False + fails = me.grade(case, expected, ["wrap"], parsed, result) + assert result.symbol_recall == round(2 / 3, 4) + assert result.symbol_precision == round(2 / 3, 4) + assert result.signature_completeness == 0.5 + assert result.range_accuracy == 0.5 + assert result.must_not_contain_hits == ["wrap"] + assert result.missing == ["K MAX [L1-L1]"] and result.extra == ["F wrap [L18-L19]"] + heads = {f.split(" ")[0].rstrip(":") for f in fails} + assert heads == {"symbol_recall", "must_not_contain", "determinism", "signature_completeness", + "range_accuracy", "max_map_tokens"} + + def test_chrome_share_counts_emoji_and_padding_only(self): + assert me.chrome_share("F main() [L1-L2]\n") == 0.0 + text = " 12-12 \U0001F48E constant MAX\n" + share = me.chrome_share(text) + # 5 padding spaces after "12-12" + 1 emoji, over the line's characters + assert share == round(6 / len(text), 4) + # leading indentation is not chrome + assert me.chrome_share(" M A.b() [L1-L1]") == 0.0 + + +class TestRenderer: + def test_resolves_todays_renderer_and_renders_a_fixture(self, tmp_path): + store = me.build_store(tmp_path) + renderer, name = me.resolve_renderer(store) + assert name in ("services.file_memory.FileMemoryStore.get_or_build_map", "services.file_map.render_map") + (tmp_path / "py_basic.py").write_bytes((me.FIXTURES_DIR / "py_basic.py").read_bytes()) + text = renderer("py_basic.py") + parsed = me.parse_map(text) + assert parsed.grammar in ("legacy", "canonical") + names = {s.bare for s in parsed.symbols} + assert {"Worker", "Scheduler", "fetch", "gather_all", "render"} <= names + + def test_prefers_file_map_render_map_when_it_exists(self, tmp_path, monkeypatch): + """Phase C1 ships services/file_map.py; the harness must pick it up + by import and report it as the renderer, with the canonical grammar + graded from the same fixtures.""" + import services + + def render_map(record, *, include_docs=False, max_tokens=None): + lines = [f"# {record['path']} ({record.get('lines', 0)}L {record.get('language', '')})"] + kinds = {"class": "C", "function": "F", "method": "M", "constant": "K"} + + def sig(sec): + m = re.search(r"\(([^()]*)\)", sec.get("signature", ""), re.S) + return f"({me.normalize_ws(m.group(1))})" if m else "" + for sec in record["sections"]: + t = sec["type"] + if t == "import": + continue + k = kinds.get(t, "V") + params = sig(sec) if k in ("F", "M") else "" + lines.append(f"{k} {sec['name']}{params} [L{sec['line_start']}-L{sec['line_end']}]") + for ch in sec.get("children", []): + ck = kinds.get(ch["type"], "M") + lines.append(f" {ck} {sec['name']}.{ch['name']}{sig(ch)} [L{ch['line_start']}-L{ch['line_end']}]") + return "\n".join(lines) + + fake = types.ModuleType("services.file_map") + fake.render_map = render_map + monkeypatch.setitem(sys.modules, "services.file_map", fake) + monkeypatch.setattr(services, "file_map", fake, raising=False) + + store = me.build_store(tmp_path) + renderer, name = me.resolve_renderer(store) + assert name == "services.file_map.render_map" + (tmp_path / "js_basic.js").write_bytes((me.FIXTURES_DIR / "js_basic.js").read_bytes()) + text = renderer("js_basic.js") + parsed = me.parse_map(text) + assert parsed.grammar == "canonical" + _, cases = me.load_suite(SUITE) + case = next(c for c in cases if c.id == "js_basic") + result = me.CaseResult(id=case.id, file=case.file, gate=case.gate, phase=None, status="error") + result.determinism = True + fails = me.grade(case, case.expected, case.must_not_contain, parsed, result) + assert result.symbol_recall == 1.0 and result.range_accuracy == 1.0, fails + assert result.signature_completeness == 1.0, result.signature_misses + + +class TestFixtureGate: + def test_every_case_executed(self, report): + assert report.aggregates["n_errors"] == 0, report.aggregates["errors"] + assert report.aggregates["n_executed"] == report.aggregates["n_cases"] + assert all(r.determinism is True for r in report.results), [ + r.id for r in report.results if r.determinism is not True] + + def test_must_pass_cases_pass(self, report): + failed = report.aggregates["must_pass_failed"] + detail = "\n".join(f"{r.id}: {r.reason}" for r in report.results if r.id in failed) + assert not failed, f"must_pass regressions:\n{detail}\n\n{report.render()}" + + def test_aggregates_meet_floors_and_ceilings(self, report): + limits = [v for v in report.baseline_violations if "floor" in v or "ceiling" in v] + assert not limits, "\n".join(limits) + "\n\n" + report.render() + + def test_no_violations(self, report): + assert not report.baseline_violations, "\n".join(report.baseline_violations) + "\n\n" + report.render() + + def test_xfail_and_regressions_are_visible(self, report): + for w in report.baseline_warnings: + warnings.warn(w, stacklevel=1) + for cid in report.aggregates["xfail_passing"]: + warnings.warn(f"{cid} is xfail but passes; promote it", stacklevel=1) + + def test_report_renders_and_serialises(self, report): + text = report.render() + assert "verdict:" in text and "renderer=" in text and "grammars=" in text + assert report.suite == "fixture" + data = report.to_dict() + assert len(data["results"]) == report.aggregates["n_cases"] + json.dumps(data) # must be JSON-serialisable for --json + assert Path(report.suite_path).name == "fixture_suite.jsonl" + + def test_baseline_roundtrip_and_token_gate(self, report, tmp_path): + path = tmp_path / "baseline.json" + me.write_baseline(report, path, floors={"pass_rate_must_pass": 1.0}, ceilings={"tokens_p50": 10}) + data = me.load_baseline(path) + assert data["floors"] == {"pass_rate_must_pass": 1.0} + violations, _ = me.compare_to_baseline(report, data) + assert any("above ceiling" in v for v in violations) + # keep_limits: a refresh without explicit limits keeps the hand-set ones + me.write_baseline(report, path) + assert me.load_baseline(path)["ceilings"] == {"tokens_p50": 10} + # a complete must_pass map that grows is a violation; a lost symbol too + data = me.load_baseline(path) + data["ceilings"] = {} + pc = data["per_case"]["js_basic"] + pc["map_tokens"] = 1 + pc["symbol_recall"] = 1.0 + violations, warnings_ = me.compare_to_baseline(report, data) + assert any("js_basic map_tokens" in v for v in violations) + pc = data["per_case"]["py_basic"] + pc["symbol_recall"] = 1.0 + r = next(x for x in report.results if x.id == "py_basic") + r.symbol_recall = 0.5 + try: + _, warnings_ = me.compare_to_baseline(report, data) + assert any("py_basic symbol_recall 0.5" in w for w in warnings_) + finally: + r.symbol_recall = 1.0 diff --git a/tests/test_map_telemetry_c0.py b/tests/test_map_telemetry_c0.py new file mode 100644 index 0000000..458d998 --- /dev/null +++ b/tests/test_map_telemetry_c0.py @@ -0,0 +1,272 @@ +"""C0 of the c3_compress remediation: every map is measured. + +Measured 2026-09-06 over this repo's .c3/tool_telemetry.jsonl: c3_compress +rows said WHICH file (target, since 2.111.0) but never which backend built +the map, which parser extracted the symbols, whether the record was already +fresh, or whether the map led to a targeted read. These tests pin: + +- file_memory records carry `parser` (tree_sitter | regex | generic) and a + pre-2.120.0 record without it is re-extracted on the next update; +- handle_compress / handle_read hand a flat `detail` to the telemetry row for + every map they serve (file_memory maps, compressor modes, batches, the two + c3_read map fallbacks) and for source reads; +- aggregate_tool_telemetry folds those into `map_by_backend` and measures + map → read adjacency in `map_read_chain` from `target` alone, so the chain + works on rows written before this release too. +""" +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from cli.tools.compress import handle_compress # noqa: E402 +from cli.tools.read import handle_read # noqa: E402 +from services.compressor import CodeCompressor # noqa: E402 +from services.file_memory import FileMemoryStore # noqa: E402 +from services.session_manager import SessionManager # noqa: E402 +from services.telemetry import ( # noqa: E402 + MAP_CHAIN_WINDOW, + aggregate_tool_telemetry, + append_telemetry_record, +) + +PY_SRC = '''"""module doc""" +import os +import sys + + +CONST = 1 + + +class Thing: + def run(self, a: int, b: str = "x") -> bool: + return True + + +def helper(x): + return x +''' + "".join( + f''' + +def worker_{i}(value: int, scale: float = 1.0, *, label: str = "w{i}") -> dict: + """Worker {i}: multiply value by scale and label the result.""" + total = value * scale + if total > {i * 10}: + total = total - {i} + return {{"label": label, "total": total, "index": {i}}} +''' for i in range(12)) + + +def _project(tmp: str) -> SimpleNamespace: + root = Path(tmp) + (root / "pkg").mkdir() + (root / "pkg" / "mod.py").write_text(PY_SRC, encoding="utf-8") + (root / "notes.r").write_text("f <- function(a) a\n", encoding="utf-8") + sm = SessionManager(tmp) + sm.start_session("t") + svc = SimpleNamespace( + project_path=tmp, + file_memory=FileMemoryStore(tmp), + compressor=CodeCompressor(str(root / ".c3" / "cache"), project_root=tmp), + session_mgr=sm, + hybrid_config={}, + activity_log=None, + edit_ledger=None, + ) + return svc + + +def _finalize_via(sm): + def finalize(name, args, resp, summ, **kw): + sm.log_tool_call(name, args, summ) + sm.track_response(name, resp, kw.get("response_tokens", 0)) + return resp + return finalize + + +def _rows(tmp: str) -> list: + path = Path(tmp) / ".c3" / "tool_telemetry.jsonl" + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() + if line.strip()] + + +def _no_facts(*_a, **_k): + return "" + + +class TestParserAttribution(unittest.TestCase): + def test_record_names_its_parser(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + self.assertEqual(svc.file_memory.update("pkg/mod.py")["parser"], "tree_sitter") + self.assertEqual(svc.file_memory.update("notes.r")["parser"], "regex") + Path(tmp, "x.java").write_text("class A {}\n", encoding="utf-8") + self.assertEqual(svc.file_memory.update("x.java")["parser"], "generic") + + def test_pre_2_120_record_is_re_extracted(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + rec = svc.file_memory.update("pkg/mod.py") + rec.pop("parser") + svc.file_memory._save("pkg/mod.py", rec) + self.assertNotIn("parser", svc.file_memory.get("pkg/mod.py")) + self.assertEqual(svc.file_memory.update("pkg/mod.py")["parser"], "tree_sitter") + + +class TestCompressDetail(unittest.TestCase): + def test_map_mode_detail(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + fin = _finalize_via(svc.session_mgr) + handle_compress("pkg/mod.py", "map", svc, fin, _no_facts) + handle_compress("pkg/mod.py", "map", svc, fin, _no_facts) + rows = [r for r in _rows(tmp) if r["tool"] == "c3_compress"] + self.assertEqual(len(rows), 2) + first, second = (r["detail"] for r in rows) + self.assertEqual(first["backend"], "file_memory") + self.assertEqual(first["requested_mode"], "map") + self.assertEqual(first["parser"], "tree_sitter") + self.assertFalse(first["cache_hit"]) + self.assertGreater(first["sections"], 0) + self.assertTrue(second["cache_hit"]) + self.assertEqual(rows[0]["target"], "pkg/mod.py") + self.assertGreater(rows[0]["raw_tokens"], rows[0]["optimized_tokens"]) + + def test_compressor_mode_detail_and_cache_hit(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + fin = _finalize_via(svc.session_mgr) + handle_compress("pkg/mod.py", "smart", svc, fin, _no_facts) + handle_compress("pkg/mod.py", "smart", svc, fin, _no_facts) + rows = [r["detail"] for r in _rows(tmp) if r["tool"] == "c3_compress"] + self.assertEqual(rows[0]["backend"], "compressor") + self.assertEqual(rows[0]["requested_mode"], "smart") + self.assertFalse(rows[0]["cache_hit"]) + self.assertTrue(rows[1]["cache_hit"]) + self.assertIn(rows[0]["actual_mode"], ("full", "outline", "structure", "smart")) + + def test_batch_detail(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + fin = _finalize_via(svc.session_mgr) + handle_compress("pkg/mod.py,notes.r,missing.py", "map", svc, fin, _no_facts) + rows = [r for r in _rows(tmp) if r["tool"] == "c3_compress"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["detail"], + {"requested_mode": "map", "backend": "batch", "files": 3, "ok": 2}) + + +class TestReadDetail(unittest.TestCase): + def test_source_read_detail(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + fin = _finalize_via(svc.session_mgr) + out = handle_read("pkg/mod.py", ["helper"], None, True, svc, fin) + self.assertIn("def helper(x):", out) + row = [r for r in _rows(tmp) if r["tool"] == "c3_read"][-1] + d = row["detail"] + self.assertEqual(d["backend"], "source") + self.assertEqual(d["symbols"], 1) + self.assertEqual(d["ranges"], 1) + self.assertFalse(d["by_lines"]) + self.assertEqual(d["lines_served"], 2) + self.assertEqual(d["file_lines"], PY_SRC.count("\n")) + self.assertLess(row["optimized_tokens"], row["raw_tokens"]) + + def test_map_only_fallback_detail(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + fin = _finalize_via(svc.session_mgr) + handle_read("pkg/mod.py", None, None, True, svc, fin) + d = [r for r in _rows(tmp) if r["tool"] == "c3_read"][-1]["detail"] + self.assertEqual(d["backend"], "file_memory") + self.assertEqual(d["fallback"], "map_only") + self.assertEqual(d["parser"], "tree_sitter") + + def test_symbols_not_found_fallback_detail(self): + with tempfile.TemporaryDirectory() as tmp: + svc = _project(tmp) + fin = _finalize_via(svc.session_mgr) + handle_read("pkg/mod.py", ["nope_zz"], None, True, svc, fin) + d = [r for r in _rows(tmp) if r["tool"] == "c3_read"][-1]["detail"] + self.assertEqual(d["fallback"], "symbols_not_found") + self.assertEqual(d["symbols"], 1) + + +class TestAggregate(unittest.TestCase): + def _write(self, tmp, tool, target, detail=None, raw=None, opt=None, sid="s1"): + append_telemetry_record(tmp, { + "session_id": sid, "tool": tool, "target": target, + "response_tokens": opt or 5, "raw_tokens": raw, "optimized_tokens": opt, + **({"detail": detail} if detail else {}), + }) + + def test_map_by_backend_folds_detail(self): + with tempfile.TemporaryDirectory() as tmp: + self._write(tmp, "c3_compress", "a.py", raw=1000, opt=100, + detail={"requested_mode": "map", "backend": "file_memory", + "parser": "tree_sitter", "cache_hit": True, "sections": 4}) + self._write(tmp, "c3_compress", "b.py", raw=1000, opt=300, + detail={"requested_mode": "map", "backend": "file_memory", + "parser": "regex", "cache_hit": False, "sections": 2}) + self._write(tmp, "c3_compress", "c.py", raw=500, opt=50, + detail={"requested_mode": "smart", "backend": "compressor", + "actual_mode": "outline", "cache_hit": False}) + self._write(tmp, "c3_read", "a.py", raw=1000, opt=120, + detail={"backend": "file_memory", "parser": "tree_sitter", + "cache_hit": True, "fallback": "map_only", "sections": 4}) + self._write(tmp, "c3_read", "a.py", raw=1000, opt=40, + detail={"backend": "source", "symbols": 1, "ranges": 1}) + agg = aggregate_tool_telemetry(tmp, days=0)["map_by_backend"] + ts = agg["file_memory/tree_sitter"] + self.assertEqual(ts["calls"], 2) + self.assertEqual(ts["cache_hits"], 2) + self.assertEqual(ts["map_fallbacks"], 1) + self.assertEqual(ts["measured_calls"], 2) + self.assertAlmostEqual(ts["ratio_p50"], 0.12, places=2) + self.assertEqual(agg["file_memory/regex"]["calls"], 1) + self.assertEqual(agg["compressor/outline"]["calls"], 1) + self.assertEqual(agg["source"]["symbol_reads"], 1) + + def test_map_read_chain_from_targets_alone(self): + with tempfile.TemporaryDirectory() as tmp: + # map a.py -> read a.py (followed); map b.py -> nothing; read c.py cold + self._write(tmp, "c3_compress", "a.py") + self._write(tmp, "c3_read", "a.py") + self._write(tmp, "c3_compress", "b.py") + self._write(tmp, "c3_read", "c.py") + # another session: a map far outside the window does not count + self._write(tmp, "c3_compress", "d.py", sid="s2") + for i in range(MAP_CHAIN_WINDOW): + self._write(tmp, "c3_read", f"z{i}.py", sid="s2") + self._write(tmp, "c3_read", "d.py", sid="s2") + chain = aggregate_tool_telemetry(tmp, days=0)["map_read_chain"] + self.assertEqual(chain["window"], MAP_CHAIN_WINDOW) + self.assertEqual(chain["maps"], 3) + self.assertEqual(chain["maps_followed_by_read"], 1) + self.assertEqual(chain["reads"], 2 + MAP_CHAIN_WINDOW + 1) + self.assertEqual(chain["reads_preceded_by_map"], 1) + self.assertAlmostEqual(chain["map_follow_rate"], 1 / 3, places=3) + + def test_read_map_fallback_counts_as_a_map(self): + with tempfile.TemporaryDirectory() as tmp: + self._write(tmp, "c3_read", "a.py", detail={"backend": "file_memory", + "fallback": "map_only"}) + self._write(tmp, "c3_read", "a.py", detail={"backend": "source", "symbols": 1}) + chain = aggregate_tool_telemetry(tmp, days=0)["map_read_chain"] + self.assertEqual(chain["maps"], 1) + self.assertEqual(chain["maps_followed_by_read"], 1) + self.assertEqual(chain["reads"], 1) + self.assertEqual(chain["reads_preceded_by_map"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_windows_reliability.py b/tests/test_windows_reliability.py index 9bdb237..e443511 100644 --- a/tests/test_windows_reliability.py +++ b/tests/test_windows_reliability.py @@ -15,7 +15,10 @@ _SKIP_DIRS = {".venv", "__pycache__", ".pytest_cache", ".c3", # Build artifacts: gitignored copies of sources we already scan. - "build", "dist", ".eggs"} + "build", "dist", ".eggs", + # Sibling worktrees (.claude/worktrees) hold their own copy of + # the tree; each is scanned by its own test run. + ".claude"} _SKIP_FRAGMENTS = (".tmp.",) # Sequence produced when UTF-8 text is written then re-read as cp1252 and
+

Map Eval

+ +