diff --git a/archaeology/metrics.py b/archaeology/metrics.py index 246403c..8fe9d9b 100644 --- a/archaeology/metrics.py +++ b/archaeology/metrics.py @@ -4,6 +4,7 @@ import json import sqlite3 from collections import Counter +from contextlib import closing from pathlib import Path from .utils import _parse_date, atomic_write @@ -16,7 +17,7 @@ def write_metrics(project_root: Path, db_path: Path) -> dict: with csv_path.open(encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) hashes = [row["hash"] for row in rows] - with sqlite3.connect(db_path) as conn: + with closing(sqlite3.connect(db_path)) as conn: stored = [row[0] for row in conn.execute("SELECT hash FROM commits")] if len(set(hashes)) != len(hashes) or sorted(hashes) != sorted(stored): raise ValueError("CSV/SQLite commit identities differ or contain duplicates") @@ -34,6 +35,10 @@ def write_metrics(project_root: Path, db_path: Path) -> dict: indent=2, ), ) + from .provenance import bind_derived_artifact + + bind_derived_artifact(project_root, project_root / "deliverables" / "canonical-metrics.json") + bind_derived_artifact(project_root, project_root / "deliverables" / "data.json") return metrics diff --git a/archaeology/provenance.py b/archaeology/provenance.py index c4041ac..b1a4125 100644 --- a/archaeology/provenance.py +++ b/archaeology/provenance.py @@ -14,6 +14,7 @@ "data/archaeology.db", "data/detected-signals.json", "data/commit-eras.json", + "data/gap-clusters.json", "deliverables/canonical-metrics.json", "deliverables/data.json", ) @@ -91,3 +92,112 @@ def verify_artifact(project, artifact): current = {str(p.relative_to(project)): digest(p) for p in analysis_paths(project)} if binding["analysis"] != current: raise ValueError(f"Stale analysis inputs for artifact: {artifact.name}; re-export") + + +def _load_coverage(project): + coverage_path = Path(project) / "data" / "coverage.json" + if not coverage_path.is_file(): + raise ValueError("Missing data/coverage.json for bound mining project") + try: + coverage = json.loads(coverage_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError("Invalid data/coverage.json") from exc + if not isinstance(coverage, dict): + raise ValueError("Invalid data/coverage.json") + return coverage + + +def verify_mining_inputs(project): + """Fail when mined CSV/stats no longer match coverage.json artifact hashes.""" + project = Path(project) + if not requires_binding(project): + return + coverage = _load_coverage(project) + hashes = coverage.get("artifact_sha256") + if not isinstance(hashes, dict) or not hashes: + raise ValueError("Mining coverage missing artifact_sha256 bindings") + for name, expected in hashes.items(): + path = project / "data" / name + actual = digest(path) + if actual != expected: + raise ValueError(f"Stale or modified mining artifact: {name}; remine before export") + + +def verify_derived_stage(project): + """Refuse export when DB/metrics/signals belong to a different or incomplete mining generation. + + A failed remine may refresh coverage.json + raw CSV hashes while leaving an older + archaeology.db / metrics / signals tree in place. Export must not publish that mix. + + Legacy CLI coverage (no mining_status / mining_generation) only requires that the + SQLite DB exist. Plugin mining always stamps generation + derived_sha256 and must + pass the full binding checks. + """ + project = Path(project) + if not requires_binding(project): + return + coverage = _load_coverage(project) + status = coverage.get("mining_status") + generation = coverage.get("mining_generation") + db_path = project / "data" / "archaeology.db" + + # Legacy manifests from the framework CLI mine path. + if status is None and generation is None: + if not db_path.is_file(): + raise ValueError("Missing archaeology.db; remine before export") + return + + if status != "complete": + raise ValueError( + "Mining rebuild incomplete or unbound; remine successfully before export " + f"(mining_status={status!r})" + ) + if not generation: + raise ValueError("Mining coverage missing mining_generation; remine before export") + derived = coverage.get("derived_sha256") + if not isinstance(derived, dict) or not derived: + raise ValueError("Mining coverage missing derived_sha256 bindings; remine before export") + expected_db = derived.get("archaeology.db") + actual_db = digest(db_path) + if not expected_db or actual_db != expected_db: + raise ValueError( + "Stale or unbound archaeology.db for current mining generation; remine before export" + ) + # Optional derived JSON must declare the same generation when present. + for rel in ( + "deliverables/canonical-metrics.json", + "data/detected-signals.json", + "data/gap-clusters.json", + "deliverables/data.json", + ): + path = project / rel + if not path.is_file(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"Invalid derived artifact: {rel}") from exc + if not isinstance(payload, dict) or payload.get("mining_generation") != generation: + raise ValueError(f"Unbound derived artifact: {rel}; rebuild signals/metrics") + sidecar = path.with_name(path.name + ".generation.json") + try: + binding = json.loads(sidecar.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"Missing derived content binding: {rel}") from exc + if binding != {"mining_generation": generation, "sha256": digest(path)}: + raise ValueError(f"Modified derived artifact: {rel}; rebuild signals/metrics") + + +def bind_derived_artifact(project, artifact): + project, artifact = Path(project), Path(artifact) + coverage_path = project / "data" / "coverage.json" + if not coverage_path.is_file(): + return + generation = _load_coverage(project).get("mining_generation") + if not generation: + return + payload = json.loads(artifact.read_text(encoding="utf-8")) + payload["mining_generation"] = generation + atomic_write(artifact, json.dumps(payload, indent=2)) + binding = {"mining_generation": generation, "sha256": digest(artifact)} + atomic_write(str(artifact) + ".generation.json", json.dumps(binding, indent=2)) diff --git a/archaeology/report.py b/archaeology/report.py index dbd9b7a..a2887c3 100644 --- a/archaeology/report.py +++ b/archaeology/report.py @@ -49,14 +49,27 @@ def export_markdown_report( """Export a concise Markdown report from canonical metrics + analysis JSON.""" project_root = Path(project_root) deliverables = project_root / "deliverables" - from .provenance import verify_analyses + from .provenance import ( + analysis_paths, + verify_analyses, + verify_derived_stage, + verify_mining_inputs, + ) - verify_analyses(project_root, require=True) + # Fail closed when mined CSV/stats diverge from coverage.json hashes. + verify_mining_inputs(project_root) + # Refuse mixed-generation export after a failed remine (old DB/metrics under new coverage). + verify_derived_stage(project_root) + # Require bound analyses only when analysis artifacts exist; signals-only + # reports remain valid after mine → signals. + verify_analyses(project_root, require=bool(analysis_paths(project_root))) analysis_dir = deliverables / "analysis" data_dir = project_root / "data" project = _load_json(project_root / "project.json") or {} canonical = _load_json(deliverables / "canonical-metrics.json") or {} eras = _load_json(data_dir / "commit-eras.json") or {} + gap_clusters = _load_json(data_dir / "gap-clusters.json") or {} + detected = _load_json(data_dir / "detected-signals.json") or {} # Look for analysis files in analysis/ subdirectory first, then fall back to deliverables root analysis_search = [analysis_dir, deliverables] analyses = {} @@ -92,14 +105,91 @@ def export_markdown_report( out.append("\n") era_list = eras.get("eras") if isinstance(eras, dict) else [] - if era_list: + cluster_file_list = gap_clusters.get("eras") if isinstance(gap_clusters, dict) else [] + era_semantics = "" + if isinstance(gap_clusters, dict) and gap_clusters.get("era_semantics"): + era_semantics = str(gap_clusters.get("era_semantics") or "") + elif isinstance(eras, dict) and eras.get("era_semantics"): + era_semantics = str(eras.get("era_semantics") or "") + out.append("## Era Semantics\n\n") + if era_semantics: + out.append(f"{era_semantics}\n\n") + else: + out.append( + "Automated archaeology emits **signals** and **gap-separated clusters**. " + "Clusters are heuristic groupings, not curated human development eras. " + "A `commit-eras.json` narrative era list is optional and human-authored; " + "generated clusters live in `gap-clusters.json`.\n\n" + ) + + # Prefer curated human eras from commit-eras.json; never treat gap clusters as eras. + curated = [ + e for e in (era_list or []) if isinstance(e, dict) and e.get("kind") != "gap_cluster" + ] + legacy_clusters_in_eras = [ + e for e in (era_list or []) if isinstance(e, dict) and e.get("kind") == "gap_cluster" + ] + if curated: out.append("## Development Eras\n\n") - for era in era_list: + for era in curated: + out.append( + _bullet( + f"**Era {era.get('id')}: {era.get('name')}** — {era.get('dates', 'unknown dates')}; " + f"{era.get('commits', 'unknown')} commits. " + f"{era.get('description') or era.get('narrative_arc') or ''}" + ) + ) + out.append("\n") + + display_clusters = list(cluster_file_list or []) or list(legacy_clusters_in_eras or []) + if display_clusters: + out.append("## Gap-Separated Clusters\n\n") + for era in display_clusters: + if not isinstance(era, dict): + continue + out.append( + _bullet( + f"**Cluster {era.get('id')}: {era.get('name')}** — {era.get('dates', 'unknown dates')}; " + f"{era.get('commits', 'unknown')} commits. " + f"{era.get('description') or era.get('narrative_arc') or ''}" + ) + ) + out.append("\n") + + # Measured signals from detect_signals (always prefer evidence over silence). + signal_list = [] + cluster_list = [] + if isinstance(detected, dict): + signal_list = detected.get("signals") or [] + cluster_list = detected.get("cluster_summary") or [] + if signal_list or cluster_list: + out.append("## Detected Signals\n\n") + out.append( + f"Measured {len(signal_list)} signal(s) and {len(cluster_list)} gap cluster(s). " + "These are heuristic investigation leads, not verified conclusions.\n\n" + ) + for sig in list(signal_list)[:25]: + if not isinstance(sig, dict): + out.append(_bullet(str(sig)[:240])) + continue + detail = sig.get("detail") or sig.get("description") or sig.get("message") or "" out.append( _bullet( - f"**Era {era.get('id')}: {era.get('name')}** — {era.get('dates', 'unknown dates')}; {era.get('commits', 'unknown')} commits. {era.get('description') or era.get('narrative_arc') or ''}" + f"**{sig.get('type') or sig.get('kind') or 'signal'}** " + f"@ {sig.get('date') or sig.get('when') or 'unknown'}: {detail}" ) ) + if cluster_list and not display_clusters: + out.append("\n### Clusters (from signals)\n\n") + for cluster in list(cluster_list)[:25]: + if not isinstance(cluster, dict): + continue + out.append( + _bullet( + f"{cluster.get('start_date')} → {cluster.get('end_date')} " + f"({cluster.get('commit_count')} commits, {cluster.get('active_days')} active days)" + ) + ) out.append("\n") sdlc = analyses.get("sdlc-gap-finder") or {} diff --git a/plugins/devarch-hermes/.gitignore b/plugins/devarch-hermes/.gitignore new file mode 100644 index 0000000..b908d4c --- /dev/null +++ b/plugins/devarch-hermes/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.DS_Store diff --git a/plugins/devarch-hermes/LICENSE b/plugins/devarch-hermes/LICENSE new file mode 100644 index 0000000..57bc88a --- /dev/null +++ b/plugins/devarch-hermes/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/plugins/devarch-hermes/README.md b/plugins/devarch-hermes/README.md new file mode 100644 index 0000000..6451d5b --- /dev/null +++ b/plugins/devarch-hermes/README.md @@ -0,0 +1,111 @@ +# DevArch for Hermes + +**Disclosure first:** this Apache-2.0 plugin vendors DevArch v0.4.1 from +`b44430e198e45bfaee1edb2f3473f7d50ce17bc2` plus the patches in VENDOR.md. +Installation downloads the exact `click`, `sqlite-utils`, and `datasette` versions +in plugin.yaml from PyPI. Invocation uses the DevArch Python API directly, not +`devarch` or its MCP server. Framework internals launch local Git and +`python -m sqlite_utils` subprocesses. No account, API key, model call or paid +service is needed. Child processes inherit the host environment; this is **not** +a credential sandbox. Mining disables Git transports and lazy fetches, refuses +shallow/promisor/missing-object repositories, and never fetches remotes. + +Reads target Git history; writes SQLite, extracted data, reports, lock files and +a summary ledger only under `/plugin-data/devarch/`. No telemetry. +Optional HTML reports reference shared design-system web fonts when opened in a +browser; use the Markdown report for an entirely offline viewing path. + +## Why native, rather than an MCP wrapper? + +The `pre_llm_call` hook recognizes history/evolution questions and injects an +archaeology brief before Hermes guesses at an answer. Tools measure history; +`post_tool_call` records a profile-local summary ledger. `/devarch` and a bundled +skill complete the same workflow without starting an MCP server. + +| Surface | Behavior | +|---|---| +| `devarch_mine` | Extract local commits into SQLite; capped at 500 by default | +| `devarch_signals` | Measured gaps, velocity/author shifts and gap clusters | +| `devarch_report` | Markdown in the session, optional HTML path | +| `devarch_pipeline` | Mine → signals → six best-effort analysis vectors → report | +| `devarch_list`, `devarch_version` | Project inventory and vendor source pin | +| `/devarch` | mine, signals, report, pipeline, list, version, status, help | +| Hooks | archaeology brief and mine/report summary ledger | + +## Install and run + +Requires Git, Python >=3.10 and Hermes >=0.21.6. Catalog installation must pin the +**accepted merge SHA**, not a moving branch. Before catalog submission, use the +exact reviewed checkout locally: + +```sh +hermes plugins validate /path/to/devarch-framework/plugins/devarch-hermes --install-deps --json +hermes plugins install KyaniteLabs/devarch-framework --subdir plugins/devarch-hermes +hermes plugins enable devarch +``` + +The unpinned repository install above is for local development only. Follow +`submission/CLICK-PACKET.md` for pinned catalog preparation. + +```text +/devarch pipeline /path/to/full/local/checkout demo +/devarch report demo +/devarch status +``` + +Ask: “How did this codebase evolve?” The hook supplies the evidence-first brief; +Hermes still needs a local repository path before mining. + +## Integrity and concurrency + +- Reports disclose full local ref coverage **and the extracted sample count**. + Capped results are not a complete-history claim. Deleted, inaccessible and + unfetched refs remain outside coverage; shallow history is rejected, not + relabeled as sampling. +- A per-project reentrant transaction covers mine, signals, export and the whole + pipeline, with thread and OS file locks. Separate projects remain independent. +- Raw files, DB and present metrics/signals/clusters are checked against their + mining generation and hashes. Failed remining cannot export stale evidence. + This is consistency checking, not signed/tamper-proof forensic custody. +- The patched vendor has an isolated namespace. An installed `archaeology` package + neither replaces it nor gets replaced by it. +- Gap clusters are **not curated eras**. Human `commit-eras.json` survives mining + and signal detection. Vectors are keyword investigation leads, never causal + conclusions or productivity scores. +- Gateway slash mine/pipeline/report are disabled by default. Agent tools retain + local filesystem authority; the slash restriction is not tool authorization. + +## Settings (`plugins.entries.devarch.settings`) + +| Key | Default | Meaning | +|---|---|---| +| `inject_on_archaeology` | true | Hook brief | +| `allow_repo_slash_on_gateway` | false | Explicit slash opt-in | +| `max_commits` | 500 | Positive hard extraction cap across local refs | +| `max_commits_warn` | 5000 | Warn about full local history size | +| `default_project` | session | Project used when omitted | + +## Verification + +From the framework root, with plugin dependencies and pytest/ruff installed: + +```sh +PYTHONPATH="$PWD" python -m pytest -q --tb=short +ruff check . && ruff format --check . +hermes plugins validate plugins/devarch-hermes --json +``` + +Fresh-process plugin-only tests run with and without an incompatible installed +framework. Deterministic schedules cover mine/mine, signals/mine and report/mine; +content/generation/binding mutations are rejected. See the card's evidence packet +for native PluginManager/tool-registry/hooks/slash execution, network tracing, +hermes-agent demo timing, and installed-wheel P16/U7 replay. A native API harness +is not a live deployed chat session or a post-merge Product acceptance. + +## Rollback + +Disable with `hermes plugins disable devarch`. Target repositories need no +rollback because they are read-only. Preserve plugin-data if you need its +reports; deleting that directory is a separate deliberate data-loss decision. + +Demo/capture and Simon-only catalog steps: `submission/`. diff --git a/plugins/devarch-hermes/VENDOR.md b/plugins/devarch-hermes/VENDOR.md new file mode 100644 index 0000000..d33171c --- /dev/null +++ b/plugins/devarch-hermes/VENDOR.md @@ -0,0 +1,34 @@ +# Vendored DevArch Framework package + +`archaeology/` is copied from KyaniteLabs/devarch-framework @ `b44430e198e45bfaee1edb2f3473f7d50ce17bc2` (v0.4.1). + +Omitted from the vendor copy (not used by this Hermes wrap): +- `mcp_server/` (optional `mcp` extra) +- `templates/` and large `visualization/*.html` dashboards (HTML comment scanner caution; wrap uses markdown report + `_markdown_to_html`) + +Refresh from repo root: + +```bash +rm -rf plugins/devarch-hermes/archaeology +mkdir -p plugins/devarch-hermes/archaeology +tar -C archaeology -cf - \ + --exclude='__pycache__' --exclude='*.pyc' --exclude='mcp_server' \ + --exclude='templates' \ + --exclude='visualization/template.html' \ + --exclude='visualization/global-template.html' \ + --exclude='visualization/multi-project-dashboard.html' \ + . | tar -C plugins/devarch-hermes/archaeology -xf - +``` + +## Plugin-local patches (KLH-328 Kai review) + +Applied on top of the pinned vendor copy for the Hermes wrap (kept in sync with matching framework `archaeology/` fixes where noted): + +- `extractors/git.py` — `GIT_NO_LAZY_FETCH=1` on all git invocations; `incomplete_object_reason()` refuses promisor/partial/missing-object repos (local-only contract). +- `report.py` — include measured `detected-signals.json` + `gap-clusters.json`; honest era/cluster semantics; curated `commit-eras.json` preserved; `verify_mining_inputs` + `verify_derived_stage`; `verify_analyses(require=…)` only when analysis artifacts exist. +- `provenance.py`, `metrics.py` — present derived metrics/signals/clusters require both the current generation and a content-hash sidecar (also synchronized to root framework). Optional absence does not exempt a present artifact. +- `extractors/git.py` — positive commit cap for both raw/stats extraction, disabled transports and optional Git locks; full ref coverage remains separately reported. +- `report.py`, `visualization/dashboard.py`, `visualization/agent_benchmark.py` — package-relative imports support the isolated `_devarch_vendor` namespace. The wrapper never prefers an arbitrary installed framework. +- Wrapper project transactions serialize full operations across threads/processes; complete pipelines retain the same lock across nested stages. + +**Refresh warning:** the raw copy commands above do not preserve these patches. Reapply and rerun the full framework + plugin suite, native runtime/validator and deterministic concurrency/content-integrity probes after any refresh. diff --git a/plugins/devarch-hermes/__init__.py b/plugins/devarch-hermes/__init__.py new file mode 100644 index 0000000..499bd49 --- /dev/null +++ b/plugins/devarch-hermes/__init__.py @@ -0,0 +1,1425 @@ +"""DevArch — native Hermes plugin (git repository archaeology). + +Surfaces: + pre_llm_call -> when a message asks about repo history/evolution, inject an archaeology brief + post_tool_call -> record mine/report summaries in a profile-local ledger + /devarch -> mine | signals | report | pipeline | list | version | status | help + tools -> devarch_mine / _signals / _report / _pipeline / _list / _version + skill -> skills/devarch/SKILL.md + +Calls the DevArch Python API in-process (no CLI shell-out). Project DBs, signals, +reports and the ledger live under /plugin-data/devarch/ via +plugins.plugin_storage. Only public ctx.* APIs. Mining is local-only and rejects +shallow and incomplete-object (promisor) clones. Analysis vectors are investigation +leads, not verified conclusions. Process cwd / os.environ are never mutated for +workspace routing — absolute project paths only. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +import shlex +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional + +# Load only our patched vendor, without replacing an installed archaeology package. +_PLUGIN_DIR = Path(__file__).resolve().parent + + +def _load_vendor() -> None: + import importlib.util + + name = "_devarch_vendor" + root = _PLUGIN_DIR / "archaeology" + existing = sys.modules.get(name) + if existing is not None: + if Path(existing.__file__).resolve() != (root / "__init__.py").resolve(): + raise ImportError("DevArch vendor namespace belongs to another installation") + return + spec = importlib.util.spec_from_file_location( + name, root / "__init__.py", submodule_search_locations=[str(root)] + ) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(name, None) + raise + + +_load_vendor() + +logger = logging.getLogger(__name__) + +PLUGIN_NAME = "devarch" +PINNED_SHA = "b44430e198e45bfaee1edb2f3473f7d50ce17bc2" +DEFAULTS: Dict[str, Any] = { + "inject_on_archaeology": True, + "allow_repo_slash_on_gateway": False, + "max_commits_warn": 5000, + "max_commits": 500, + "default_project": "session", +} +LEDGER_MAX = 200 +MAX_OUT = 20000 +_PROJECT_RE = re.compile(r"^[a-zA-Z0-9._-]{1,100}$") +_LOCAL_SURFACES = frozenset( + { + "", + "api_server", + "cli", + "codex", + "desktop", + "gateway", + "kanban", + "local", + "msgraph_webhook", + "tool", + "tui", + "webhook", + "acp", + } +) + +_ARCH_RE = re.compile( + r"\b(?:" + r"git\s+history|commit\s+history|repo(?:sitory)?\s+history|" + r"codebase\s+evol(?:ve|ution)|how\s+(?:this|the)\s+(?:code|repo|codebase)\s+evolved|" + r"archaeolog|devarch|era(?:s)?\b|velocity\s+shift|author\s+change|" + r"development\s+signals?|mine\s+(?:the\s+)?(?:repo|history|commits)" + r")\b", + re.IGNORECASE, +) + +_ctx = None +_ledger_lock = threading.Lock() +_project_locks = {} +_project_locks_guard = threading.Lock() +_transaction_state = threading.local() + + +def _project_transaction(function): + from functools import wraps + from inspect import signature + + call_signature = signature(function) + + @wraps(function) + def serialized(*args, **kwargs): + arguments = call_signature.bind(*args, **kwargs) + project = _project_name(arguments.arguments.get("project")) + key = str(_project_dir(project)) + with _project_locks_guard: + lock = _project_locks.setdefault(key, threading.RLock()) + with lock: + active = getattr(_transaction_state, "active", set()) + if key in active: + return function(*args, **kwargs) + # Keep lock files outside mutable project generations. OS locks also + # serialize independent Hermes processes sharing a profile data dir. + lock_dir = _workspace() / ".locks" + lock_dir.mkdir(exist_ok=True) + with (lock_dir / f"{project}.lock").open("a+b") as handle: + if os.name == "nt": + import msvcrt + + handle.write(b"\0") + handle.flush() + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + else: + import fcntl + + fcntl.flock(handle, fcntl.LOCK_EX) + _transaction_state.active = active | {key} + try: + return function(*args, **kwargs) + finally: + _transaction_state.active = active + if os.name == "nt": + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(handle, fcntl.LOCK_UN) + + return serialized + + +# --- config / storage ------------------------------------------------------- + + +def _cfg(key: str) -> Any: + try: + val = _ctx.get_config(key, None) if _ctx is not None else None + except Exception: + val = None + return DEFAULTS[key] if val is None else val + + +def _data_dir() -> Path: + try: + from plugins.plugin_storage import plugin_data_dir + + return plugin_data_dir(PLUGIN_NAME) + except Exception: + home = os.environ.get("HERMES_HOME") or str(Path.home() / ".hermes") + p = Path(home) / "plugin-data" / PLUGIN_NAME + p.mkdir(parents=True, exist_ok=True) + return p + + +def _workspace() -> Path: + """DevArch project root — always under plugin-data (never the target repo).""" + ws = _data_dir() / "workspace" + ws.mkdir(parents=True, exist_ok=True) + (ws / "projects").mkdir(parents=True, exist_ok=True) + return ws + + +def _ledger_path() -> Path: + return _data_dir() / "report_ledger.json" + + +def _load_json(path: Path, default: Any) -> Any: + try: + if path.is_file(): + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + logger.debug("devarch: failed reading %s", path, exc_info=True) + return default + + +def _save_json(path: Path, payload: Any) -> None: + """Atomic write with a unique temp name (safe under concurrent callers).""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=str(path.parent), + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(json.dumps(payload, indent=2, ensure_ascii=False) + "\n") + os.replace(tmp_name, path) + except BaseException: + try: + os.unlink(tmp_name) + except OSError: + pass + raise + + +def _on_gateway() -> bool: + """True when the active Hermes session is a messaging/gateway surface. + + Hermes >=0.21.6 stores session platform in task-local ContextVars, not only + os.environ. Prefer gateway.session_context helpers; fall back to env. + """ + try: + from gateway.session_context import get_session_env + + plat = ( + ( + os.environ.get("HERMES_PLATFORM") + or get_session_env("HERMES_SESSION_PLATFORM", "") + or get_session_env("HERMES_SESSION_SOURCE", "") + or "" + ) + .strip() + .lower() + ) + except Exception: + plat = ( + (os.environ.get("HERMES_PLATFORM") or os.environ.get("HERMES_SESSION_PLATFORM") or "") + .strip() + .lower() + ) + return bool(plat) and plat not in _LOCAL_SURFACES + + +# --- helpers ---------------------------------------------------------------- + + +def _project_name(raw: Optional[str]) -> str: + name = str(raw if raw not in (None, "") else _cfg("default_project")).strip() or "session" + if not _PROJECT_RE.match(name) or ".." in name: + raise ValueError( + f"invalid project name {name!r}: use 1-100 chars of [A-Za-z0-9._-], no '..'" + ) + return name + + +def _project_dir(project: str) -> Path: + """Resolve /projects/ with traversal checks.""" + projects = _workspace() / "projects" + projects.mkdir(parents=True, exist_ok=True) + candidate = (projects / project).resolve() + try: + candidate.relative_to(projects.resolve()) + except ValueError as exc: + raise ValueError(f"project path escape detected: {project!r}") from exc + return candidate + + +def _resolve_repo(repo_path: str) -> Path: + raw = str(repo_path or "").strip() + if not raw: + raise ValueError("repo_path is required") + p = Path(raw).expanduser() + p = (Path.cwd() / p).resolve() if not p.is_absolute() else p.resolve() + if not p.exists() or not p.is_dir(): + raise ValueError(f"repository path does not exist: {p}") + git_meta = p / ".git" + if not git_meta.exists() and not (p / "HEAD").exists(): + raise ValueError(f"not a git repository: {p}") + return p + + +def _coverage(repo: Path) -> Dict[str, Any]: + from _devarch_vendor.extractors.git import repository_coverage + + return repository_coverage(str(repo)) + + +def _reject_shallow(cov: Dict[str, Any]) -> Optional[str]: + if cov.get("shallow"): + return ( + "Refusing to mine a shallow clone (evidence integrity 0.4.x). " + "Run `git fetch --unshallow` (or clone with full history) and retry." + ) + return None + + +def _incomplete_object_reason(repo_path: str) -> Optional[str]: + """Refuse promisor/partial clones that would lazy-fetch missing blobs. + + Implemented in the plugin so framework archaeology need not be shadowed just + to pick up the vendor-only helper during the monorepo test suite. + """ + try: + from _devarch_vendor.extractors.git import incomplete_object_reason as vendor_fn + + return vendor_fn(repo_path) + except ImportError: + pass + + repo = Path(repo_path).expanduser() + try: + raw = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "--git-dir"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=60, + env={**os.environ, "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"}, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + if raw.returncode != 0: + return None + git_dir = Path(raw.stdout.strip()) + if not git_dir.is_absolute(): + git_dir = (repo / git_dir).resolve() + pack = git_dir / "objects" / "pack" + if pack.is_dir() and list(pack.glob("*.promisor")): + return ( + "Refusing incomplete-object / partial (promisor) clone — mining is local-only " + "and must not fetch missing blobs. Re-clone with full objects and retry." + ) + env = {**os.environ, "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} + + def _cfg(*args: str) -> str: + r = subprocess.run( + ["git", "-C", str(repo), "config", *args], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=60, + env=env, + ) + return r.stdout if r.returncode == 0 else "" + + filt = _cfg("--get", "remote.origin.partialclonefilter").strip() + promisor_cfg = _cfg("--get-regexp", r"\.promisor$").strip() + if filt or promisor_cfg: + return ( + "Refusing partial clone (promisor/filter configured) — mining is local-only " + "and must not fetch missing blobs. Re-clone without `--filter`/`--partial` and retry." + ) + return None + + +def _reject_incomplete(repo: Path) -> Optional[str]: + """Refuse promisor/partial clones that would lazy-fetch missing blobs.""" + return _incomplete_object_reason(str(repo)) + + +def version() -> Dict[str, Any]: + try: + from _devarch_vendor import __version__ as ver + except Exception as exc: + return { + "ok": False, + "error": ( + f"devarch-framework not importable ({type(exc).__name__}: {exc}). " + "Run `hermes plugins install …` / `hermes plugins validate --install-deps`." + ), + } + return { + "ok": True, + "version": ver, + "pinned_sha": PINNED_SHA, + "workspace": str(_workspace()), + "integrity": ( + "rejects shallow + incomplete/promisor objects; " + "vectors are leads not conclusions; absolute project paths (no cwd mutation)" + ), + } + + +def list_projects() -> Dict[str, Any]: + projects_dir = _workspace() / "projects" + projects: List[Dict[str, Any]] = [] + if projects_dir.exists(): + for project_dir in sorted(projects_dir.iterdir()): + if not project_dir.is_dir() or project_dir.name.startswith((".", "_")): + continue + config = _load_json(project_dir / "project.json", {}) or {} + projects.append( + { + "name": project_dir.name, + "path": str(project_dir), + "has_data": (project_dir / "data").exists(), + "has_deliverables": (project_dir / "deliverables").exists(), + "has_database": (project_dir / "data" / "archaeology.db").exists(), + "description": config.get("description", ""), + "repo_url": config.get("repo_url", ""), + } + ) + return { + "ok": True, + "projects": projects, + "count": len(projects), + "workspace": str(_workspace()), + } + + +def _init_project(project: str, description: str = "", repo_url: str = "") -> Dict[str, Any]: + project_dir = _project_dir(project) + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "data").mkdir(exist_ok=True) + (project_dir / "deliverables").mkdir(exist_ok=True) + config_path = project_dir / "project.json" + if config_path.exists(): + return {"status": "exists", "project": project, "path": str(project_dir)} + config = { + "name": project, + "description": description, + "repo_url": repo_url, + "developer": {}, + "timeline": {}, + "overrides": {}, + "visualization": {}, + "data_sources": {}, + } + _save_json(config_path, config) + return {"status": "created", "project": project, "path": str(project_dir)} + + +_DERIVED_REL_PATHS = ( + "data/archaeology.db", + "data/detected-signals.json", + "data/gap-clusters.json", + "deliverables/canonical-metrics.json", + "deliverables/data.json", +) + + +def _remove_file(path: Path) -> None: + """Best-effort delete that tolerates Windows SQLite file locks (WinError 32).""" + if not path.is_file(): + return + try: + path.unlink() + return + except PermissionError: + pass + except OSError: + pass + # Rename aside so a new archaeology.db can be created at the canonical path. + trash = path.with_name(f".{path.name}.stale-{uuid.uuid4().hex}") + try: + path.replace(trash) + except OSError: + try: + path.write_bytes(b"") + except OSError: + logger.debug("devarch: could not clear locked file %s", path, exc_info=True) + return + try: + trash.unlink() + except OSError: + # Leftover trash is harmless; next remine uses a fresh uuid name. + logger.debug("devarch: deferred delete of %s", trash, exc_info=True) + + +def _invalidate_derived_artifacts(project_dir: Path) -> None: + """Drop derived evidence so a failed remine cannot export a mixed generation. + + Preserves human-authored commit-eras.json. Generated clusters live in gap-clusters.json. + """ + for rel in _DERIVED_REL_PATHS: + target = project_dir / rel + _remove_file(target) + _remove_file(Path(str(target) + ".provenance.json")) + for pattern in ( + "deliverables/analysis-*.json", + "deliverables/analysis/analysis-*.json", + "deliverables/analysis-*.json.provenance.json", + "deliverables/analysis/*.provenance.json", + "deliverables/reports/*", + "deliverables/visuals/*", + ): + for target in project_dir.glob(pattern): + _remove_file(target) + + +def _bind_mining_manifest( + project_dir: Path, + repo: Path, + cov: Dict[str, Any], + csv_path: Path, + stats_path: Path, + *, + generation: str, + status: str = "incomplete", + derived_sha256: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: + """Mirror the canonical CLI: coverage.json + mined_history_manifest_required + hashes.""" + from _devarch_vendor.extractors.git import repository_coverage + + post = repository_coverage(str(repo)) + + # Compare identity fields; ignore any future artifact_sha256 on either side. + def _identity(c: Dict[str, Any]) -> Dict[str, Any]: + return { + "head": c.get("head"), + "scope": c.get("scope"), + "shallow": c.get("shallow"), + "bare": c.get("bare"), + "commit_count": c.get("commit_count"), + "refs": c.get("refs"), + "roots": c.get("roots"), + } + + if _identity(post) != _identity(cov): + raise RuntimeError("Repository refs changed during mining; retry before analysis") + + coverage = dict(cov) + coverage["artifact_sha256"] = { + csv_path.name: hashlib.sha256(csv_path.read_bytes()).hexdigest(), + stats_path.name: hashlib.sha256(stats_path.read_bytes()).hexdigest(), + } + coverage["mining_generation"] = generation + coverage["mining_status"] = status + if derived_sha256 is not None: + coverage["derived_sha256"] = derived_sha256 + elif "derived_sha256" in coverage: + # Incomplete rebuild must not keep prior derived bindings. + coverage.pop("derived_sha256", None) + _save_json(project_dir / "data" / "coverage.json", coverage) + + config_path = project_dir / "project.json" + config = _load_json(config_path, {}) or {} + config["repo_path"] = str(repo) + config["mined_history_manifest_required"] = True + _save_json(config_path, config) + return coverage + + +@_project_transaction +def mine(repo_path: str, project: Optional[str] = None) -> Dict[str, Any]: + """Mine local git history into a DevArch project under plugin-data.""" + proj = _project_name(project) + repo = _resolve_repo(repo_path) + cov = _coverage(repo) + cap = int(_cfg("max_commits")) + if cap < 1: + return {"ok": False, "error": "max_commits must be a positive integer"} + cov["max_commits"] = cap + cov["extracted_count"] = min(int(cov.get("commit_count") or 0), cap) + cov["sampled"] = int(cov.get("commit_count") or 0) > cap + shallow_err = _reject_shallow(cov) + if shallow_err: + return {"ok": False, "error": shallow_err, "coverage": cov} + incomplete_err = _reject_incomplete(repo) + if incomplete_err: + return {"ok": False, "error": incomplete_err, "coverage": cov} + + warn = None + count = int(cov.get("commit_count") or 0) + threshold = int(_cfg("max_commits_warn") or 0) + if threshold and count > threshold: + warn = ( + f"Local commit count {count} exceeds max_commits_warn={threshold}; " + "mining still runs — expect longer wall-clock." + ) + + _init_project(proj, description=f"Hermes archaeology of {repo}", repo_url=str(repo)) + + from _devarch_vendor.db.builder import build_db + from _devarch_vendor.extractors.git import extract_git_log, extract_git_log_with_stats + + project_dir = _project_dir(proj) + data_dir = project_dir / "data" + data_dir.mkdir(parents=True, exist_ok=True) + csv_path = data_dir / "github-commits.csv" + stats_path = data_dir / "github-commits-with-stats.txt" + generation = uuid.uuid4().hex + # Invalidate before writing a new coverage manifest so a failed remine cannot + # leave old DB/metrics/signals paired with new raw hashes. + _invalidate_derived_artifacts(project_dir) + try: + extracted = extract_git_log(str(repo), str(csv_path), max_commits=cap) + extract_git_log_with_stats(str(repo), str(stats_path), max_commits=cap) + coverage = _bind_mining_manifest( + project_dir, + repo, + cov, + csv_path, + stats_path, + generation=generation, + status="incomplete", + ) + except Exception as exc: + return {"ok": False, "error": f"{type(exc).__name__}: {exc}", "coverage": cov} + + try: + # Quiet via verbose=False only — never mutate process-wide stdout/stderr. + build_db(project_root=project_dir, verbose=False) + db_path = project_dir / "data" / "archaeology.db" + if not db_path.exists(): + return { + "ok": False, + "error": "Database build produced no archaeology.db", + "status": "failed", + "project": proj, + "coverage": { + "commit_count": coverage.get("commit_count"), + "shallow": coverage.get("shallow"), + "head": coverage.get("head"), + "mining_status": "incomplete", + "mining_generation": generation, + }, + "commits_extracted": extracted, + } + derived = { + "archaeology.db": hashlib.sha256(db_path.read_bytes()).hexdigest(), + } + coverage = _bind_mining_manifest( + project_dir, + repo, + cov, + csv_path, + stats_path, + generation=generation, + status="complete", + derived_sha256=derived, + ) + db_status = "complete" + except Exception as exc: + return { + "ok": False, + "error": f"database build failed: {type(exc).__name__}: {exc}", + "status": "failed", + "project": proj, + "failed_at": "build_db", + "commits_extracted": extracted, + "coverage": { + "commit_count": coverage.get("commit_count"), + "shallow": coverage.get("shallow"), + "head": coverage.get("head"), + "mining_status": "incomplete", + "mining_generation": generation, + }, + "database": f"failed: {type(exc).__name__}: {exc}", + } + + res: Dict[str, Any] = { + "ok": True, + "status": "complete", + "project": proj, + "repo": str(repo), + "commits_extracted": extracted, + "coverage": { + "extracted_count": extracted, + "sampled": cov["sampled"], + "max_commits": cap, + "commit_count": coverage.get("commit_count"), + "shallow": coverage.get("shallow"), + "scope": coverage.get("scope"), + "head": coverage.get("head"), + "mining_generation": generation, + "mining_status": "complete", + }, + "database": db_status, + "evidence_binding": True, + "workspace": str(_workspace()), + } + if warn: + res["warning"] = warn + return res + + +def _write_gap_clusters(project_dir: Path, project: str, detection: Dict[str, Any]) -> None: + """Persist gap-based clusters to gap-clusters.json — never overwrite curated eras. + + Signal detection does not invent narrative eras. Clusters are gap-separated + commit groupings stored separately from human-authored commit-eras.json. + """ + clusters = detection.get("cluster_summary") or [] + signals = detection.get("signals") or [] + total = sum(int(c.get("commit_count") or 0) for c in clusters) if clusters else 0 + eras = [] + for idx, cluster in enumerate(clusters, start=1): + start = cluster.get("start_date") or "?" + end = cluster.get("end_date") or start + dates = start if start == end else f"{start} to {end}" + eras.append( + { + "id": idx, + "name": f"Cluster {idx}", + "dates": dates, + "commits": cluster.get("commit_count"), + "description": ( + f"Gap-separated activity cluster ({cluster.get('active_days', '?')} active days; " + f"primary author={cluster.get('primary_author') or 'unknown'})." + ), + "narrative_arc": ( + "Automated cluster from commit-date gaps — not a curated human era narrative." + ), + "kind": "gap_cluster", + } + ) + coverage = _load_json(project_dir / "data" / "coverage.json", {}) or {} + generation = coverage.get("mining_generation") + payload = { + "project": project, + "total_commits": total, + "eras": eras, + "era_semantics": ( + "gap_clusters: clusters are heuristic groupings separated by multi-day gaps. " + "They are not curated development eras. Curated eras require human narrative judgment " + "and live in commit-eras.json (never overwritten by signals)." + ), + "signal_count": len(signals), + "cluster_count": len(clusters), + "mining_generation": generation, + } + _save_json(project_dir / "data" / "gap-clusters.json", payload) + + +@_project_transaction +def signals(project: Optional[str] = None, min_gap_days: Optional[int] = None) -> Dict[str, Any]: + proj = _project_name(project) + from _devarch_vendor.classifiers.era_detector import SignalDetector + + project_dir = _project_dir(proj) + db_path = project_dir / "data" / "archaeology.db" + if not db_path.exists(): + return {"ok": False, "error": "Database not found. Run devarch_mine first."} + config: Dict[str, Any] = {} + if min_gap_days is not None: + config["min_gap_days"] = int(min_gap_days) + try: + from _devarch_vendor.provenance import ( + bind_derived_artifact, + verify_derived_stage, + verify_mining_inputs, + ) + + verify_mining_inputs(project_dir) + verify_derived_stage(project_dir) + detector = SignalDetector(db_path, config=config or None) + out = detector.detect() + out_path = project_dir / "data" / "detected-signals.json" + detector.save(out_path, out) + if isinstance(out, dict): + coverage = _load_json(project_dir / "data" / "coverage.json", {}) or {} + generation = coverage.get("mining_generation") + if generation and "mining_generation" not in out: + out = dict(out) + out["mining_generation"] = generation + _save_json(out_path, out) + _write_gap_clusters(project_dir, proj, out if isinstance(out, dict) else {}) + bind_derived_artifact(project_dir, out_path) + bind_derived_artifact(project_dir, project_dir / "data" / "gap-clusters.json") + except Exception as exc: + return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + signal_list = out.get("signals", []) if isinstance(out, dict) else (out or []) + clusters = out.get("cluster_summary", []) if isinstance(out, dict) else [] + summary = [] + for s in list(signal_list)[:25]: + if isinstance(s, dict): + summary.append( + { + "type": s.get("type") or s.get("kind") or s.get("signal"), + "date": s.get("date") or s.get("when"), + "detail": str(s.get("description") or s.get("detail") or s.get("message") or s)[ + :240 + ], + } + ) + else: + summary.append({"detail": str(s)[:240]}) + cluster_summary = [] + for c in list(clusters)[:25]: + if isinstance(c, dict): + cluster_summary.append( + { + "start_date": c.get("start_date"), + "end_date": c.get("end_date"), + "commit_count": c.get("commit_count"), + "active_days": c.get("active_days"), + } + ) + return { + "ok": True, + "status": "complete", + "project": proj, + "signal_count": len(signal_list), + "cluster_count": len(clusters), + "signals": summary, + "clusters": cluster_summary, + "era_semantics": ( + "Automated output is signals + gap-separated clusters. " + "Clusters are not curated human eras." + ), + "note": "Signals are heuristic investigation leads, not verified conclusions.", + "output_path": str(project_dir / "data" / "detected-signals.json"), + } + + +@_project_transaction +def report(project: Optional[str] = None, fmt: str = "markdown") -> Dict[str, Any]: + """Export a markdown (default) or html report; return a truncated body for the session.""" + proj = _project_name(project) + fmt = (fmt or "markdown").lower().strip() + if fmt not in {"markdown", "html", "md"}: + return {"ok": False, "error": "fmt must be markdown or html"} + if fmt == "md": + fmt = "markdown" + + from _devarch_vendor.report import _markdown_to_html, export_markdown_report + + project_dir = _project_dir(proj) + if not project_dir.exists(): + return {"ok": False, "error": f"Project '{proj}' not found. Run devarch_mine first."} + try: + md_path = export_markdown_report(proj, str(project_dir)) + except Exception as exc: + return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + md_text = Path(md_path).read_text(encoding="utf-8") if Path(md_path).is_file() else "" + output_path = str(md_path) + if fmt == "html": + title = proj + config = _load_json(project_dir / "project.json", {}) or {} + title = (config.get("visualization") or {}).get("title", proj) + html = _markdown_to_html(md_text, title) + out_dir = project_dir / "deliverables" / "visuals" + out_dir.mkdir(parents=True, exist_ok=True) + html_path = out_dir / "report.html" + html_path.write_text(html, encoding="utf-8") + output_path = str(html_path) + + return { + "ok": True, + "status": "complete", + "project": proj, + "format": fmt, + "output_path": output_path, + "markdown": md_text[:12000], + "truncated": len(md_text) > 12000, + "note": ( + "Vectors/signals in this report are investigation leads based on keyword " + "heuristics — not verified source conclusions or productivity measurements. " + "Gap clusters are not curated eras." + ), + } + + +def _classify_analysis(result: Any) -> Dict[str, Any]: + """Separate successful vector paths from ERROR: values.""" + if not isinstance(result, dict): + return { + "ok": False, + "vectors_run": [], + "vector_errors": {"_": f"unexpected analysis result type: {type(result).__name__}"}, + "note": "Keyword-based leads, not verified conclusions.", + } + errors: Dict[str, str] = {} + ok_vecs: List[str] = [] + for name, value in result.items(): + if isinstance(value, str) and value.startswith("ERROR:"): + errors[name] = value + else: + ok_vecs.append(name) + if errors and not ok_vecs: + status_ok = False + partial = False + elif errors and ok_vecs: + status_ok = False + partial = True + else: + status_ok = True + partial = False + return { + "ok": status_ok, + "partial": partial, + "vectors_run": ok_vecs, + "vector_errors": errors, + "note": "Keyword-based leads, not verified conclusions.", + } + + +@_project_transaction +def pipeline(repo_path: str, project: Optional[str] = None) -> Dict[str, Any]: + """mine → signals → markdown report (analyze vectors optional / best-effort).""" + proj = _project_name(project) + mined = mine(repo_path, proj) + if not mined.get("ok"): + return {"ok": False, "status": "failed", "failed_at": "mine", **mined} + sig = signals(proj) + if not sig.get("ok"): + return {"ok": False, "status": "failed", "failed_at": "signals", "mine": mined, **sig} + + analysis: Dict[str, Any] + try: + from _devarch_vendor.analysis_runner import AnalysisRunner + + project_dir = _project_dir(proj) + runner = AnalysisRunner(proj, str(project_dir), verbose=False) + result = runner.run_all(vectors=None) + analysis = _classify_analysis(result) + except Exception as exc: + analysis = { + "ok": False, + "partial": False, + "error": f"{type(exc).__name__}: {exc}", + "vectors_run": [], + "vector_errors": {}, + } + + rep = report(proj, fmt="markdown") + stages_ok = bool(rep.get("ok")) + # Pipeline is complete only when mine+signals+report succeed; analysis may be partial. + if not stages_ok: + status = "failed" + ok = False + failed_at = "report" + elif analysis.get("ok"): + status = "complete" + ok = True + failed_at = None + elif analysis.get("partial"): + status = "partial" + ok = True + failed_at = None + else: + status = "partial" + ok = True # mine+signals+report succeeded; analysis best-effort failed + failed_at = None + + out: Dict[str, Any] = { + "ok": ok, + "status": status, + "project": proj, + "mine": { + "commits_extracted": mined.get("commits_extracted"), + "coverage": mined.get("coverage"), + "warning": mined.get("warning"), + "evidence_binding": mined.get("evidence_binding"), + }, + "signals": { + "signal_count": sig.get("signal_count"), + "cluster_count": sig.get("cluster_count"), + "signals": sig.get("signals"), + "clusters": sig.get("clusters"), + "era_semantics": sig.get("era_semantics"), + }, + "analysis": analysis, + "report": { + "output_path": rep.get("output_path"), + "markdown": rep.get("markdown"), + "truncated": rep.get("truncated"), + "note": rep.get("note"), + "ok": rep.get("ok"), + "error": rep.get("error"), + }, + } + if failed_at: + out["failed_at"] = failed_at + if not rep.get("ok"): + out["error"] = rep.get("error") + return out + + +# --- hooks ------------------------------------------------------------------ + +_BRIEF = ( + "[DevArch archaeology] This message asks about repository history or how the codebase " + "evolved. Before guessing from a few recent commits: (1) call `devarch_pipeline` (or " + "`devarch_mine` → `devarch_signals` → `devarch_report`) on the local git checkout, " + "(2) lead with measured signals and gap-separated clusters from the report (clusters are " + "not curated human eras), (3) label every automated vector as an investigation lead — " + "not a verified conclusion or productivity measurement, " + "(4) refuse shallow / incomplete-object clones (integrity rule). " + "Data stays under plugin-data/devarch/." +) + + +def _on_pre_llm(user_message: str = "", **_: Any) -> Optional[Dict[str, str]]: + try: + if _cfg("inject_on_archaeology") and user_message and _ARCH_RE.search(user_message): + return {"context": _BRIEF} + except Exception: + logger.debug("devarch: pre_llm_call failed", exc_info=True) + return None + + +def _append_ledger(entry: Dict[str, Any]) -> None: + with _ledger_lock: + ledger = _load_json(_ledger_path(), []) + if not isinstance(ledger, list): + ledger = [] + ledger.append(entry) + _save_json(_ledger_path(), ledger[-LEDGER_MAX:]) + + +def _on_post_tool( + tool_name: str = "", args: Optional[Dict[str, Any]] = None, result: Any = None, **_: Any +) -> None: + try: + if tool_name not in { + "devarch_mine", + "devarch_signals", + "devarch_report", + "devarch_pipeline", + }: + return None + payload = result + if isinstance(payload, str): + try: + payload = json.loads(payload) + except Exception: + payload = {} + if not isinstance(payload, dict): + return None + entry = { + "ts": int(time.time()), + "tool": tool_name, + "project": str((args or {}).get("project") or payload.get("project") or "")[:80], + "repo": str((args or {}).get("repo_path") or "")[:300], + "ok": bool(payload.get("ok")), + "commits": payload.get("commits_extracted") + or (payload.get("mine") or {}).get("commits_extracted"), + "signals": payload.get("signal_count") + or (payload.get("signals") or {}).get("signal_count"), + "output_path": payload.get("output_path") + or (payload.get("report") or {}).get("output_path"), + } + _append_ledger(entry) + except Exception: + logger.debug("devarch: post_tool_call failed", exc_info=True) + return None + + +# --- tools ------------------------------------------------------------------ + +MINE_SCHEMA = { + "name": "devarch_mine", + "description": ( + "Mine a local git repository's full commit history into a DevArch project under " + "plugin-data. Rejects shallow and incomplete/promisor clones. Returns commit count + " + "coverage. Local-only (no network fetch; GIT_NO_LAZY_FETCH)." + ), + "parameters": { + "type": "object", + "properties": { + "repo_path": {"type": "string", "description": "Path to a local git checkout"}, + "project": {"type": "string", "description": "DevArch project name (default: session)"}, + }, + "required": ["repo_path"], + }, +} +SIGNALS_SCHEMA = { + "name": "devarch_signals", + "description": ( + "Detect development signals (gaps, velocity shifts, author changes) for a mined " + "DevArch project. Heuristic leads — not verified conclusions. Also records " + "gap-separated clusters (not curated eras)." + ), + "parameters": { + "type": "object", + "properties": { + "project": {"type": "string"}, + "min_gap_days": {"type": "integer"}, + }, + }, +} +REPORT_SCHEMA = { + "name": "devarch_report", + "description": ( + "Export an eras/signals engineering report (markdown by default) for a mined " + "project and return a truncated body for the session. Includes measured signals " + "and gap clusters; states era semantics honestly." + ), + "parameters": { + "type": "object", + "properties": { + "project": {"type": "string"}, + "fmt": {"type": "string", "enum": ["markdown", "html"], "default": "markdown"}, + }, + }, +} +PIPELINE_SCHEMA = { + "name": "devarch_pipeline", + "description": ( + "One-shot archaeology: mine → signals → (best-effort) analysis vectors → markdown " + "report. Prefer this when the user asks how a local repo evolved. Failed stages " + "are reported accurately (ok/status/failed_at)." + ), + "parameters": { + "type": "object", + "properties": { + "repo_path": {"type": "string"}, + "project": {"type": "string"}, + }, + "required": ["repo_path"], + }, +} +LIST_SCHEMA = { + "name": "devarch_list", + "description": "List DevArch projects under the plugin-data workspace.", + "parameters": {"type": "object", "properties": {}}, +} +VERSION_SCHEMA = { + "name": "devarch_version", + "description": "Report the DevArch framework version this plugin imported.", + "parameters": {"type": "object", "properties": {}}, +} + + +def _dumps_capped(payload: Any, max_out: int = MAX_OUT) -> str: + """Serialize to JSON, bounding bulky fields so the envelope stays valid JSON.""" + text = json.dumps(payload, ensure_ascii=False) + if len(text) <= max_out: + return text + if not isinstance(payload, dict): + minimal = { + "ok": False, + "truncated": True, + "error": "output exceeded cap", + "truncation": {"limit": max_out, "original_chars": len(text)}, + } + return json.dumps(minimal, ensure_ascii=False) + + bounded: Dict[str, Any] = dict(payload) + original_chars = len(text) + + def _fits(obj: Dict[str, Any]) -> bool: + return len(json.dumps(obj, ensure_ascii=False)) <= max_out + + # Prefer trimming list/string fields that blow the budget. + for key in ("projects", "signals", "clusters", "markdown"): + value = bounded.get(key) + if isinstance(value, list) and value: + lo, hi = 0, len(value) + best = 0 + while lo <= hi: + mid = (lo + hi) // 2 + candidate = dict(bounded) + candidate[key] = value[:mid] + candidate["truncated"] = True + candidate["truncation"] = { + "field": key, + "returned": mid, + "total": len(value), + "limit_chars": max_out, + "original_chars": original_chars, + } + if _fits(candidate): + best = mid + lo = mid + 1 + else: + hi = mid - 1 + bounded[key] = value[:best] + bounded["truncated"] = True + bounded["truncation"] = { + "field": key, + "returned": best, + "total": len(value), + "limit_chars": max_out, + "original_chars": original_chars, + } + if _fits(bounded): + return json.dumps(bounded, ensure_ascii=False) + elif isinstance(value, str) and len(value) > 256: + # Binary-search string length + lo, hi = 0, len(value) + best = 0 + while lo <= hi: + mid = (lo + hi) // 2 + candidate = dict(bounded) + candidate[key] = value[:mid] + candidate["truncated"] = True + candidate["truncation"] = { + "field": key, + "returned_chars": mid, + "total_chars": len(value), + "limit_chars": max_out, + "original_chars": original_chars, + } + if _fits(candidate): + best = mid + lo = mid + 1 + else: + hi = mid - 1 + bounded[key] = value[:best] + bounded["truncated"] = True + bounded["truncation"] = { + "field": key, + "returned_chars": best, + "total_chars": len(value), + "limit_chars": max_out, + "original_chars": original_chars, + } + if _fits(bounded): + return json.dumps(bounded, ensure_ascii=False) + + # Nested report.markdown etc. + report_obj = bounded.get("report") + if isinstance(report_obj, dict) and isinstance(report_obj.get("markdown"), str): + nested = dict(report_obj) + md = nested["markdown"] + lo, hi = 0, len(md) + best = 0 + while lo <= hi: + mid = (lo + hi) // 2 + candidate_report = dict(nested) + candidate_report["markdown"] = md[:mid] + candidate = dict(bounded) + candidate["report"] = candidate_report + candidate["truncated"] = True + candidate["truncation"] = { + "field": "report.markdown", + "returned_chars": mid, + "total_chars": len(md), + "limit_chars": max_out, + "original_chars": original_chars, + } + if _fits(candidate): + best = mid + lo = mid + 1 + else: + hi = mid - 1 + nested["markdown"] = md[:best] + bounded["report"] = nested + bounded["truncated"] = True + bounded["truncation"] = { + "field": "report.markdown", + "returned_chars": best, + "total_chars": len(md), + "limit_chars": max_out, + "original_chars": original_chars, + } + if _fits(bounded): + return json.dumps(bounded, ensure_ascii=False) + + minimal = { + "ok": bounded.get("ok", True), + "truncated": True, + "error": "output exceeded cap; narrow the query or read output_path on disk", + "truncation": {"limit": max_out, "original_chars": original_chars}, + "project": bounded.get("project"), + "count": bounded.get("count"), + "workspace": bounded.get("workspace"), + "output_path": ( + bounded.get("output_path") + or ( + (bounded.get("report") or {}).get("output_path") + if isinstance(bounded.get("report"), dict) + else None + ) + ), + } + return json.dumps(minimal, ensure_ascii=False) + + +def _wrap(fn): + def handler(args: Optional[Dict[str, Any]] = None, **_: Any) -> str: + try: + return _dumps_capped(fn(args or {})) + except ValueError as exc: + return json.dumps({"ok": False, "error": str(exc)}) + except Exception as exc: + logger.debug("devarch tool failed", exc_info=True) + return json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"}) + + return handler + + +_tool_mine = _wrap(lambda a: mine(a.get("repo_path", ""), a.get("project"))) +_tool_signals = _wrap(lambda a: signals(a.get("project"), a.get("min_gap_days"))) +_tool_report = _wrap(lambda a: report(a.get("project"), a.get("fmt", "markdown"))) +_tool_pipeline = _wrap(lambda a: pipeline(a.get("repo_path", ""), a.get("project"))) +_tool_list = _wrap(lambda a: list_projects()) +_tool_version = _wrap(lambda a: version()) + + +# --- slash ------------------------------------------------------------------ + +_SAFE_SLASH = {"mine", "signals", "report", "pipeline", "list", "version", "status", "help"} +_REPO_SLASH = {"mine", "pipeline", "report"} # touch local FS / produce artifacts +_HELP = ( + "/devarch — git repository archaeology (local-only)\n" + " /devarch mine [project] Mine capped local history (rejects shallow/promisor)\n" + " /devarch signals [project] Detect development signals (leads)\n" + " /devarch report [project] [markdown|html]\n" + " /devarch pipeline [project] Mine → signals → report\n" + " /devarch list Projects under plugin-data\n" + " /devarch version Framework version + pin\n" + " /devarch status Recent ledger entries\n" + "Vectors are investigation leads, not verified conclusions. Clusters ≠ curated eras.\n" + "Data under plugin-data/devarch/.\n" + "On gateway platforms, mine/pipeline/report are refused unless allow_repo_slash_on_gateway: true." +) + + +def _fmt(res: Dict[str, Any]) -> str: + if res.get("error") and not res.get("ok"): + return f"DevArch failed: {res['error']}" + if res.get("markdown"): + head = [ + f"DevArch report — project={res.get('project')} path={res.get('output_path')}", + res.get("note") or "", + "", + str(res["markdown"])[:3500], + ] + return "\n".join(head) + return json.dumps(res, indent=2, ensure_ascii=False)[:3500] + + +def _slash(raw_args: str) -> str: + try: + parts = shlex.split((raw_args or "").strip()) + except ValueError as exc: + return f"Bad arguments: {exc}\n\n{_HELP}" + cmd = parts[0].lower() if parts else "help" + rest = parts[1:] + if cmd in {"help", "-h", "--help"}: + return _HELP + if cmd not in _SAFE_SLASH: + return f"Unknown or blocked subcommand `{cmd}`. Allowed: {', '.join(sorted(_SAFE_SLASH))}." + if cmd in _REPO_SLASH and _on_gateway() and not _cfg("allow_repo_slash_on_gateway"): + return ( + f"/devarch {cmd} is disabled on gateway platforms (local git/filesystem). " + "Ask the agent to use the tool instead, or set allow_repo_slash_on_gateway: true." + ) + try: + if cmd == "status": + return json.dumps(_load_json(_ledger_path(), [])[-10:], indent=2) + if cmd == "version": + return _fmt(version()) + if cmd == "list": + return _fmt(list_projects()) + if cmd == "signals": + return _fmt(signals(rest[0] if rest else None)) + if cmd == "report": + proj = rest[0] if rest and rest[0] not in {"markdown", "html", "md"} else None + fmt = "markdown" + for tok in rest: + if tok in {"markdown", "html", "md"}: + fmt = tok + return _fmt(report(proj, fmt)) + if cmd == "mine": + if not rest: + return "Usage: /devarch mine [project]" + res = mine(rest[0], rest[1] if len(rest) > 1 else None) + _on_post_tool( + "devarch_mine", + {"repo_path": rest[0], "project": rest[1] if len(rest) > 1 else None}, + res, + ) + return _fmt(res) + if cmd == "pipeline": + if not rest: + return "Usage: /devarch pipeline [project]" + res = pipeline(rest[0], rest[1] if len(rest) > 1 else None) + _on_post_tool( + "devarch_pipeline", + {"repo_path": rest[0], "project": rest[1] if len(rest) > 1 else None}, + res, + ) + return _fmt(res) + except ValueError as exc: + return f"{exc}\n\n{_HELP}" + except Exception as exc: + logger.debug("devarch slash failed", exc_info=True) + return f"DevArch error: {type(exc).__name__}: {exc}" + return _HELP + + +# --- register --------------------------------------------------------------- + + +def register(ctx) -> None: + global _ctx + _ctx = ctx + ctx.register_hook("pre_llm_call", _on_pre_llm) + ctx.register_hook("post_tool_call", _on_post_tool) + ctx.register_command( + "devarch", + handler=_slash, + description="DevArch git archaeology (mine/signals/report/pipeline/list/version/status).", + args_hint="mine [project] | signals | report | pipeline | list | version | status", + ) + for schema, handler, emoji in ( + (MINE_SCHEMA, _tool_mine, "⛏️"), + (SIGNALS_SCHEMA, _tool_signals, "📡"), + (REPORT_SCHEMA, _tool_report, "📜"), + (PIPELINE_SCHEMA, _tool_pipeline, "🏺"), + (LIST_SCHEMA, _tool_list, "📂"), + (VERSION_SCHEMA, _tool_version, "ℹ️"), + ): + ctx.register_tool( + name=schema["name"], + toolset="devarch", + schema=schema, + handler=handler, + description=schema["description"], + emoji=emoji, + ) + skills_dir = Path(__file__).parent / "skills" + if skills_dir.is_dir(): + for child in sorted(skills_dir.iterdir()): + skill_md = child / "SKILL.md" + if child.is_dir() and skill_md.exists(): + try: + ctx.register_skill(child.name, skill_md) + except Exception: + logger.debug("devarch: register_skill failed for %s", child.name, exc_info=True) diff --git a/plugins/devarch-hermes/archaeology/__init__.py b/plugins/devarch-hermes/archaeology/__init__.py new file mode 100644 index 0000000..4ca5ae6 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/__init__.py @@ -0,0 +1,3 @@ +"""DevArch Framework - forensic mining of software development history.""" + +__version__ = "0.4.1" diff --git a/plugins/devarch-hermes/archaeology/analysis_runner.py b/plugins/devarch-hermes/archaeology/analysis_runner.py new file mode 100644 index 0000000..adfbb8f --- /dev/null +++ b/plugins/devarch-hermes/archaeology/analysis_runner.py @@ -0,0 +1,573 @@ +"""Automated analysis vector execution for DevArch Framework. + +This module executes the six built-in analysis vectors against a project's +SQLite database and local JSON artifacts. The outputs are deterministic, +structured JSON summaries intended to be useful without a manual LLM handoff. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Callable + +from .utils import _load_json, atomic_write + + +class AnalysisRunner: + """Runs all six analysis vectors against a project database.""" + + VECTORS = [ + "sdlc-gap-finder", + "ml-pattern-mapper", + "agentic-workflow", + "formal-terms-mapper", + "source-archaeologist", + "youtube-correlator", + ] + + def __init__(self, project_name: str, project_dir: str, verbose: bool = False): + self.project_name = project_name + self.project_dir = Path(project_dir) + self.verbose = verbose + self.data_dir = self.project_dir / "data" + self.deliverables_dir = self.project_dir / "deliverables" + self.db_path = self.data_dir / "archaeology.db" + + def _log(self, msg: str) -> None: + if self.verbose: + print(f" [analysis] {msg}") + + def _query_db(self, query: str, params: tuple = ()) -> list[dict]: + """Execute SQL query against archaeology database.""" + if not self.db_path.exists(): + raise ValueError("Analysis database missing; run build-db first") + conn = sqlite3.connect(str(self.db_path), timeout=30) + conn.row_factory = sqlite3.Row + try: + cursor = conn.execute(query, params) + return [dict(row) for row in cursor.fetchall()] + except sqlite3.Error as e: + raise ValueError(f"Analysis query failed: {e}") from e + finally: + conn.close() + + def _load_json(self, rel_path: str) -> Any | None: + """Load JSON from project directory (wrapper for utils._load_json).""" + return _load_json(self.project_dir / rel_path) + + def _like_commits(self, keywords: list[str], limit: int | None = 100) -> list[dict]: + if not keywords: + return [] + clauses = " OR ".join("LOWER(message) LIKE ?" for _ in keywords) + params = tuple(f"%{kw.lower()}%" for kw in keywords) + (-1 if limit is None else limit,) + return self._query_db( + f"SELECT hash, date, message, author FROM commits WHERE {clauses} ORDER BY date DESC LIMIT ?", + params, + ) + + def _commit_count(self) -> int: + rows = self._query_db("SELECT COUNT(*) as cnt FROM commits") + return int(rows[0]["cnt"]) if rows else 0 + + def run_sdlc_gap_finder(self) -> dict[str, Any]: + """Analyze SDLC practices and gaps.""" + self._log("Running SDLC Gap Finder...") + total_commits = self._commit_count() + ci_cd = self._like_commits(["github action", "ci", "workflow", "deploy", "pipeline"], None) + tests = self._like_commits(["test", "spec", "coverage", "vitest", "pytest"], None) + refactor = self._like_commits(["refactor", "clean", "simplify"], None) + security = self._like_commits(["security", "cve", "xss", "injection", "secret"], None) + docs = self._like_commits(["docs", "readme", "documentation"], None) + + def status(count: int, low: float, high: float) -> str: + ratio = count / total_commits if total_commits else 0 + if ratio < low: + return "UNVERIFIED" + if ratio < high: + return "EMERGING" + return "PRESENT" + + practices = [ + ("CI/CD Pipeline", ci_cd, 0.01, 0.04, "Run local/GitHub quality gates automatically"), + ("Test Coverage", tests, 0.05, 0.15, "Keep behavior tests above the agreed threshold"), + ( + "Refactoring Discipline", + refactor, + 0.02, + 0.08, + "Reserve explicit simplification cycles", + ), + ( + "Security Review", + security, + 0.005, + 0.025, + "Keep security findings tied to a verification gate", + ), + ( + "Documentation Hygiene", + docs, + 0.02, + 0.08, + "Synchronize public claims with canonical metrics", + ), + ] + gaps = [] + for practice, rows, low, high, recommendation in practices: + practice_status = status(len(rows), low, high) + severity = ( + "MEDIUM" + if practice_status == "UNVERIFIED" + else "MEDIUM" + if practice_status == "EMERGING" + else "LOW" + ) + gaps.append( + { + "practice": practice, + "status": practice_status, + "confidence": "LOW", + "interpretation": "Commit-keyword frequency only; not verified presence, absence or coverage", + "evidence": [ + { + "sample": rows[:5], + "result_count": len(rows), + "ratio": f"{(len(rows) / total_commits if total_commits else 0):.1%}", + } + ], + "severity": severity, + "effort_to_implement": 3 if severity == "HIGH" else 2, + "expected_impact": 5 if severity == "HIGH" else 3, + "roi": round( + (5 if severity == "HIGH" else 3) / (3 if severity == "HIGH" else 2), 2 + ), + "recommendation": recommendation, + } + ) + + return { + "project": self.project_name, + "analysis_date": datetime.now().isoformat(), + "gaps": gaps, + "summary": { + "total_gaps": len(gaps), + "critical_gaps": sum(1 for g in gaps if g["severity"] == "CRITICAL"), + "top_3_roi": [ + g["practice"] + for g in sorted(gaps, key=lambda row: row["roi"], reverse=True)[:3] + ], + }, + } + + def run_ml_pattern_mapper(self) -> dict[str, Any]: + """Map intuitive code/commit language to formal ML patterns.""" + self._log("Running ML Pattern Mapper...") + patterns = [ + ( + "scoring system", + "Weighted Multi-Criteria Decision Analysis", + ["score", "rank", "weight", "threshold"], + False, + None, + ), + ( + "evolution loop", + "Evolutionary Strategy / Quality-Diversity Search", + ["evolve", "mutate", "fitness", "diversity", "map-elites"], + True, + "DEAP or pymoo", + ), + ( + "model routing", + "Contextual Bandit / Mixture-of-Experts Routing", + ["router", "route", "model", "provider"], + False, + None, + ), + ( + "critic ensemble", + "Ensemble Evaluation / Multi-Critic Reward Modeling", + ["critic", "aesthetic", "evaluator", "judge"], + False, + None, + ), + ( + "retrieval memory", + "Retrieval-Augmented Generation", + ["rag", "retrieval", "archive", "memory", "semantic"], + False, + None, + ), + ] + mappings = [] + for intuitive, formal, keywords, reinvention, library in patterns: + evidence = self._like_commits(keywords, 8) + if not evidence: + continue + mappings.append( + { + "intuitive_name": intuitive, + "formal_term": formal, + "confidence": "LOW", + "status": "UNVERIFIED keyword candidate; inspect source before assigning an algorithm", + "similarity_to_canonical": None, + "is_reinvention": None, + "library_alternative": library, + "estimated_token_waste": None, + "evidence": evidence[:5], + } + ) + return { + "project": self.project_name, + "analysis_date": datetime.now().isoformat(), + "mappings": mappings, + "reinventions": [m for m in mappings if m.get("is_reinvention")], + "summary": { + "total_patterns_found": len(mappings), + "reinventions_detected": sum(1 for m in mappings if m.get("is_reinvention")), + }, + } + + def _approximate_sessions(self) -> list[dict]: + """Approximate sessions from commits when sessions table is absent. + + Groups commits into sessions using a 2-hour inactivity gap heuristic. + Falls back to daily grouping if timestamps lack time components. + """ + tables = { + r["name"] for r in self._query_db("SELECT name FROM sqlite_master WHERE type='table'") + } + if "sessions" in tables: + return self._query_db("SELECT session_id, timestamp FROM sessions ORDER BY timestamp") + + commits = self._query_db("SELECT date FROM commits ORDER BY date") + if not commits: + return [] + + from datetime import datetime as dt + + GAP_HOURS = 2 + sessions: list[dict] = [] + _session_start = None + prev_ts = None + + for row in commits: + raw = row.get("date", "") + try: + ts = dt.fromisoformat(raw[:19]) + except (ValueError, TypeError): + ts = None + + if ts is None: + day = raw[:10] + if day != (prev_ts or ""): + sessions.append({"session_id": day, "timestamp": day}) + prev_ts = day + continue + + if prev_ts is None or (ts - prev_ts).total_seconds() > GAP_HOURS * 3600: + session_id = ts.strftime("%Y%m%d-%H%M%S") + sessions.append({"session_id": session_id, "timestamp": ts.isoformat()}) + _session_start = ts + + prev_ts = ts + + return sessions + + def run_agentic_workflow(self) -> dict[str, Any]: + """Analyze AI agent interaction patterns.""" + self._log("Running Agentic Workflow Analyzer...") + hooks = self._like_commits(["hook", "pre-commit", "post-commit", "automation"], 50) + authors = self._query_db( + "SELECT author, COUNT(*) as cnt FROM commits GROUP BY author ORDER BY cnt DESC" + ) + return { + "project": self.project_name, + "analysis_date": datetime.now().isoformat(), + "session_depth_distribution": None, + "session_taxonomy": None, + "hook_effectiveness": [], + "hook_commit_evidence": hooks, + "author_attribution": authors, + "limitations": "Commit authors are not verified agent identities. Session depth, autonomy and hook effectiveness are unmeasured; no fabricated estimates.", + "summary": {"total_sessions_analyzed": 0, "dominant_session_type": None}, + } + + def run_formal_terms_mapper(self) -> dict[str, Any]: + """Map project-specific terms to formal engineering vocabulary.""" + self._log("Running Formal Terms Mapper...") + terms = [ + ("CompostMill", "Content Processing Pipeline / Creative Memory Store", ["compost"]), + ("RalphLoop", "Generate-Evaluate-Improve Control Loop", ["ralph", "loop", "iterate"]), + ("Swarm", "Multi-Agent Ensemble / Debate", ["swarm", "agent", "collaboration"]), + ( + "Quality Gate", + "Verification Gate / Acceptance Criterion", + ["quality gate", "guardrail", "validation"], + ), + ("Archive", "Event-Sourced Knowledge Store", ["archive", "event", "sqlite"]), + ] + dictionary = [] + for code_name, formal, keywords in terms: + evidence = self._like_commits(keywords, 5) + if evidence: + dictionary.append( + { + "code_name": code_name, + "formal_term": formal, + "category": "ARCHITECTURE", + "similarity_score": "UNVERIFIED", + "confidence": "LOW", + "evidence": evidence, + } + ) + return { + "project": self.project_name, + "analysis_date": datetime.now().isoformat(), + "term_dictionary": dictionary, + "naming_trajectory": "Unmeasured; keyword candidates require source validation.", + "learning_opportunities": [ + "Control theory", + "Quality-diversity algorithms", + "Event sourcing", + "Multi-agent evaluation", + ], + "summary": { + "terms_mapped": len(dictionary), + "high_confidence": sum(1 for t in dictionary if t["similarity_score"] == "CLOSE"), + }, + } + + def run_source_archaeologist(self) -> dict[str, Any]: + """Mine commit history for code quality trajectory and hotspots.""" + self._log("Running Source Code Archaeologist...") + quality = self._like_commits(["fix", "test", "refactor", "security", "lint", "type"], None) + large_change = self._like_commits( + ["split", "extract", "monolith", "decompose", "simplify"], 100 + ) + todo = self._like_commits(["todo", "stub", "placeholder", "not implemented"], 100) + by_month: Counter[str] = Counter() + for row in quality: + date = str(row.get("date", ""))[:7] + if date: + by_month[date] += 1 + hotspots = self._query_db( + "SELECT message, COUNT(*) as cnt FROM commits GROUP BY message ORDER BY cnt DESC LIMIT 10" + ) + improvements = self._derive_improvements(quality, large_change, todo, hotspots) + return { + "analysis_metadata": { + "timestamp": datetime.now().isoformat(), + "analyst": "Automated Source Code Archaeologist", + "project": self.project_name, + "commit_count": self._commit_count(), + }, + "quality_trajectory": { + "assessment": "UNVERIFIED keyword activity; no quality direction established", + "evidence_count": len(quality), + "by_month": dict(sorted(by_month.items())), + }, + "architecture_drift": { + "large_change_signals": large_change[:10], + "todo_or_stub_signals": todo[:10], + }, + "hotspots": hotspots, + "improvements": improvements, + "summary": { + "quality_signal_count": len(quality), + "large_change_signal_count": len(large_change), + "todo_signal_count": len(todo), + }, + } + + def _derive_improvements( + self, + quality: list[dict], + large_change: list[dict], + todo: list[dict], + hotspots: list[dict], + ) -> list[dict]: + """Derive prioritized remediation recommendations from actual commit data.""" + items: list[tuple[int, str, str, str]] = [] # (score, title, effort, impact) + + # Flapping issues: repeated commit messages signal unresolved root causes + flapping = [h for h in hotspots if h.get("cnt", 0) >= 3] + if flapping: + top_msg = str(flapping[0].get("message", ""))[:60] + items.append( + ( + 100, + f"Investigate repeated message (may be merge/cherry-pick duplication): {top_msg}", + "M", + "HIGH", + ) + ) + + # Unresolved stubs / TODOs + if todo: + items.append( + ( + 90 if len(todo) >= 5 else 70, + f"Check whether {len(todo)} historical stub/placeholder mentions remain unresolved", + "S", + "HIGH" if len(todo) >= 5 else "MEDIUM", + ) + ) + + # Decomposition momentum: carry it through + if large_change: + items.append( + ( + 60, + f"Review {len(large_change)} historical decomposition signals before proposing more splits", + "L", + "MEDIUM", + ) + ) + + # Quality signal density: low fix/test ratio suggests coverage gaps + commit_count = self._commit_count() or 1 + quality_ratio = len(quality) / commit_count + if quality_ratio < 0.10: + items.append( + ( + 80, + f"Boost quality signal density — fix/test ratio at {quality_ratio:.0%} (target ≥10%)", + "M", + "HIGH", + ) + ) + elif quality_ratio < 0.20: + items.append( + ( + 50, + f"Maintain quality signal density — currently at {quality_ratio:.0%}", + "S", + "LOW", + ) + ) + + # No issues found: project is healthy + if not items: + items.append( + (10, "No keyword-derived candidates; source review still required", "S", "LOW") + ) + + items.sort(key=lambda x: x[0], reverse=True) + return [ + { + "rank": i + 1, + "title": title, + "effort": effort, + "impact": impact, + "status": "UNVERIFIED investigation candidate", + "evidence": ( + todo[:3] + if "placeholder" in title + else large_change[:3] + if "decomposition" in title + else hotspots[:3] + if "repeated" in title + else quality[:3] + ), + } + for i, (_, title, effort, impact) in enumerate(items) + ] + + def run_youtube_correlator(self) -> dict[str, Any]: + """Summarize YouTube/watch-history correlation artifacts when available.""" + self._log("Running YouTube Correlator...") + yt_corr = self._load_json("data/youtube-ai-correlation.json") or {} + yt_creators = self._load_json("data/youtube-creators.json") or {} + yt_topics = self._load_json("data/youtube-topic-classification.json") or {} + canonical = self._load_json("deliverables/canonical-metrics.json") or {} + correlations = yt_corr.get("key_correlations") or yt_corr.get("correlations") or [] + creators = ( + yt_creators.get("creators") + if isinstance(yt_creators, dict) + else yt_creators + if isinstance(yt_creators, list) + else [] + ) + topics = yt_topics.get("categories") if isinstance(yt_topics, dict) else [] + smoking_guns = [ + row for row in correlations if isinstance(row, dict) and row.get("is_smoking_gun") + ] + return { + "project": self.project_name, + "analysis_date": datetime.now().isoformat(), + "commit_count": canonical.get("total_commits", self._commit_count()), + "active_days": canonical.get("active_days"), + "date_range_days": canonical.get("span_days"), + "correlations": correlations[:20] if isinstance(correlations, list) else [], + "creator_influence": creators[:20] if isinstance(creators, list) else [], + "lag_analysis": yt_corr.get("lag_analysis", {}), + "topic_overlap": topics[:20] if isinstance(topics, list) else [], + "smoking_guns": smoking_guns[:10], + "summary": { + "correlation_count": len(correlations) if isinstance(correlations, list) else 0, + "creator_count": len(creators) if isinstance(creators, list) else 0, + "smoking_gun_count": len(smoking_guns), + "data_available": bool(yt_corr or yt_creators or yt_topics), + }, + } + + def run_all(self, vectors: list[str] | None = None) -> dict[str, str]: + """Execute selected analysis vectors and save JSON outputs.""" + results: dict[str, str] = {} + runners: dict[str, Callable[[], dict[str, Any]]] = { + "sdlc-gap-finder": self.run_sdlc_gap_finder, + "ml-pattern-mapper": self.run_ml_pattern_mapper, + "agentic-workflow": self.run_agentic_workflow, + "formal-terms-mapper": self.run_formal_terms_mapper, + "source-archaeologist": self.run_source_archaeologist, + "youtube-correlator": self.run_youtube_correlator, + } + target = vectors or self.VECTORS + unknown = [vector for vector in target if vector not in runners] + if unknown: + raise ValueError(f"Unknown analysis vector(s): {', '.join(unknown)}") + self.deliverables_dir.mkdir(parents=True, exist_ok=True) + analysis_dir = self.deliverables_dir / "analysis" + analysis_dir.mkdir(parents=True, exist_ok=True) + from .provenance import bind_artifact, snapshot + + inputs = snapshot(self.project_dir) + for vector_name in target: + runner_func = runners[vector_name] + try: + output_path = analysis_dir / f"analysis-{vector_name}.json" + result = runner_func() + if snapshot(self.project_dir) != inputs: + raise ValueError("Analysis inputs changed during execution; retry") + result["evidence_binding"] = inputs + result["methodology"] = { + "basis": "commit-message heuristics", + "source_inspection": False, + "causal_inference": False, + "limitations": "Requires source/PR validation. Missing keyword evidence does not establish absence. Repeated commits across refs are not necessarily recurring defects.", + } + atomic_write(output_path, json.dumps(result, indent=2, ensure_ascii=False) + "\n") + bind_artifact(self.project_dir, output_path) + results[vector_name] = str(output_path) + print(f" [analysis] {vector_name}: {output_path}") + except (OSError, ValueError, KeyError, sqlite3.OperationalError) as exc: + print(f" [analysis] {vector_name} ERROR: {exc}") + results[vector_name] = f"ERROR: {exc}" + return results + + +def run_analysis_vectors( + project_name: str, verbose: bool = False, vectors: list[str] | None = None +) -> dict[str, str]: + """Public entry point to run analysis vectors.""" + project_dir = os.path.join("projects", project_name) + if not os.path.isdir(project_dir): + raise ValueError(f"Project '{project_name}' not found") + runner = AnalysisRunner(project_name, project_dir, verbose) + return runner.run_all(vectors=vectors) diff --git a/plugins/devarch-hermes/archaeology/api.py b/plugins/devarch-hermes/archaeology/api.py new file mode 100644 index 0000000..1f28326 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/api.py @@ -0,0 +1,892 @@ +"""JSON API for dev-archaeology strategic insights. + +Lightweight API that serves strategic analysis data to external consumers. +Stdlib-only — no Flask/FastAPI dependency. + +Endpoints: + GET /api/health — System status + GET /api/projects — All projects with summary metrics + GET /api/insights/ — All strategic analyses for a project + GET /api/swot/ — SWOT analysis + GET /api/wardley/ — Wardley Map + GET /api/value-chain/ — Porter's Value Chain + GET /api/bridge — Full bridge file for Factory consumption +""" + +import json +import re +import sqlite3 +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] # dev-archaeology/ +PROJECTS_DIR = ROOT / "projects" +BRIDGE_PATH = ROOT / "global" / "data" / "factory-bridge.json" + +__version__ = "1.0.0" + + +def _validate_project_name(name): + """Reject path traversal and invalid characters in project names.""" + if not name or not re.match(r"^[a-zA-Z0-9._-]+$", name): + return False + # Double-check no path traversal + resolved = (PROJECTS_DIR / name).resolve() + try: + resolved.relative_to(PROJECTS_DIR.resolve()) + except ValueError: + return False + return True + + +def _json_response(handler, data, status=200): + handler.send_response(status) + handler.send_header("Content-Type", "application/json") + payload = json.dumps(data, indent=2, default=str).encode() + handler.send_header("Content-Length", str(len(payload))) + handler.end_headers() + handler.wfile.write(payload) + + +def _error_response(handler, message, status=404): + _json_response(handler, {"error": message, "status": status}, status) + + +def _load_json(path): + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError, OSError): + return None + + +def _discover_projects(): + """Return list of project directories with data.""" + projects = [] + if not PROJECTS_DIR.exists(): + return projects + for d in sorted(PROJECTS_DIR.iterdir()): + if not d.is_dir(): + continue + eras_path = d / "data" / "commit-eras.json" + if eras_path.exists(): + projects.append(d) + return projects + + +def _project_metrics(project_dir): + """Extract core metrics for a project.""" + name = project_dir.name + eras_data = _load_json(project_dir / "data" / "commit-eras.json") or {} + metrics = _load_json(project_dir / "deliverables" / "canonical-metrics.json") or {} + config = _load_json(project_dir / "project.json") or {} + + return { + "name": name, + "description": config.get("description", ""), + "total_commits": eras_data.get("total_commits", 0), + "era_count": len(eras_data.get("eras", [])), + "active_days": metrics.get("active_days", 0), + "span_days": metrics.get("span_days", 0), + "contributors": len(eras_data.get("contributors", [])), + "commit_types": eras_data.get("commit_types", {}), + "lifespan": eras_data.get("lifespan", ""), + "repo_url": config.get("repo_url", ""), + } + + +def _parse_md_sections(md_path): + """Parse a markdown file into {heading: [lines]} dict.""" + if not md_path.exists(): + return {} + text = md_path.read_text(encoding="utf-8") + sections = {} + current_heading = "header" + current_lines = [] + + for line in text.splitlines(): + if line.startswith("## "): + if current_lines: + sections[current_heading] = "\n".join(current_lines) + current_heading = line[3:].strip() + current_lines = [] + else: + current_lines.append(line) + if current_lines: + sections[current_heading] = "\n".join(current_lines) + return sections + + +def _parse_list_items(text): + """Extract numbered or bulleted list items from text.""" + items = [] + for line in text.splitlines(): + line = line.strip() + if re.match(r"^(\d+\.|[-*])\s+", line): + clean = re.sub(r"^(\d+\.|[-*])\s+", "", line) + clean = re.sub(r"\*\*([^*]+)\*\*", r"\1", clean) + if clean: + items.append(clean) + return items + + +def _extract_score(text, label): + """Extract a score like 'Margin: 58/100' from text.""" + pattern = rf"{re.escape(label)}[^0-9]*(\d+)/100" + m = re.search(pattern, text, re.IGNORECASE) + return int(m.group(1)) if m else None + + +def _parse_swot(project_dir): + """Parse SWOT-ANALYSIS.md into structured data.""" + md_path = project_dir / "deliverables" / "strategy" / "SWOT-ANALYSIS.md" + sections = _parse_md_sections(md_path) + if not sections: + return None + + result = {"project": project_dir.name} + for key in ("Strengths", "Weaknesses", "Opportunities", "Threats"): + section = sections.get(key, "") + result[key.lower()] = _parse_list_items(section) + + # Extract matrix counts + matrix_text = sections.get("header", "") + for quadrant in ("strengths", "weaknesses", "opportunities", "threats"): + pattern = rf"{quadrant}.*?(\d+)\s+found" + m = re.search(pattern, matrix_text, re.IGNORECASE) + if m: + result.setdefault(f"{quadrant}_count", int(m.group(1))) + + return result + + +def _parse_wardley(project_dir): + """Parse WARDLEY-MAP.md into structured data.""" + md_path = project_dir / "deliverables" / "strategy" / "WARDLEY-MAP.md" + sections = _parse_md_sections(md_path) + if not sections: + return None + + overview = sections.get("Overview", "") + result = { + "project": project_dir.name, + "maturity": None, + "avg_evolution": None, + } + + m = re.search(r"Maturity:\s+(\w+)", overview) + if m: + result["maturity"] = m.group(1) + m = re.search(r"avg evolution:\s+([\d.]+)", overview) + if m: + result["avg_evolution"] = float(m.group(1)) + + # Parse component table + table_text = sections.get("Component Evolution Table", "") + components = [] + for line in table_text.splitlines(): + if ( + "|" in line + and not line.strip().startswith("|-") + and not line.strip().startswith("| Component") + ): + cells = [c.strip() for c in line.split("|") if c.strip()] + if len(cells) >= 3: + components.append( + { + "component": cells[0], + "stage": cells[1], + "evidence": cells[2] if len(cells) > 2 else "", + } + ) + result["components"] = components + + # Parse recommendations + recs_text = sections.get("Movement Recommendations", "") + result["recommendations"] = _parse_list_items(recs_text) + + return result + + +def _parse_value_chain(project_dir): + """Parse VALUE-CHAIN-ANALYSIS.md into structured data.""" + md_path = project_dir / "deliverables" / "strategy" / "VALUE-CHAIN-ANALYSIS.md" + sections = _parse_md_sections(md_path) + if not sections: + return None + + summary = sections.get("Value Chain Summary", sections.get("header", "")) + result = { + "project": project_dir.name, + "margin_score": _extract_score(summary, "MARGIN"), + "primary_activities": {}, + "support_activities": {}, + } + + # Primary activities with scores + primary = [ + ("Inbound Logistics", "inbound_logistics"), + ("Operations", "operations"), + ("Outbound Logistics", "outbound_logistics"), + ("Marketing & Sales", "marketing_sales"), + ("Service", "service"), + ] + for heading, key in primary: + section_text = sections.get(heading, "") + result["primary_activities"][key] = { + "score": _extract_score(section_text, heading.split()[0]), + "details": _parse_list_items(section_text)[:5], + } + + # Support activities + support = [ + ("Infrastructure", "infrastructure"), + ("Technology Development", "technology"), + ("Human Resource Management", "hr"), + ("Procurement", "procurement"), + ] + for heading, key in support: + section_text = sections.get(heading, "") + result["support_activities"][key] = { + "score": _extract_score(section_text, heading.split()[0]), + "details": _parse_list_items(section_text)[:3], + } + + # Recommendations + recs = sections.get("Recommendations", "") + result["recommendations"] = _parse_list_items(recs) + + return result + + +def _parse_bcg(project_dir): + """Parse BCG-MATRIX.md into structured data.""" + md_path = project_dir / "deliverables" / "strategy" / "BCG-MATRIX.md" + sections = _parse_md_sections(md_path) + if not sections: + return None + + overview = sections.get("Overview", "") + result = {"project": project_dir.name} + + m = re.search(r"Total commits\*\*: (\d+)", overview) + if m: + result["total_commits"] = int(m.group(1)) + m = re.search(r"Velocity trend\*\*: (\w+)", overview) + if m: + result["velocity_trend"] = m.group(1) + + components = [] + table_text = sections.get("Component Classification", "") + for line in table_text.splitlines(): + if ( + "|" in line + and not line.strip().startswith("|-") + and not line.strip().startswith("| Component") + ): + cells = [c.strip() for c in line.split("|") if c.strip()] + if len(cells) >= 4: + components.append( + { + "component": cells[0], + "commits": cells[1], + "share": cells[2], + "quadrant": cells[3], + } + ) + result["components"] = components + + for quadrant_key in ("Stars", "Cash Cows", "Question Marks", "Dogs"): + section = sections.get(f"{quadrant_key}", "") + _count_match = re.search(r"\((\d+)\)", sections.get("Quadrant Analysis", "")) + result[quadrant_key.lower().replace(" ", "_")] = _parse_list_items(section) + + result["recommendations"] = _parse_list_items(sections.get("Strategic Recommendations", "")) + return result + + +def _parse_ansoff(project_dir): + """Parse ANSOFF-MATRIX.md into structured data.""" + md_path = project_dir / "deliverables" / "strategy" / "ANSOFF-MATRIX.md" + sections = _parse_md_sections(md_path) + if not sections: + return None + + overview = sections.get("Overview", "") + result = {"project": project_dir.name} + + m = re.search(r"Primary strategy\*\*: ([\w\s]+)", overview) + if m: + result["primary_strategy"] = m.group(1).strip() + m = re.search(r"Secondary strategy\*\*: ([\w\s]+)", overview) + if m: + result["secondary_strategy"] = m.group(1).strip() + + scores = {} + table_text = sections.get("Quadrant Scores", "") + for line in table_text.splitlines(): + if "|" in line and "Strategy" not in line and not line.strip().startswith("|-"): + cells = [c.strip() for c in line.split("|") if c.strip()] + if len(cells) >= 2: + strategy = cells[0] + score_match = re.match(r"(\d+)/100", cells[1]) + if score_match: + scores[strategy] = int(score_match.group(1)) + result["scores"] = scores + + result["recommendations"] = _parse_list_items(sections.get("Recommendations", "")) + return result + + +def _parse_blue_ocean(project_dir): + """Parse BLUE-OCEAN.md into structured data.""" + md_path = project_dir / "deliverables" / "strategy" / "BLUE-OCEAN.md" + sections = _parse_md_sections(md_path) + if not sections: + return None + + overview = sections.get("Overview", "") + result = {"project": project_dir.name} + + m = re.search(r"Average value score\*\*: ([\d.]+)/10", overview) + if m: + result["avg_value_score"] = float(m.group(1)) + m = re.search(r"Critical gaps\*\*: (\d+)", overview) + if m: + result["critical_gaps"] = int(m.group(1)) + + factors = [] + table_text = sections.get("Strategy Canvas", "") + for line in table_text.splitlines(): + if "|" in line and "Value Factor" not in line and not line.strip().startswith("|-"): + cells = [c.strip() for c in line.split("|") if c.strip()] + if len(cells) >= 2: + score_match = re.match(r"(\d+)/10", cells[1]) + if score_match: + factors.append({"factor": cells[0], "score": int(score_match.group(1))}) + result["factors"] = factors + + for action in ("Eliminate", "Reduce", "Raise", "Create"): + section_key = f"{action} — What to {'stop doing' if action == 'Eliminate' else 'do less of' if action == 'Reduce' else 'do more of' if action == 'Raise' else 'build that doesn'}" + section = sections.get(section_key, "") + if not section: + section = sections.get(action, "") + result[action.lower()] = _parse_list_items(section) + + result["recommendations"] = _parse_list_items(sections.get("Recommendations", "")) + return result + + +def _get_pipeline_status(project_dir): + """Get latest pipeline run status from archaeology.db.""" + db_path = project_dir / "data" / "archaeology.db" + if not db_path.exists(): + return None + try: + with sqlite3.connect(str(db_path)) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute( + "SELECT run_timestamp, status FROM pipeline_runs " + "ORDER BY run_timestamp DESC LIMIT 1" + ).fetchone() + if row: + return {"last_run": row["run_timestamp"], "status": row["status"]} + except sqlite3.OperationalError: + pass + return None + + +def _compute_health_score(metrics, swot, wardley, value_chain): + """Compute a 0-100 health score from available data.""" + score = 50 # baseline + + # Positive signals + ct = metrics.get("commit_types", {}) + if ct.get("test", 0) > 0: + score += 10 + if ct.get("ci", 0) > 0: + score += 10 + if ct.get("docs", 0) > 5: + score += 5 + if metrics.get("era_count", 0) >= 3: + score += 5 + if metrics.get("active_days", 0) > 10: + score += 5 + + # Negative signals + if ct.get("test", 0) == 0: + score -= 10 + if ct.get("ci", 0) == 0: + score -= 10 + + # SWOT balance (more strengths than weaknesses = healthy) + if swot: + s = len(swot.get("strengths", [])) + w = len(swot.get("weaknesses", [])) + if s > w: + score += 5 + elif w > s: + score -= 5 + + # Value chain margin + if value_chain and value_chain.get("margin_score"): + margin = value_chain["margin_score"] + if margin > 60: + score += 5 + elif margin < 30: + score -= 5 + + return max(0, min(100, score)) + + +# ── Endpoint handlers ────────────────────────────────────────── + + +def handle_health(handler): + """GET /api/health""" + projects = _discover_projects() + _json_response( + handler, + { + "status": "ok", + "version": __version__, + "projects": len(projects), + "bridge_available": BRIDGE_PATH.exists(), + }, + ) + + +def handle_projects(handler): + """GET /api/projects""" + projects = [] + for pdir in _discover_projects(): + metrics = _project_metrics(pdir) + pipeline = _get_pipeline_status(pdir) + if pipeline: + metrics["last_pipeline"] = pipeline + projects.append(metrics) + _json_response(handler, {"projects": projects, "count": len(projects)}) + + +def handle_insights(handler, project_name): + """GET /api/insights/""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + + metrics = _project_metrics(pdir) + swot = _parse_swot(pdir) + wardley = _parse_wardley(pdir) + value_chain = _parse_value_chain(pdir) + bcg = _parse_bcg(pdir) + ansoff = _parse_ansoff(pdir) + blue_ocean = _parse_blue_ocean(pdir) + pipeline = _get_pipeline_status(pdir) + health = _compute_health_score(metrics, swot, wardley, value_chain) + + _json_response( + handler, + { + "project": project_name, + "health_score": health, + "metrics": metrics, + "swot": swot, + "wardley": wardley, + "value_chain": value_chain, + "bcg": bcg, + "ansoff": ansoff, + "blue_ocean": blue_ocean, + "pipeline": pipeline, + }, + ) + + +def handle_swot(handler, project_name): + """GET /api/swot/""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + data = _parse_swot(pdir) + if not data: + return _error_response(handler, f"SWOT analysis not found for '{project_name}'") + _json_response(handler, data) + + +def handle_wardley(handler, project_name): + """GET /api/wardley/""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + data = _parse_wardley(pdir) + if not data: + return _error_response(handler, f"Wardley Map not found for '{project_name}'") + _json_response(handler, data) + + +def handle_value_chain(handler, project_name): + """GET /api/value-chain/""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + data = _parse_value_chain(pdir) + if not data: + return _error_response(handler, f"Value Chain not found for '{project_name}'") + _json_response(handler, data) + + +def handle_bcg(handler, project_name): + """GET /api/bcg/""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + data = _parse_bcg(pdir) + if not data: + return _error_response(handler, f"BCG Matrix not found for '{project_name}'") + _json_response(handler, data) + + +def handle_ansoff(handler, project_name): + """GET /api/ansoff/""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + data = _parse_ansoff(pdir) + if not data: + return _error_response(handler, f"Ansoff Matrix not found for '{project_name}'") + _json_response(handler, data) + + +def handle_blue_ocean(handler, project_name): + """GET /api/blue-ocean/""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + data = _parse_blue_ocean(pdir) + if not data: + return _error_response(handler, f"Blue Ocean analysis not found for '{project_name}'") + _json_response(handler, data) + + +def handle_bridge(handler): + """GET /api/bridge — Serve pre-generated bridge file or generate on the fly.""" + if BRIDGE_PATH.exists(): + data = _load_json(BRIDGE_PATH) + if data: + return _json_response(handler, data) + + # Fallback: generate on the fly + bridge = _generate_bridge() + _json_response(handler, bridge) + + +def handle_health_trend(handler, project_name): + """GET /api/health-trend/ — Pipeline history from SQLite.""" + db_path = PROJECTS_DIR / project_name / "data" / "archaeology.db" + if not db_path.exists(): + return _error_response(handler, f"No database for '{project_name}'") + try: + with sqlite3.connect(str(db_path)) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + "SELECT r.run_timestamp, r.status, pr.issues_count, pr.fixes_applied " + "FROM pipeline_runs r " + "JOIN pipeline_repo_results pr ON r.id = pr.run_id " + "WHERE pr.repo_name LIKE '%' || ? || '%' " + "ORDER BY r.run_timestamp DESC LIMIT 50", + (project_name,), + ).fetchall() + trend = [dict(r) for r in rows] + _json_response(handler, {"project": project_name, "trend": trend}) + except sqlite3.OperationalError: + _json_response(handler, {"project": project_name, "trend": []}) + + +# ── Bridge generation ────────────────────────────────────────── + + +def _generate_bridge(): + """Generate the bridge data structure.""" + projects_data = {} + for pdir in _discover_projects(): + metrics = _project_metrics(pdir) + swot = _parse_swot(pdir) + wardley = _parse_wardley(pdir) + value_chain = _parse_value_chain(pdir) + bcg = _parse_bcg(pdir) + ansoff = _parse_ansoff(pdir) + blue_ocean = _parse_blue_ocean(pdir) + pipeline = _get_pipeline_status(pdir) + health = _compute_health_score(metrics, swot, wardley, value_chain) + + projects_data[metrics["name"]] = { + "health_score": health, + "total_commits": metrics["total_commits"], + "active_days": metrics["active_days"], + "era_count": metrics["era_count"], + "swot_summary": { + "strengths": len(swot.get("strengths", [])) if swot else 0, + "weaknesses": len(swot.get("weaknesses", [])) if swot else 0, + "opportunities": len(swot.get("opportunities", [])) if swot else 0, + "threats": len(swot.get("threats", [])) if swot else 0, + } + if swot + else None, + "wardley_maturity": wardley.get("maturity") if wardley else None, + "value_chain_margin": value_chain.get("margin_score") if value_chain else None, + "bcg_velocity_trend": bcg.get("velocity_trend") if bcg else None, + "ansoff_primary": ansoff.get("primary_strategy") if ansoff else None, + "blue_ocean_avg_score": blue_ocean.get("avg_value_score") if blue_ocean else None, + "last_pipeline_status": pipeline.get("status") if pipeline else None, + "recommendations": ( + (wardley.get("recommendations", []) if wardley else []) + + (value_chain.get("recommendations", []) if value_chain else []) + )[:5], + } + + return { + "generated_at": datetime.now().isoformat(), + "version": __version__, + "projects": projects_data, + "cross_repo": { + "total_repos": len(projects_data), + "total_commits": sum(p["total_commits"] for p in projects_data.values()), + "frameworks_available": [ + "swot", + "wardley", + "value-chain", + "bcg", + "ansoff", + "blue-ocean", + ], + "opportunity_features": [ + "learning-velocity", + "frustration-to-automation", + "knowledge-gap", + "token-efficiency", + "session-quality", + "ai-agent-mastery", + "creative-dna", + "neurodivergent-profile", + "model-selection-advisor", + "before-after-snapshot", + "cross-repo-transfer", + "youtube-learning-graph", + "architecture-timelapse", + "commit-cognitive-load", + ], + }, + } + + +def generate_bridge_file(): + """Generate and write bridge file to disk. Called from pipeline.""" + bridge = _generate_bridge() + BRIDGE_PATH.parent.mkdir(parents=True, exist_ok=True) + BRIDGE_PATH.write_text(json.dumps(bridge, indent=2, default=str) + "\n", encoding="utf-8") + print(f"Bridge generated: {len(bridge['projects'])} projects → {BRIDGE_PATH}") + return len(bridge["projects"]) + + +# ── Opportunity endpoints ──────────────────────────────────────── + +OPPORTUNITY_FEATURES = [ + "learning-velocity", + "frustration-to-automation", + "knowledge-gap", + "token-efficiency", + "session-quality", + "ai-agent-mastery", + "creative-dna", + "neurodivergent-profile", + "model-selection-advisor", + "before-after-snapshot", + "cross-repo-transfer", + "youtube-learning-graph", + "architecture-timelapse", + "commit-cognitive-load", +] + + +def _load_opportunity(project_name, feature): + """Load a single opportunity analysis JSON.""" + path = ( + PROJECTS_DIR / project_name / "deliverables" / "opportunity" / f"opportunity-{feature}.json" + ) + return _load_json(path) + + +def handle_opportunity_feature(handler, project_name, feature): + """GET /api/opportunity//""" + if feature not in OPPORTUNITY_FEATURES: + return _error_response(handler, f"Unknown opportunity feature: {feature}", 404) + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + data = _load_opportunity(project_name, feature) + if not data: + return _error_response( + handler, f"Opportunity '{feature}' not generated for '{project_name}'", 404 + ) + _json_response(handler, data) + + +def handle_opportunity_all(handler, project_name): + """GET /api/opportunity/all/ — All 14 opportunity analyses.""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + results = {} + for feature in OPPORTUNITY_FEATURES: + data = _load_opportunity(project_name, feature) + results[feature] = data + _json_response( + handler, + { + "project": project_name, + "features": OPPORTUNITY_FEATURES, + "data": results, + "generated_at": datetime.now().isoformat(), + }, + ) + + +def handle_opportunity_index(handler, project_name): + """GET /api/opportunity/ — Index of available features with status.""" + pdir = PROJECTS_DIR / project_name + if not pdir.exists(): + return _error_response(handler, f"Project '{project_name}' not found") + available = [] + for feature in OPPORTUNITY_FEATURES: + data = _load_opportunity(project_name, feature) + available.append( + { + "feature": feature, + "available": data is not None, + "analysis_type": data.get("analysis_type") if data else None, + } + ) + _json_response( + handler, + { + "project": project_name, + "total_features": len(OPPORTUNITY_FEATURES), + "available_count": sum(1 for f in available if f["available"]), + "features": available, + }, + ) + + +# ── Router ───────────────────────────────────────────────────── + + +# Route table: (pattern, handler, needs_project) +ROUTES = [ + (r"^/api/health$", handle_health, False), + (r"^/api/projects$", handle_projects, False), + (r"^/api/bridge$", handle_bridge, False), + (r"^/api/insights/(.+)$", handle_insights, True), + (r"^/api/swot/(.+)$", handle_swot, True), + (r"^/api/wardley/(.+)$", handle_wardley, True), + (r"^/api/value-chain/(.+)$", handle_value_chain, True), + (r"^/api/bcg/(.+)$", handle_bcg, True), + (r"^/api/ansoff/(.+)$", handle_ansoff, True), + (r"^/api/blue-ocean/(.+)$", handle_blue_ocean, True), + (r"^/api/health-trend/(.+)$", handle_health_trend, True), + # Opportunity endpoints (14 features) — specific routes FIRST, generic LAST + (r"^/api/opportunity/all/(.+)$", lambda h, p: handle_opportunity_all(h, p), True), + ( + r"^/api/opportunity/learning-velocity/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "learning-velocity"), + True, + ), + ( + r"^/api/opportunity/frustration-to-automation/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "frustration-to-automation"), + True, + ), + ( + r"^/api/opportunity/knowledge-gap/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "knowledge-gap"), + True, + ), + ( + r"^/api/opportunity/token-efficiency/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "token-efficiency"), + True, + ), + ( + r"^/api/opportunity/session-quality/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "session-quality"), + True, + ), + ( + r"^/api/opportunity/ai-agent-mastery/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "ai-agent-mastery"), + True, + ), + ( + r"^/api/opportunity/creative-dna/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "creative-dna"), + True, + ), + ( + r"^/api/opportunity/neurodivergent-profile/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "neurodivergent-profile"), + True, + ), + ( + r"^/api/opportunity/model-selection-advisor/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "model-selection-advisor"), + True, + ), + ( + r"^/api/opportunity/before-after-snapshot/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "before-after-snapshot"), + True, + ), + ( + r"^/api/opportunity/cross-repo-transfer/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "cross-repo-transfer"), + True, + ), + ( + r"^/api/opportunity/youtube-learning-graph/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "youtube-learning-graph"), + True, + ), + ( + r"^/api/opportunity/architecture-timelapse/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "architecture-timelapse"), + True, + ), + ( + r"^/api/opportunity/commit-cognitive-load/(.+)$", + lambda h, p: handle_opportunity_feature(h, p, "commit-cognitive-load"), + True, + ), + (r"^/api/opportunity/(.+)$", lambda h, p: handle_opportunity_index(h, p), True), +] + + +def route(handler): + """Route an incoming /api/* request to the appropriate handler.""" + if handler.command != "GET": + return _error_response(handler, "Method not allowed", 405) + + path = urlparse(handler.path).path + + for pattern, handler_func, needs_project in ROUTES: + m = re.match(pattern, path) + if m: + if needs_project: + project_name = m.group(1) + if not _validate_project_name(project_name): + return _error_response(handler, "Invalid project name", 400) + handler_func(handler, project_name) + else: + handler_func(handler) + return + + _error_response(handler, f"Unknown endpoint: {path}", 404) diff --git a/plugins/devarch-hermes/archaeology/audit.py b/plugins/devarch-hermes/archaeology/audit.py new file mode 100644 index 0000000..aa89797 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/audit.py @@ -0,0 +1,599 @@ +"""Forensic audit gate for dev-archaeology projects. + +The audit module intentionally checks the product's credibility surface rather +than only whether individual scripts run. It catches drift between canonical +metrics, project config, generated data, the SQLite database, and publishable +claims. +""" + +from __future__ import annotations + +import re +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +from .utils import _load_json + +SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} +PUBLISHABLE_SUFFIXES = {".md", ".html", ".json", ".j2"} +SENSITIVE_NAME_PATTERNS = [ + re.compile(r"raw-sessions", re.I), + re.compile(r"human-messages", re.I), + re.compile(r"gpt-conversations", re.I), + re.compile(r"youtube-search-history", re.I), + re.compile(r"developer-resume", re.I), +] +SECRET_PATTERNS = [ + re.compile(r"\bsk-(?:proj|live|test|ant|svcacct)-[A-Za-z0-9_-]{20,}"), + re.compile(r"ghp_[A-Za-z0-9_]{20,}"), + re.compile(r"github_pat_[A-Za-z0-9_]{40,}"), + re.compile(r"AKIA[0-9A-Z]{16}"), + re.compile(r"-----BEGIN (?:RSA |OPENSSH |EC )?PRIVATE KEY-----"), +] + + +@dataclass(frozen=True) +class AuditFinding: + severity: str + code: str + message: str + path: str | None = None + detail: str | None = None + + def format(self) -> str: + location = f" [{self.path}]" if self.path else "" + detail = f"\n {self.detail}" if self.detail else "" + return f"{self.severity}: {self.code}: {self.message}{location}{detail}" + + +def _project_dir(project_name: str, root: Path) -> Path: + return root / "projects" / project_name + + +def _rel(path: Path, root: Path) -> str: + try: + return str(path.relative_to(root)) + except ValueError: + return str(path) + + +def _iter_publishable_files(paths: Iterable[Path]) -> Iterable[Path]: + for path in paths: + if path.is_file() and path.suffix.lower() in PUBLISHABLE_SUFFIXES: + yield path + elif path.is_dir(): + for child in path.rglob("*"): + if child.is_file() and child.suffix.lower() in PUBLISHABLE_SUFFIXES: + yield child + + +def _as_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, str): + cleaned = value.replace(",", "").strip() + if cleaned.isdigit(): + return int(cleaned) + return None + + +def _db_count(db_path: Path, table: str) -> int | None: + if not db_path.exists(): + return None + conn = sqlite3.connect(str(db_path), timeout=30) + try: + return int(conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) + except sqlite3.Error: + return None + finally: + conn.close() + + +def _table_exists(db_path: Path, table: str) -> bool: + if not db_path.exists(): + return False + conn = sqlite3.connect(str(db_path), timeout=30) + try: + row = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", + (table,), + ).fetchone() + return row is not None + except sqlite3.Error: + return False + finally: + conn.close() + + +def check_canonical_consistency(project_name: str, root: Path) -> list[AuditFinding]: + findings: list[AuditFinding] = [] + project_root = _project_dir(project_name, root) + project_json_path = project_root / "project.json" + canonical_path = project_root / "deliverables" / "canonical-metrics.json" + data_json_path = project_root / "deliverables" / "data.json" + eras_path = project_root / "data" / "commit-eras.json" + db_path = project_root / "data" / "archaeology.db" + + project = _load_json(project_json_path) or {} + canonical = _load_json(canonical_path) or {} + data = _load_json(data_json_path) or {} + eras = _load_json(eras_path) or {} + + if not canonical: + findings.append( + AuditFinding( + "CRITICAL", + "CANONICAL_MISSING", + "canonical-metrics.json is missing or invalid", + _rel(canonical_path, root), + ) + ) + return findings + + expected_commits = _as_int(canonical.get("total_commits")) + expected_days = _as_int(canonical.get("span_days")) + expected_active = _as_int(canonical.get("active_days")) + + project_overrides = project.get("overrides", {}) if isinstance(project, dict) else {} + project_timeline = project.get("timeline", {}) if isinstance(project, dict) else {} + checks = [ + ( + "total_commits", + expected_commits, + project_overrides.get("total_commits"), + project_json_path, + ), + ("span_days", expected_days, project_timeline.get("total_days"), project_json_path), + ("active_days", expected_active, project_overrides.get("active_days"), project_json_path), + ] + for metric, expected, observed, path in checks: + observed_int = _as_int(observed) + if expected is not None and observed_int is not None and expected != observed_int: + findings.append( + AuditFinding( + "HIGH", + "PROJECT_DRIFT", + f"project.json {metric} does not match canonical metrics", + _rel(path, root), + f"canonical={expected}, project={observed_int}", + ) + ) + + tv_meta = ( + data.get("telemetry_visualizations", {}).get("meta", {}) if isinstance(data, dict) else {} + ) + data_checks = [ + ("total_commits", expected_commits, tv_meta.get("total_commits")), + ("span_days", expected_days, tv_meta.get("lifespan_days") or tv_meta.get("span_days")), + ("active_days", expected_active, tv_meta.get("active_days")), + ] + for metric, expected, observed in data_checks: + observed_int = _as_int(observed) + if expected is not None and observed_int is not None and expected != observed_int: + findings.append( + AuditFinding( + "HIGH", + "DATA_DRIFT", + f"data.json {metric} does not match canonical metrics", + _rel(data_json_path, root), + f"canonical={expected}, data={observed_int}", + ) + ) + + if isinstance(eras, dict): + era_commits = _as_int(eras.get("total_commits")) + if ( + expected_commits is not None + and era_commits is not None + and era_commits != expected_commits + ): + findings.append( + AuditFinding( + "HIGH", + "ERA_DRIFT", + "commit-eras.json total_commits does not match canonical metrics", + _rel(eras_path, root), + f"canonical={expected_commits}, commit-eras={era_commits}", + ) + ) + canonical_eras = _as_int(project_overrides.get("era_count")) + era_count = len(eras.get("eras", [])) if isinstance(eras.get("eras"), list) else None + if canonical_eras is not None and era_count is not None and canonical_eras != era_count: + findings.append( + AuditFinding( + "HIGH", + "ERA_COUNT_DRIFT", + "commit-eras.json era count does not match project override", + _rel(eras_path, root), + f"project={canonical_eras}, commit-eras={era_count}", + ) + ) + + db_commits = _db_count(db_path, "commits") + if expected_commits is not None and db_commits is not None and db_commits != expected_commits: + findings.append( + AuditFinding( + "HIGH", + "DB_COMMIT_DRIFT", + "SQLite commits table does not match canonical metrics", + _rel(db_path, root), + f"canonical={expected_commits}, db={db_commits}", + ) + ) + db_eras = _db_count(db_path, "eras") + canonical_eras = _as_int(project_overrides.get("era_count")) + if canonical_eras is not None and db_eras is not None and db_eras != canonical_eras: + findings.append( + AuditFinding( + "HIGH", + "DB_ERA_DRIFT", + "SQLite eras table does not match project era count", + _rel(db_path, root), + f"project={canonical_eras}, db={db_eras}", + ) + ) + if db_path.exists() and not _table_exists(db_path, "pipeline_runs"): + findings.append( + AuditFinding( + "MEDIUM", + "PIPELINE_TABLE_MISSING", + "pipeline_runs table is absent; pipeline history queries cannot work", + _rel(db_path, root), + ) + ) + + return findings + + +def check_placeholder_data(project_name: str, root: Path) -> list[AuditFinding]: + findings: list[AuditFinding] = [] + data_json_path = _project_dir(project_name, root) / "deliverables" / "data.json" + data = _load_json(data_json_path) + if not isinstance(data, dict): + return findings + + def walk(obj: Any, path: str = "", excluded: bool = False) -> Iterable[tuple[str, Any, bool]]: + current_excluded = excluded + if isinstance(obj, dict): + provenance = obj.get("provenance") + if isinstance(provenance, dict): + status = str(provenance.get("status", "")).lower() + if ( + status in {"placeholder_excluded", "excluded", "historical_raw"} + or provenance.get("publishable") is False + ): + current_excluded = True + yield path, obj, current_excluded + if isinstance(obj, dict): + for key, value in obj.items(): + yield from walk(value, f"{path}.{key}" if path else str(key), current_excluded) + elif isinstance(obj, list): + for idx, value in enumerate(obj): + yield from walk(value, f"{path}[{idx}]", current_excluded) + + zero_total_paths: list[str] = [] + excluded_zero_total_paths: list[str] = [] + mpc_paths: list[str] = [] + excluded_mpc_paths: list[str] = [] + for path, value, excluded in walk(data): + if isinstance(value, dict): + if value.get("co_authored") == 0 and value.get("total") == 0: + (excluded_zero_total_paths if excluded else zero_total_paths).append(path) + if value.get("messages_per_commit") == 1.0: + (excluded_mpc_paths if excluded else mpc_paths).append(path) + + if len(zero_total_paths) >= 3: + findings.append( + AuditFinding( + "HIGH", + "PLACEHOLDER_COAUTHORSHIP", + "Repeated all-zero co-authorship rows look placeholder-derived", + _rel(data_json_path, root), + f"examples={zero_total_paths[:5]}", + ) + ) + elif excluded_zero_total_paths: + findings.append( + AuditFinding( + "INFO", + "PLACEHOLDER_COAUTHORSHIP_EXCLUDED", + "Co-authorship placeholder rows are explicitly marked non-publishable", + _rel(data_json_path, root), + f"count={len(excluded_zero_total_paths)}", + ) + ) + if len(mpc_paths) >= 3: + findings.append( + AuditFinding( + "MEDIUM", + "PLACEHOLDER_SESSION_DEPTH", + "Repeated messages_per_commit=1.0 rows look placeholder-derived", + _rel(data_json_path, root), + f"examples={mpc_paths[:5]}", + ) + ) + elif excluded_mpc_paths: + findings.append( + AuditFinding( + "INFO", + "PLACEHOLDER_SESSION_DEPTH_EXCLUDED", + "Session-depth placeholder rows are explicitly marked non-publishable", + _rel(data_json_path, root), + f"count={len(excluded_mpc_paths)}", + ) + ) + return findings + + +def check_sensitive_artifacts(project_name: str, root: Path) -> list[AuditFinding]: + findings: list[AuditFinding] = [] + project_root = _project_dir(project_name, root) + sensitive_paths = [] + for path in project_root.rglob("*"): + if not path.is_file(): + continue + rel = _rel(path, root) + if any(pattern.search(path.name) for pattern in SENSITIVE_NAME_PATTERNS): + sensitive_paths.append(rel) + if sensitive_paths: + manifest = project_root / "PRIVACY-MANIFEST.md" + severity = "INFO" if manifest.exists() else "MEDIUM" + message = ( + "Private/raw data artifacts are present and governed by the project privacy manifest" + if manifest.exists() + else "Private/raw data artifacts are present in the project tree" + ) + findings.append( + AuditFinding( + severity, + "SENSITIVE_ARTIFACTS", + message, + _rel(project_root, root), + "examples=" + ", ".join(sensitive_paths[:8]), + ) + ) + + for path in _iter_publishable_files([project_root / "data", project_root / "deliverables"]): + try: + text = path.read_text(errors="ignore") + except OSError: + continue + for pattern in SECRET_PATTERNS: + if pattern.search(text): + findings.append( + AuditFinding( + "CRITICAL", + "SECRET_PATTERN", + "Potential secret/private key pattern found", + _rel(path, root), + ) + ) + break + return findings + + +def check_project_config(project_name: str, root: Path) -> list[AuditFinding]: + findings: list[AuditFinding] = [] + project_json_path = _project_dir(project_name, root) / "project.json" + project = _load_json(project_json_path) + if not isinstance(project, dict): + findings.append( + AuditFinding( + "CRITICAL", + "PROJECT_JSON_INVALID", + "project.json is missing or invalid", + _rel(project_json_path, root), + ) + ) + return findings + for key in ("name", "description", "repo_url"): + if not str(project.get(key, "")).strip(): + findings.append( + AuditFinding( + "MEDIUM", + "PROJECT_FIELD_EMPTY", + f"project.json field '{key}' is empty", + _rel(project_json_path, root), + ) + ) + if project.get("repo_url") and not str(project["repo_url"]).startswith("https://github.com/"): + findings.append( + AuditFinding( + "MEDIUM", + "PROJECT_REPO_URL", + "repo_url should be a GitHub HTTPS URL", + _rel(project_json_path, root), + ) + ) + return findings + + +def check_era_references(project_name: str, root: Path) -> list[AuditFinding]: + """Scan deliverables for stale era references.""" + from .era_mapper import load_eras + from .era_scanner import scan_deliverables + + findings: list[AuditFinding] = [] + project_root = _project_dir(project_name, root) + eras_path = project_root / "data" / "commit-eras.json" + + if not eras_path.exists(): + return findings + + eras = load_eras(eras_path) + if not eras: + return findings + + result = scan_deliverables(project_root, eras) + + for ref in result.refs: + severity = "MEDIUM" + code = "ERA_STALE_REF" + if ref.kind == "era_json_field": + severity = "HIGH" + code = "ERA_STALE_NUMBER" + elif ref.kind == "era_name": + severity = "HIGH" + code = "ERA_STALE_NAME" + elif ref.kind == "era_css_var": + severity = "HIGH" + code = "ERA_STALE_CSS" + elif ref.kind == "era_count": + severity = "MEDIUM" + code = "ERA_STALE_COUNT" + + findings.append( + AuditFinding( + severity, + code, + f"Stale era reference: {ref.old_value} (expected: {ref.expected})", + path=_rel(ref.file, root), + detail=f"line {ref.line}, kind={ref.kind}", + ) + ) + + return findings + + +def run_audit(project_name: str, root: str | Path = ".") -> list[AuditFinding]: + root_path = Path(root).resolve() + project_root = _project_dir(project_name, root_path) + findings: list[AuditFinding] = [] + if not project_root.exists(): + return [ + AuditFinding( + "CRITICAL", + "PROJECT_MISSING", + f"Project '{project_name}' does not exist", + _rel(project_root, root_path), + ) + ] + + for check in ( + check_project_config, + check_mined_history, + check_derived_evidence, + check_canonical_consistency, + check_placeholder_data, + check_sensitive_artifacts, + check_era_references, + ): + findings.extend(check(project_name, root_path)) + + return sorted( + findings, key=lambda f: (SEVERITY_ORDER.get(f.severity, 99), f.code, f.path or "") + ) + + +def has_blocking_findings(findings: Iterable[AuditFinding], fail_on: str = "HIGH") -> bool: + threshold = SEVERITY_ORDER[fail_on] + return any(SEVERITY_ORDER.get(f.severity, 99) <= threshold for f in findings) + + +def summarize(findings: Iterable[AuditFinding]) -> dict[str, int]: + summary = {severity: 0 for severity in SEVERITY_ORDER} + for finding in findings: + summary[finding.severity] = summary.get(finding.severity, 0) + 1 + return summary + + +def check_mined_history(project_name: str, root: Path) -> list[AuditFinding]: + """Reconcile extraction identities and byte bindings when mining evidence exists.""" + import csv + import hashlib + + project = _project_dir(project_name, root) + coverage_path = project / "data" / "coverage.json" + coverage = _load_json(coverage_path) + config = _load_json(project / "project.json") or {} + if not coverage_path.exists() and not config.get("mined_history_manifest_required"): + return [] # Legacy imported datasets never claimed a mining manifest. + if not isinstance(coverage, dict) or not isinstance(coverage.get("artifact_sha256"), dict): + return [ + AuditFinding( + "HIGH", "MINING_MANIFEST_INVALID", "Mining coverage manifest missing or invalid" + ) + ] + findings = [] + expected_names = {"github-commits.csv", "github-commits-with-stats.txt"} + if set(coverage["artifact_sha256"]) != expected_names: + findings.append( + AuditFinding("HIGH", "MINING_MANIFEST_INVALID", "Mining artifact bindings incomplete") + ) + for name, expected in coverage["artifact_sha256"].items(): + if name not in expected_names: + findings.append( + AuditFinding("HIGH", "MINING_MANIFEST_INVALID", "Unexpected artifact name") + ) + continue + path = project / "data" / name + if not path.exists() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + findings.append( + AuditFinding("HIGH", "MINING_ARTIFACT_DRIFT", f"Mined artifact changed: {name}") + ) + csv_path = project / "data" / "github-commits.csv" + db_path = project / "data" / "archaeology.db" + try: + with csv_path.open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + keys = ("hash", "date", "message", "author") + source = [tuple(row[key] for key in keys) for row in rows] + hashes = [row["hash"] for row in rows] + if not db_path.exists(): + raise ValueError("Mined history database is missing") + with sqlite3.connect(db_path) as conn: + stored = list(conn.execute("SELECT hash,date,message,author FROM commits")) + if ( + len(hashes) != coverage.get("commit_count") + or len(set(hashes)) != len(hashes) + or sorted(source) != sorted(stored) + ): + raise ValueError("Git manifest, CSV and SQLite commit records do not reconcile") + from .metrics import calculate_metrics + + measured = calculate_metrics(rows) + canonical = _load_json(project / "deliverables" / "canonical-metrics.json") or {} + if any(canonical.get(key) != value for key, value in measured.items()): + raise ValueError("Canonical metrics differ from source commit measurements") + visual = _load_json(project / "deliverables" / "data.json") or {} + meta = visual.get("telemetry_visualizations", {}).get("meta", {}) + if any( + visual.get(key) != value or meta.get(key) != value for key, value in measured.items() + ): + raise ValueError("Visualization metrics differ from source commit measurements") + except (OSError, ValueError, KeyError, TypeError, sqlite3.Error) as exc: + findings.append(AuditFinding("HIGH", "MINING_HISTORY_DRIFT", str(exc))) + return findings + + +def check_derived_evidence(project_name: str, root: Path) -> list[AuditFinding]: + from .provenance import requires_binding, verify_analyses, verify_artifact + + project = _project_dir(project_name, root) + try: + if not requires_binding(project): + return [] + verify_analyses(project) + deliverables = project / "deliverables" + artifacts = { + Path(str(p)[: -len(".provenance.json")]) + for p in deliverables.rglob("*.provenance.json") + } + artifacts.update( + p + for p in [ + deliverables / "reports/ARCHAEOLOGY-REPORT.md", + deliverables / "visuals/report.html", + deliverables / "visuals/archaeology.html", + ] + if p.exists() + ) + for path in artifacts: + verify_artifact(project, path) + except (OSError, ValueError, KeyError, TypeError) as exc: + return [AuditFinding("HIGH", "DERIVED_EVIDENCE_STALE", str(exc))] + return [] diff --git a/plugins/devarch-hermes/archaeology/classifiers/__init__.py b/plugins/devarch-hermes/archaeology/classifiers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/devarch-hermes/archaeology/classifiers/era_detector.py b/plugins/devarch-hermes/archaeology/classifiers/era_detector.py new file mode 100644 index 0000000..10bb8d2 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/classifiers/era_detector.py @@ -0,0 +1,607 @@ +"""Signal detection module for development archaeology. + +Detects development signals from commit history stored in a SQLite database +using 5 heuristics: gap detection, velocity shifts, author changes, scope +changes, and cross-repo activation. + +This module does NOT define eras. It surfaces patterns and signals that a +human (or LLM with full analysis context) uses to define narrative eras. +Era definitions live in a hand-curated eras.json file. +""" + +import argparse +import json +import sqlite3 +from collections import Counter +from datetime import datetime +from pathlib import Path + +from ..utils import _parse_date, _script_dir + +# Default config values +DEFAULTS = { + "min_gap_days": 3, + "velocity_shift_factor": 2.0, + "scope_change_keywords": [ + "refactor", + "rewrite", + "restructure", + "migration", + "architecture", + ], + "cross_repo_activation_threshold": 3, +} + + +def _load_defaults_json() -> dict: + """Load settings from config/defaults.json if available.""" + defaults_path = _script_dir().parent.parent / "config" / "defaults.json" + if defaults_path.exists(): + try: + with open(defaults_path, encoding="utf-8") as f: + data = json.load(f) + return data.get("signal_detection", {}) + except (json.JSONDecodeError, OSError): + pass + return {} + + +class SignalDetector: + """Detect development signals from commit history. + + Surfaces patterns (gaps, velocity shifts, author changes, scope keywords, + new repo activation) as signals. Does NOT define eras — that is a human + narrative judgment. + + Args: + db_path: Path to the SQLite database containing a 'commits' table. + config: Optional dict overriding default settings. + """ + + def __init__(self, db_path: str | Path, config: dict | None = None): + self.db_path = Path(db_path) + merged = {**DEFAULTS, **_load_defaults_json()} + if config: + merged.update(config) + self.config = merged + + self.min_gap_days: int = merged["min_gap_days"] + self.velocity_shift_factor: float = merged["velocity_shift_factor"] + self.scope_change_keywords: list[str] = merged["scope_change_keywords"] + self.cross_repo_threshold: int = merged["cross_repo_activation_threshold"] + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def detect(self) -> dict: + """Run all heuristics and return structured signals + clusters. + + Returns: + Dict with keys: project_meta, signals, cluster_summary, + active_days_summary. + """ + commits = self._load_commits() + if not commits: + return {"signals": [], "cluster_summary": []} + + gap_signals = self._detect_gaps(commits) + velocity_signals = self._detect_velocity_shifts(commits) + author_signals = self._detect_author_changes(commits) + scope_signals = self._detect_scope_changes(commits) + repo_signals = self._detect_cross_repo(commits) + + all_signals = gap_signals + velocity_signals + author_signals + scope_signals + repo_signals + all_signals.sort(key=lambda s: (s["index"], s["type"])) + + clusters = self._build_clusters(commits) + + return { + "commit_count": len(commits), + "date_range": { + "first": commits[0]["date"][:10] if commits[0]["date"] else "", + "last": commits[-1]["date"][:10] if commits[-1]["date"] else "", + }, + "active_days": len({c["date"][:10] for c in commits if len(c["date"]) >= 10}), + "signals": all_signals, + "cluster_summary": clusters, + } + + def save(self, output_path: str | Path, result: dict | None = None) -> Path: + """Write detected signals to a JSON file. + + Args: + output_path: Path to write the JSON file. + result: Signal dict (if None, runs detect() first). + + Returns: + Path to the written file. + """ + if result is None: + result = self.detect() + from ..utils import atomic_write + + output_path = Path(output_path) + atomic_write(output_path, json.dumps(result, indent=2, ensure_ascii=False)) + return output_path + + # ------------------------------------------------------------------ + # Data loading + # ------------------------------------------------------------------ + + def _load_commits(self) -> list[dict]: + """Load commits from the SQLite database, ordered by date.""" + if not self.db_path.exists(): + return [] + + try: + conn = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True, timeout=30) + conn.row_factory = sqlite3.Row + try: + cursor = conn.execute( + "SELECT date, author, message, repo FROM commits ORDER BY date ASC" + ) + has_repo = True + except sqlite3.OperationalError: + cursor = conn.execute("SELECT date, author, message FROM commits ORDER BY date ASC") + has_repo = False + rows = cursor.fetchall() + except sqlite3.OperationalError: + return [] + finally: + try: + conn.close() + except NameError: + pass + + commits = [] + for row in rows: + commits.append( + { + "date": str(row["date"]) if row["date"] else "", + "author": str(row["author"]) if row["author"] else "", + "message": str(row["message"]) if row["message"] else "", + "repo": (str(row["repo"]) if has_repo and row["repo"] else ""), + } + ) + return commits + + # ------------------------------------------------------------------ + # Signal: Gap detection + # ------------------------------------------------------------------ + + def _detect_gaps(self, commits: list[dict]) -> list[dict]: + """Find positions where the gap between consecutive commits exceeds + min_gap_days.""" + if len(commits) < 2: + return [] + + signals = [] + for i in range(1, len(commits)): + prev_date = _parse_date(commits[i - 1]["date"]) + curr_date = _parse_date(commits[i]["date"]) + if prev_date is None or curr_date is None: + continue + gap = (curr_date - prev_date).days + if gap >= self.min_gap_days: + signals.append( + { + "index": i, + "date": commits[i]["date"][:10] if len(commits[i]["date"]) >= 10 else "", + "type": "gap", + "detail": f"{gap}-day gap since previous commit", + "strength": "strong" if gap >= 7 else "moderate", + } + ) + return signals + + # ------------------------------------------------------------------ + # Signal: Velocity shift + # ------------------------------------------------------------------ + + def _detect_velocity_shifts(self, commits: list[dict]) -> list[dict]: + """Find day-level velocity shifts between clusters. + + Compares average daily commit rate in consecutive active-day windows. + Only flags shifts between days, not within them. + """ + if len(commits) < 10: + return [] + + # Build day-level data + day_counts: dict[str, int] = {} + for c in commits: + day = c["date"][:10] if len(c["date"]) >= 10 else "" + if day: + day_counts[day] = day_counts.get(day, 0) + 1 + + days_sorted = sorted(day_counts.keys()) + if len(days_sorted) < 3: + return [] + + # Compare consecutive day windows (3-day rolling average) + window = 3 + signals = [] + flagged_days = set() + + for i in range(window, len(days_sorted)): + before_avg = sum(day_counts.get(d, 0) for d in days_sorted[i - window : i]) / window + after_avg = sum(day_counts.get(d, 0) for d in days_sorted[i : i + window]) / max( + 1, min(window, len(days_sorted) - i) + ) + + if before_avg == 0: + continue + + ratio = after_avg / before_avg + day = days_sorted[i] + + if ( + ratio >= self.velocity_shift_factor or ratio <= 1.0 / self.velocity_shift_factor + ) and day not in flagged_days: + flagged_days.add(day) + direction = "up" if ratio > 1 else "down" + display_ratio = max(ratio, 1.0 / ratio) + # Find the commit index for this day + idx = next( + (j for j, c in enumerate(commits) if c["date"][:10] == day), + 0, + ) + signals.append( + { + "index": idx, + "date": day, + "type": "velocity", + "detail": f"{display_ratio:.1f}x {direction}shift ({before_avg:.0f}→{after_avg:.0f} commits/day avg)", + "strength": "strong" if display_ratio >= 4.0 else "moderate", + } + ) + + return signals + + # ------------------------------------------------------------------ + # Signal: Author change + # ------------------------------------------------------------------ + + def _detect_author_changes(self, commits: list[dict]) -> list[dict]: + """Find positions where the primary author changes.""" + if len(commits) < 10: + return [] + + window_size = max(10, len(commits) // 5) + raw_boundaries = [] + + def primary_author(start: int, end: int) -> str: + authors = Counter( + commits[j]["author"] + for j in range(start, min(end, len(commits))) + if commits[j]["author"] + ) + return authors.most_common(1)[0][0] if authors else "" + + for i in range(window_size, len(commits) - window_size): + before_author = primary_author(i - window_size, i) + after_author = primary_author(i, i + window_size) + + if before_author and after_author and before_author != after_author: + raw_boundaries.append((i, before_author, after_author)) + + deduped = self._deduplicate(raw_boundaries, min_gap=10) + + signals = [] + for idx, before, after in deduped: + signals.append( + { + "index": idx, + "date": commits[idx]["date"][:10] if len(commits[idx]["date"]) >= 10 else "", + "type": "author", + "detail": f"primary author shifts from {before} to {after}", + "strength": "strong", + } + ) + return signals + + # ------------------------------------------------------------------ + # Signal: Scope change keywords + # ------------------------------------------------------------------ + + def _detect_scope_changes(self, commits: list[dict]) -> list[dict]: + """Find concentrated bursts of scope-change keywords. + + Only flags positions where 3+ scope-change commits cluster within + a 20-commit window. Isolated refactors don't indicate an era boundary. + """ + if not self.scope_change_keywords or len(commits) < 10: + return [] + + window = 20 + threshold = 3 + + # Pre-compute which commits match + matches = [] + for i, commit in enumerate(commits): + message_lower = commit["message"].lower() + matched = [kw for kw in self.scope_change_keywords if kw in message_lower] + matches.append(matched) + + # Find windows with concentrated scope changes + signals = [] + flagged = set() + for i in range(len(commits) - window + 1): + count = sum(1 for j in range(i, i + window) if matches[j]) + if count >= threshold: + # Flag the center of the burst + center = i + window // 2 + keywords = set() + for j in range(i, i + window): + if matches[j]: + keywords.update(matches[j]) + + # Round to nearest day boundary to avoid flagging every commit + day = commits[center]["date"][:10] if len(commits[center]["date"]) >= 10 else "" + if day not in flagged: + flagged.add(day) + signals.append( + { + "index": center, + "date": day, + "type": "scope", + "detail": f"concentrated {', '.join(sorted(keywords))} burst ({count} in {window} commits)", + "strength": "strong" if count >= 6 else "moderate", + } + ) + + return signals + + # ------------------------------------------------------------------ + # Signal: Cross-repo activation + # ------------------------------------------------------------------ + + def _detect_cross_repo(self, commits: list[dict]) -> list[dict]: + """Find positions where new repos accumulate significant commits.""" + if not commits: + return [] + + seen_repos: dict[str, int] = {} + signals = [] + + for i, commit in enumerate(commits): + repo = commit["repo"] + if not repo: + continue + + if repo not in seen_repos: + seen_repos[repo] = 0 + seen_repos[repo] += 1 + + if seen_repos[repo] == self.cross_repo_threshold and i > 0: + first_appearance = next( + (j for j in range(i, -1, -1) if commits[j]["repo"] == repo), + i, + ) + if first_appearance > 0: + signals.append( + { + "index": first_appearance, + "date": commits[first_appearance]["date"][:10] + if len(commits[first_appearance]["date"]) >= 10 + else "", + "type": "repo_activation", + "detail": f"repo {repo} reaches {self.cross_repo_threshold} commits", + "strength": "moderate", + } + ) + + return signals + + # ------------------------------------------------------------------ + # Cluster summary (gap-based natural groupings) + # ------------------------------------------------------------------ + + def _build_clusters(self, commits: list[dict]) -> list[dict]: + """Group commits into natural clusters separated by multi-day gaps. + + These are factual groupings, not narrative eras. They provide the + raw material for era definition. + """ + if not commits: + return [] + + # Find day-level gaps + day_commits: dict[str, list[int]] = {} + for i, c in enumerate(commits): + day = c["date"][:10] if len(c["date"]) >= 10 else "" + if day: + day_commits.setdefault(day, []).append(i) + + days_sorted = sorted(day_commits.keys()) + + if not days_sorted: + return [] + + # Split into clusters at gaps >= min_gap_days + clusters = [] + cluster_days = [days_sorted[0]] + + for i in range(1, len(days_sorted)): + prev = datetime.strptime(days_sorted[i - 1], "%Y-%m-%d") + curr = datetime.strptime(days_sorted[i], "%Y-%m-%d") + gap = (curr - prev).days + + if gap >= self.min_gap_days: + # Close current cluster + clusters.append(self._summarize_cluster(commits, cluster_days, day_commits)) + cluster_days = [days_sorted[i]] + else: + cluster_days.append(days_sorted[i]) + + # Close last cluster + if cluster_days: + clusters.append(self._summarize_cluster(commits, cluster_days, day_commits)) + + return clusters + + def _summarize_cluster( + self, + commits: list[dict], + days: list[str], + day_commits: dict[str, list[int]], + ) -> dict: + """Build a summary dict for a cluster of active days.""" + indices = [] + for d in days: + indices.extend(day_commits.get(d, [])) + indices.sort() + + cluster_commits = [commits[i] for i in indices] + authors = Counter(c["author"] for c in cluster_commits if c["author"]) + repos = Counter(c["repo"] for c in cluster_commits if c["repo"]) + + return { + "start_date": days[0], + "end_date": days[-1], + "active_days": len(days), + "commit_count": len(cluster_commits), + "primary_author": authors.most_common(1)[0][0] if authors else "", + "dominant_repo": repos.most_common(1)[0][0] if repos else "", + "daily_breakdown": {d: len(day_commits.get(d, [])) for d in days}, + } + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + @staticmethod + def _deduplicate(boundaries: list[tuple], min_gap: int = 5) -> list[tuple]: + """Remove boundary entries whose indices are too close together.""" + if not boundaries: + return [] + + sorted_bounds = sorted(boundaries, key=lambda b: b[0]) + deduped = [sorted_bounds[0]] + for b in sorted_bounds[1:]: + if b[0] - deduped[-1][0] >= min_gap: + deduped.append(b) + return deduped + + +# ---------------------------------------------------------------------- +# Convenience function (called by CLI) +# ---------------------------------------------------------------------- + + +def detect_signals(project_name: str, config: dict | None = None) -> dict: + """Detect signals for a project by name. + + Resolves the project database at projects//data/archaeology.db, + runs detection, and saves results to projects//data/detected-signals.json. + + Args: + project_name: Name of the project (directory under projects/). + config: Optional config overrides. + + Returns: + Signal detection result dict. + """ + repo_root = Path.cwd() + project_dir = repo_root / "projects" / project_name + db_path = project_dir / "data" / "archaeology.db" + + if not db_path.exists(): + print(f"ERROR: Database not found at {db_path}", flush=True) + return {"signals": [], "cluster_summary": []} + + detector = SignalDetector(db_path, config=config) + result = detector.detect() + + output_path = project_dir / "data" / "detected-signals.json" + detector.save(output_path, result) + + signals = result.get("signals", []) + clusters = result.get("cluster_summary", []) + print(f"Detected {len(signals)} signals, {len(clusters)} clusters") + for sig_type in sorted(set(s["type"] for s in signals)): + count = sum(1 for s in signals if s["type"] == sig_type) + print(f" {sig_type}: {count} signals") + for cluster in clusters: + print( + f" Cluster: {cluster['start_date']} -> {cluster['end_date']} " + f"({cluster['commit_count']} commits, {cluster['active_days']} days)" + ) + print(f"Saved to {output_path}") + + return result + + +# ---------------------------------------------------------------------- +# CLI entry point +# ---------------------------------------------------------------------- + + +def main() -> None: + """CLI entry point for standalone signal detection. + + Usage: + python -m archaeology.classifiers.era_detector --project + python -m archaeology.classifiers.era_detector --db [--output ] + """ + parser = argparse.ArgumentParser(description="Detect development signals from commit history") + parser.add_argument( + "--project", + help="Project name (resolves to projects//data/archaeology.db)", + ) + parser.add_argument( + "--db", + help="Direct path to the SQLite database", + ) + parser.add_argument( + "--output", + help="Output path for detected-signals.json", + ) + parser.add_argument( + "--config", + help="Path to a JSON config file with overrides", + ) + parser.add_argument( + "--min-gap-days", + type=int, + help="Minimum gap in days to flag as a signal", + ) + args = parser.parse_args() + + config_overrides: dict = {} + if args.config: + config_path = Path(args.config) + if config_path.exists(): + with open(config_path, encoding="utf-8") as f: + config_overrides = json.load(f) + + if args.min_gap_days is not None: + config_overrides["min_gap_days"] = args.min_gap_days + + if args.project: + detect_signals(args.project, config=config_overrides or None) + return + + if not args.db: + parser.error("Either --project or --db is required") + + db_path = Path(args.db) + if not db_path.exists(): + print(f"ERROR: Database not found: {db_path}", flush=True) + return + + detector = SignalDetector(db_path, config=config_overrides or None) + result = detector.detect() + + output_path = Path(args.output) if args.output else db_path.parent / "detected-signals.json" + detector.save(output_path, result) + + signals = result.get("signals", []) + print(f"Detected {len(signals)} signals from {db_path}") + for sig in signals: + print(f" [{sig['type']}] {sig['date']}: {sig['detail']}") + + +if __name__ == "__main__": + main() diff --git a/plugins/devarch-hermes/archaeology/cli.py b/plugins/devarch-hermes/archaeology/cli.py new file mode 100644 index 0000000..7302bbf --- /dev/null +++ b/plugins/devarch-hermes/archaeology/cli.py @@ -0,0 +1,1189 @@ +"""DevArch Framework CLI.""" + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import click + +from .analysis_runner import run_analysis_vectors +from .classifiers.era_detector import detect_signals +from .extractors.git import extract_git_log, extract_git_log_with_stats, repository_coverage + + +def _project_dir(project_name): + """Resolve project directory path.""" + d = os.path.join("projects", project_name) + if not os.path.isdir(d): + click.echo(f"Project '{project_name}' not found at {d}/", err=True) + click.echo("Run 'devarch init {project_name}' first.", err=True) + sys.exit(1) + return d + + +def _viz_template_path(filename): + """Resolve a visualization template against the installed package. + + Templates ship as package data, so resolving them against the CWD only + works when the CLI happens to run from the repository source root. + """ + return Path(__file__).parent / "visualization" / filename + + +def _is_nonempty_string(value): + return isinstance(value, str) and bool(value.strip()) + + +def _invalid_value_desc(value): + if value is None: + return "null" + if isinstance(value, str): + return "an empty or whitespace-only string" + return type(value).__name__ + + +@click.group() +@click.version_option(package_name="devarch-framework") +def main(): + """DevArch Framework - forensic mining of software development history.""" + pass + + +@main.command() +@click.argument("project_name") +@click.option( + "--description", default="Draft archaeology project", help="Human-readable project description" +) +@click.option( + "--repo-url", default="https://github.com/example/example", help="GitHub repository URL" +) +def init(project_name, description, repo_url): + """Create a new project directory with default config.""" + project_dir = os.path.join("projects", project_name) + os.makedirs(os.path.join(project_dir, "data"), exist_ok=True) + os.makedirs(os.path.join(project_dir, "deliverables"), exist_ok=True) + + config_path = os.path.join(project_dir, "project.json") + if os.path.exists(config_path): + click.echo(f"Project '{project_name}' already exists at {config_path}") + return + + config = { + "name": project_name, + "description": description, + "repo_url": repo_url, + "developer": {}, + "timeline": {}, + "overrides": {}, + "visualization": {}, + "data_sources": {}, + } + + with open(config_path, "w", encoding="utf-8") as f: + json.dump(config, f, indent=2) + + click.echo(f"Created project '{project_name}' at {project_dir}/") + + +@main.command() +@click.option( + "--project", "project_name", default="demo-archaeology", help="Demo project name to create" +) +@click.option("--force", is_flag=True, help="Overwrite an existing demo project") +@click.option( + "--build-db", is_flag=True, help="Build the demo SQLite database after creating files" +) +def demo(project_name, force, build_db): + """Create a sanitized demo archaeology project.""" + from .demo import create_demo_project + + try: + project_root = create_demo_project(Path.cwd(), project_name=project_name, force=force) + except FileExistsError as exc: + click.echo(str(exc), err=True) + raise click.exceptions.Exit(1) + click.echo(f"Created sanitized demo project at {project_root}") + click.echo(f"Try: devarch build-db {project_name}") + click.echo(f"Then: devarch audit {project_name} --fail-on HIGH") + if build_db: + cmd = [sys.executable, "-m", "archaeology.db.builder", "--project-root", str(project_root)] + _env = os.environ.copy() + _pkg_root = str(Path(__file__).parent.parent) + _env["PYTHONPATH"] = _pkg_root + ( + (os.pathsep + _env["PYTHONPATH"]) if _env.get("PYTHONPATH") else "" + ) + result = subprocess.run(cmd, check=True, timeout=300, env=_env) + if result.returncode != 0: + raise click.exceptions.Exit(result.returncode) + + +@main.command() +@click.argument("repo_path") +@click.option("--project", "-p", required=True, help="Project name to extract into") +@click.option("--verbose", "-v", is_flag=True) +def mine(repo_path, project, verbose): + """Phase 1: Extract data from a git repository.""" + from .extractors.git import extract_git_log, extract_git_log_with_stats + + project_dir = _project_dir(project) + data_dir = os.path.join(project_dir, "data") + + if not os.path.isdir(os.path.expanduser(repo_path)): + click.echo(f"Repository not found: {repo_path}", err=True) + sys.exit(1) + + repo_path = str(Path(repo_path).expanduser().resolve()) + try: + coverage = repository_coverage(repo_path) + except RuntimeError as exc: + raise click.ClickException(str(exc)) from exc + if coverage["shallow"]: + raise click.ClickException( + "Shallow history: fetch complete history before mining; no fetch performed" + ) + + click.echo(f"Extracting git log from {repo_path}...") + + csv_path = os.path.join(data_dir, "github-commits.csv") + try: + count = extract_git_log(repo_path, csv_path, verbose=verbose) + click.echo(f" Extracted {count} commits to {csv_path}") + except (RuntimeError, Exception) as e: + click.echo(f"Error: Git extraction failed: {e}", err=True) + sys.exit(1) + + stats_path = os.path.join(data_dir, "github-commits-with-stats.txt") + try: + extract_git_log_with_stats(repo_path, stats_path, verbose=verbose) + except (RuntimeError, Exception) as e: + click.echo(f"Error: Git stats extraction failed: {e}", err=True) + sys.exit(1) + + if repository_coverage(repo_path) != coverage: + raise click.ClickException("Repository refs changed during mining; retry before analysis") + import hashlib + + from .utils import atomic_write + + coverage["artifact_sha256"] = { + Path(path).name: hashlib.sha256(Path(path).read_bytes()).hexdigest() + for path in (csv_path, stats_path) + } + atomic_write(Path(data_dir) / "coverage.json", json.dumps(coverage, indent=2)) + config_path = Path(project_dir) / "project.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["repo_path"] = repo_path + config["mined_history_manifest_required"] = True + atomic_write(config_path, json.dumps(config, indent=2)) + + click.echo(f"Phase 1 complete for '{project}'.") + + +@main.command() +@click.argument("project_name") +@click.option("--verbose", "-v", is_flag=True, help="Verbose output") +def build_db(project_name, verbose): + """Phase 1.5: Build SQLite database from extracted data.""" + project_dir = _project_dir(project_name) + db_path = os.path.join(project_dir, "data", "archaeology.db") + + cmd = [sys.executable, "-m", "archaeology.db.builder", "--project-root", project_dir] + if verbose: + cmd.append("--verbose") + + _env = os.environ.copy() + _pkg_root = str(Path(__file__).parent.parent) + _env["PYTHONPATH"] = _pkg_root + ( + (os.pathsep + _env["PYTHONPATH"]) if _env.get("PYTHONPATH") else "" + ) + result = subprocess.run(cmd, check=True, timeout=300, env=_env) + if result.returncode == 0 and os.path.exists(db_path): + click.echo(f"Database built at {db_path}") + else: + click.echo(f"Build failed with exit code {result.returncode}", err=True) + sys.exit(result.returncode) + + +@main.command() +@click.argument("project_name") +@click.option("--port", default=8001, help="Port for Datasette server") +@click.option( + "--unsafe-cors", + is_flag=True, + help="Enable Datasette CORS headers. Off by default for local data safety.", +) +def serve(project_name, port, unsafe_cors): + """Launch Datasette for a project.""" + project_dir = _project_dir(project_name) + db_path = os.path.join(project_dir, "data", "archaeology.db") + + if not os.path.exists(db_path): + click.echo(f"Database not found at {db_path}. Run 'devarch build-db {project_name}' first.") + sys.exit(1) + + # Load project config for display name + project_config = {} + config_path = os.path.join(project_dir, "project.json") + if os.path.exists(config_path): + try: + with open(config_path, encoding="utf-8") as f: + project_config = json.load(f) + except json.JSONDecodeError as e: + click.echo(f"Error: Invalid JSON in {config_path}: {e}", err=True) + sys.exit(1) + + display_name = project_config.get("visualization", {}).get("title", project_name.upper()) + + # Use project-specific metadata if it exists, otherwise default + project_metadata = os.path.join(project_dir, "datasette-metadata.yaml") + default_metadata = os.path.join("config", "datasette-metadata.yaml") + + metadata_src = None + if os.path.exists(project_metadata): + metadata_src = project_metadata + elif os.path.exists(default_metadata): + # Create a temp metadata with project name injected + with open(default_metadata, encoding="utf-8") as f: + content = f.read() + content = content.replace( + 'title: "Archaeology Database"', + f'title: "{display_name} Archaeology Database"', + ) + tmp = tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False, prefix="arch-metadata-" + ) + tmp.write(content) + tmp.close() + metadata_src = tmp.name + + # datasette's --setting takes "NAME VALUE" as two arguments; the + # comma-joined single-argument form is rejected by every datasette in the + # supported range (>=0.64.0), which left `serve` unable to start. + cmd = [ + "datasette", + db_path, + "--port", + str(port), + "--setting", + "sql_time_limit_ms", + "5000", + ] + if unsafe_cors: + cmd.append("--cors") + if metadata_src: + cmd.extend(["--metadata", metadata_src]) + + click.echo(f"Launching Datasette at http://localhost:{port}") + try: + subprocess.run(cmd, timeout=300) + finally: + if metadata_src and metadata_src.startswith(tempfile.gettempdir()): + os.unlink(metadata_src) + + +@main.command() +@click.argument("project_name") +@click.option("--config", "config_path", help="Path to custom config JSON") +@click.option("--min-gap-days", type=int, help="Override min gap days for signal detection") +@click.option("--verbose", "-v", is_flag=True) +def signals(project_name, config_path, min_gap_days, verbose): + """Phase 2: Detect development signals (gaps, velocity shifts, etc). + + Signals are patterns in the data, NOT era definitions. Eras are + hand-curated narrative judgments based on these signals. + """ + from .classifiers.era_detector import detect_signals + + config = {} + if config_path: + try: + with open(config_path) as f: + config = json.load(f) + except json.JSONDecodeError as e: + click.echo(f"Error: Invalid JSON in {config_path}: {e}", err=True) + sys.exit(1) + if min_gap_days is not None: + config["min_gap_days"] = min_gap_days + + db_path = os.path.join(_project_dir(project_name), "data", "archaeology.db") + if not os.path.exists(db_path): + click.echo(f"No database found. Run 'devarch build-db {project_name}' first.", err=True) + sys.exit(1) + + result = detect_signals(project_name, config=config or None) + if result.get("signals"): + click.echo( + f"Detected {len(result['signals'])} signals " + f"across {len(result['cluster_summary'])} clusters." + ) + else: + click.echo("No significant patterns detected in the commit history.") + + +@main.command() +@click.argument("project_name") +@click.option("--sessions-dir", help="Directory containing .jsonl session files") +@click.option("--verbose", "-v", is_flag=True) +def extract_sessions(project_name, sessions_dir, verbose): + """Extract Claude Code session data.""" + project_dir = _project_dir(project_name) + output_path = os.path.join(project_dir, "data", "raw-sessions.md") + + cmd = [sys.executable, "-m", "archaeology.extractors.sessions", "--output", output_path] + if sessions_dir: + cmd.extend(["--sessions-dir", sessions_dir]) + cmd.extend(["--project", project_name]) + + result = subprocess.run(cmd, check=True, timeout=300) + if result.returncode == 0: + click.echo(f"Sessions extracted to {output_path}") + else: + click.echo("Session extraction failed", err=True) + sys.exit(result.returncode) + + +@main.command() +@click.argument("project_name") +@click.option( + "--vector", + "-v", + "vectors", + multiple=True, + help="Run specific analysis vector(s). Repeat for multiple.", +) +@click.option( + "--prompts", + is_flag=True, + help="Show legacy prompt-template instructions instead of running automation", +) +@click.option("--verbose", is_flag=True, help="Print vector execution detail") +def analyze(project_name, vectors, prompts, verbose): + """Phase 3: Run automated analysis vectors against project data.""" + from .analysis_runner import AnalysisRunner, run_analysis_vectors + + available = AnalysisRunner.VECTORS + unknown = set(vectors) - set(available) if vectors else set() + if unknown: + for vector in sorted(unknown): + click.echo(f"Unknown vector: {vector}", err=True) + raise click.exceptions.Exit(1) + + target = list(vectors) if vectors else list(available) + project_dir = _project_dir(project_name) + deliverables_dir = os.path.join(project_dir, "deliverables") + analysis_dir = os.path.join(deliverables_dir, "analysis") + os.makedirs(analysis_dir, exist_ok=True) + + if prompts: + vectors_dir = os.path.join(os.path.dirname(__file__), "..", "analysis-vectors") + click.echo(f"Analysis prompt templates for '{project_name}'") + for vec_name in target: + prompt_path = os.path.join(vectors_dir, f"{vec_name}.md") + output_path = os.path.join(analysis_dir, f"analysis-{vec_name}.md") + click.echo(f" [{vec_name}] prompt={prompt_path} output={output_path}") + return + + click.echo(f"Running automated analysis vectors for '{project_name}'") + click.echo(f"Vectors: {', '.join(target)}") + results = run_analysis_vectors(project_name, verbose=verbose, vectors=target) + failed = {name: path for name, path in results.items() if str(path).startswith("ERROR")} + for name, path in results.items(): + click.echo(f" {name}: {path}") + if failed: + raise click.exceptions.Exit(1) + + +@main.command() +@click.argument("project_name") +@click.option( + "--analyzers", "-a", multiple=True, help="Specific analyzer(s) to run (default: all 14)" +) +@click.option("--verbose", "-v", is_flag=True, help="Show detailed output") +def opportunity(project_name, analyzers, verbose): + """Disabled pending evidence and data-exposure hardening.""" + raise click.ClickException( + "opportunity is disabled pending readiness hardening. Use init, mine, build-db, signals, analyze, visualize, export-report and audit." + ) + + +@main.command("public-case-study") +@click.option( + "--output", + "output_dir", + default="public-case-study", + help="Output directory for the sanitized public case study", +) +@click.option( + "--project", + "project_name", + default="demo-archaeology", + help="Temporary/generated sanitized demo project name", +) +@click.option( + "--force", is_flag=True, default=True, help="Overwrite existing generated demo project" +) +def public_case_study(output_dir, project_name, force): + """Generate a sanitized public case-study showroom.""" + from .report import export_public_case_study + + path = export_public_case_study( + Path.cwd(), output_dir=output_dir, project_name=project_name, force=force + ) + click.echo(f"Public case study exported to {path}") + + +@main.command("local-pipeline") +@click.option( + "--repo", + "repo_name", + default="dev-archaeology", + help="Repository name or owner/name to inspect", +) +@click.option( + "--pipeline-dir", + default=None, + help="Path to the local GITHUB_pipeline workspace (defaults to ARCHAEOLOGY_PIPELINE_ROOT)", +) +@click.option( + "--repos-dir", + default=None, + help="Directory containing local repositories (defaults to ARCHAEOLOGY_REPOS_DIR)", +) +@click.option( + "--top-repos", default=20, type=int, help="Number of active repos to review when --run is used" +) +@click.option( + "--review-days", default=30, type=int, help="Commit lookback window when --run is used" +) +@click.option( + "--run", "run_first", is_flag=True, help="Run the local pipeline before reading latest.json" +) +@click.option( + "--fail-on-issues", + is_flag=True, + help="Exit nonzero if the repo has any local-pipeline findings", +) +def local_pipeline( + repo_name, pipeline_dir, repos_dir, top_repos, review_days, run_first, fail_on_issues +): + """Read or run the local GITHUB_pipeline verification status.""" + from .local_pipeline import read_local_pipeline_status, run_local_pipeline, status_lines + + # Resolve pipeline_dir if not provided + if pipeline_dir is None: + pipeline_dir = os.environ.get("ARCHAEOLOGY_PIPELINE_ROOT", "") + if not pipeline_dir: + raise click.UsageError( + "ARCHAEOLOGY_PIPELINE_ROOT environment variable not set. " + "Please set it to your local GITHUB_pipeline workspace path, " + "or use --pipeline-dir option." + ) + + if run_first: + # repos_dir only needed for running the pipeline + if repos_dir is None: + repos_dir = os.environ.get("ARCHAEOLOGY_REPOS_DIR", "") + if not repos_dir: + raise click.UsageError( + "ARCHAEOLOGY_REPOS_DIR environment variable not set. " + "Please set it to the directory containing your local repositories, " + "or use --repos-dir option." + ) + run_local_pipeline( + pipeline_dir=pipeline_dir, + repos_dir=repos_dir, + top_repos=top_repos, + review_days=review_days, + ) + status = read_local_pipeline_status(pipeline_dir, repo_name) + for line in status_lines(status): + click.echo(line) + if fail_on_issues and status.issue_total > 0: + raise click.exceptions.Exit(1) + + +@main.command("export-report") +@click.argument("project_name") +@click.option( + "--format", + "fmt", + type=click.Choice(["markdown", "md", "html"]), + default="markdown", + help="Report format to export", +) +@click.option( + "--output", + "output_path", + help="Output path. Defaults to project deliverables/ARCHAEOLOGY-REPORT.", +) +def export_report_cmd(project_name, fmt, output_path): + """Export an archaeology report from analysis outputs.""" + from .report import export_report + + project_dir = _project_dir(project_name) + try: + path = export_report(project_name, project_dir, output_path=output_path, fmt=fmt) + except (OSError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + click.echo(f"Report exported to {path}") + + +@main.command() +@click.argument("project_name") +def visualize(project_name): + """Phase 4: Generate visualization HTML from template.""" + from .visualization.history import render_history + + try: + output = render_history(Path(_project_dir(project_name))) + except (OSError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + click.echo(f"Visualization generated at {output}") + + +@main.command() +@click.argument("project_name") +@click.option("--logs-dir", help="Path to pipeline logs directory (default: auto-detect)") +@click.option("--verbose", "-v", is_flag=True) +def ingest_pipeline(project_name, logs_dir, verbose): + """Ingest GITHUB_pipeline run logs into archaeology database. + + Imports pipeline JSON logs from .omc/logs/repo-pipeline/ into + pipeline_runs and pipeline_repo_results tables for historical tracking. + """ + from .db.pipeline_ingest import ingest_directory + + project_dir = _project_dir(project_name) + db_path = os.path.join(project_dir, "data", "archaeology.db") + + if not os.path.exists(db_path): + click.echo(f"Database not found. Run 'devarch build-db {project_name}' first.", err=True) + sys.exit(1) + + # Auto-detect pipeline logs dir + if not logs_dir: + candidates = [ + os.path.expanduser("~/workspaces/GITHUB_pipeline/.omc/logs/repo-pipeline"), + os.path.expanduser("~/dev/GITHUB_pipeline/.omc/logs/repo-pipeline"), + os.path.expanduser("~/Desktop/GITHUB_pipeline/.omc/logs/repo-pipeline"), + os.path.expanduser("~/Desktop/OMC/.omc/logs/repo-pipeline"), + ] + for c in candidates: + if os.path.isdir(c): + logs_dir = c + break + + if not logs_dir or not os.path.isdir(logs_dir): + click.echo("Pipeline logs directory not found. Use --logs-dir to specify path.", err=True) + sys.exit(1) + + click.echo(f"Ingesting pipeline logs from {logs_dir}...") + stats = ingest_directory(Path(db_path), Path(logs_dir), verbose=verbose) + click.echo( + f" Ingested: {stats['ingested']}, Skipped: {stats['skipped']}, Errors: {len(stats['errors'])}" + ) + for err in stats["errors"]: + click.echo(f" ERROR: {err}", err=True) + + +@main.command() +@click.argument("project_name") +@click.option("--dry-run", is_flag=True, help="Show what would change without writing") +@click.option("--skip-mine", is_flag=True, help="Skip git mining (use existing data)") +def cascade(project_name, dry_run, skip_mine): + """Disabled pending evidence and data-exposure hardening.""" + raise click.ClickException( + "cascade is disabled pending readiness hardening. Use init, mine, build-db, signals, analyze, visualize, export-report and audit." + ) + + +@main.command() +@click.argument("project_name") +@click.option( + "--fail-on", + type=click.Choice(["CRITICAL", "HIGH", "MEDIUM", "LOW"]), + default="HIGH", + help="Lowest severity that causes nonzero exit", +) +def audit(project_name, fail_on): + """Run forensic audit quality gate.""" + from .audit import has_blocking_findings, run_audit, summarize + + findings = run_audit(project_name, root=Path.cwd()) + summary = summarize(findings) + click.echo(f"Forensic audit for '{project_name}'") + click.echo("Summary: " + ", ".join(f"{k}={v}" for k, v in summary.items() if v)) + if not findings: + click.echo("PASS: no findings") + return + for finding in findings: + click.echo(finding.format()) + if has_blocking_findings(findings, fail_on=fail_on): + raise click.exceptions.Exit(1) + + +@main.command() +@click.argument("project_name") +def validate(project_name): + """Validate the generated history HTML and its evidence binding.""" + from .validators.history import validate_history + + try: + validate_history(Path(_project_dir(project_name))) + except (OSError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + click.echo("PASS: generated history HTML and evidence binding") + + +def _aggregate_global(targets, profile, verbose=False): + """Merge per-project data into global/ for cross-project narrative.""" + import csv + import sqlite3 + + global_dir = os.path.join("global", "data") + os.makedirs(global_dir, exist_ok=True) + + all_commits = [] + all_eras = [] + project_summaries = [] + + for proj in targets: + proj_name = proj["name"] + proj_dir = os.path.join("projects", proj_name) + db_path = os.path.join(proj_dir, "data", "archaeology.db") + + # Collect commits from CSV (faster than DB) + csv_path = os.path.join(proj_dir, "data", "github-commits.csv") + if os.path.exists(csv_path): + with open(csv_path, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + # Normalize: drop None keys from malformed rows + clean = {k: v for k, v in row.items() if k is not None} + clean["_project"] = proj_name + all_commits.append(clean) + + # Collect signals from detected-signals.json + signals_path = os.path.join(proj_dir, "data", "detected-signals.json") + if os.path.exists(signals_path): + try: + with open(signals_path, encoding="utf-8") as f: + proj_signals = json.load(f) + proj_signals["_project"] = proj_name + all_eras.append(proj_signals) + except json.JSONDecodeError as e: + click.echo(f"Warning: Invalid JSON in {signals_path}: {e}", err=True) + continue + + # Build per-project summary from DB + if os.path.exists(db_path): + conn = sqlite3.connect(db_path, timeout=30) + conn.row_factory = sqlite3.Row + try: + row = conn.execute( + "SELECT COUNT(*) as cnt, MIN(date) as first, MAX(date) as last FROM commits" + ).fetchone() + project_summaries.append( + { + "name": proj_name, + "total_commits": row["cnt"], + "first_commit": row["first"], + "last_commit": row["last"], + } + ) + except sqlite3.OperationalError: + project_summaries.append({"name": proj_name, "total_commits": 0}) + finally: + conn.close() + + # Write global commits CSV + if all_commits: + # Collect union of all field names for consistent schema + all_fields = set() + for row in all_commits: + all_fields.update(row.keys()) + fields = sorted(all_fields) + commit_path = os.path.join(global_dir, "global-commits.csv") + with open(commit_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + writer.writerows(all_commits) + click.echo( + f" {len(all_commits)} commits across {len(targets)} projects → global-commits.csv" + ) + + # Write global signals JSON + if all_eras: + signals_path = os.path.join(global_dir, "global-signals.json") + with open(signals_path, "w", encoding="utf-8") as f: + json.dump(all_eras, f, indent=2) + click.echo( + f" {len(all_eras)} signal reports across {len(targets)} projects → global-signals.json" + ) + + # Write project summaries + summaries_path = os.path.join(global_dir, "project-summaries.json") + with open(summaries_path, "w", encoding="utf-8") as f: + json.dump(project_summaries, f, indent=2) + click.echo(f" {len(project_summaries)} project summaries → project-summaries.json") + + # Build global DB + global_db = os.path.join(global_dir, "global.db") + Path(global_db).unlink(missing_ok=True) + + if all_commits: + # Use sqlite-utils CLI if available, otherwise sqlite3 + tmp_csv = os.path.join(global_dir, "global-commits.csv") + try: + subprocess.run( + ["sqlite-utils", "insert", global_db, "commits", tmp_csv, "--csv"], + capture_output=True, + check=True, + timeout=300, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + conn = sqlite3.connect(global_db, timeout=30) + try: + with open(tmp_csv, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + cols = reader.fieldnames + conn.execute(f"CREATE TABLE commits ({', '.join(c + ' TEXT' for c in cols)})") + for row in reader: + placeholders = ", ".join("?" for _ in cols) + conn.execute( + f"INSERT INTO commits VALUES ({placeholders})", + [row[c] for c in cols], + ) + conn.commit() + finally: + conn.close() + + click.echo(" Global DB → global.db") + + +@main.command() +@click.option( + "--project", + "-p", + "projects", + multiple=True, + help="Sync specific project(s) only. Defaults to all in profile.json.", +) +@click.option("--skip-mine", is_flag=True, help="Skip git extraction (use cached data)") +@click.option("--skip-signals", is_flag=True, help="Skip signal detection") +@click.option("--verbose", "-v", is_flag=True) +def sync(projects, skip_mine, skip_signals, verbose): + """Sync registered projects and aggregate into global narrative.""" + profile_path = os.path.join(os.path.dirname(__file__), "..", "config", "profile.json") + if not os.path.exists(profile_path): + profile_path = "config/profile.json" + if not os.path.exists(profile_path): + click.echo( + "No profile.json found. Create one at config/profile.json with your project list.", + err=True, + ) + sys.exit(1) + + try: + with open(profile_path, encoding="utf-8") as f: + profile = json.load(f) + except json.JSONDecodeError as e: + click.echo(f"Error: Invalid JSON in {profile_path}: {e}", err=True) + sys.exit(1) + + if not isinstance(profile, dict): + click.echo( + f"Error: profile.json must contain a JSON object, got {type(profile).__name__}.", + err=True, + ) + sys.exit(1) + + registered = profile.get("projects", []) + sync_cfg = profile.get("sync", {}) + + if not isinstance(registered, list): + click.echo( + f"Error: 'projects' in profile.json must be a list, got {type(registered).__name__}.", + err=True, + ) + sys.exit(1) + + if not registered: + click.echo("No projects registered in profile.json.") + sys.exit(0) + + # Validate entries and accept the README-documented "path" alias for the + # canonical "repo_path" key before any code reads a project entry, so a + # malformed profile fails with an actionable message instead of a KeyError + # or TypeError. Names and effective locations must be nonempty strings, + # checked before project filtering and before any filesystem writes. + problems = [] + for i, proj in enumerate(registered): + if not isinstance(proj, dict): + problems.append(f"projects[{i}] must be an object, got {type(proj).__name__}") + continue + name = proj.get("name") + label = name if _is_nonempty_string(name) else f"projects[{i}]" + if "name" not in proj: + problems.append(f"{label}: missing required key 'name'") + elif not _is_nonempty_string(name): + problems.append( + f"{label}: 'name' must be a nonempty string, got {_invalid_value_desc(name)}" + ) + if "repo_path" in proj: + location_key, location = "repo_path", proj["repo_path"] + elif "path" in proj: + location_key, location = "path", proj["path"] + else: + location_key, location = None, None + if location_key is None: + problems.append(f"{label}: missing required key 'repo_path' (or 'path')") + elif not _is_nonempty_string(location): + problems.append( + f"{label}: '{location_key}' must be a nonempty string, " + f"got {_invalid_value_desc(location)}" + ) + if problems: + click.echo("Invalid project entries in profile.json:", err=True) + for problem in problems: + click.echo(f" - {problem}", err=True) + click.echo( + "Each project needs 'name' and 'repo_path' ('path' is accepted as an alias).", + err=True, + ) + sys.exit(1) + registered = [ + {**proj, "repo_path": proj["repo_path"] if "repo_path" in proj else proj["path"]} + for proj in registered + ] + + # Filter to requested projects + if projects: + names = set(projects) + targets = [p for p in registered if p["name"] in names] + unknown = names - {p["name"] for p in registered} + for u in unknown: + click.echo(f"Unknown project: {u} (not in profile.json)", err=True) + else: + targets = registered + + if not targets: + click.echo("No matching projects to sync.") + sys.exit(1) + + click.echo(f"Syncing {len(targets)} project(s)...\n") + + # Ensure all project dirs exist + for proj in targets: + proj_dir = os.path.join("projects", proj["name"]) + if not os.path.isdir(proj_dir): + click.echo(f" Initializing {proj['name']}...") + os.makedirs(os.path.join(proj_dir, "data"), exist_ok=True) + os.makedirs(os.path.join(proj_dir, "deliverables"), exist_ok=True) + config = { + "name": proj["name"], + "repo_path": os.path.expanduser(proj["repo_path"]), + "repo_url": proj.get("repo_url", ""), + "developer": profile.get("developer", {}), + } + with open(os.path.join(proj_dir, "project.json"), "w", encoding="utf-8") as f: + json.dump(config, f, indent=2) + + # Phase 1: Mine each project + if not skip_mine: + for proj in targets: + repo_path = os.path.expanduser(proj["repo_path"]) + if not os.path.isdir(repo_path): + click.echo(f" SKIP {proj['name']}: repo not found at {repo_path}", err=True) + continue + if not os.path.isdir(os.path.join(repo_path, ".git")): + click.echo(f" SKIP {proj['name']}: not a git repo", err=True) + continue + + click.echo(f" Mining {proj['name']}...") + + data_dir = os.path.join("projects", proj["name"], "data") + csv_path = os.path.join(data_dir, "github-commits.csv") + try: + count = extract_git_log(repo_path, csv_path, verbose=verbose) + click.echo(f" {count} commits extracted") + except (RuntimeError, Exception) as e: + click.echo(f" Error: Git extraction failed: {e}", err=True) + continue + + stats_path = os.path.join(data_dir, "github-commits-with-stats.txt") + try: + extract_git_log_with_stats(repo_path, stats_path, verbose=verbose) + except (RuntimeError, Exception) as e: + click.echo(f" Error: Git stats extraction failed: {e}", err=True) + continue + + # Phase 1.5: Build DBs + for proj in targets: + proj_name = proj["name"] + db_path = os.path.join("projects", proj_name, "data", "archaeology.db") + click.echo(f" Building DB for {proj_name}...") + + cmd = [ + sys.executable, + "-m", + "archaeology.db.builder", + "--project-root", + os.path.join("projects", proj_name), + ] + if verbose: + cmd.append("--verbose") + + _env = os.environ.copy() + _pkg_root = str(Path(__file__).parent.parent) + _env["PYTHONPATH"] = _pkg_root + ( + (os.pathsep + _env["PYTHONPATH"]) if _env.get("PYTHONPATH") else "" + ) + result = subprocess.run(cmd, capture_output=not verbose, check=True, timeout=300, env=_env) + if result.returncode == 0 and os.path.exists(db_path): + click.echo(" DB built") + else: + click.echo(f" DB build failed (exit {result.returncode})", err=True) + + # Phase 2: Detect signals + should_detect_signals = sync_cfg.get("run_signals", True) and not skip_signals + if should_detect_signals: + min_gap = sync_cfg.get("default_min_gap_days", 3) + for proj in targets: + click.echo(f" Detecting signals for {proj['name']}...") + result = detect_signals(proj["name"], config={"min_gap_days": min_gap}) + sig_count = len(result.get("signals", [])) + cluster_count = len(result.get("cluster_summary", [])) + click.echo(f" {sig_count} signals, {cluster_count} clusters") + + # Phase 3: Run automated analysis vectors + run_auto_analysis = sync_cfg.get("run_analysis", True) + if run_auto_analysis: + click.echo("\nRunning automated analysis vectors...") + for proj in targets: + click.echo(f" Analyzing {proj['name']}...") + try: + results = run_analysis_vectors(proj["name"], verbose=verbose) + for vector, path in results.items(): + if not path.startswith("ERROR"): + click.echo(f" ✓ {vector}") + else: + click.echo(f" ✗ {vector}: {path}") + except Exception as e: + click.echo(f" Analysis failed: {e}", err=True) + + # Phase 4: Aggregate into global/ + click.echo("\nAggregating global data...") + _aggregate_global(targets, profile, verbose) + click.echo("Sync complete.") + + +@main.command("global-viz") +@click.option( + "--output", + "output_dir", + default="global/deliverables", + help="Output directory for the global visualization", +) +@click.option("--top", "top_n", type=int, help="Limit to top N repos by commit count") +@click.option("--year", type=int, help="Only include repos updated in this year") +@click.option("--verbose", "-v", is_flag=True) +def global_viz(output_dir, top_n, year, verbose): + """Generate cross-repo visualization from synced global data.""" + from .visualization.global_data_builder import prepare_global_visualization_data + + global_dir = "global" + data_dir = os.path.join(global_dir, "data") + github_json = os.path.join(data_dir, "github-repos.json") + commits_csv = os.path.join(data_dir, "global-commits.csv") + + if not os.path.exists(commits_csv) and not os.path.exists(github_json): + click.echo( + "No global data found. Run 'devarch fetch-github' or 'devarch sync' first.", + err=True, + ) + sys.exit(1) + + # Build visualization data + click.echo("Building global visualization data...") + viz_data = prepare_global_visualization_data(global_dir, top_n=top_n, year=year) + + # Write viz data JSON + viz_json_path = os.path.join(data_dir, "global-viz-data.json") + with open(viz_json_path, "w") as f: + json.dump(viz_data, f, indent=2) + click.echo(f" Data written to {viz_json_path}") + + # Hydrate template + template_path = _viz_template_path("global-template.html") + if not os.path.exists(template_path): + click.echo(f"Template not found at {template_path}", err=True) + sys.exit(1) + + with open(template_path, encoding="utf-8") as f: + html = f.read() + + # Inline the data JSON + safe_data = ( + json.dumps(viz_data).replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + ) + + # Replace the placeholder lines inside the script block + # Template has: "// GLOBAL_DATA_PLACEHOLDER\nwindow.GLOBAL_DATA = {};" + old_placeholder = "// GLOBAL_DATA_PLACEHOLDER\nwindow.GLOBAL_DATA = {};" + new_inline = f"window.GLOBAL_DATA = {safe_data};\n window.dispatchEvent(new Event('global-data-loaded'));" + if old_placeholder in html: + html = html.replace(old_placeholder, new_inline) + elif "window.GLOBAL_DATA = {};" in html: + html = html.replace("window.GLOBAL_DATA = {};", f"window.GLOBAL_DATA = {safe_data};") + else: + click.echo("Warning: could not find GLOBAL_DATA placeholder in template", err=True) + + # Write output + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, "global.html") + with open(output_path, "w", encoding="utf-8") as f: + f.write(html) + + click.echo(f"Global visualization generated at {output_path}") + meta = viz_data.get("meta", {}) + click.echo( + f" {meta.get('total_commits', '?')} commits across {meta.get('total_repos', '?')} repos" + ) + + +@main.command("multi-project-dashboard") +@click.option( + "--output", + "output_dir", + default="global/deliverables", + help="Output directory for the dashboard", +) +@click.option("--top", "top_n", type=int, help="Limit to top N repos by commit count") +@click.option("--year", type=int, help="Only include repos updated in this year") +@click.option("--verbose", "-v", is_flag=True) +def multi_project_dashboard(output_dir, top_n, year, verbose): + """Generate comprehensive multi-project dashboard visualization.""" + from .visualization.global_data_builder import prepare_dashboard_data + + global_dir = "global" + data_dir = os.path.join(global_dir, "data") + github_json = os.path.join(data_dir, "github-repos.json") + + if not os.path.exists(github_json): + click.echo("No GitHub data found. Run 'devarch fetch-github' first.", err=True) + sys.exit(1) + + # Build dashboard data + click.echo("Building dashboard data...") + dashboard_data = prepare_dashboard_data(global_dir, top_n=top_n, year=year) + + # Write dashboard data JSON + dashboard_json_path = os.path.join(data_dir, "dashboard-data.json") + with open(dashboard_json_path, "w") as f: + json.dump(dashboard_data, f, indent=2) + if verbose: + click.echo(f" Data written to {dashboard_json_path}") + + # Hydrate template + template_path = _viz_template_path("multi-project-dashboard.html") + if not os.path.exists(template_path): + click.echo(f"Template not found at {template_path}", err=True) + sys.exit(1) + + with open(template_path, encoding="utf-8") as f: + html = f.read() + + # Inline the data JSON + safe_data = ( + json.dumps(dashboard_data) + .replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + ) + + # Replace the placeholder + old_placeholder = "// DATA_PLACEHOLDER\nwindow.DASHBOARD_DATA = {};" + new_inline = f"window.DASHBOARD_DATA = {safe_data};\n window.dispatchEvent(new Event('dashboard-data-loaded'));" + if old_placeholder in html: + html = html.replace(old_placeholder, new_inline) + elif "window.DASHBOARD_DATA = {};" in html: + html = html.replace("window.DASHBOARD_DATA = {};", f"window.DASHBOARD_DATA = {safe_data};") + else: + click.echo("Warning: could not find DASHBOARD_DATA placeholder in template", err=True) + + # Write output + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, "dashboard.html") + with open(output_path, "w", encoding="utf-8") as f: + f.write(html) + + click.echo(f"Multi-project dashboard generated at {output_path}") + meta = dashboard_data.get("meta", {}) + click.echo( + f" {meta.get('total_commits', '?')} commits across {meta.get('total_repos', '?')} repos" + ) + click.echo(f" Period: {meta.get('first_date', '?')} to {meta.get('last_date', '?')}") + + +@main.command("fetch-github") +@click.option( + "--owner", default=os.environ.get("ARCHAEOLOGY_GITHUB_OWNER", ""), help="GitHub username/org" +) +@click.option( + "--output", "output_path", default="global/data/github-repos.json", help="Output JSON path" +) +def fetch_github(owner, output_path): + """Fetch repo metadata from GitHub API for all repos (no cloning).""" + from .visualization.github_fetcher import save_github_data + + click.echo(f"Fetching repos for {owner} from GitHub...") + data = save_github_data(output_path, owner=owner) + click.echo(f" {data['total_repos']} repos, {data['total_commits']} total commits") + + +@main.command() +@click.argument("project_name") +def benchmark(project_name): + """Generate agent performance benchmark visualization.""" + from .visualization.agent_benchmark import run_benchmark_analysis + + project_dir = _project_dir(project_name) + + click.echo(f"Analyzing agent performance for '{project_name}'...") + + try: + output_path = run_benchmark_analysis(project_dir) + click.echo(f"Benchmark visualization generated at {output_path}") + except FileNotFoundError as e: + click.echo(str(e), err=True) + sys.exit(1) + except Exception as e: + click.echo(f"Error generating benchmark: {e}", err=True) + sys.exit(1) + + +@main.command("dashboard") +@click.option("--port", default=8080, help="Port to serve on") +@click.option("--no-open", is_flag=True, help="Don't open browser automatically") +def dashboard(port, no_open): + """Disabled pending evidence and data-exposure hardening.""" + raise click.ClickException( + "dashboard is disabled pending readiness hardening. Use init, mine, build-db, signals, analyze, visualize, export-report and audit." + ) + + +@main.command("publish-static") +@click.option("--output", "output_dir", default="site", help="Output directory for the static site") +def publish_static(output_dir): + """Disabled pending evidence and data-exposure hardening.""" + raise click.ClickException( + "publish-static is disabled pending readiness hardening. Use init, mine, build-db, signals, analyze, visualize, export-report and audit." + ) + + +if __name__ == "__main__": + main() diff --git a/plugins/devarch-hermes/archaeology/db/__init__.py b/plugins/devarch-hermes/archaeology/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/devarch-hermes/archaeology/db/builder.py b/plugins/devarch-hermes/archaeology/db/builder.py new file mode 100644 index 0000000..2f60735 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/db/builder.py @@ -0,0 +1,752 @@ +#!/usr/bin/env python3 +"""Convert archaeology data files into a SQLite database for Datasette exploration. + +Project-agnostic: point --project or --project-root at any project +directory with a data/ folder and this script auto-detects CSV and JSON files, +flattens nested structures into separate tables, creates indexes, and enables +FTS5 for full-text search. + +Table registry is loaded from: + 1. /project.json -> "db_tables" key + 2. config/defaults.json -> "db_tables" key (relative to script location) + 3. DEFAULT_TABLE_REGISTRY constant (hardcoded fallback) + +Requires: sqlite-utils CLI on PATH. +""" + +import argparse +import csv +import json +import re +import sqlite3 +import subprocess +import sys +import tempfile +from pathlib import Path + +from ..utils import _script_dir + +# --------------------------------------------------------------------------- +# Default table registry (fallback when no project.json or defaults.json) +# --------------------------------------------------------------------------- + +DEFAULT_TABLE_REGISTRY: dict[str, dict] = { + "commits": {"file": "github-commits.csv", "format": "csv"}, + "sessions": {"file": "human-messages.json", "format": "json_array"}, + "youtube_searches": {"file": "youtube-search-history.json", "format": "json_array"}, + "eras": {"file": "commit-eras.json", "format": "json_special"}, + "derived_patterns": {"file": "derived-patterns.json", "format": "json_special"}, + "cross_repo_analysis": {"file": "cross-repo-analysis.json", "format": "json_nested"}, + "model_adoption": {"file": "model-adoption-analysis.json", "format": "json_nested"}, + "lunar_phases": {"file": "lunar-phases.json", "format": "json_nested"}, + "youtube_correlation": {"file": "youtube-ai-correlation.json", "format": "json_nested"}, + "youtube_creators": {"file": "youtube-creators.json", "format": "json_special_creators"}, + "telemetry_git": {"file": "metrics-git.json", "format": "json_nested"}, + "telemetry_agents": {"file": "metrics-agents.json", "format": "json_nested"}, + "telemetry_codebase": {"file": "metrics-codebase.json", "format": "json_nested"}, + "telemetry_cross_repo": {"file": "metrics-cross-repo.json", "format": "json_nested"}, + "telemetry_github_full": {"file": "metrics-github-full.json", "format": "json_nested"}, + "telemetry_repo_depth": {"file": "metrics-repo-depth.json", "format": "json_nested"}, + "telemetry_visualizations": {"file": "metrics-visualizations.json", "format": "json_nested"}, + "youtube_topic_classification": { + "file": "youtube-topic-classification.json", + "format": "json_nested", + }, + "youtube_engagement": {"file": "youtube-engagement-heuristics.json", "format": "json"}, + "youtube_transcript_analysis": {"file": "youtube-transcript-analysis.json", "format": "json"}, + "context_management": {"file": "context-management-analysis.json", "format": "json"}, +} + +# Nested key-to-table mappings for json_nested format files. +# Keys within each JSON file that should be extracted into separate tables. +NESTED_KEY_MAPPINGS: dict[str, dict[str, str]] = { + "cross-repo-analysis.json": { + "monthly_velocity": "monthly_velocity", + "top_repos": "repos", + "hourly_pattern": "hourly_activity", + "day_of_week": "weekly_activity", + "language_evolution": "languages", + }, + "model-adoption-analysis.json": { + "model_releases": "model_releases", + "first_mentions": "model_mentions", + "adoption_lag": "adoption_lag", + "timeline": "model_timeline", + }, + "lunar-phases.json": {"daily_phases": "lunar_phases", "key_events": "lunar_events"}, + "youtube-ai-correlation.json": { + "monthly_summary": "yt_monthly", + "key_correlations": "yt_correlations", + "creator_influence_map": "yt_creator_influence", + }, + "youtube-topic-classification.json": { + "classified_videos": "yt_classified", + "categories": "yt_categories", + }, + "metrics-git.json": { + "commits_by_hour": "commits_by_hour", + "commits_by_day_of_week": "commits_by_weekday", + "author_breakdown": "authors", + "co_authored_by": "co_authors", + }, + "metrics-agents.json": { + "agent_comparison": "agent_comparison", + "co_authorship_patterns": "co_authorship_patterns", + }, + "metrics-codebase.json": { + "file_growth_timeline": "file_growth", + "language_evolution": "codebase_languages", + "module_emergence_timeline": "module_emergence", + }, + "metrics-cross-repo.json": { + "timeline": "cross_repo_timeline", + "concurrent_repos": "concurrent_repos", + }, + "metrics-github-full.json": { + "repos": "github_repos", + "activity_heatmap": "github_heatmap", + }, + "metrics-repo-depth.json": { + "repos": "repo_depth", + "domain_map": "domain_map", + "feeder_repos": "feeder_repos", + }, +} + +# Default indexes to create (table -> list of columns). +DEFAULT_INDEXES: list[tuple[str, list[str]]] = [ + ("commits", ["date", "author", "repo"]), + ("eras", ["name", "frustration_category", "dominant_intent"]), + ("sessions", ["timestamp", "session_id"]), + ("yt_classified", ["category"]), + ("lunar_phases", ["date"]), +] + +# Default FTS configurations (table -> columns to index). +DEFAULT_FTS: list[tuple[str, list[str]]] = [ + ("commits", ["message"]), + ("sessions", ["messages"]), + ("eras", ["description", "narrative_arc"]), +] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def log(msg: str, verbose: bool = False) -> None: + if verbose: + print(f" {msg}") + + +class SqliteUtilsError(RuntimeError): + """Raised when the sqlite-utils CLI fails. + + Failing loud here is deliberate: a swallowed failure (e.g. sqlite-utils not + installed) leaves tables empty and makes the whole pipeline emit a + confidently wrong all-zero report (every SDLC practice reported 'ABSENT'). + """ + + +def run_su(args: list[str], verbose: bool = False) -> subprocess.CompletedProcess: + """Call sqlite-utils CLI and return the result. + + Raises SqliteUtilsError on any non-zero exit so build-db never silently + produces an empty/garbage database. + """ + cmd = [sys.executable, "-m", "sqlite_utils"] + args + if verbose: + print(f" $ {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + stderr = result.stderr.strip() + if "No module named sqlite_utils" in stderr: + raise SqliteUtilsError( + f"sqlite-utils is not installed for this interpreter ({sys.executable}). " + "Install it with `pip install -e .` (it is a declared dependency) or " + "`pip install sqlite-utils`. Without it build-db cannot populate any " + "table, so every analysis vector silently returns zero." + ) + raise SqliteUtilsError(f"sqlite-utils {args[0] if args else ''} failed: {stderr}") + return result + + +def load_json(path: Path, verbose: bool = False) -> dict | list | None: + if not path.exists(): + log(f"SKIP {path.name} (not found)", verbose) + return None + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + log(f"LOADED {path.name}", verbose) + return data + except (json.JSONDecodeError, OSError) as exc: + print(f" WARNING: Failed to read {path}: {exc}", file=sys.stderr) + return None + + +def import_csv(db: Path, table: str, csv_path: Path, verbose: bool = False) -> int: + if not csv_path.exists(): + log(f"SKIP {csv_path.name} (not found)", verbose) + return 0 + # Count rows during initial read to avoid re-reading the file + with open(csv_path, encoding="utf-8") as f: + count = sum(1 for _ in csv.reader(f)) - 1 + run_su(["insert", str(db), table, str(csv_path), "--csv", "--alter"], verbose) + log(f"IMPORTED {csv_path.name} -> {table} ({count} rows)", verbose) + return max(count, 0) + + +def write_temp(records: list[dict]) -> Path: + tmp = tempfile.NamedTemporaryFile(suffix=".json", delete=False, mode="w", encoding="utf-8") + json.dump(records, tmp, default=str) + tmp.close() + return Path(tmp.name) + + +def import_list(db: Path, table: str, records: list[dict], verbose: bool = False) -> int: + if not records: + log(f"SKIP {table} (empty)", verbose) + return 0 + tmp = write_temp(records) + try: + run_su(["insert", str(db), table, str(tmp), "--alter"], verbose) + log(f"IMPORTED {table} ({len(records)} rows)", verbose) + finally: + tmp.unlink(missing_ok=True) + return len(records) + + +def extract_nested(data: dict, key: str) -> list[dict] | None: + value = data.get(key) + if value is None: + return None + if isinstance(value, list): + return value + if isinstance(value, dict): + return [ + {"_key": k, **(v if isinstance(v, dict) else {"value": v})} for k, v in value.items() + ] + return None + + +def flatten_dict(data: dict, prefix: str = "") -> dict: + flat: dict = {} + for k, v in data.items(): + key = f"{prefix}_{k}" if prefix else k + flat[key] = json.dumps(v, default=str) if isinstance(v, (dict, list)) else v + return flat + + +def _flat(records: list) -> list[dict]: + return [flatten_dict(r) if isinstance(r, dict) else {"value": r} for r in records] + + +def _import_mapping(db: Path, data: dict, mapping: dict[str, str], verbose: bool = False) -> int: + total = 0 + for key, table in mapping.items(): + records = extract_nested(data, key) + if records: + total += import_list(db, table, _flat(records), verbose) + return total + + +# --------------------------------------------------------------------------- +# Config loading +# --------------------------------------------------------------------------- + + +def load_table_registry(project_root: Path, verbose: bool = False) -> dict[str, dict]: + """Load table registry from project.json, then defaults.json, then fallback. + + Priority: + 1. /project.json -> "db_tables" + 2. /../../config/defaults.json -> "db_tables" + 3. DEFAULT_TABLE_REGISTRY constant + """ + # Try project.json first + project_json = project_root / "project.json" + if project_json.exists(): + try: + with open(project_json, encoding="utf-8") as f: + data = json.load(f) + tables = data.get("db_tables") + if tables: + log(f"Loaded table registry from project.json ({len(tables)} tables)", verbose) + return tables + except (json.JSONDecodeError, OSError) as exc: + print(f" WARNING: Failed to read {project_json}: {exc}", file=sys.stderr) + + # Try defaults.json + defaults_json = _script_dir().parent.parent / "config" / "defaults.json" + if defaults_json.exists(): + try: + with open(defaults_json, encoding="utf-8") as f: + data = json.load(f) + tables = data.get("db_tables") + if tables: + log(f"Loaded table registry from defaults.json ({len(tables)} tables)", verbose) + return tables + except (json.JSONDecodeError, OSError) as exc: + print(f" WARNING: Failed to read {defaults_json}: {exc}", file=sys.stderr) + + # Fallback to hardcoded defaults + log(f"Using DEFAULT_TABLE_REGISTRY ({len(DEFAULT_TABLE_REGISTRY)} tables)", verbose) + return DEFAULT_TABLE_REGISTRY.copy() + + +def load_nested_key_mappings( + project_root: Path, verbose: bool = False +) -> dict[str, dict[str, str]]: + """Load nested key-to-table mappings from project config. + + Priority: + 1. /project.json -> "nested_key_mappings" + 2. NESTED_KEY_MAPPINGS constant + """ + project_json = project_root / "project.json" + if project_json.exists(): + try: + with open(project_json, encoding="utf-8") as f: + data = json.load(f) + mappings = data.get("nested_key_mappings") + if mappings: + log(f"Loaded nested mappings from project.json ({len(mappings)} files)", verbose) + return mappings + except (json.JSONDecodeError, OSError): + pass + return NESTED_KEY_MAPPINGS.copy() + + +def resolve_project_root(args: argparse.Namespace) -> Path: + """Resolve the project root directory from CLI arguments. + + Accepts either --project (resolves to projects//) or + --project-root (direct path). + """ + repo_root = _script_dir().parent.parent + + if args.project: + project_path = repo_root / "projects" / args.project + if not project_path.exists(): + print(f"ERROR: Project directory not found: {project_path}", file=sys.stderr) + sys.exit(1) + return project_path.resolve() + + return Path(args.project_root).resolve() + + +# --------------------------------------------------------------------------- +# Specialized importers +# --------------------------------------------------------------------------- + + +def import_commit_eras( + db: Path, data_dir: Path, era_file: str = "commit-eras.json", verbose: bool = False +) -> int: + data = load_json(data_dir / era_file, verbose) + if data is None: + return 0 + total = import_list(db, "eras", _flat(data.get("eras", [])), verbose) + meta_keys = [k for k in data if k != "eras"] + if meta_keys: + meta = [{"key": k, "value": json.dumps(data[k], default=str)} for k in meta_keys] + total += import_list(db, "project_meta", meta, verbose) + return total + + +def import_derived_patterns( + db: Path, data_dir: Path, patterns_file: str = "derived-patterns.json", verbose: bool = False +) -> int: + data = load_json(data_dir / patterns_file, verbose) + if data is None: + return 0 + total = 0 + named = { + "frustration_to_automation_latency": "frustration_patterns", + "co_authorship_gap_analysis": "co_authorship_gaps", + } + for key, table in named.items(): + section = data.get(key) + if section is None: + continue + if isinstance(section, dict): + for sub_val in section.values(): + if isinstance(sub_val, list) and sub_val: + total += import_list(db, table, _flat(sub_val), verbose) + break + elif isinstance(section, list): + total += import_list(db, table, section, verbose) + for key, val in data.items(): + if key not in named and isinstance(val, list) and val: + total += import_list(db, key.replace("-", "_")[:60], val, verbose) + return total + + +def import_telemetry_sessions( + db: Path, data_dir: Path, sessions_file: str = "metrics-sessions.json", verbose: bool = False +) -> int: + data = load_json(data_dir / sessions_file, verbose) + if data is None: + return 0 + total = import_list(db, "sessions_per_era", data.get("sessions_per_era", []), verbose) + for key in ("frustration_analysis", "intent_analysis"): + section = data.get(key) + if isinstance(section, dict): + rows = [flatten_dict({k: v}) for k, v in section.items()] + total += import_list(db, key, rows, verbose) + elif isinstance(section, list): + total += import_list(db, key, section, verbose) + return total + + +def import_audit_files(db: Path, data_dir: Path, verbose: bool = False) -> int: + total = 0 + for path in sorted(data_dir.glob("audit-*.json")): + table = path.stem.replace("-", "_") + data = load_json(path, verbose) + if data is None: + continue + if isinstance(data, list): + total += import_list(db, table, data, verbose) + elif isinstance(data, dict): + rows = [] + for k, v in data.items(): + if isinstance(v, list) and v: + rows.extend(_flat(v)) + else: + rows.append({"key": k, "value": json.dumps(v, default=str)}) + total += import_list(db, table, rows, verbose) + return total + + +# --------------------------------------------------------------------------- +# Registry-driven import (generalized) +# --------------------------------------------------------------------------- + + +def import_from_registry( + db: Path, + data_dir: Path, + registry: dict[str, dict], + nested_mappings: dict[str, dict[str, str]], + verbose: bool = False, +) -> int: + """Import all tables defined in the registry. + + Format handling: + - csv: import via CSV + - json_array: top-level JSON array -> single table + - json_special: specialized importer (eras, derived_patterns, etc.) + - json_special_creators: youtube-creators specific handling + - json_nested: use nested_key_mappings to extract sub-tables + - json: generic dict -> flatten and import + """ + total = 0 + for table_name, entry in registry.items(): + filename = entry["file"] + fmt = entry.get("format", "json") + filepath = data_dir / filename + + if fmt == "csv": + total += import_csv(db, table_name, filepath, verbose) + + elif fmt == "json_array": + data = load_json(filepath, verbose) + if isinstance(data, list): + total += import_list(db, table_name, data, verbose) + + elif fmt == "json_special": + # Delegate to specialized importers + if filename == "commit-eras.json": + total += import_commit_eras(db, data_dir, filename, verbose) + elif filename == "derived-patterns.json": + total += import_derived_patterns(db, data_dir, filename, verbose) + + elif fmt == "json_nested": + data = load_json(filepath, verbose) + if isinstance(data, dict): + mapping = nested_mappings.get(filename, {}) + if mapping: + total += _import_mapping(db, data, mapping, verbose) + else: + # Generic: flatten top-level keys as separate tables + for k, v in data.items(): + if isinstance(v, list) and v: + total += import_list(db, f"{table_name}_{k}"[:60], _flat(v), verbose) + + elif fmt == "json": + data = load_json(filepath, verbose) + if isinstance(data, list): + total += import_list(db, table_name, _flat(data), verbose) + elif isinstance(data, dict): + rows = [] + for k, v in data.items(): + if isinstance(v, (dict, list)): + rows.append({"key": k, "value": json.dumps(v, default=str)}) + else: + rows.append(flatten_dict({k: v})) + total += import_list(db, table_name, rows, verbose) + + return total + + +# --------------------------------------------------------------------------- +# Indexes & FTS +# --------------------------------------------------------------------------- + + +def _validate_table_name(table: str) -> str: + """Reject table names that could enable SQL injection.""" + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", table): + raise ValueError(f"Invalid table name: {table!r}") + return table + + +def table_exists(db: Path, table: str) -> bool: + table = _validate_table_name(table) + result = run_su( + ["query", str(db), f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table}'"] + ) + if result.returncode == 0 and result.stdout.strip(): + try: + rows = json.loads(result.stdout) + return len(rows) > 0 + except json.JSONDecodeError: + pass + return False + + +def table_columns(db: Path, table: str) -> set[str]: + table = _validate_table_name(table) + result = run_su(["query", str(db), f"PRAGMA table_info([{table}])"]) + if result.returncode != 0 or not result.stdout.strip(): + return set() + try: + rows = json.loads(result.stdout) + except json.JSONDecodeError: + return set() + return {row.get("name") for row in rows if row.get("name")} + + +def create_indexes( + db: Path, indexes: list[tuple[str, list[str]]] | None = None, verbose: bool = False +) -> None: + if indexes is None: + indexes = DEFAULT_INDEXES + for table, columns in indexes: + if not table_exists(db, table): + log(f"SKIP indexes on {table} (table not found)", verbose) + continue + available = table_columns(db, table) + for col in columns: + if col not in available: + log(f"SKIP index {table}.{col} (column not found)", verbose) + continue + run_su( + [ + "create-index", + str(db), + table, + col, + "--name", + f"idx_{table}_{col}", + "--if-not-exists", + ], + verbose, + ) + + +def create_fts( + db: Path, fts_config: list[tuple[str, list[str]]] | None = None, verbose: bool = False +) -> None: + if fts_config is None: + fts_config = DEFAULT_FTS + for table, columns in fts_config: + if not table_exists(db, table): + log(f"SKIP FTS on {table} (table not found)", verbose) + continue + fts_table = f"{table}_fts" + if table_exists(db, fts_table): + log(f"SKIP FTS on {table} (already exists)", verbose) + continue + run_su(["enable-fts", str(db), table] + columns + ["--fts4", "--create-triggers"], verbose) + + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + + +def print_summary(db: Path, verbose: bool = False) -> None: + result = run_su(["tables", str(db), "--counts"], verbose) + if result.returncode == 0 and result.stdout.strip(): + print("\nTables (with row counts):") + for line in result.stdout.strip().splitlines(): + print(f" {line}") + idx = run_su(["indexes", str(db)], verbose) + if idx.returncode == 0 and idx.stdout.strip(): + print("\nIndexes:") + for line in idx.stdout.strip().splitlines(): + print(f" {line}") + fts = run_su( + [ + "query", + str(db), + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%_fts%'", + ], + verbose, + ) + if fts.returncode == 0 and fts.stdout.strip(): + try: + tables = json.loads(fts.stdout) + if tables: + print("\nFTS tables:") + for t in tables: + print(f" {t.get('name', t)}") + except json.JSONDecodeError: + pass + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def _assert_commits_ingested(db_path: Path, data_dir: Path) -> None: + """Fail loud if github-commits.csv has rows but the commits table is empty. + + Backstop for a silent ingest failure: without it the pipeline emits a + confidently wrong all-zero report (every SDLC practice reported 'ABSENT'). + """ + commits_csv = data_dir / "github-commits.csv" + if not commits_csv.exists(): + return + with open(commits_csv, encoding="utf-8") as f: + csv_rows = max(sum(1 for _ in csv.reader(f)) - 1, 0) + if csv_rows == 0: + return + conn = sqlite3.connect(db_path) + try: + tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} + db_rows = ( + conn.execute("SELECT COUNT(*) FROM commits").fetchone()[0] if "commits" in tables else 0 + ) + finally: + conn.close() + if db_rows != csv_rows: + print( + f"ERROR: github-commits.csv has {csv_rows} commit row(s) but the 'commits' " + "table count differs after build. Commit ingestion failed (is sqlite-utils " + "installed?). Aborting so the pipeline does not emit a false all-zero report.", + file=sys.stderr, + ) + sys.exit(1) + + +def build_db(project_root: Path, output: Path | None = None, verbose: bool = False) -> None: + """Build archaeology SQLite database from data files. + + Args: + project_root: Path to the project root directory. + output: Optional output DB path (default: /data/archaeology.db). + verbose: If True, print detailed progress. + """ + data_dir = project_root / "data" + + if output: + db_path = Path(output) + if not db_path.is_absolute(): + db_path = project_root / output + else: + db_path = data_dir / "archaeology.db" + + if not data_dir.exists(): + print(f"ERROR: Data directory not found: {data_dir}", file=sys.stderr) + sys.exit(1) + + # Load configuration + registry = load_table_registry(project_root, verbose) + nested_mappings = load_nested_key_mappings(project_root, verbose) + + # Load project-specific index/fts config if available + project_json = project_root / "project.json" + indexes = DEFAULT_INDEXES + fts_config = DEFAULT_FTS + if project_json.exists(): + try: + with open(project_json, encoding="utf-8") as f: + pdata = json.load(f) + if "indexes" in pdata: + indexes = [(item["table"], item["columns"]) for item in pdata["indexes"]] + if "fts" in pdata: + fts_config = [(item["table"], item["columns"]) for item in pdata["fts"]] + except (json.JSONDecodeError, OSError): + pass + + db_path.unlink(missing_ok=True) + log(f"Rebuilding DB: {db_path}", verbose) + db_path.parent.mkdir(parents=True, exist_ok=True) + db_path.touch() + + print(f"Building archaeology DB: {db_path}") + print(f"Project root: {project_root}") + print(f"Data directory: {data_dir}") + + # Registry-driven import (handles all formats) + print("\n--- Importing tables ---") + import_from_registry(db_path, data_dir, registry, nested_mappings, verbose) + + # Backstop: a populated commits CSV must yield a populated commits table. + _assert_commits_ingested(db_path, data_dir) + + # Audit files (auto-discovered from glob) + print("\n--- Audit files ---") + import_audit_files(db_path, data_dir, verbose) + + # Pipeline history tables are part of the stable query surface even when + # no pipeline logs have been ingested yet. Create them empty so audit and + # query helpers can distinguish "no runs" from "schema missing". + try: + from .pipeline_ingest import ensure_tables + + ensure_tables(db_path) + except (OSError, sqlite3.Error) as exc: # pragma: no cover - defensive CLI guard + print(f" WARNING: Failed to ensure pipeline tables: {exc}", file=sys.stderr) + + # Indexes & FTS + print("\n--- Indexes ---") + create_indexes(db_path, indexes, verbose) + print("\n--- Full-text search ---") + create_fts(db_path, fts_config, verbose) + + from ..metrics import write_metrics + + write_metrics(project_root, db_path) + print_summary(db_path, verbose) + print(f"\nDone. Database: {db_path}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Build archaeology SQLite database from data files" + ) + parser.add_argument("--project", help="Project name (resolves to projects//)") + parser.add_argument( + "--project-root", default=".", help="Direct path to project root (default: .)" + ) + parser.add_argument( + "--output", + default=None, + help="Output DB path (default: /data/archaeology.db)", + ) + parser.add_argument("--verbose", action="store_true", help="Print detailed progress") + args = parser.parse_args() + + project_root = resolve_project_root(args) + build_db(project_root, args.output, args.verbose) + + +if __name__ == "__main__": + main() diff --git a/plugins/devarch-hermes/archaeology/db/pipeline_ingest.py b/plugins/devarch-hermes/archaeology/db/pipeline_ingest.py new file mode 100644 index 0000000..b68dbc2 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/db/pipeline_ingest.py @@ -0,0 +1,290 @@ +"""Ingest GITHUB_pipeline run logs into an archaeology SQLite database. + +Reads pipeline JSON files (from .omc/logs/repo-pipeline/) and inserts them +into pipeline_runs and pipeline_repo_results tables for historical tracking +and cross-referencing with commit/session data. + +Pipeline JSON format (expected keys): + { + "run_timestamp": "2026-04-09T22:17:10Z", + "status": "COMPLETE|NEEDS_ATTENTION|RUNNING", + "duration_seconds": 120, + "repos": [ + { + "name": "repo-name", + "tier": 1, + "issues": {...}, + "fixes_applied": 2, + "status": "clean" + } + ], + "supervision": { + "missions": [ + { + "repo": "owner/repo", + "status": "supervising|needs_attention", + ... + } + ] + }, + "summary": { ... } + } +""" + +import json +import sqlite3 +from datetime import datetime +from pathlib import Path +from typing import Optional + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS pipeline_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_timestamp TEXT NOT NULL, + status TEXT, + duration_seconds INTEGER, + agents_used TEXT, + summary_json TEXT, + source_file TEXT, + ingested_at TEXT DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS pipeline_repo_results ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + repo_name TEXT NOT NULL, + tier INTEGER, + status TEXT, + issues_count INTEGER DEFAULT 0, + fixes_applied INTEGER DEFAULT 0, + issues_json TEXT, + FOREIGN KEY (run_id) REFERENCES pipeline_runs(id) +); + +CREATE INDEX IF NOT EXISTS idx_pipeline_runs_timestamp ON pipeline_runs(run_timestamp); +CREATE INDEX IF NOT EXISTS idx_pipeline_runs_status ON pipeline_runs(status); +CREATE INDEX IF NOT EXISTS idx_pipeline_repo_results_repo ON pipeline_repo_results(repo_name); +CREATE INDEX IF NOT EXISTS idx_pipeline_repo_results_run ON pipeline_repo_results(run_id); +""" + + +def _normalize_status(raw_status: str) -> str: + """Normalize new pipeline status values to legacy format for backward compatibility.""" + if not raw_status or not isinstance(raw_status, str): + return "unknown" + + status_map = { + "COMPLETE": "pass", + "SUCCESS": "pass", + "NEEDS_ATTENTION": "partial", + "RUNNING": "partial", + "FAILED": "fail", + "ERROR": "fail", + } + return status_map.get(raw_status.upper(), raw_status.lower()) + + +def _extract_agents(run_json: dict) -> str: + """Extract agent list from either supervision.missions or agents_used field.""" + # Try new format: supervision.missions + supervision = run_json.get("supervision", {}) + if isinstance(supervision, dict): + missions = supervision.get("missions", []) + if missions and isinstance(missions, list): + # Extract unique agent types from mission assignments + agents = set() + for mission in missions: + if isinstance(mission, dict): + assigned_layers = mission.get("assigned_layers", []) + if assigned_layers and isinstance(assigned_layers, list): + agents.update(assigned_layers) + if agents: + return json.dumps(sorted(agents)) + + # Fall back to old format: agents_used + agents_used = run_json.get("agents_used", []) + if agents_used: + return json.dumps(agents_used) + + # Default: empty list + return "[]" + + +def ensure_tables(db_path: Path) -> None: + """Create pipeline tables if they don't exist.""" + conn = sqlite3.connect(str(db_path), timeout=30) + try: + conn.executescript(SCHEMA) + conn.commit() + finally: + conn.close() + + +def ingest_run(db_path: Path, run_json: dict, source_file: str = "") -> int: + """Ingest a single pipeline run JSON. Returns the run_id.""" + ensure_tables(db_path) + conn = sqlite3.connect(str(db_path), timeout=30) + try: + # Support both old (timestamp) and new (run_timestamp) field names + ts = run_json.get("run_timestamp") or run_json.get( + "timestamp", datetime.utcnow().isoformat() + ) + # Map new status values to old format for backward compatibility + raw_status = run_json.get("status", "unknown") + status = _normalize_status(raw_status) + duration = run_json.get("duration_seconds") + # Extract agents from supervision.missions if available, else use agents_used + agents = _extract_agents(run_json) + summary = json.dumps(run_json.get("summary", {})) + + cursor = conn.execute( + "INSERT INTO pipeline_runs (run_timestamp, status, duration_seconds, agents_used, summary_json, source_file) " + "VALUES (?, ?, ?, ?, ?, ?)", + (ts, status, duration, agents, summary, source_file), + ) + run_id = cursor.lastrowid + + for repo in run_json.get("repos", []): + issues = repo.get("issues", []) + issues_count = ( + len(issues) + if isinstance(issues, list) + else sum(issues.values()) + if isinstance(issues, dict) + else 0 + ) + issues_json = json.dumps(issues) if isinstance(issues, (list, dict)) else "[]" + fixes_raw = repo.get("fixes_applied", 0) + fixes_count = ( + len(fixes_raw) + if isinstance(fixes_raw, list) + else fixes_raw + if isinstance(fixes_raw, int) + else 0 + ) + + # Extract repo name from multiple possible fields + repo_name = repo.get("full_name") or repo.get("path") or repo.get("name", "unknown") + + conn.execute( + "INSERT INTO pipeline_repo_results (run_id, repo_name, tier, status, issues_count, fixes_applied, issues_json) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + run_id, + repo_name, + repo.get("tier"), + repo.get("status", "unknown"), + issues_count, + fixes_count, + issues_json, + ), + ) + + conn.commit() + finally: + conn.close() + return run_id + + +def ingest_directory(db_path: Path, logs_dir: Path, verbose: bool = False) -> dict: + """Ingest all pipeline run JSONs from a directory. + + Returns stats: {"ingested": int, "skipped": int, "errors": list[str]} + """ + ensure_tables(db_path) + + # Get already-ingested source files to avoid duplicates + conn = sqlite3.connect(str(db_path), timeout=30) + try: + existing = { + row[0] + for row in conn.execute( + "SELECT source_file FROM pipeline_runs WHERE source_file != ''" + ).fetchall() + } + finally: + conn.close() + + stats = {"ingested": 0, "skipped": 0, "errors": []} + + for json_file in sorted(logs_dir.glob("*.json")): + if json_file.name == "latest.json" or json_file.name in existing: + if verbose: + print(f" SKIP {json_file.name} (already ingested or symlink)") + stats["skipped"] += 1 + continue + + try: + with open(json_file, encoding="utf-8") as f: + data = json.load(f) + + # Validate it looks like a pipeline run (support both old and new formats) + has_repos = "repos" in data or "supervision" in data + has_timestamp = "run_timestamp" in data or "timestamp" in data + if not has_repos and not has_timestamp: + if verbose: + print(f" SKIP {json_file.name} (not a pipeline run)") + stats["skipped"] += 1 + continue + + run_id = ingest_run(db_path, data, source_file=json_file.name) + if verbose: + print(f" INGESTED {json_file.name} -> run_id={run_id}") + stats["ingested"] += 1 + + except (json.JSONDecodeError, OSError) as exc: + stats["errors"].append(f"{json_file.name}: {exc}") + + return stats + + +def get_pipeline_history( + db_path: Path, repo_name: Optional[str] = None, limit: int = 50 +) -> list[dict]: + """Query pipeline run history, optionally filtered by repo.""" + conn = sqlite3.connect(str(db_path), timeout=30) + conn.row_factory = sqlite3.Row + try: + if repo_name: + rows = conn.execute( + """ + SELECT r.*, pr.repo_name, pr.tier, pr.status as repo_status, + pr.issues_count, pr.fixes_applied + FROM pipeline_runs r + JOIN pipeline_repo_results pr ON r.id = pr.run_id + WHERE pr.repo_name = ? + ORDER BY r.run_timestamp DESC LIMIT ? + """, + (repo_name, limit), + ).fetchall() + else: + rows = conn.execute( + "SELECT * FROM pipeline_runs ORDER BY run_timestamp DESC LIMIT ?", + (limit,), + ).fetchall() + + results = [dict(r) for r in rows] + finally: + conn.close() + return results + + +def get_repo_quality_trend(db_path: Path, repo_name: str, limit: int = 30) -> list[dict]: + """Get quality trend for a repo across pipeline runs (issues/fixes over time).""" + conn = sqlite3.connect(str(db_path), timeout=30) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + """ + SELECT r.run_timestamp, pr.status, pr.issues_count, pr.fixes_applied, pr.tier + FROM pipeline_runs r + JOIN pipeline_repo_results pr ON r.id = pr.run_id + WHERE pr.repo_name = ? + ORDER BY r.run_timestamp DESC LIMIT ? + """, + (repo_name, limit), + ).fetchall() + results = [dict(r) for r in rows] + finally: + conn.close() + return results diff --git a/plugins/devarch-hermes/archaeology/db/queries.py b/plugins/devarch-hermes/archaeology/db/queries.py new file mode 100644 index 0000000..9884912 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/db/queries.py @@ -0,0 +1,177 @@ +"""Common query helpers for archaeology SQLite databases.""" + +import re +import sqlite3 +from pathlib import Path +from typing import Optional + +# Allowed table names for FTS queries (whitelist validation) +_ALLOWED_FTS_TABLES = {"commits", "sessions", "eras"} + + +def _validate_table_name(table: str) -> str: + """Validate table name to prevent SQL injection. + + Only allows alphanumeric characters and underscores. + Raises ValueError if invalid. + """ + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", table): + raise ValueError(f"Invalid table name: {table}") + return table + + +def _validate_order_by(col: str) -> str: + """Validate ORDER BY column name to prevent SQL injection.""" + col = col.strip() + parts = col.split() + if len(parts) > 2: + raise ValueError(f"Invalid order_by: {col!r}") + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", parts[0]): + raise ValueError(f"Invalid column name in order_by: {parts[0]!r}") + if len(parts) == 2 and parts[1].upper() not in ("ASC", "DESC"): + raise ValueError(f"Invalid sort direction: {parts[1]!r}") + return col + + +def get_connection(db_path: str | Path) -> sqlite3.Connection: + """Get a connection to the archaeology database.""" + conn = sqlite3.connect(str(db_path), timeout=30) + conn.row_factory = sqlite3.Row + return conn + + +def get_commits(db_path: str, filters: Optional[dict] = None, limit: int = 1000) -> list[dict]: + """Query commits with optional filters. Filters: repo, author, date_from, date_to.""" + conn = get_connection(db_path) + try: + query = "SELECT * FROM commits WHERE 1=1" + params = [] + if filters: + if "repo" in filters: + query += " AND repo = ?" + params.append(filters["repo"]) + if "author" in filters: + query += " AND author = ?" + params.append(filters["author"]) + if "date_from" in filters: + query += " AND date >= ?" + params.append(filters["date_from"]) + if "date_to" in filters: + query += " AND date <= ?" + params.append(filters["date_to"]) + query += " ORDER BY date LIMIT ?" + params.append(limit) + rows = conn.execute(query, params).fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +def get_eras(db_path: str) -> list[dict]: + """Get all era definitions. + + Older archaeology databases do not have a start_date column; they usually + expose id and/or dates instead. Prefer start_date when available and fall + back to stable available columns so query helpers do not break on + databases with varying schema versions. + """ + conn = get_connection(db_path) + try: + cols = {row[1] for row in conn.execute("PRAGMA table_info(eras)").fetchall()} + if "start_date" in cols: + order_by = "start_date" + elif "id" in cols: + order_by = "id" + elif "dates" in cols: + order_by = "dates" + else: + order_by = "rowid" + order_by = _validate_order_by(order_by) + rows = conn.execute(f"SELECT * FROM eras ORDER BY {order_by}").fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +def get_sessions(db_path: str, filters: Optional[dict] = None, limit: int = 500) -> list[dict]: + """Query sessions with optional filters.""" + conn = get_connection(db_path) + try: + query = "SELECT * FROM sessions WHERE 1=1" + params = [] + if filters: + if "session_id" in filters: + query += " AND session_id = ?" + params.append(filters["session_id"]) + query += " ORDER BY timestamp LIMIT ?" + params.append(limit) + rows = conn.execute(query, params).fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +def get_fts_results(db_path: str, table: str, query_text: str, limit: int = 50) -> list[dict]: + """Full-text search on FTS-enabled tables. + + Args: + db_path: Path to SQLite database + table: Base table name (must be in allowed FTS tables whitelist) + query_text: FTS search query + limit: Maximum results to return + + Raises: + ValueError: If table name is not in the allowed whitelist + """ + # Validate table name against whitelist to prevent SQL injection + if table not in _ALLOWED_FTS_TABLES: + raise ValueError( + f"Table '{table}' not allowed for FTS queries. Allowed: {sorted(_ALLOWED_FTS_TABLES)}" + ) + + conn = get_connection(db_path) + try: + fts_table = f"{table}_fts" + rows = conn.execute( + f"SELECT * FROM {fts_table} WHERE {fts_table} MATCH ? LIMIT ?", [query_text, limit] + ).fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +def get_table_list(db_path: str) -> list[str]: + """Get all table names in the database.""" + conn = get_connection(db_path) + try: + rows = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" + ).fetchall() + return [r[0] for r in rows] + finally: + conn.close() + + +def get_table_count(db_path: str, table: str) -> int: + """Get row count for a table.""" + table = _validate_table_name(table) + conn = get_connection(db_path) + try: + count = conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] + return count + finally: + conn.close() + + +def get_pipeline_runs(db_path: str, repo_name: str | None = None, limit: int = 50) -> list[dict]: + """Query pipeline run history from the pipeline_runs table.""" + from .pipeline_ingest import get_pipeline_history + + return get_pipeline_history(Path(db_path), repo_name=repo_name, limit=limit) + + +def get_repo_quality_trend(db_path: str, repo_name: str, limit: int = 30) -> list[dict]: + """Get quality trend for a repo across pipeline runs.""" + from .pipeline_ingest import get_repo_quality_trend as _trend + + return _trend(Path(db_path), repo_name=repo_name, limit=limit) diff --git a/plugins/devarch-hermes/archaeology/demo.py b/plugins/devarch-hermes/archaeology/demo.py new file mode 100644 index 0000000..40f5807 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/demo.py @@ -0,0 +1,162 @@ +"""Sanitized demo project generation for DevArch Framework.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +DEMO_PROJECT = "demo-archaeology" + + +def _write_json(path: Path, data: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def create_demo_project( + root: str | Path = ".", project_name: str = DEMO_PROJECT, force: bool = False +) -> Path: + """Create a small sanitized demo project under projects/. + + The demo uses invented commit/session data. It contains no raw private logs, + no personal telemetry, and no external behavioral exports. + """ + root = Path(root) + project_root = root / "projects" / project_name + if project_root.exists() and not force: + raise FileExistsError( + f"Demo project already exists at {project_root}. Use --force to overwrite." + ) + + data_dir = project_root / "data" + deliverables_dir = project_root / "deliverables" + data_dir.mkdir(parents=True, exist_ok=True) + deliverables_dir.mkdir(parents=True, exist_ok=True) + + _write_json( + project_root / "project.json", + { + "name": project_name, + "description": "Sanitized demo project for Dev-Archaeology", + "repo_url": "https://github.com/example/demo-archaeology", + "developer": {"name": "Demo Developer", "github": "demo"}, + "timeline": {"start_date": "2026-01-01", "end_date": "2026-01-05", "total_days": 5}, + "overrides": {"era_count": 3, "total_commits": 6, "active_days": 4}, + "visualization": { + "title": "DEMO ARCHAEOLOGY", + "subtitle": "A sanitized sample project", + "counters": [ + {"label": "commits", "value": 6}, + {"label": "eras", "value": 3}, + ], + "agent_colors": {"Human": "#74c0fc", "Agent": "#51cf66"}, + "era_colors": {"era-01": "#74c0fc", "era-02": "#51cf66", "era-03": "#ffd43b"}, + }, + "data_sources": {"github_api": False}, + }, + ) + + commits = [ + [ + "demo001", + "2026-01-01 09:00:00 +0000", + "docs: write initial product intent", + "Demo Developer", + ], + ["demo002", "2026-01-01 11:00:00 +0000", "feat: scaffold prototype", "Agent"], + ["demo003", "2026-01-02 15:30:00 +0000", "fix: wire prototype output", "Agent"], + ["demo004", "2026-01-03 10:15:00 +0000", "test: add behavior checks", "Agent"], + [ + "demo005", + "2026-01-05 13:00:00 +0000", + "refactor: extract audit boundary", + "Demo Developer", + ], + [ + "demo006", + "2026-01-05 16:45:00 +0000", + "docs: publish remediation notes", + "Demo Developer", + ], + ] + with (data_dir / "github-commits.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(["hash", "date", "message", "author"]) + writer.writerows(commits) + + _write_json( + data_dir / "human-messages.json", + [ + { + "session_id": "demo-session-1", + "timestamp": "2026-01-01T09:00:00Z", + "messages": "We need a prototype that proves the core loop.", + }, + { + "session_id": "demo-session-2", + "timestamp": "2026-01-03T10:00:00Z", + "messages": "The audit should catch wiring gaps before launch.", + }, + ], + ) + + _write_json( + data_dir / "commit-eras.json", + { + "project": "Demo Archaeology", + "lifespan": "5 days (2026-01-01 to 2026-01-05)", + "total_commits": 6, + "eras": [ + { + "id": 1, + "name": "Intent", + "dates": "2026-01-01", + "commits": 1, + "description": "The project goal is written down.", + "narrative_arc": "A clear intent appears before code.", + }, + { + "id": 2, + "name": "Prototype", + "dates": "2026-01-01 to 2026-01-02", + "commits": 2, + "description": "The prototype is scaffolded and wired.", + "narrative_arc": "Implementation pressure exposes the first integration gap.", + }, + { + "id": 3, + "name": "Hardening", + "dates": "2026-01-03 to 2026-01-05", + "commits": 3, + "description": "Tests and audit boundaries are added.", + "narrative_arc": "The project shifts from making claims to proving them.", + }, + ], + }, + ) + + _write_json( + deliverables_dir / "canonical-metrics.json", + { + "generated": "2026-01-05", + "source_scope": "sanitized demo fixture", + "total_commits": 6, + "span_days": 5, + "active_days": 4, + "peak_day": "2026-01-05", + "peak_day_commits": 2, + "session_count": 2, + "human_messages": 2, + }, + ) + + (project_root / "README.md").write_text( + "# Demo Archaeology\n\nThis is a sanitized demo fixture generated by `devarch demo`. It contains invented data only.\n", + encoding="utf-8", + ) + (project_root / "PRIVACY-MANIFEST.md").write_text( + "# Demo Privacy Manifest\n\nThis demo contains invented, sanitized fixture data only. It is safe to publish and contains no raw private sessions, behavioral exports, resumes, or personal telemetry.\n", + encoding="utf-8", + ) + return project_root diff --git a/plugins/devarch-hermes/archaeology/era_cascade.py b/plugins/devarch-hermes/archaeology/era_cascade.py new file mode 100644 index 0000000..6b289a8 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/era_cascade.py @@ -0,0 +1,366 @@ +"""Era cascade: automated propagation of era structure changes. + +Reads canonical era definitions from commit-eras.json and propagates +changes across all deliverable files. This is the fix engine that the +scanner detects and the mapper calculates. + +Usage: + from archaeology.era_cascade import cascade + result = cascade(project_dir, eras_path, dry_run=True) +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path + +from .era_mapper import EraDef, era_count, get_current_era_names, load_eras, remap_json_era_fields +from .era_scanner import scan_deliverables + +# Known old era names that may appear in deliverables +KNOWN_OLD_NAMES = { + "The Acceleration", + "The Crusade", + "The Hardening", + "The Threshold", + "The Surface", + "The Return", +} + +# Files exempt from era name replacement (historical mapping docs) +EXEMPT_FILES: frozenset[str] = frozenset( + { + "ERA_UPDATE_SUMMARY.md", + } +) + + +@dataclass +class CascadeResult: + files_scanned: int = 0 + files_changed: int = 0 + era_fields_remapped: int = 0 + names_replaced: int = 0 + css_vars_fixed: int = 0 + ranges_capped: int = 0 + counts_fixed: int = 0 + project_json_synced: bool = False + stale_refs_remaining: int = 0 + + +def cascade( + project_dir: Path, + eras_path: Path, + dry_run: bool = False, +) -> CascadeResult: + """Run the full era cascade on a project. + + Steps: + 1. Load canonical eras + 2. Sync project.json + 3. Remap data.json era fields by date + 4. Mirror data.js + 5. Fix HTML era CSS vars and data-era-range + 6. Fix markdown era names and numbers + 7. Verify with scanner + """ + result = CascadeResult() + + eras = load_eras(eras_path) + if not eras: + return result + n_eras = era_count(eras) + current_names = get_current_era_names(eras) + + # Step 2: Sync project.json + _sync_project_json(project_dir, eras, n_eras, dry_run, result) + + # Step 3: Remap data.json era fields + data_json = project_dir / "deliverables" / "data.json" + if data_json.exists(): + changed = _remap_data_json(data_json, eras, dry_run) + result.era_fields_remapped += changed + + # Step 4: Mirror data.js + data_js = project_dir / "deliverables" / "data.js" + if data_json.exists() and data_js.exists(): + _mirror_data_js(data_json, data_js, dry_run) + + # Step 5: Fix HTML files (check both root and visuals/ subdirectory) + deliverables = project_dir / "deliverables" + html_files = list(deliverables.glob("*.html")) + list(deliverables.glob("visuals/*.html")) + for html_file in html_files: + if html_file.name in EXEMPT_FILES: + continue + _fix_html_file(html_file, eras, n_eras, dry_run, result) + + # Step 6: Fix markdown files + for md_file in deliverables.rglob("*.md"): + if md_file.name in EXEMPT_FILES: + continue + _fix_markdown_file(md_file, eras, n_eras, current_names, dry_run, result) + + # Step 7: Verify + scan_result = scan_deliverables(project_dir, eras) + result.files_scanned = scan_result.files_scanned + result.stale_refs_remaining = len(scan_result.refs) + + return result + + +def _sync_project_json( + project_dir: Path, + eras: list[EraDef], + n_eras: int, + dry_run: bool, + result: CascadeResult, +) -> None: + """Sync era_count and era_colors in project.json.""" + pj_path = project_dir / "project.json" + if not pj_path.exists(): + return + + pj = json.loads(pj_path.read_text()) + changed = False + + # Fix era_count + overrides = pj.setdefault("overrides", {}) + if overrides.get("era_count") != n_eras: + overrides["era_count"] = n_eras + changed = True + + # Fix era_colors — trim to n_eras entries + viz = pj.setdefault("visualization", {}) + colors = viz.setdefault("era_colors", {}) + trimmed = { + f"era-{i + 1:02d}": colors.get(f"era-{i + 1:02d}", _default_color(i)) for i in range(n_eras) + } + if len(colors) != len(trimmed) or colors != trimmed: + viz["era_colors"] = trimmed + changed = True + + if changed and not dry_run: + pj_path.write_text(json.dumps(pj, indent=2) + "\n") + result.project_json_synced = True + result.files_changed += 1 + + +def _default_color(index: int) -> str: + """Default era color palette.""" + palette = [ + "#4ade80", + "#f87171", + "#fb923c", + "#60a5fa", + "#a78bfa", + "#34d399", + "#fbbf24", + "#f472b6", + "#c084fc", + "#9ca3af", + ] + return palette[index % len(palette)] + + +def _remap_data_json(data_json: Path, eras: list[EraDef], dry_run: bool) -> int: + """Remap era fields in data.json using date-based calculation.""" + data = json.loads(data_json.read_text()) + changes = remap_json_era_fields(data, eras) + if changes and not dry_run: + data_json.write_text(json.dumps(data, indent=2) + "\n") + return len(changes) + + +def _mirror_data_js(data_json: Path, data_js: Path, dry_run: bool) -> None: + """Mirror data.json into data.js with JS wrapper.""" + if dry_run: + return + data = json.loads(data_json.read_text()) + js_content = f"const DATA = {json.dumps(data, indent=2)};\n" + data_js.write_text(js_content) + + +def _fix_html_file( + html_path: Path, + eras: list[EraDef], + n_eras: int, + dry_run: bool, + result: CascadeResult, +) -> None: + """Fix era references in HTML deliverables.""" + content = html_path.read_text(errors="ignore") + original = content + lines = content.splitlines(keepends=True) + + new_lines = [] + for line in lines: + # Fix CSS vars: --era-NN where NN > n_eras + line = _fix_css_vars(line, n_eras, result) + + # Fix data-era-range: cap upper bound + line = _fix_era_range(line, n_eras, result) + + # Fix era count text: "14 Eras", "Ten Eras" + line = _fix_era_count_text(line, n_eras, result) + + # Fix "Era N" text references + line = _fix_era_number_text(line, n_eras, result) + + new_lines.append(line) + + content = "".join(new_lines) + + # Fix embedded JSON era fields + content = _fix_embedded_json_eras(content, eras, result) + + if content != original and not dry_run: + html_path.write_text(content) + result.files_changed += 1 + + +def _fix_css_vars(line: str, n_eras: int, result: CascadeResult) -> str: + """Remove or fix era CSS variables beyond current count.""" + + def _replace(m: re.Match) -> str: + num = int(m.group(1)) + if num > n_eras: + result.css_vars_fixed += 1 + return f"/* removed: era-{m.group(1)} */" + return m.group(0) + + return re.sub(r"--era-(\d{2})", _replace, line) + + +def _fix_era_range(line: str, n_eras: int, result: CascadeResult) -> str: + """Cap data-era-range upper bound to n_eras.""" + + def _replace(m: re.Match) -> str: + low = int(m.group(1)) + high = int(m.group(2)) + if high > n_eras: + new_high = n_eras + if low > new_high: + low = 1 + result.ranges_capped += 1 + return f'data-era-range="{low}-{new_high}"' + return m.group(0) + + return re.sub(r'data-era-range="(\d+)-(\d+)"', _replace, line) + + +def _fix_era_count_text(line: str, n_eras: int, result: CascadeResult) -> str: + """Fix era count references like '14 Eras', 'Ten Eras'.""" + # Skip historical/comparison lines + if "→" in line or "->" in line: + return line + + word_map = { + 1: "One", + 2: "Two", + 3: "Three", + 4: "Four", + 5: "Five", + 6: "Six", + 7: "Seven", + 8: "Eight", + 9: "Nine", + 10: "Ten", + 11: "Eleven", + 12: "Twelve", + 13: "Thirteen", + 14: "Fourteen", + 15: "Fifteen", + 16: "Sixteen", + } + target_word = word_map.get(n_eras, str(n_eras)) + + # "14 Eras" → "7 Eras" + def _replace_num(m: re.Match) -> str: + num = int(m.group(1)) + if num != n_eras and num > n_eras: + result.counts_fixed += 1 + return f"{n_eras} Eras" + return m.group(0) + + line = re.sub(r"\b(\d+)\s+Eras\b", _replace_num, line) + + # "Ten Eras" → "Seven Eras" + for word, num in word_map.items(): + if num != n_eras and f"{word} Eras" in line: + line = line.replace(f"{word} Eras", f"{target_word} Eras") + result.counts_fixed += 1 + + return line + + +def _fix_era_number_text(line: str, n_eras: int, result: CascadeResult) -> str: + """Fix 'Era N' text where N > n_eras. Limited context — cannot auto-remap.""" + # We can only flag these, not auto-fix without date context + return line + + +def _fix_embedded_json_eras(content: str, eras: list[EraDef], result: CascadeResult) -> str: + """Fix 'era': N fields in embedded HTML data using nearby date context.""" + lines = content.splitlines(keepends=True) + _new_lines = [] + + era_line_indices = [] + for i, line in enumerate(lines): + if re.search(r'"era":\s*(\d+)', line): + m = re.search(r'"era":\s*(\d+)', line) + num = int(m.group(1)) + if num > len(eras): + era_line_indices.append(i) + + for idx in era_line_indices: + # Search backwards up to 20 lines for a date field + found_date = None + for j in range(max(0, idx - 20), idx + 1): + dm = re.search( + r'"(?:date|first_expression|estimated_hook_commit)":\s*"(\d{4}-\d{2}-\d{2})', + lines[j], + ) + if dm: + found_date = dm.group(1) + break + + if found_date: + from .era_mapper import era_from_date + + new_era = era_from_date(eras, found_date) + if new_era is not None: + lines[idx] = re.sub(r'"era":\s*\d+', f'"era": {new_era}', lines[idx]) + result.era_fields_remapped += 1 + + return "".join(lines) + + +def _fix_markdown_file( + md_path: Path, + eras: list[EraDef], + n_eras: int, + current_names: set[str], + dry_run: bool, + result: CascadeResult, +) -> None: + """Fix era names in markdown files.""" + content = md_path.read_text(errors="ignore") + original = content + + # Skip historical/comparison lines + lines = content.splitlines(keepends=True) + new_lines = [] + for line in lines: + if re.search(r"→|->|Original Claim|originally reported", line): + new_lines.append(line) + continue + line = _fix_era_count_text(line, n_eras, result) + new_lines.append(line) + + content = "".join(new_lines) + + if content != original and not dry_run: + md_path.write_text(content) + result.files_changed += 1 diff --git a/plugins/devarch-hermes/archaeology/era_mapper.py b/plugins/devarch-hermes/archaeology/era_mapper.py new file mode 100644 index 0000000..7468dbb --- /dev/null +++ b/plugins/devarch-hermes/archaeology/era_mapper.py @@ -0,0 +1,165 @@ +"""Date-based era mapping and remapping for era cascade. + +The key insight: era numbers in data files must be calculated from dates, +not assumed from previous structure. When eras merge, split, or renumber, +the only reliable remapping source is the timestamp. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class EraDef: + id: int + name: str + start: datetime + end: datetime + commits: int + + +def _infer_year(raw: dict) -> int: + """Infer the year from commit-eras.json data.""" + # Try first_commit_date or timeline metadata + first = raw.get("first_commit_date", "") + if first and len(first) >= 4: + try: + return int(first[:4]) + except (ValueError, TypeError): + pass + # Try daily data — find earliest date key + for era in raw.get("eras", []): + daily = era.get("daily", {}) + if isinstance(daily, dict): + for date_key in sorted(daily.keys()): + if len(date_key) >= 4: + try: + return int(date_key[:4]) + except (ValueError, TypeError): + continue + # Default to current year + return datetime.now().year + + +def _parse_era_date(date_str: str, year: int, reference: datetime | None = None) -> datetime | None: + """Parse a single date string supporting multiple formats.""" + s = date_str.strip() + # ISO: 2026-01-15 + for fmt in ("%Y-%m-%d", "%Y-%m", "%b %d %Y", "%b %d"): + try: + if fmt == "%b %d": + dt = datetime.strptime(f"{s} {year}", "%b %d %Y") + else: + dt = datetime.strptime(s, fmt) + return dt + except ValueError: + continue + return None + + +def load_eras(eras_path: Path) -> list[EraDef]: + """Load era definitions from commit-eras.json. + + Handles date formats: "Jan 1 - Jan 5", "2026-01-01 to 2026-01-05", + ISO single dates (era spans to next day), and month-only ranges. + """ + if not eras_path.exists(): + return [] + import re as _re + + raw = json.loads(eras_path.read_text()) + year = _infer_year(raw) + eras = [] + for era in raw.get("eras", []): + dates = era.get("dates", "") + # Split on " - ", " – ", " to " (ISO range), or handle single dates + for sep in (" - ", " – ", " to "): + if sep in dates: + parts = dates.split(sep, 1) + break + else: + parts = [dates, dates] # single date → era spans that day + + start = _parse_era_date(parts[0], year) + end = _parse_era_date(parts[1], year) if len(parts) > 1 else start + if start is None or end is None: + continue + # If end is earlier than start, assume it wraps to next year + if end < start: + end = _parse_era_date(parts[1], year + 1) or end + + commits = era.get("commits", 0) + if isinstance(commits, str): + m = _re.search(r"(\d+)", commits) + commits = int(m.group(1)) if m else 0 + eras.append( + EraDef( + id=era["id"], + name=era["name"], + start=start, + end=end, + commits=commits, + ) + ) + return eras + + +def era_from_date(eras: list[EraDef], date_str: str) -> int | None: + """Given a date string like '2026-03-30', return the era id it belongs to.""" + if not date_str or not isinstance(date_str, str): + return None + try: + d = datetime.strptime(date_str[:10], "%Y-%m-%d") + except (ValueError, TypeError): + return None + for era in eras: + if era.start <= d <= era.end: + return era.id + return None + + +def remap_json_era_fields(data: Any, eras: list[EraDef]) -> list[tuple[int, int, str]]: + """Walk JSON structure and remap all 'era' fields based on their date fields. + + Returns list of (old_era, new_era, date_string) for each change made. + Modifies data in place. + """ + changed: list[tuple[int, int, str]] = [] + _remap_walk(data, eras, changed) + return changed + + +def _remap_walk(obj: Any, eras: list[EraDef], changed: list[tuple[int, int, str]]) -> None: + """Recursively walk and remap era fields.""" + if isinstance(obj, dict): + if "era" in obj: + # Try multiple date field names + date_val = ( + obj.get("date") or obj.get("first_expression") or obj.get("estimated_hook_commit") + ) + if date_val: + new_era = era_from_date(eras, date_val) + if new_era is not None and new_era != obj["era"]: + old = obj["era"] + obj["era"] = new_era + changed.append((old, new_era, date_val)) + for v in obj.values(): + _remap_walk(v, eras, changed) + elif isinstance(obj, list): + for item in obj: + _remap_walk(item, eras, changed) + + +def get_current_era_names(eras: list[EraDef]) -> set[str]: + """Return set of all current era names.""" + return {era.name for era in eras} + + +def era_count(eras: list[EraDef]) -> int: + """Return the number of eras.""" + return len(eras) diff --git a/plugins/devarch-hermes/archaeology/era_scanner.py b/plugins/devarch-hermes/archaeology/era_scanner.py new file mode 100644 index 0000000..1feebb7 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/era_scanner.py @@ -0,0 +1,464 @@ +"""Stale era reference scanner for deliverable files. + +Detects era numbers, names, CSS variables, count text, and semantic +drift (wrong canonical values, stale day counts, per-era commit mismatches) +that don't match the current era structure. Returns findings for audit +reporting and cascade fix decisions. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path + +from .era_mapper import EraDef, era_count, get_current_era_names + +# Files that legitimately contain historical era references (mapping docs) +HISTORICAL_FILES: frozenset[str] = frozenset( + { + "ERA_UPDATE_SUMMARY.md", + } +) + + +def _load_canonical_metrics(project_dir: Path) -> dict: + """Load canonical metrics from project.json for semantic drift detection.""" + import json + + pjson = project_dir / "project.json" + if not pjson.exists(): + return {} + try: + raw = json.loads(pjson.read_text()) + except (json.JSONDecodeError, OSError): + return {} + return { + "total_commits": raw.get("total_commits"), + "span_days": raw.get("span_days"), + "active_days": raw.get("active_days"), + "era_count": raw.get("era_count"), + } + + +@dataclass(frozen=True) +class EraRef: + file: Path + line: int + kind: str # "era_number" | "era_name" | "era_css_var" | "era_count" | "era_json_field" + old_value: str + expected: str # what it should be, or "N/A" if unmappable + + +@dataclass +class ScanResult: + refs: list[EraRef] = field(default_factory=list) + files_scanned: int = 0 + lines_scanned: int = 0 + + @property + def has_findings(self) -> bool: + return len(self.refs) > 0 + + +def scan_deliverables(project_dir: Path, eras: list[EraDef]) -> ScanResult: + """Scan all deliverable files for stale era references.""" + deliverables_dir = project_dir / "deliverables" + if not deliverables_dir.exists(): + return ScanResult() + + current_names = get_current_era_names(eras) + n_eras = era_count(eras) + metrics = _load_canonical_metrics(project_dir) + result = ScanResult() + + for path in deliverables_dir.rglob("*"): + if not path.is_file(): + continue + if path.suffix not in {".md", ".html", ".json", ".js"}: + continue + if path.name in HISTORICAL_FILES: + continue + + _scan_file(path, eras, current_names, n_eras, metrics, result) + + return result + + +def _scan_file( + path: Path, + eras: list[EraDef], + current_names: set[str], + n_eras: int, + metrics: dict, + result: ScanResult, +) -> None: + """Scan a single file for stale era references and semantic drift.""" + result.files_scanned += 1 + try: + lines = path.read_text(errors="ignore").splitlines() + except OSError: + return + + rel = path.relative_to(path.parents[2]) # relative to project dir + _is_historical = any(hf in str(rel) for hf in HISTORICAL_FILES) + + # Build per-era commit count map from canonical source + era_commits = {e.id: e.commits for e in eras} + + for i, line in enumerate(lines, start=1): + result.lines_scanned += 1 + + # Skip historical/context lines + if re.search( + r"Original Claim|originally reported|was \d+ eras|Corrected To", + line, + re.I, + ): + continue + + # --- Structural checks (existing) --- + + # Check "Era N" where N > n_eras + for m in re.finditer(r"\bEra\s+(\d+)\b", line): + num = int(m.group(1)) + if num > n_eras: + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_number", + old_value=f"Era {num}", + expected=f"Era 1-{n_eras} (remap by date)", + ) + ) + + # Check "era-NN" CSS variables where NN > n_eras + for m in re.finditer(r"era-(\d{2})", line): + num = int(m.group(1)) + if num > n_eras: + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_css_var", + old_value=f"era-{m.group(1)}", + expected=f"era-01 through era-{n_eras:02d}", + ) + ) + + # Check "N eras" count text — only flag if clearly claiming to be the total + # Known stale total counts from previous structures: 10, 14, 15, 16 + known_stale_totals = {10, 14, 15, 16} + for m in re.finditer(r"\b(\d+)\s+eras\b", line, re.I): + num = int(m.group(1)) + if num in known_stale_totals: + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_count", + old_value=f"{num} eras", + expected=f"{n_eras} eras", + ) + ) + + # Check "Ten Eras" / "Fourteen Eras" style + word_map = { + "Ten": 10, + "Eleven": 11, + "Twelve": 12, + "Thirteen": 13, + "Fourteen": 14, + "Fifteen": 15, + "Sixteen": 16, + "Seven": 7, + "Eight": 8, + "Nine": 9, + } + for m in re.finditer( + r"\b(Ten|Eleven|Twelve|Thirteen|Fourteen|Fifteen|Sixteen)\s+Eras\b", line + ): + num = word_map.get(m.group(1), 0) + if num != n_eras: + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_count", + old_value=f"{m.group(1)} Eras", + expected=f"{_number_word(n_eras)} Eras", + ) + ) + + # Check "era": N in JSON/JS where N > n_eras + for m in re.finditer(r'"era":\s*(\d+)', line): + num = int(m.group(1)) + if num > n_eras or num == 0: + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_json_field", + old_value=f'"era": {num}', + expected=f'"era": 1-{n_eras} (remap by date)', + ) + ) + + # Check old era names (skip era names inside quotes that are + # clearly part of a mapping table, sub-phase names, or blog titles) + known_old_names = { + "The Acceleration", + "The Crusade", + "The Hardening", + "The Return", + } + for old_name in known_old_names: + if old_name in line and old_name not in current_names: + if "→" in line or "->" in line: + continue + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_name", + old_value=old_name, + expected=", ".join(sorted(current_names)), + ) + ) + + # --- Semantic drift checks (new) --- + + # Note: THE_BIBLE, Plus Ultra, The Outer Loop are event/concept names, + # NOT era names. They appear legitimately in narrative text and should + # NOT be flagged. Only actual old era names (The Acceleration, The Crusade, + # etc.) are flagged above in the known_old_names check. + + # Check "N chapters" / "N Chapters" where N != n_eras + + # Check "N chapters" / "N Chapters" where N != n_eras + for m in re.finditer(r"\b(\d+)\s+[Cc]hapters?\b", line): + num = int(m.group(1)) + if num != n_eras: + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_count", + old_value=f"{num} chapters", + expected=f"{n_eras} (matches era count)", + ) + ) + + # Check "N-day development" / "N days" against canonical span_days + canonical_span = metrics.get("span_days") + if canonical_span: + known_stale_spans = {canonical_span - 1, canonical_span + 1} + for m in re.finditer(r"\b(\d+)[\s-]*day", line, re.I): + num = int(m.group(1)) + if num in known_stale_spans and num != canonical_span: + result.refs.append( + EraRef( + file=path, + line=i, + kind="semantic_drift", + old_value=f"{num} day", + expected=f"{canonical_span} days (from project.json)", + ) + ) + + # Check "1,050 commits" / "1050 commits" (old Cluster 4 count) + for m in re.finditer(r"\b1,?050\s+commits", line): + result.refs.append( + EraRef( + file=path, + line=i, + kind="semantic_drift", + old_value="1,050 commits", + expected="972 commits (Eras 3-7) or era-specific count", + ) + ) + + # Check per-era commit count drift: only direct "Era N: X commits" patterns + # Avoid matching combined counts like "Era 3-4: 691 commits" or sub-periods + for m in re.finditer(r"\bEra\s+(\d+)\s*[:\(]\s*(\d[\d,]*)\s+commits", line, re.I): + era_num = int(m.group(1)) + count_str = m.group(2).replace(",", "") + try: + count = int(count_str) + except ValueError: + continue + if era_num in era_commits and count != era_commits[era_num]: + result.refs.append( + EraRef( + file=path, + line=i, + kind="semantic_drift", + old_value=f"Era {era_num}: {m.group(2)} commits", + expected=f"Era {era_num}: {era_commits[era_num]} commits", + ) + ) + + # Check "N development eras" / "across N eras" against canonical + for m in re.finditer(r"across\s+(\d+)\s+(?:development\s+)?eras?\b", line, re.I): + num = int(m.group(1)) + if num != n_eras: + result.refs.append( + EraRef( + file=path, + line=i, + kind="era_count", + old_value=f"across {num} eras", + expected=f"across {n_eras} eras", + ) + ) + + # Check JS/HTML script blocks for oversized era arrays + if path.suffix in {".html", ".js"}: + _scan_js_era_arrays(path, eras, current_names, n_eras, result) + + +def _number_word(n: int) -> str: + """Convert a number to its English word form for era counts.""" + words = { + 1: "One", + 2: "Two", + 3: "Three", + 4: "Four", + 5: "Five", + 6: "Six", + 7: "Seven", + 8: "Eight", + 9: "Nine", + 10: "Ten", + 11: "Eleven", + 12: "Twelve", + 13: "Thirteen", + 14: "Fourteen", + 15: "Fifteen", + 16: "Sixteen", + } + return words.get(n, str(n)) + + +def _scan_js_era_arrays( + path: Path, + eras: list[EraDef], + current_names: set[str], + n_eras: int, + result: ScanResult, +) -> None: + """Detect oversized or stale-named era arrays in JS/HTML script blocks. + + Catches patterns like: + - const modelTimeline = [{era:1,...}, {era:2,...}, ... {era:12,...}] + - const eraProfiles = [{era:'Seed',...}, {era:'Explosion',...}, ...] + """ + try: + content = path.read_text(errors="ignore") + except OSError: + return + + lines = content.splitlines() + + # Pattern 1: JS arrays with { era: N } where N > n_eras + # Find array boundaries that contain era entries + _check_era_number_arrays(path, lines, n_eras, result) + + # Pattern 2: JS arrays with {era:'Name'} using non-current era names + _check_era_name_arrays(path, lines, current_names, n_eras, result) + + +def _check_era_number_arrays(path: Path, lines: list[str], n_eras: int, result: ScanResult) -> None: + """Find JS arrays containing { era: N } entries where N exceeds n_eras.""" + # Track array start lines and collect era numbers within them + in_array = False + array_start = 0 + era_numbers: list[int] = [] + _brace_depth = 0 + + for i, line in enumerate(lines, start=1): + stripped = line.strip() + + if not in_array: + # Detect array start that will contain era entries + # Look for = [...] or const xyz = [ + if re.search(r"=\s*\[", stripped) and not stripped.startswith("//"): + in_array = True + array_start = i + era_numbers = [] + _brace_depth = 0 + + if in_array: + # Collect era: N entries + for m in re.finditer(r"\bera:\s*(\d+)", line): + era_numbers.append(int(m.group(1))) + + # Track if array closes + if "]" in stripped: + # Check if this is the closing bracket (rough heuristic) + open_brackets = stripped.count("[") + close_brackets = stripped.count("]") + if close_brackets > open_brackets: + in_array = False + # Evaluate collected era numbers + if era_numbers and max(era_numbers) > n_eras: + stale = [n for n in era_numbers if n > n_eras] + result.refs.append( + EraRef( + file=path, + line=array_start, + kind="js_era_array", + old_value=f"array with era entries {era_numbers} ({len(era_numbers)} entries, max={max(era_numbers)})", + expected=f"max {n_eras} entries (stale: {stale})", + ) + ) + + +def _check_era_name_arrays( + path: Path, + lines: list[str], + current_names: set[str], + n_eras: int, + result: ScanResult, +) -> None: + """Find JS arrays with {era:'Name'} using names not in current eras.""" + in_array = False + array_start = 0 + era_names: list[str] = [] + + for i, line in enumerate(lines, start=1): + stripped = line.strip() + + if not in_array: + if re.search(r"=\s*\[", stripped) and not stripped.startswith("//"): + in_array = True + array_start = i + era_names = [] + + if in_array: + # Collect era:'Name' or era: 'Name' entries + for m in re.finditer(r"\bera:\s*['\"]([^'\"]+)['\"]", line): + era_names.append(m.group(1)) + + if "]" in stripped: + open_brackets = stripped.count("[") + close_brackets = stripped.count("]") + if close_brackets > open_brackets: + in_array = False + if era_names and len(era_names) > n_eras: + # Also check for non-current names + non_current = [n for n in era_names if n not in current_names] + if non_current or len(era_names) != n_eras: + desc = f"array with {len(era_names)} era profiles (expected {n_eras})" + if non_current: + desc += f", non-current names: {non_current}" + result.refs.append( + EraRef( + file=path, + line=array_start, + kind="js_era_array", + old_value=desc, + expected=f"{n_eras} entries with names: {', '.join(sorted(current_names))}", + ) + ) diff --git a/plugins/devarch-hermes/archaeology/extractors/__init__.py b/plugins/devarch-hermes/archaeology/extractors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/devarch-hermes/archaeology/extractors/git.py b/plugins/devarch-hermes/archaeology/extractors/git.py new file mode 100644 index 0000000..e09e47b --- /dev/null +++ b/plugins/devarch-hermes/archaeology/extractors/git.py @@ -0,0 +1,249 @@ +"""Git log extraction for archaeology pipeline.""" + +import csv +import os +import subprocess +from pathlib import Path + +# Never allow promisor/partial clones to pull missing blobs during mining. +_LOCAL_GIT_ENV = { + "GIT_NO_LAZY_FETCH": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ALLOW_PROTOCOL": "", + "GIT_OPTIONAL_LOCKS": "0", +} + + +def _git_env() -> dict: + env = os.environ.copy() + env.update(_LOCAL_GIT_ENV) + return env + + +def _git(repo_path: str, *args: str, check: bool = True) -> str: + """Read Git metadata without invoking a shell or repository hooks.""" + try: + result = subprocess.run( + ["git", "-C", str(Path(repo_path).expanduser()), *args], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=300, + env=_git_env(), + ) + except FileNotFoundError as exc: + raise RuntimeError("git binary not found. Install git and ensure it's on PATH.") from exc + except subprocess.TimeoutExpired as exc: + raise RuntimeError("git read timed out after 300s") from exc + if check and result.returncode: + raise RuntimeError(f"git {args[0]} failed: {result.stderr.strip()}") + return result.stdout + + +def _git_dir(repo_path: str) -> Path: + raw = _git(repo_path, "rev-parse", "--git-dir").strip() + p = Path(raw) + if not p.is_absolute(): + p = Path(repo_path).expanduser().resolve() / p + return p.resolve() + + +def incomplete_object_reason(repo_path: str) -> str | None: + """Return an actionable error if the repo would lazy-fetch missing objects.""" + git_dir = _git_dir(repo_path) + promisor_packs = ( + list((git_dir / "objects" / "pack").glob("*.promisor")) + if (git_dir / "objects" / "pack").is_dir() + else [] + ) + if promisor_packs: + return ( + "Refusing incomplete-object / partial (promisor) clone — mining is local-only " + "and must not fetch missing blobs. Re-clone with full objects " + "(`git clone --no-partial` / omit `--filter`) or run " + "`git fetch --refetch` / `git sparse-checkout disable` as appropriate, then retry." + ) + # remote.*.promisor config (exit 1 = none) + out = _git(repo_path, "config", "--get-regexp", r"^remote\..*\.promisor$", check=False) + # Also probe filter config + filt = _git( + repo_path, "config", "--get", "remote.origin.partialclonefilter", check=False + ).strip() + promisor_cfg = _git(repo_path, "config", "--get-regexp", r"\.promisor$", check=False).strip() + if filt or promisor_cfg or out.strip(): + return ( + "Refusing partial clone (promisor/filter configured) — mining is local-only " + "and must not fetch missing blobs. Re-clone without `--filter`/`--partial` and retry." + ) + # With GIT_NO_LAZY_FETCH=1, missing blobs make shortstat fail rather than fetch. + # Probe HEAD tree readability when commits exist. + count = int(_git(repo_path, "rev-list", "--all", "--count").strip() or "0") + if count: + probe = subprocess.run( + [ + "git", + "-C", + str(Path(repo_path).expanduser()), + "rev-list", + "--objects", + "--all", + "--missing=print", + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=120, + env=_git_env(), + ) + if probe.returncode == 0: + missing = [ln for ln in probe.stdout.splitlines() if ln.startswith("?")] + if missing: + return ( + f"Refusing incomplete-object repository ({len(missing)} missing object(s)). " + "Mining is local-only and will not fetch. Restore full objects and retry." + ) + # Some git versions lack --missing=print; fall through. + return None + + +def repository_coverage(repo_path: str) -> dict: + """Record local reachability, never imply that remote/deleted refs were fetched.""" + _git(repo_path, "rev-parse", "--git-dir") + refs = _git(repo_path, "for-each-ref", "--format=%(refname)%09%(objectname)") + count = int(_git(repo_path, "rev-list", "--all", "--count").strip()) + return { + "head": _git(repo_path, "rev-parse", "--verify", "HEAD").strip() if count else None, + "scope": "all locally available refs plus HEAD; no automatic remote fetch", + "shallow": _git(repo_path, "rev-parse", "--is-shallow-repository").strip() == "true", + "bare": _git(repo_path, "rev-parse", "--is-bare-repository").strip() == "true", + "commit_count": count, + "refs": [dict(zip(("name", "object"), line.split("\t", 1))) for line in refs.splitlines()], + "roots": _git(repo_path, "rev-list", "--all", "--max-parents=0").splitlines(), + "gaps": [ + "Deleted, inaccessible and unfetched remote refs are outside this local snapshot." + ], + } + + +def extract_git_log( + repo_path: str, output_path: str, verbose: bool = False, max_commits: int = 0 +) -> int: + """Extract every reachable commit, with NUL-delimited fields and UTC dates. + + Subjects can contain tabs, pipes and unit separators. Git NUL delimiters + preserve those values rather than silently shifting or dropping CSV fields. + Empty histories write a header, replacing any stale previous extraction. + """ + import io + from datetime import datetime, timezone + + from ..utils import atomic_write + + reason = incomplete_object_reason(repo_path) + if reason: + raise RuntimeError(reason) + + limit = [f"--max-count={max_commits}"] if max_commits > 0 else [] + raw = _git(repo_path, "log", "-z", "--all", *limit, "--format=%H%x00%aI%x00%s%x00%an") + fields = raw.split("\0") + if fields[-1] == "": + fields.pop() + if len(fields) % 4: + raise RuntimeError("Malformed Git extraction; refusing partial history") + stream = io.StringIO(newline="") + writer = csv.writer(stream, lineterminator="\n") + writer.writerow(["hash", "date", "message", "author"]) + for i in range(0, len(fields), 4): + sha, date, subject, author = fields[i : i + 4] + normalized = ( + datetime.fromisoformat(date.replace("Z", "+00:00")).astimezone(timezone.utc).isoformat() + ) + writer.writerow([sha, normalized, subject, author]) + count = len(fields) // 4 + expected = int(_git(repo_path, "rev-list", "--all", *limit, "--count").strip()) + if count != expected: + raise RuntimeError("Repository changed during extraction; retry against frozen refs") + atomic_write(output_path, stream.getvalue()) + if verbose: + print(f"Extracted {count} commits from {repo_path}") + return count + + +def extract_git_log_with_stats( + repo_path: str, output_path: str, verbose: bool = False, max_commits: int = 0 +) -> int: + """Extract git log with file change stats (local-only; no lazy fetch).""" + reason = incomplete_object_reason(repo_path) + if reason: + raise RuntimeError(reason) + + limit = [f"--max-count={max_commits}"] if max_commits > 0 else [] + cmd = [ + "git", + "-C", + repo_path, + "log", + "--format=%H%x1f%ai%x1f%s%x1f%an", + "--shortstat", + "--all", + *limit, + ] + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=300, + env=_git_env(), + ) + except FileNotFoundError: + raise RuntimeError("git binary not found. Install git and ensure it's on PATH.") + except subprocess.TimeoutExpired: + raise RuntimeError("git log timed out after 300s. Repository may be too large.") + + if result.returncode != 0: + err = (result.stderr or "").strip() + if "promisor" in err.lower() or "lazy" in err.lower() or "missing" in err.lower(): + raise RuntimeError( + "git log --shortstat failed because objects are missing; " + "refusing to fetch. Restore a complete local clone and retry. " + f"Detail: {err}" + ) + raise RuntimeError(f"git log failed: {err}") + + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + if verbose: + print(f"Extracted git log with stats from {repo_path}") + return int(_git(repo_path, "rev-list", "--all", *limit, "--count").strip()) + + +def get_repo_list(repo_path: str) -> list[str]: + """Get list of all repos accessible from the given path (for multi-repo extraction).""" + # If it's a GitHub user/org, use gh API + try: + result = subprocess.run( + ["gh", "repo", "list", "--limit", "100", "--json", "name,url"], + capture_output=True, + text=True, + cwd=repo_path, + timeout=60, + env=_git_env(), + ) + except FileNotFoundError: + raise RuntimeError("gh CLI not found. Install GitHub CLI and ensure it's on PATH.") + except subprocess.TimeoutExpired: + raise RuntimeError("gh repo list timed out after 60s.") + + if result.returncode == 0: + import json + + repos = json.loads(result.stdout) + return [r["name"] for r in repos] + return [] diff --git a/plugins/devarch-hermes/archaeology/extractors/sessions.py b/plugins/devarch-hermes/archaeology/extractors/sessions.py new file mode 100644 index 0000000..e5d2ca4 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/extractors/sessions.py @@ -0,0 +1,736 @@ +#!/usr/bin/env python3 +# Extracted from Liminal's archaeology pipeline (narrative/extract_sessions.py) +""" +Extract narrative material from all Claude Code JSONL session files. +V2: Better filtering of tool-result noise, focus on genuine dialogue. +V3: Project-agnostic -- accepts --sessions-dir, --output, --config, --project args. +""" + +import argparse +import glob +import json +import os +import re +import sys +from datetime import datetime + +# --------------------------------------------------------------------------- +# Default values (used when no config file is provided) +# --------------------------------------------------------------------------- + +DEFAULT_EMOTIONAL_KEYWORDS = [ + "frustrat", + "excit", + "breakthrough", + "finally", + "aha", + "damn", + "hell", + "love", + "hate", + "annoy", + "stuck", + "confus", + "wow", + "amazing", + "beautiful", + "ugly", + "terrible", + "horrible", + "incredible", + "magic", + "magical", + "inspired", + "inspiration", + "creative", + "creativity", + "art", + "artist", + "philosophy", + "philosophical", + "meaning", + "purpose", + "vision", + "dream", + "passion", + "proud", + "disappoint", + "surpris", + "shock", + "satisfy", + "satisfying", + "elegant", + "inelegant", + "kludge", + "hack", + "eureka", + "awesome", + "disgust", + "delight", + "joy", + "rage", + "anger", + "happy", + "sad", + "nervous", + "anxious", + "worried", + "relief", + "celebrate", + "ship", + "shipped", + "done", + "works", + "working", + "fixed", + "broke", + "painful", + "pain", + "hard", + "easy", + "simple", + "complicated", +] + +DEFAULT_PHILOSOPHICAL_KEYWORDS = [ + "why are we", + "what is the point", + "the whole idea", + "the vision", + "i want to", + "i believe", + "the goal is", + "the dream", + "the mission", + "this project is", + "what i really", + "the truth is", + "honestly", + "at the end of the day", + "the real question", + "fundamentally", + "the deeper", + "meta", + "recursive", + "self-", + "emergent", + "emergence", + "consciousness", + "intelligence", + "creative coding", + "agent", + "ai should", + "ai could", + "ai is", + "what if", + "imagine", + "i think we should", + "let's think about", + "bigger picture", + "the story", + "the narrative", + "the journey", + "this is about", + "the whole point", + "it's not about", + "we're building", + "we are building", + "end goal", +] + +DEFAULT_REDACT_PATTERNS = [ + (r"sk-[a-zA-Z0-9]{20,}", "[REDACTED_API_KEY]"), + (r'key["\s:=]+[a-zA-Z0-9]{32,}', "[REDACTED_KEY]"), + (r'token["\s:=]+[a-zA-Z0-9]{20,}', "[REDACTED_TOKEN]"), + (r'password["\s:=]+\S+', "[REDACTED_PASSWORD]"), + (r"[\w.+-]+@[\w.-]+\.\w+", "[REDACTED_EMAIL]"), + (r"ghp_[a-zA-Z0-9]{36}", "[REDACTED_GITHUB_TOKEN]"), + (r"gho_[a-zA-Z0-9]{36}", "[REDACTED_GITHUB_TOKEN]"), + (r"github_pat_[a-zA-Z0-9_]{82}", "[REDACTED_GITHUB_TOKEN]"), + (r"AKIA[0-9A-Z]{16}", "[REDACTED_AWS_KEY]"), +] + +# Label used for config-provided patterns that don't specify one. +_UNLABELLED_REDACT = "[REDACTED]" + + +# --------------------------------------------------------------------------- +# Config loading helpers +# --------------------------------------------------------------------------- + + +def load_config(config_path): + """Load a JSON config file and return the session_extraction section.""" + if not config_path or not os.path.exists(config_path): + return {} + with open(config_path, "r", encoding="utf-8") as f: + data = json.load(f) + return data.get("session_extraction", {}) + + +def resolve_config(args): + """Build effective keyword lists and redaction patterns from args + config. + + Resolution order: + 1. If --project is given (and no --config), load projects//project.json + 2. If --config is given, load that file + 3. Fall back to module-level defaults + """ + config_path = args.config + + if args.project and not config_path: + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + candidate = os.path.join(repo_root, "projects", args.project, "project.json") + if os.path.exists(candidate): + config_path = candidate + + cfg = load_config(config_path) + + emotional = cfg.get("emotional_keywords", DEFAULT_EMOTIONAL_KEYWORDS) + philosophical = cfg.get("philosophical_keywords", DEFAULT_PHILOSOPHICAL_KEYWORDS) + + # Redaction patterns: config may provide raw regex strings or [pattern, label] pairs. + raw_patterns = cfg.get("redaction_patterns", None) + if raw_patterns is None: + redact_patterns = list(DEFAULT_REDACT_PATTERNS) + else: + redact_patterns = [] + for entry in raw_patterns: + if isinstance(entry, list) and len(entry) >= 2: + redact_patterns.append((entry[0], entry[1])) + elif isinstance(entry, str): + redact_patterns.append((entry, _UNLABELLED_REDACT)) + + return emotional, philosophical, redact_patterns + + +# --------------------------------------------------------------------------- +# Text processing helpers +# --------------------------------------------------------------------------- + + +def redact_text(text, redact_patterns=None): + if redact_patterns is None: + redact_patterns = DEFAULT_REDACT_PATTERNS + if not isinstance(text, str): + return str(text) + for pattern, replacement in redact_patterns: + text = re.sub(pattern, replacement, text, flags=re.IGNORECASE) + return text + + +def extract_human_text(content): + """Extract only the human-typed text from user message content. + Filters out tool_result blocks and file content dumps.""" + if isinstance(content, str): + return content + + if isinstance(content, list): + texts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + texts.append(block.get("text", "")) + # Skip tool_result blocks entirely - those are tool outputs, not human text + # Skip tool_use blocks - those are agent tool calls + elif isinstance(block, str): + texts.append(block) + return "\n".join(texts) + return str(content) + + +def extract_assistant_text(content): + """Extract only text blocks from assistant content.""" + if isinstance(content, str): + return content + + if isinstance(content, list): + texts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + texts.append(block.get("text", "")) + # Skip tool_use blocks - we just want the narrative text + elif isinstance(block, str): + texts.append(block) + return "\n".join(texts) + return str(content) + + +def is_tool_result_noise(text): + """Check if a human message is just tool result noise.""" + if not text or not text.strip(): + return True + + stripped = text.strip() + + # Pure file content dumps (start with line numbers) + if re.match(r"^\s*\d+[→|]", stripped): + return True + + # Pure diff output + if ( + stripped.startswith("diff --git") + or stripped.startswith("--- a/") + or stripped.startswith("+++ b/") + ): + return True + if re.match(r"^\+[^+]", stripped) and len(stripped.split("\n")) > 5: + return True + if re.match(r"^-[^-]", stripped) and len(stripped.split("\n")) > 5: + return True + + # Pure file path listings + if ( + stripped.startswith("/") + and len(stripped.split("\n")) > 5 + and all(line.startswith("/") for line in stripped.split("\n")) + ): + return True + + # Mostly code (over 60% looks like code) + lines = stripped.split("\n") + code_lines = 0 + for line in lines: + if re.match( + r"^\s*(import |export |const |let |var |function |class |interface |type |if |for |while |return |}\s*$|{\s*$|\)\s*$|^\s*\d+[→|])", + line, + ): + code_lines += 1 + if len(lines) > 3 and code_lines / len(lines) > 0.6: + return True + + # Very short file update confirmations + if re.match(r"^The file .+ has been (updated|created|deleted) successfully\.?$", stripped): + return True + + # Task notification XML blocks + if stripped.startswith(""): + return True + if "" in stripped and "" in stripped: + return True + + # Subagent completion summaries (auto-generated) + if re.match(r'^Agent ".+" (completed|failed)', stripped): + return True + + # Messages that are mostly task-notification content + if stripped.count(" 3: + return True + + # Skill/plugin content dumps (Claude Code plugins injecting instructions) + if stripped.startswith("Base directory for this skill:") or "skills/brainstorming" in stripped: + return True + if stripped.startswith("# Brainstorming Ideas") or "HARD-GATE" in stripped: + return True + if "" in stripped and len(stripped) > 500: + return True + + # Very long plan dumps (>2000 chars, mostly formatted plans not dialogue) + # But keep shorter messages that are plans since they show intent + if len(stripped) > 3000 and stripped.count("\n") > 30: + # Check if it's mostly structural content (headers, lists, code blocks) + structural = stripped.count("\n#") + stripped.count("\n-") + stripped.count("\n```") + if structural > 15: + return True + + return False + + +def truncate(text, max_len=600): + if len(text) <= max_len: + return text + return text[:max_len] + "..." + + +def is_interesting(text, keywords): + text_lower = text.lower() + return any(kw in text_lower for kw in keywords) + + +# --------------------------------------------------------------------------- +# Session parsing +# --------------------------------------------------------------------------- + + +def parse_session(filepath): + session_id = os.path.basename(filepath).replace(".jsonl", "") + messages = [] + ai_title = None + session_timestamp = None + + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + d = json.loads(line) + except json.JSONDecodeError: + continue + + msg_type = d.get("type", "") + + if session_timestamp is None and "timestamp" in d: + session_timestamp = d.get("timestamp") + + if msg_type == "ai-title": + ai_title = d.get("aiTitle", d.get("title", d.get("message", ""))) + + elif msg_type == "user": + msg = d.get("message", {}) + content = extract_human_text(msg.get("content", "")) + if content and content.strip(): + messages.append( + { + "type": "user", + "role": "user", + "content": content, + "timestamp": d.get("timestamp", ""), + } + ) + + elif msg_type == "assistant": + msg = d.get("message", {}) + content = extract_assistant_text(msg.get("content", "")) + if content and content.strip(): + messages.append( + { + "type": "assistant", + "role": "assistant", + "content": content, + "timestamp": d.get("timestamp", ""), + } + ) + except (json.JSONDecodeError, KeyError, TypeError) as e: + return { + "session_id": session_id, + "error": str(e), + "ai_title": ai_title, + "timestamp": session_timestamp, + } + + # Filter human messages to only genuine human dialogue + genuine_human = [] + for m in messages: + if m["role"] != "user": + continue + text = m["content"].strip() + # Skip IDE notifications + if text.startswith("") or text.startswith(""): + continue + # Skip pure tool result noise + if is_tool_result_noise(text): + continue + genuine_human.append(m) + + assistant_msgs = [m for m in messages if m["role"] == "assistant"] + + # Generate a fallback title from first substantive human message + fallback_title = None + for m in genuine_human: + text = m["content"].strip() + if len(text) > 10 and not text.startswith("[Request interrupted"): + first_line = text.split("\n")[0].strip() + fallback_title = first_line[:80] + break + + return { + "session_id": session_id, + "ai_title": ai_title, + "fallback_title": fallback_title, + "timestamp": session_timestamp, + "human_messages": genuine_human, + "assistant_messages": assistant_msgs, + "total_human": len(genuine_human), + "total_assistant": len(assistant_msgs), + } + + +# --------------------------------------------------------------------------- +# Narrative extraction +# --------------------------------------------------------------------------- + + +def extract_narrative( + session_data, emotional_keywords=None, philosophical_keywords=None, redact_patterns=None +): + if emotional_keywords is None: + emotional_keywords = DEFAULT_EMOTIONAL_KEYWORDS + if philosophical_keywords is None: + philosophical_keywords = DEFAULT_PHILOSOPHICAL_KEYWORDS + if redact_patterns is None: + redact_patterns = DEFAULT_REDACT_PATTERNS + + sid = session_data["session_id"] + title = session_data.get("ai_title") or session_data.get("fallback_title") or "Untitled Session" + timestamp = session_data.get("timestamp", "") + human_msgs = session_data.get("human_messages", []) + assistant_msgs = session_data.get("assistant_messages", []) + + if session_data.get("error"): + return f"## Session: Error\n**ID**: `{sid}`\n**Error**: {session_data['error']}\n\n---\n\n" + + if not human_msgs and not assistant_msgs: + return f"## Session: Empty\n**ID**: `{sid}`\n*No messages found.*\n\n---\n\n" + + ts_display = "" + if timestamp: + try: + dt = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + ts_display = dt.strftime("%Y-%m-%d %H:%M") + except Exception: + ts_display = timestamp[:16] + + md = f"## Session: {redact_text(title, redact_patterns)}\n" + md += f"**ID**: `{sid}` \n" + md += f"**Date**: {ts_display} \n" + md += f"**Messages**: {len(human_msgs)} human, {len(assistant_msgs)} assistant \n\n" + + # --- HUMAN INTENT --- + md += "### Human Intent\n\n" + first_real = None + for m in human_msgs: + text = m["content"].strip() + if len(text) > 20: + first_real = text + break + + if first_real: + md += f"> {redact_text(truncate(first_real, 500), redact_patterns)}\n\n" + + # --- ALL GENUINE HUMAN MESSAGES --- + md += "### Human Messages\n\n" + for i, m in enumerate(human_msgs): + text = m["content"].strip() + if not text: + continue + redacted = redact_text(text, redact_patterns) + md += f"**[{i + 1}]** {truncate(redacted, 500)}\n\n" + + # --- KEY ASSISTANT MOMENTS --- + md += "### Key Assistant Responses\n\n" + + interesting_assistant = [] + for idx, m in enumerate(assistant_msgs): + text = m["content"] + if is_interesting(text, emotional_keywords) or is_interesting(text, philosophical_keywords): + interesting_assistant.append(idx) + + indices_to_show = set() + if assistant_msgs: + indices_to_show.add(0) + if len(assistant_msgs) > 1: + indices_to_show.add(len(assistant_msgs) - 1) + for idx in interesting_assistant[:5]: + indices_to_show.add(idx) + if len(indices_to_show) >= 7: + break + + for idx in sorted(indices_to_show): + m = assistant_msgs[idx] + text = redact_text(truncate(m["content"], 500), redact_patterns) + md += f"**[Assistant #{idx + 1}]** {text}\n\n" + + # --- EMOTIONAL MOMENTS --- + emotional_human = [m for m in human_msgs if is_interesting(m["content"], emotional_keywords)] + emotional_assistant = [ + m for m in assistant_msgs if is_interesting(m["content"], emotional_keywords) + ] + + if emotional_human or emotional_assistant: + md += "### Emotional Moments\n\n" + for m in emotional_human[:4]: + text = redact_text(truncate(m["content"], 400), redact_patterns) + md += f"**[Human]** {text}\n\n" + for m in emotional_assistant[:4]: + text = redact_text(truncate(m["content"], 400), redact_patterns) + md += f"**[Agent]** {text}\n\n" + + # --- PHILOSOPHICAL MOMENTS --- + phil_human = [m for m in human_msgs if is_interesting(m["content"], philosophical_keywords)] + phil_assistant = [ + m for m in assistant_msgs if is_interesting(m["content"], philosophical_keywords) + ] + + if phil_human or phil_assistant: + md += "### Philosophical Moments\n\n" + for m in phil_human[:4]: + text = redact_text(truncate(m["content"], 400), redact_patterns) + md += f"**[Human]** {text}\n\n" + for m in phil_assistant[:4]: + text = redact_text(truncate(m["content"], 400), redact_patterns) + md += f"**[Agent]** {text}\n\n" + + # --- CREATIVE DECISIONS --- + decision_keywords = [ + "let's call", + "named", + "rename", + "should we", + "i think", + "what about", + "how about", + "instead of", + "better to", + "let's use", + "let's go with", + "the name", + "naming", + "i prefer", + "design decision", + "architectural", + "i want", + "i don't want", + "make it", + "this should", + "this needs to", + "the idea is", + "concept here", + "the approach", + ] + decision_msgs = [m for m in human_msgs if is_interesting(m["content"], decision_keywords)] + if decision_msgs: + md += "### Creative/Design Decisions\n\n" + for m in decision_msgs[:5]: + text = redact_text(truncate(m["content"], 400), redact_patterns) + md += f"**[Human]** {text}\n\n" + + # --- TECHNICAL BREAKTHROUGHS --- + breakthrough_keywords = [ + "finally works", + "got it working", + "this works", + "test passes", + "all tests pass", + "build passes", + "success", + "breakthrough", + "figured out", + "solved", + "the fix", + "working now", + "it's alive", + "that did it", + "nailed it", + "perfect", + ] + tech_msgs = [ + m + for m in human_msgs + assistant_msgs + if is_interesting(m["content"], breakthrough_keywords) + ] + if tech_msgs: + md += "### Technical Breakthroughs\n\n" + for m in tech_msgs[:4]: + role = "Human" if m["role"] == "user" else "Agent" + text = redact_text(truncate(m["content"], 400), redact_patterns) + md += f"**[{role}]** {text}\n\n" + + md += "---\n\n" + return md + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def parse_args(argv=None): + parser = argparse.ArgumentParser( + description="Extract narrative material from Claude Code JSONL session files." + ) + parser.add_argument( + "--sessions-dir", + default=os.path.expanduser("~/.claude/projects/"), + help="Directory containing .jsonl session files (default: ~/.claude/projects/)", + ) + parser.add_argument( + "--output", + default=None, + help="Output markdown file (default: /data/raw-sessions.md)", + ) + parser.add_argument( + "--config", default=None, help="Path to a JSON config file with session_extraction settings" + ) + parser.add_argument( + "--project", + default=None, + help="Project name; loads projects//project.json for config", + ) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + + sessions_dir = os.path.expanduser(args.sessions_dir) + if not os.path.isdir(sessions_dir): + print(f"Error: sessions directory not found: {sessions_dir}", file=sys.stderr) + sys.exit(1) + + # Resolve output path + output_file = args.output + if output_file is None: + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + output_file = os.path.join(repo_root, "data", "raw-sessions.md") + + os.makedirs(os.path.dirname(output_file), exist_ok=True) + + # Resolve config (keywords + redaction patterns) + emotional_kw, philosophical_kw, redact_pat = resolve_config(args) + + files = sorted(glob.glob(os.path.join(sessions_dir, "*.jsonl"))) + print(f"Found {len(files)} session files in {sessions_dir}") + + sessions = [] + for f in files: + session_id = os.path.basename(f).replace(".jsonl", "") + print(f" Parsing {session_id}...") + data = parse_session(f) + sessions.append(data) + + # Sort by timestamp + sessions.sort(key=lambda s: s.get("timestamp") or "") + + # Derive project name for the header (from --project arg or cwd) + project_name = args.project or os.path.basename(os.path.abspath(".")) + + output_parts = [] + output_parts.append(f"# {project_name.title()} Session Narratives\n\n") + output_parts.append(f"Extracted from {len(files)} Claude Code session logs.\n") + output_parts.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}\n") + output_parts.append("Note: Tool-result noise (file contents, diffs, code dumps) filtered out. ") + output_parts.append("Only genuine human dialogue and agent narrative text included.\n\n") + output_parts.append("---\n\n") + + for data in sessions: + print( + f" Extracting narrative for {data['session_id']} ({data['total_human']} human msgs)..." + ) + narrative = extract_narrative( + data, + emotional_keywords=emotional_kw, + philosophical_keywords=philosophical_kw, + redact_patterns=redact_pat, + ) + output_parts.append(narrative) + + full_output = "".join(output_parts) + with open(output_file, "w", encoding="utf-8") as f: + f.write(full_output) + + print(f"\nDone! Written to {output_file}") + print(f"Total size: {len(full_output):,} characters") + + total_human = sum(s.get("total_human", 0) for s in sessions) + total_assistant = sum(s.get("total_assistant", 0) for s in sessions) + sessions_with_content = sum(1 for s in sessions if s.get("total_human", 0) > 0) + print(f"Sessions with human dialogue: {sessions_with_content}/{len(sessions)}") + print(f"Total genuine human messages: {total_human}") + print(f"Total assistant messages: {total_assistant}") + + +if __name__ == "__main__": + main() diff --git a/plugins/devarch-hermes/archaeology/local_pipeline.py b/plugins/devarch-hermes/archaeology/local_pipeline.py new file mode 100644 index 0000000..32692d2 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/local_pipeline.py @@ -0,0 +1,174 @@ +"""Helpers for the local GITHUB_pipeline verification authority.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +def _get_dir(env_var: str, label: str) -> Path: + """Resolve a directory from an environment variable.""" + env_val = os.environ.get(env_var, "") + if env_val: + return Path(env_val) + raise OSError(f"{env_var} environment variable not set. Please set it to {label}.") + + +DEFAULT_PIPELINE_DIR: Path | None = None +DEFAULT_REPOS_DIR: Path | None = None + + +@dataclass(frozen=True) +class LocalPipelineStatus: + run_timestamp: str | None + overall_health: str | None + repo: str + repo_health: str | None + repo_verdict: str | None + issues: dict[str, Any] + open_prs: int | None + open_issues: int | None + latest_json: Path + + @property + def issue_total(self) -> int: + try: + return int(self.issues.get("total", 0)) + except (TypeError, ValueError, AttributeError): + return 0 + + +def latest_json_path(pipeline_dir: str | Path) -> Path: + return Path(pipeline_dir) / ".omc" / "logs" / "repo-pipeline" / "latest.json" + + +def run_local_pipeline( + pipeline_dir: str | Path = DEFAULT_PIPELINE_DIR, + repos_dir: str | Path = DEFAULT_REPOS_DIR, + top_repos: int = 20, + review_days: int = 30, +) -> None: + """Run the deterministic local pipeline producer.""" + pipeline_dir = Path(pipeline_dir) + cmd = [ + sys.executable, + "scripts/run-pipeline-once.py", + "--top-repos", + str(top_repos), + "--review-days", + str(review_days), + ] + env = os.environ.copy() + env.update( + { + "PIPELINE_REPOS_DIR": str(repos_dir), + "PIPELINE_TOP_REPOS": str(top_repos), + "PIPELINE_REVIEW_DAYS": str(review_days), + } + ) + subprocess.run(cmd, cwd=pipeline_dir, env=env, check=True, timeout=300) + + +def read_local_pipeline_status(pipeline_dir: str | Path, repo_name: str) -> LocalPipelineStatus: + """Read latest local pipeline JSON and extract one repo's status.""" + path = latest_json_path(pipeline_dir) + if not path.exists(): + raise FileNotFoundError(f"Local pipeline latest.json not found: {path}") + payload = json.loads(path.read_text(encoding="utf-8")) + + # Try to find repo in supervision.missions first (new format) + target = None + supervision = payload.get("supervision", {}) + if isinstance(supervision, dict): + missions = supervision.get("missions", []) + if missions and isinstance(missions, list): + for mission in missions: + if isinstance(mission, dict): + mission_repo = mission.get("repo", "") + # Match against owner/repo or just repo name + if ( + mission_repo == repo_name + or mission_repo.endswith("/" + repo_name) + or repo_name.endswith("/" + mission_repo.split("/")[-1]) + ): + target = mission + break + + # Fall back to repos array (old format) + if target is None: + for repo in payload.get("repos", []): + names = { + str(repo.get("name", "")), + str(repo.get("full_name", "")), + str(repo.get("path", "")), + } + if repo_name in names or repo_name.endswith("/" + str(repo.get("name", ""))): + target = repo + break + + if target is None: + # Build helpful error message with available repos + reviewed_repos = [] + supervision = payload.get("supervision", {}) + if isinstance(supervision, dict): + missions = supervision.get("missions", []) + if missions: + reviewed_repos = [m.get("repo", "") for m in missions if isinstance(m, dict)] + if not reviewed_repos: + reviewed_repos = [repo.get("name", "") for repo in payload.get("repos", [])] + reviewed = ", ".join(str(r) for r in reviewed_repos) + raise ValueError( + f"Repo '{repo_name}' not found in latest local pipeline reviewed repos. Reviewed: {reviewed}" + ) + + summary = payload.get("summary", {}) + # Normalize issues to dict - pipeline may return list or dict + raw_issues = target.get("issues") + if isinstance(raw_issues, list): + issues = {"items": raw_issues, "total": len(raw_issues)} + else: + issues = raw_issues or {} + + # Extract repo name from multiple possible fields + repo_full_name = ( + target.get("repo") # new format + or target.get("full_name") # old format + or target.get("path") + or target.get("name", "") + ) + + # Extract health/verdict from mission or repo data + repo_health = target.get("health") or target.get("status") + repo_verdict = target.get("verdict") + + return LocalPipelineStatus( + run_timestamp=payload.get("run_timestamp"), + overall_health=summary.get("overall_health"), + repo=str(repo_full_name), + repo_health=repo_health, + repo_verdict=repo_verdict, + issues=issues, + open_prs=target.get("open_prs"), + open_issues=target.get("open_issues"), + latest_json=path, + ) + + +def status_lines(status: LocalPipelineStatus) -> list[str]: + """Human-readable status lines for CLI output.""" + return [ + f"run_timestamp: {status.run_timestamp}", + f"summary_overall_health: {status.overall_health}", + f"repo: {status.repo}", + f"health: {status.repo_health}", + f"verdict: {status.repo_verdict}", + f"issues: {json.dumps(status.issues, sort_keys=True)}", + f"open_prs: {status.open_prs}", + f"open_issues: {status.open_issues}", + f"latest_json: {status.latest_json}", + ] diff --git a/plugins/devarch-hermes/archaeology/metrics.py b/plugins/devarch-hermes/archaeology/metrics.py new file mode 100644 index 0000000..8fe9d9b --- /dev/null +++ b/plugins/devarch-hermes/archaeology/metrics.py @@ -0,0 +1,67 @@ +"""Canonical metrics from the mined commits, reconciled against SQLite.""" + +import csv +import json +import sqlite3 +from collections import Counter +from contextlib import closing +from pathlib import Path + +from .utils import _parse_date, atomic_write + + +def write_metrics(project_root: Path, db_path: Path) -> dict: + csv_path = project_root / "data" / "github-commits.csv" + if not csv_path.exists(): + return {} + with csv_path.open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + hashes = [row["hash"] for row in rows] + with closing(sqlite3.connect(db_path)) as conn: + stored = [row[0] for row in conn.execute("SELECT hash FROM commits")] + if len(set(hashes)) != len(hashes) or sorted(hashes) != sorted(stored): + raise ValueError("CSV/SQLite commit identities differ or contain duplicates") + metrics = calculate_metrics(rows) + atomic_write( + project_root / "deliverables" / "canonical-metrics.json", json.dumps(metrics, indent=2) + ) + atomic_write( + project_root / "deliverables" / "data.json", + json.dumps( + { + **metrics, + "telemetry_visualizations": {"meta": metrics}, + }, + indent=2, + ), + ) + from .provenance import bind_derived_artifact + + bind_derived_artifact(project_root, project_root / "deliverables" / "canonical-metrics.json") + bind_derived_artifact(project_root, project_root / "deliverables" / "data.json") + return metrics + + +def calculate_metrics(rows: list[dict]) -> dict: + """Derive canonical values from source rows without writing artifacts.""" + days = Counter() + for row in rows: + parsed = _parse_date(row["date"]) + if parsed is None: + raise ValueError(f"Invalid commit date for {row['hash']}") + days[parsed.date().isoformat()] += 1 + dates = sorted(days) + peak = min(days, key=lambda d: (-days[d], d)) if days else None + metrics = { + "total_commits": len(rows), + "active_days": len(days), + "daily_commits": dict(sorted(days.items())), + "first_commit_date": dates[0] if dates else None, + "last_commit_date": dates[-1] if dates else None, + "span_days": (_parse_date(dates[-1]) - _parse_date(dates[0])).days + 1 if dates else 0, + "peak_day": peak, + "peak_day_commits": days[peak] if peak else 0, + "date_basis": "author timestamps normalized to UTC; inclusive calendar span", + "source_scope": "all locally mined refs, not proof of all remote history", + } + return metrics diff --git a/plugins/devarch-hermes/archaeology/opportunity_analyzers.py b/plugins/devarch-hermes/archaeology/opportunity_analyzers.py new file mode 100644 index 0000000..14b9d3d --- /dev/null +++ b/plugins/devarch-hermes/archaeology/opportunity_analyzers.py @@ -0,0 +1,14 @@ +"""Legacy opportunity inference is disabled: commit history cannot establish personal profiles.""" + + +class OpportunityAnalyzer: + ANALYZERS = [] + + def __init__(self, *args, **kwargs): + raise RuntimeError( + "opportunity is disabled: unsupported personal profiles and measurements" + ) + + +def run_opportunity_analyzers(*args, **kwargs): + raise RuntimeError("opportunity is disabled: unsupported personal profiles and measurements") diff --git a/plugins/devarch-hermes/archaeology/provenance.py b/plugins/devarch-hermes/archaeology/provenance.py new file mode 100644 index 0000000..b1a4125 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/provenance.py @@ -0,0 +1,203 @@ +"""Bind derived evidence to the exact local inputs, without claiming signed provenance.""" + +import hashlib +import json +from pathlib import Path + +from .utils import atomic_write + +INPUTS = ( + "project.json", + "data/coverage.json", + "data/github-commits.csv", + "data/github-commits-with-stats.txt", + "data/archaeology.db", + "data/detected-signals.json", + "data/commit-eras.json", + "data/gap-clusters.json", + "deliverables/canonical-metrics.json", + "deliverables/data.json", +) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + + +def snapshot(project): + project = Path(project) + return {name: digest(project / name) for name in INPUTS} + + +def requires_binding(project): + project = Path(project) + config_path = project / "project.json" + config = json.loads(config_path.read_text(encoding="utf-8")) if config_path.exists() else {} + return bool( + config.get("mined_history_manifest_required") or (project / "data/coverage.json").exists() + ) + + +def analysis_paths(project): + deliverables = Path(project) / "deliverables" + paths = set(deliverables.glob("analysis-*.json")) | set( + (deliverables / "analysis").glob("analysis-*.json") + ) + return sorted(p for p in paths if not p.name.endswith(".provenance.json")) + + +def verify_analyses(project, require=False): + if not requires_binding(project): + return + paths = analysis_paths(project) + if require and not paths: + raise ValueError("Missing bound analysis; run analyze before exporting") + current = snapshot(project) + for path in paths: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("evidence_binding") != current: + raise ValueError(f"Stale or unbound analysis: {path.name}; rerun signals and analyze") + verify_artifact(project, path) + + +def bind_artifact(project, artifact, include_analysis=False): + project, artifact = Path(project), Path(artifact) + if not requires_binding(project): + return + binding = {"snapshot": snapshot(project), "artifact": artifact.name, "sha256": digest(artifact)} + if include_analysis: + binding["analysis"] = { + str(p.relative_to(project)): digest(p) for p in analysis_paths(project) + } + atomic_write(str(artifact) + ".provenance.json", json.dumps(binding, indent=2)) + + +def verify_artifact(project, artifact): + project, artifact = Path(project), Path(artifact) + if not requires_binding(project): + return + sidecar = Path(str(artifact) + ".provenance.json") + try: + binding = json.loads(sidecar.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"Missing or invalid evidence binding: {artifact.name}") from exc + if ( + not isinstance(binding, dict) + or binding.get("artifact") != artifact.name + or binding.get("sha256") != digest(artifact) + or binding.get("snapshot") != snapshot(project) + ): + raise ValueError(f"Stale or modified artifact: {artifact.name}; regenerate it") + if "analysis" in binding: + current = {str(p.relative_to(project)): digest(p) for p in analysis_paths(project)} + if binding["analysis"] != current: + raise ValueError(f"Stale analysis inputs for artifact: {artifact.name}; re-export") + + +def _load_coverage(project): + coverage_path = Path(project) / "data" / "coverage.json" + if not coverage_path.is_file(): + raise ValueError("Missing data/coverage.json for bound mining project") + try: + coverage = json.loads(coverage_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError("Invalid data/coverage.json") from exc + if not isinstance(coverage, dict): + raise ValueError("Invalid data/coverage.json") + return coverage + + +def verify_mining_inputs(project): + """Fail when mined CSV/stats no longer match coverage.json artifact hashes.""" + project = Path(project) + if not requires_binding(project): + return + coverage = _load_coverage(project) + hashes = coverage.get("artifact_sha256") + if not isinstance(hashes, dict) or not hashes: + raise ValueError("Mining coverage missing artifact_sha256 bindings") + for name, expected in hashes.items(): + path = project / "data" / name + actual = digest(path) + if actual != expected: + raise ValueError(f"Stale or modified mining artifact: {name}; remine before export") + + +def verify_derived_stage(project): + """Refuse export when DB/metrics/signals belong to a different or incomplete mining generation. + + A failed remine may refresh coverage.json + raw CSV hashes while leaving an older + archaeology.db / metrics / signals tree in place. Export must not publish that mix. + + Legacy CLI coverage (no mining_status / mining_generation) only requires that the + SQLite DB exist. Plugin mining always stamps generation + derived_sha256 and must + pass the full binding checks. + """ + project = Path(project) + if not requires_binding(project): + return + coverage = _load_coverage(project) + status = coverage.get("mining_status") + generation = coverage.get("mining_generation") + db_path = project / "data" / "archaeology.db" + + # Legacy manifests from the framework CLI mine path. + if status is None and generation is None: + if not db_path.is_file(): + raise ValueError("Missing archaeology.db; remine before export") + return + + if status != "complete": + raise ValueError( + "Mining rebuild incomplete or unbound; remine successfully before export " + f"(mining_status={status!r})" + ) + if not generation: + raise ValueError("Mining coverage missing mining_generation; remine before export") + derived = coverage.get("derived_sha256") + if not isinstance(derived, dict) or not derived: + raise ValueError("Mining coverage missing derived_sha256 bindings; remine before export") + expected_db = derived.get("archaeology.db") + actual_db = digest(db_path) + if not expected_db or actual_db != expected_db: + raise ValueError( + "Stale or unbound archaeology.db for current mining generation; remine before export" + ) + # Optional derived JSON must declare the same generation when present. + for rel in ( + "deliverables/canonical-metrics.json", + "data/detected-signals.json", + "data/gap-clusters.json", + "deliverables/data.json", + ): + path = project / rel + if not path.is_file(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"Invalid derived artifact: {rel}") from exc + if not isinstance(payload, dict) or payload.get("mining_generation") != generation: + raise ValueError(f"Unbound derived artifact: {rel}; rebuild signals/metrics") + sidecar = path.with_name(path.name + ".generation.json") + try: + binding = json.loads(sidecar.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"Missing derived content binding: {rel}") from exc + if binding != {"mining_generation": generation, "sha256": digest(path)}: + raise ValueError(f"Modified derived artifact: {rel}; rebuild signals/metrics") + + +def bind_derived_artifact(project, artifact): + project, artifact = Path(project), Path(artifact) + coverage_path = project / "data" / "coverage.json" + if not coverage_path.is_file(): + return + generation = _load_coverage(project).get("mining_generation") + if not generation: + return + payload = json.loads(artifact.read_text(encoding="utf-8")) + payload["mining_generation"] = generation + atomic_write(artifact, json.dumps(payload, indent=2)) + binding = {"mining_generation": generation, "sha256": digest(artifact)} + atomic_write(str(artifact) + ".generation.json", json.dumps(binding, indent=2)) diff --git a/plugins/devarch-hermes/archaeology/report.py b/plugins/devarch-hermes/archaeology/report.py new file mode 100644 index 0000000..a3eca63 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/report.py @@ -0,0 +1,571 @@ +"""Report export utilities for DevArch Framework.""" + +from __future__ import annotations + +import html +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from .visualization.design_system import ( + THEME_SWITCHER_HTML, + body_end_bundle, + head_bundle, +) + +ANALYSIS_FILES = [ + "analysis-sdlc-gap-finder.json", + "analysis-ml-pattern-mapper.json", + "analysis-agentic-workflow.json", + "analysis-formal-terms-mapper.json", + "analysis-source-archaeologist.json", + "analysis-youtube-correlator.json", +] + + +def _load_json(path: Path) -> Any | None: + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + + +def _bullet(value: str) -> str: + return f"- {value}\n" + + +def _fmt_count(value: Any) -> str: + if isinstance(value, int): + return f"{value:,}" + return str(value) if value is not None else "unknown" + + +def export_markdown_report( + project_name: str, project_root: str | Path, output_path: str | Path | None = None +) -> Path: + """Export a concise Markdown report from canonical metrics + analysis JSON.""" + project_root = Path(project_root) + deliverables = project_root / "deliverables" + from .provenance import ( + analysis_paths, + verify_analyses, + verify_derived_stage, + verify_mining_inputs, + ) + + # Fail closed when mined CSV/stats diverge from coverage.json hashes. + verify_mining_inputs(project_root) + # Refuse mixed-generation export after a failed remine (old DB/metrics under new coverage). + verify_derived_stage(project_root) + # Require bound analyses only when analysis artifacts exist; signals-only + # reports remain valid after mine → signals. + verify_analyses(project_root, require=bool(analysis_paths(project_root))) + analysis_dir = deliverables / "analysis" + data_dir = project_root / "data" + project = _load_json(project_root / "project.json") or {} + canonical = _load_json(deliverables / "canonical-metrics.json") or {} + eras = _load_json(data_dir / "commit-eras.json") or {} + gap_clusters = _load_json(data_dir / "gap-clusters.json") or {} + detected = _load_json(data_dir / "detected-signals.json") or {} + # Look for analysis files in analysis/ subdirectory first, then fall back to deliverables root + analysis_search = [analysis_dir, deliverables] + analyses = {} + for name in ANALYSIS_FILES: + for search_dir in analysis_search: + loaded = _load_json(search_dir / name) + if loaded: + analyses[name.replace("analysis-", "").replace(".json", "")] = loaded + break + + title = project.get("visualization", {}).get("title") or project.get("name") or project_name + out = [] + out.append(f"# {title} Archaeology Report\n\n") + out.append(f"Generated: {datetime.now().isoformat()}\n\n") + out.append("## Executive Summary\n\n") + out.append( + f"This report summarizes the `{project_name}` development archaeology from canonical project metrics, era data, and automated analysis vectors.\n\n" + ) + + out.append( + "Automated vectors are commit-keyword investigation leads, not verified source findings, causal explanations, session measurements or implementation-quality scores. Missing evidence is not absence.\n\n" + ) + coverage = _load_json(data_dir / "coverage.json") or {} + out.append("## Local History Coverage\n\n") + out.append(f"Scope: {coverage.get('scope', 'not recorded')}\n\n") + out.append( + f"Sampled history: {coverage.get('sampled', False)}; extracted " + f"{coverage.get('extracted_count', coverage.get('commit_count', 'unknown'))} of " + f"{coverage.get('commit_count', 'unknown')} locally reachable commits; " + f"cap={coverage.get('max_commits', 'none')}. " + "Metrics, signals and clusters describe only this extraction, not all history.\n\n" + ) + out.append( + f"Local refs recorded: {len(coverage.get('refs', []))}. " + "Deleted/inaccessible/unfetched refs are outside coverage; no remote fetch.\n\n" + ) + out.append("## Canonical Metrics\n\n") + metric_rows = [ + ("Total commits", canonical.get("total_commits", eras.get("total_commits"))), + ("Span days", canonical.get("span_days")), + ("Active days", canonical.get("active_days")), + ("Peak day", canonical.get("peak_day")), + ("Peak day commits", canonical.get("peak_day_commits")), + ] + for label, value in metric_rows: + out.append(_bullet(f"**{label}:** {_fmt_count(value)}")) + out.append("\n") + + era_list = eras.get("eras") if isinstance(eras, dict) else [] + cluster_file_list = gap_clusters.get("eras") if isinstance(gap_clusters, dict) else [] + era_semantics = "" + if isinstance(gap_clusters, dict) and gap_clusters.get("era_semantics"): + era_semantics = str(gap_clusters.get("era_semantics") or "") + elif isinstance(eras, dict) and eras.get("era_semantics"): + era_semantics = str(eras.get("era_semantics") or "") + out.append("## Era Semantics\n\n") + if era_semantics: + out.append(f"{era_semantics}\n\n") + else: + out.append( + "Automated archaeology emits **signals** and **gap-separated clusters**. " + "Clusters are heuristic groupings, not curated human development eras. " + "A `commit-eras.json` narrative era list is optional and human-authored; " + "generated clusters live in `gap-clusters.json`.\n\n" + ) + + # Prefer curated human eras from commit-eras.json; never treat gap clusters as eras. + curated = [ + e for e in (era_list or []) if isinstance(e, dict) and e.get("kind") != "gap_cluster" + ] + legacy_clusters_in_eras = [ + e for e in (era_list or []) if isinstance(e, dict) and e.get("kind") == "gap_cluster" + ] + if curated: + out.append("## Development Eras\n\n") + for era in curated: + out.append( + _bullet( + f"**Era {era.get('id')}: {era.get('name')}** — {era.get('dates', 'unknown dates')}; " + f"{era.get('commits', 'unknown')} commits. " + f"{era.get('description') or era.get('narrative_arc') or ''}" + ) + ) + out.append("\n") + + display_clusters = list(cluster_file_list or []) or list(legacy_clusters_in_eras or []) + if display_clusters: + out.append("## Gap-Separated Clusters\n\n") + for era in display_clusters: + if not isinstance(era, dict): + continue + out.append( + _bullet( + f"**Cluster {era.get('id')}: {era.get('name')}** — {era.get('dates', 'unknown dates')}; " + f"{era.get('commits', 'unknown')} commits. " + f"{era.get('description') or era.get('narrative_arc') or ''}" + ) + ) + out.append("\n") + + # Measured signals from detect_signals (always prefer evidence over silence). + signal_list = [] + cluster_list = [] + if isinstance(detected, dict): + signal_list = detected.get("signals") or [] + cluster_list = detected.get("cluster_summary") or [] + if signal_list or cluster_list: + out.append("## Detected Signals\n\n") + out.append( + f"Measured {len(signal_list)} signal(s) and {len(cluster_list)} gap cluster(s). " + "These are heuristic investigation leads, not verified conclusions.\n\n" + ) + for sig in list(signal_list)[:25]: + if not isinstance(sig, dict): + out.append(_bullet(str(sig)[:240])) + continue + detail = sig.get("detail") or sig.get("description") or sig.get("message") or "" + out.append( + _bullet( + f"**{sig.get('type') or sig.get('kind') or 'signal'}** " + f"@ {sig.get('date') or sig.get('when') or 'unknown'}: {detail}" + ) + ) + if cluster_list and not display_clusters: + out.append("\n### Clusters (from signals)\n\n") + for cluster in list(cluster_list)[:25]: + if not isinstance(cluster, dict): + continue + out.append( + _bullet( + f"{cluster.get('start_date')} → {cluster.get('end_date')} " + f"({cluster.get('commit_count')} commits, {cluster.get('active_days')} active days)" + ) + ) + out.append("\n") + + sdlc = analyses.get("sdlc-gap-finder") or {} + gaps = sdlc.get("gaps") or [] + if gaps: + out.append("## SDLC / Process Gaps\n\n") + for gap in gaps[:10]: + out.append( + _bullet( + f"**{gap.get('practice')}** — {gap.get('status')} ({gap.get('severity')}). {gap.get('recommendation')}" + ) + ) + out.append("\n") + + ml = analyses.get("ml-pattern-mapper") or {} + mappings = ml.get("mappings") or [] + if mappings: + out.append("## Formal ML / Architecture Patterns\n\n") + for mapping in mappings[:10]: + out.append( + _bullet( + f"**{mapping.get('intuitive_name')}** → {mapping.get('formal_term')} (confidence: {mapping.get('confidence')})" + ) + ) + out.append("\n") + + formal = analyses.get("formal-terms-mapper") or {} + terms = formal.get("term_dictionary") or [] + if terms: + out.append("## Vocabulary Translation\n\n") + for term in terms[:10]: + out.append( + _bullet( + f"**{term.get('code_name')}** → {term.get('formal_term')} ({term.get('similarity_score')})" + ) + ) + out.append("\n") + + source = analyses.get("source-archaeologist") or {} + improvements = source.get("improvements") or [] + if improvements: + out.append("## Remediation Priorities\n\n") + for item in improvements: + out.append( + _bullet( + f"P{item.get('rank')}: **{item.get('title')}** — effort {item.get('effort')}, impact {item.get('impact')}" + ) + ) + out.append("\n") + + youtube = analyses.get("youtube-correlator") or {} + yt_summary = youtube.get("summary") or {} + out.append("## Behavioral / External Data\n\n") + out.append( + _bullet(f"YouTube/behavioral data available: {bool(yt_summary.get('data_available'))}") + ) + out.append(_bullet(f"Correlations found: {_fmt_count(yt_summary.get('correlation_count'))}")) + out.append(_bullet(f"Creator count: {_fmt_count(yt_summary.get('creator_count'))}")) + out.append("\n") + + out.append("## Provenance\n\n") + out.append(_bullet(f"Source scope: {canonical.get('source_scope', 'project data')}")) + out.append(_bullet("Generated from automated `devarch analyze` JSON outputs.")) + out.append(_bullet("Run `devarch audit --fail-on HIGH` before publishing.")) + + if output_path is None: + output_path = deliverables / "reports" / "ARCHAEOLOGY-REPORT.md" + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text("".join(out), encoding="utf-8") + from .provenance import bind_artifact + + bind_artifact(project_root, output, include_analysis=True) + return output + + +def _markdown_to_html(markdown: str, title: str) -> str: + """Render the report's constrained Markdown subset to standalone HTML using the unified design system.""" + body: list[str] = [] + in_list = False + + def close_list() -> None: + nonlocal in_list + if in_list: + body.append("") + in_list = False + + def render_inline(text: str) -> str: + """Convert **bold** and `code` inline markup to HTML.""" + import re + + text = html.escape(text) + text = re.sub(r"\*\*(.+?)\*\*", r"\1", text) + text = re.sub(r"`(.+?)`", r"\1", text) + return text + + for raw_line in markdown.splitlines(): + line = raw_line.rstrip() + if not line: + close_list() + continue + if line.startswith("# "): + close_list() + body.append(f"

{render_inline(line[2:])}

") + elif line.startswith("## "): + close_list() + body.append(f"

{render_inline(line[3:])}

") + elif line.startswith("- "): + if not in_list: + body.append("
    ") + in_list = True + body.append(f"
  • {render_inline(line[2:])}
    • ") + else: + close_list() + body.append(f"

    {render_inline(line)}

    ") + close_list() + body_html = "\n ".join(body) + escaped_title = html.escape(title) + description = f"Archaeological analysis report for {escaped_title}" + + # Use the unified design system's head_bundle + head_content = head_bundle( + title=f"{escaped_title}", + description=description, + include_charts=False, + include_d3=False, + ) + + return f""" + + +{head_content} + + + + + + + +
    +
    Archaeology Report
    + {body_html} +
    + + {body_end_bundle()} + + +""" + + +def export_html_report( + project_name: str, project_root: str | Path, output_path: str | Path | None = None +) -> Path: + """Export a standalone HTML report from the Markdown report content.""" + project_root = Path(project_root) + deliverables = project_root / "deliverables" + markdown_path = export_markdown_report(project_name, project_root) + markdown = markdown_path.read_text(encoding="utf-8") + project = _load_json(project_root / "project.json") or {} + title = project.get("visualization", {}).get("title") or project.get("name") or project_name + if output_path is None: + output_path = deliverables / "visuals" / "report.html" + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(_markdown_to_html(markdown, f"{title} Archaeology Report"), encoding="utf-8") + from .provenance import bind_artifact + + bind_artifact(project_root, output, include_analysis=True) + return output + + +def export_report( + project_name: str, + project_root: str | Path, + output_path: str | Path | None = None, + fmt: str = "markdown", +) -> Path: + """Export a report in markdown or html format.""" + if fmt in {"markdown", "md"}: + return export_markdown_report(project_name, project_root, output_path=output_path) + if fmt == "html": + return export_html_report(project_name, project_root, output_path=output_path) + raise ValueError(f"Unsupported report format: {fmt}") + + +def export_public_case_study( + root: str | Path = ".", + output_dir: str | Path = "public-case-study", + project_name: str = "demo-archaeology", + force: bool = True, +) -> Path: + """Generate a sanitized public case-study showroom from invented demo data.""" + from .analysis_runner import run_analysis_vectors + from .db.builder import build_db + from .demo import create_demo_project + + root = Path(root) + output = root / output_dir + work_project = create_demo_project(root, project_name=project_name, force=force) + + # Build database directly without mutating sys.argv + build_db(work_project, verbose=False) + + run_analysis_vectors(project_name, vectors=None) + md_report = export_markdown_report(project_name, work_project) + html_report = export_html_report(project_name, work_project) + + output.mkdir(parents=True, exist_ok=True) + data_out = output / "data" + data_out.mkdir(parents=True, exist_ok=True) + (output / "ARCHAEOLOGY-REPORT.md").write_text( + md_report.read_text(encoding="utf-8"), encoding="utf-8" + ) + (output / "index.html").write_text(html_report.read_text(encoding="utf-8"), encoding="utf-8") + for src, dst in [ + ( + work_project / "deliverables" / "canonical-metrics.json", + data_out / "canonical-metrics.json", + ), + (work_project / "data" / "commit-eras.json", data_out / "commit-eras.json"), + (work_project / "data" / "github-commits.csv", data_out / "github-commits.csv"), + ]: + dst.write_text(src.read_text(encoding="utf-8"), encoding="utf-8") + (output / "README.md").write_text( + "# Dev-Archaeology Public Case Study\n\n" + "This is a sanitized, publishable demo generated from invented fixture data. " + "It exists to show what Dev-Archaeology produces using invented fixture data only — " + "no private evidence archives, no personal telemetry.\n\n" + "## Open the case study\n\n" + "```text\npublic-case-study/index.html\n```\n\n" + "## Regenerate locally\n\n" + "```bash\narchaeology public-case-study --output public-case-study\n```\n\n" + "## Data safety\n\n" + "The files in `public-case-study/data/` are invented fixture data only:\n\n" + "- no raw session exports\n" + "- no personal watch history\n" + "- no resume or profile data\n" + "- no personal telemetry\n", + encoding="utf-8", + ) + return output diff --git a/plugins/devarch-hermes/archaeology/utils.py b/plugins/devarch-hermes/archaeology/utils.py new file mode 100644 index 0000000..dba906b --- /dev/null +++ b/plugins/devarch-hermes/archaeology/utils.py @@ -0,0 +1,89 @@ +"""Shared utility functions for DevArch Framework modules.""" + +from __future__ import annotations + +import json +import logging +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +_logger = logging.getLogger(__name__) + + +def _load_json(path: Path | str, verbose: bool = False) -> Any | None: + """Load and parse a JSON file. + + Args: + path: Path to the JSON file (can be Path or str). + verbose: If True, print loading status (not used here, kept for API compatibility). + + Returns: + Parsed JSON data, or None if the file doesn't exist or parsing fails. + """ + p = Path(path) if isinstance(path, str) else path + if not p.exists(): + return None + try: + return json.loads(p.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + _logger.warning("JSON parse error in %s: %s", p, e) + return None + except OSError as e: + _logger.warning("I/O error reading %s: %s", p, e) + return None + + +def atomic_write(path: Path | str, content: str, encoding: str = "utf-8") -> None: + """Write content to a file atomically using temp file + rename.""" + p = Path(path) if isinstance(path, str) else path + p.parent.mkdir(parents=True, exist_ok=True) + tmp = p.with_suffix(p.suffix + ".tmp") + try: + tmp.write_text(content, encoding=encoding) + os.replace(tmp, p) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def _parse_date(date_str: str) -> datetime | None: + """Parse a date string into a datetime object. + + Supports multiple date formats from git and other sources. + Returns None for unparseable dates. + + Args: + date_str: Date string to parse. + + Returns: + datetime object if parsing succeeds, None otherwise. + """ + if not date_str: + return None + date_str = str(date_str).strip() + # Normalize timezone-bearing dates to UTC, preserving a naive UTC return + # for existing date arithmetic callers. Legacy trailing annotations remain supported. + candidates = [date_str, date_str[:25], date_str[:26], date_str[:19], date_str[:10]] + for candidate in candidates: + try: + parsed = datetime.fromisoformat(candidate.replace("Z", "+00:00")) + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) + return parsed + except ValueError: + continue + return None + + +def _script_dir() -> Path: + """Return the directory containing the calling script. + + Uses Path(__file__).resolve().parent to get the directory. + Should be called from the module where __file__ is defined. + + Returns: + Path object pointing to the script's directory. + """ + return Path(__file__).resolve().parent diff --git a/plugins/devarch-hermes/archaeology/validators/__init__.py b/plugins/devarch-hermes/archaeology/validators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/devarch-hermes/archaeology/validators/history.py b/plugins/devarch-hermes/archaeology/validators/history.py new file mode 100644 index 0000000..f103a15 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/validators/history.py @@ -0,0 +1,35 @@ +"""Validate the installed CLI's self-contained measured-history output.""" + +from html.parser import HTMLParser +from pathlib import Path + +from ..provenance import verify_artifact + + +class HistoryParser(HTMLParser): + def __init__(self): + super().__init__() + self.tags = set() + self.unsafe = False + + def handle_starttag(self, tag, attrs): + self.tags.add(tag) + if tag in {"script", "iframe", "object", "embed", "form"}: + self.unsafe = True + if any( + k.lower().startswith("on") or (v and v.strip().lower().startswith("javascript:")) + for k, v in attrs + ): + self.unsafe = True + + +def validate_history(project): + path = Path(project) / "deliverables/visuals/archaeology.html" + if not path.is_file(): + raise ValueError("Generated history HTML missing; run visualize first") + verify_artifact(project, path) + parser = HistoryParser() + parser.feed(path.read_text(encoding="utf-8")) + parser.close() + if parser.unsafe or not {"html", "title", "h1", "table", "caption"}.issubset(parser.tags): + raise ValueError("Generated history HTML is incomplete or contains active content") diff --git a/plugins/devarch-hermes/archaeology/validators/validate_html.cjs b/plugins/devarch-hermes/archaeology/validators/validate_html.cjs new file mode 100644 index 0000000..015bc1c --- /dev/null +++ b/plugins/devarch-hermes/archaeology/validators/validate_html.cjs @@ -0,0 +1,211 @@ +#!/usr/bin/env node +/** + * Archaeology HTML Validator + * + * Validates an archaeology.html file for common defects before committing. + * Accepts optional --project-dir to load project-specific config. + * + * Usage: node validate_html.cjs [--project-dir ] + * + * Exit codes: + * 0 = All validations passed + * 1 = One or more validations failed + */ + +const fs = require('fs'); +const path = require('path'); + +// Parse CLI arguments +const args = process.argv.slice(2); +if (args.length === 0 || args.includes('--help') || args.includes('-h')) { + console.log('Usage: node validate_html.cjs [--project-dir ]'); + console.log(''); + console.log('Validates an archaeology HTML visualization for common defects.'); + console.log('Checks: [object Object] bugs, era colors, JSON data validity,'); + console.log('CSS variables, chart containers, placeholder text.'); + process.exit(args.includes('--help') || args.includes('-h') ? 0 : 1); +} + +const htmlPath = path.resolve(args[0]); +const errors = []; +const warnings = []; + +console.log(`Validating: ${htmlPath}\n`); + +// Check file exists +if (!fs.existsSync(htmlPath)) { + console.error(`File not found: ${htmlPath}`); + process.exit(1); +} + +const file = fs.readFileSync(htmlPath, 'utf8'); + +// Load project config if available +let projectConfig = {}; +const projectDirIdx = args.indexOf('--project-dir'); +if (projectDirIdx !== -1 && args[projectDirIdx + 1]) { + const configPath = path.join(args[projectDirIdx + 1], 'project.json'); + if (fs.existsSync(configPath)) { + try { + projectConfig = JSON.parse(fs.readFileSync(configPath, 'utf8')); + } catch (e) { + warnings.push(`Could not parse project.json: ${e.message}`); + } + } +} + +const eraCount = projectConfig.overrides?.era_count || 0; +const agentNames = Object.keys(projectConfig.visualization?.agent_colors || {}); + +// ============================================================================ +// CHECK 1: No [object Object] serialization bugs +// ============================================================================ +if (file.includes('[object Object]')) { + errors.push('Found "[object Object]" - data serialization bug'); +} else { + console.log('PASS: No [object Object] bugs found'); +} + +// ============================================================================ +// CHECK 2: Era color variables defined +// ============================================================================ +if (eraCount > 0) { + let missingEraColors = []; + for (let i = 1; i <= eraCount; i++) { + if (!file.includes(`--era${i}:`)) { + missingEraColors.push(i); + } + } + + if (missingEraColors.length > 0) { + errors.push(`Missing era color variables: ${missingEraColors.map(i => `--era${i}`).join(', ')}`); + } else { + console.log(`PASS: All ${eraCount} era colors defined`); + } +} else { + // Count how many era colors exist and just verify there are some + const eraColorMatches = file.match(/--era\d+:/g); + if (eraColorMatches && eraColorMatches.length > 0) { + console.log(`PASS: ${eraColorMatches.length} era colors found`); + } else { + warnings.push('No era color CSS variables found'); + } +} + +// ============================================================================ +// CHECK 3: JSON data validity +// ============================================================================ +const jsonMatch = file.match(/window\.(\w+_DATA)\s*=\s*(\{[\s\S]*?\});\s*\n?\s*fetch/) || + file.match(/window\.(\w+_DATA)\s*=\s*(\{[\s\S]*?\});/); +if (jsonMatch) { + const varName = jsonMatch[1]; + try { + // Try to find the end of the JSON object properly + const dataStr = jsonMatch[2]; + // Basic structural checks without full parse (data may reference variables) + const hasEras = dataStr.includes('"eras"') || dataStr.includes("'eras'"); + const hasCommits = dataStr.includes('commits') || dataStr.includes('commit'); + + const hasFetch = file.includes('fetch("data.json")') || file.includes("fetch('data.json')"); + const hasEmbeddedDataScript = file.includes('data.js') || file.includes('__EMBEDDED_DATA'); + + if (hasEras || hasCommits) { + console.log(`PASS: Data object (${varName}) has expected structure`); + } else if (dataStr.trim() === '{}' && (hasFetch || hasEmbeddedDataScript)) { + console.log(`PASS: ${varName} fallback object paired with external data load`); + } else { + warnings.push(`Data object (${varName}) may be missing expected fields`); + } + } catch (e) { + errors.push(`Invalid data object: ${e.message}`); + } +} else { + // Check if data is loaded via fetch (modern template pattern) + const hasFetch = file.includes('fetch("data.json")') || file.includes("fetch('data.json')"); + const hasEmbeddedDataScript = file.includes('data.js') || file.includes('__EMBEDDED_DATA'); + if (hasFetch && hasEmbeddedDataScript) { + console.log('PASS: Data loaded via data.js with fetch fallback'); + } else if (hasFetch) { + console.log('PASS: Data loaded via fetch (external JSON)'); + } else if (hasEmbeddedDataScript) { + console.log('PASS: Data loaded via data.js embedded-data script'); + } else { + warnings.push('Could not find data object or fetch pattern'); + } +} + +// ============================================================================ +// CHECK 4: CSS variables present +// ============================================================================ +const requiredVars = ['--bg', '--surface', '--text', '--text2']; +const missingVars = requiredVars.filter(v => !file.includes(`${v}:`)); + +if (missingVars.length > 0) { + errors.push(`Missing CSS variables: ${missingVars.join(', ')}`); +} else { + console.log('PASS: Core CSS variables present'); +} + +// Check agent color variables if configured +if (agentNames.length > 0) { + const agentVars = agentNames.map(name => `--${name.toLowerCase()}`); + const missingAgentVars = agentVars.filter(v => !file.includes(`${v}:`)); + if (missingAgentVars.length > 0) { + warnings.push(`Missing agent CSS variables: ${missingAgentVars.join(', ')}`); + } else { + console.log('PASS: All agent CSS variables present'); + } +} + +// ============================================================================ +// CHECK 5: Chart container IDs present +// ============================================================================ +const requiredCharts = [ + 'chart-commit-timeline', + 'chart-era-map', + 'chart-heatmap', +]; + +const missingCharts = requiredCharts.filter(id => !file.includes(`id="${id}"`)); + +if (missingCharts.length > 0) { + errors.push(`Missing core chart containers: ${missingCharts.join(', ')}`); +} else { + console.log('PASS: Core chart containers present'); +} + +// ============================================================================ +// CHECK 6: No placeholder text +// ============================================================================ +const placeholders = ['TODO', 'FIXME', 'XXX', 'HACK']; +const foundPlaceholders = placeholders.filter(p => + file.includes(p) +); + +if (foundPlaceholders.length > 0) { + warnings.push(`Found placeholder markers: ${foundPlaceholders.join(', ')}`); +} + +// ============================================================================ +// REPORT +// ============================================================================ +console.log('\n' + '='.repeat(60)); + +if (errors.length === 0 && warnings.length === 0) { + console.log('ALL VALIDATIONS PASSED'); + console.log('='.repeat(60)); + process.exit(0); +} else { + if (errors.length > 0) { + console.log(`${errors.length} ERROR(S) FOUND:`); + errors.forEach(e => console.log(` - ${e}`)); + } + + if (warnings.length > 0) { + console.log(`\n${warnings.length} WARNING(S):`); + warnings.forEach(w => console.log(` - ${w}`)); + } + + console.log('='.repeat(60)); + process.exit(errors.length > 0 ? 1 : 0); +} diff --git a/plugins/devarch-hermes/archaeology/visualization/__init__.py b/plugins/devarch-hermes/archaeology/visualization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/devarch-hermes/archaeology/visualization/agent_benchmark.py b/plugins/devarch-hermes/archaeology/visualization/agent_benchmark.py new file mode 100644 index 0000000..4c38a24 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/visualization/agent_benchmark.py @@ -0,0 +1,827 @@ +"""Agent performance benchmark analysis for archaeology projects. + +This module analyzes commit data to produce per-agent metrics across different +development eras, providing insights into AI agent contribution patterns. +""" + +import json +import re +import sqlite3 +from datetime import date, datetime +from pathlib import Path +from typing import Any, Dict + +from .design_system import ( + THEME_SWITCHER_HTML, + body_end_bundle, + head_bundle, +) + + +def _parse_commit_date(value: Any, commit_hash: Any) -> date: + """Parse ISO-8601 and legacy Git-log commit dates or reject the row.""" + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"Invalid commit date for commit {commit_hash!r}: {value!r}; " + "expected an ISO-8601 timestamp or Git-log date." + ) + + date_value = value.strip() + iso_value = f"{date_value[:-1]}+00:00" if date_value.endswith(("Z", "z")) else date_value + try: + # Keep the calendar date supplied by Git, matching legacy display semantics. + return datetime.fromisoformat(iso_value).date() + except ValueError as iso_error: + for date_format in ("%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d"): + try: + return datetime.strptime(date_value, date_format).date() + except ValueError: + continue + raise ValueError( + f"Invalid commit date for commit {commit_hash!r}: {value!r}; " + "expected an ISO-8601 timestamp or Git-log date." + ) from iso_error + + +def analyze_agent_benchmarks(db_path: str) -> Dict[str, Any]: + """Analyze benchmark metrics; reject malformed commit dates with row context. + + Empty databases return zero metrics and an ``Unknown`` date range. A malformed + date in a non-empty database raises ``ValueError`` instead of dropping rows. + """ + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + # Check if eras table exists + has_eras = ( + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='eras'" + ).fetchone() + is not None + ) + + # Get era information (optional) + eras = {} + era_date_ranges = {} + if has_eras: + eras_data = cursor.execute("SELECT id, name FROM eras ORDER BY id").fetchall() + eras = {row["id"]: row["name"] for row in eras_data} + _era_ids = list(eras.keys()) + + # Build era date ranges for mapping commits + era_date_ranges = {} + if has_eras: + for row in eras_data: + era_id = row["id"] + dates_str = cursor.execute("SELECT dates FROM eras WHERE id = ?", (era_id,)).fetchone()[ + "dates" + ] + era_date_ranges[era_id] = dates_str + + # Simpler approach: get all commits and map to eras in Python + commits = cursor.execute( + "SELECT hash, date, message, author FROM commits ORDER BY date" + ).fetchall() + + # Build era date mappings manually + era_mappings = [] + if has_eras: + for era_id, era_name in eras.items(): + era_row = cursor.execute("SELECT dates FROM eras WHERE id = ?", (era_id,)).fetchone() + + dates_str = era_row["dates"] if era_row else None + start_date = None + end_date = None + + # Parse the main era date range + # Format: "Feb 28 - Mar 18" + if dates_str and " - " in dates_str: + start_str, end_str = dates_str.split(" - ", 1) + start_date = f"{start_str}, 2026" + end_date = f"{end_str}, 2026" + + era_mappings.append( + {"id": era_id, "name": era_name, "start": start_date, "end": end_date} + ) + + # Map commits to eras + def parse_abbreviated_date(date_str: str) -> datetime: + """Parse dates like 'Feb 28, 2026' or 'Mar 19, 2026'.""" + # Remove weekday names if present + date_str = re.sub(r"^[A-Z][a-z]{2}\s+", "", date_str) + # Parse with format + try: + return datetime.strptime(date_str, "%b %d, %Y") + except ValueError: + # Try parsing from git log format + return datetime.strptime(date_str.split()[0], "%Y-%m-%d") + + def normalize_author(author: str) -> str: + """Normalize author names to canonical agent names.""" + author_lower = author.lower() + _ai_agents = {"claude", "kai", "cursor", "kimicode", "codex"} + if "claude" in author_lower: + return "Claude" + elif author_lower == "kai": + return "Kai" + elif author_lower == "cursor": + return "Cursor" + elif author_lower == "kimicode": + return "KimiCode" + elif author_lower == "codex": + return "Codex" + elif author_lower == "demo-project": + return "demo-project" + else: + # Any non-AI author is labeled "Human" — no personal names in framework output + return "Human" + + # Group commits by agent and era + agent_stats: Dict[str, Dict[str, Any]] = {} + + for commit in commits: + author = normalize_author(commit["author"]) + commit_date_str = commit["date"] + commit_date = _parse_commit_date(commit_date_str, commit["hash"]) + + # Find which era this commit belongs to + commit_era = None + for era_mapping in era_mappings: + if era_mapping["start"] and era_mapping["end"]: + try: + start_dt = parse_abbreviated_date(era_mapping["start"]).date() + end_dt = parse_abbreviated_date(era_mapping["end"]).date() + if start_dt <= commit_date <= end_dt: + commit_era = era_mapping["name"] + break + except ValueError: + continue + + # Initialize agent stats if needed + if author not in agent_stats: + agent_stats[author] = { + "name": author, + "total_commits": 0, + "commits_per_era": {}, + "message_lengths": [], + "rework_commits": 0, + "first_commit": commit_date, + "last_commit": commit_date, + "all_dates": [], + } + + # Update stats + agent_stats[author]["total_commits"] += 1 + agent_stats[author]["all_dates"].append(commit_date) + + if commit_era: + if commit_era not in agent_stats[author]["commits_per_era"]: + agent_stats[author]["commits_per_era"][commit_era] = 0 + agent_stats[author]["commits_per_era"][commit_era] += 1 + + # Track message length + message = commit["message"] or "" + agent_stats[author]["message_lengths"].append(len(message)) + + # Track rework (fix/revert commits) + if re.search(r"\bfix|revert|oops|undo\b", message, re.IGNORECASE): + agent_stats[author]["rework_commits"] += 1 + + # Update per-agent date range. + if commit_date < agent_stats[author]["first_commit"]: + agent_stats[author]["first_commit"] = commit_date + if commit_date > agent_stats[author]["last_commit"]: + agent_stats[author]["last_commit"] = commit_date + + # Compute final metrics per agent + agents_list = [] + for agent_name, stats in agent_stats.items(): + avg_message_length = ( + sum(stats["message_lengths"]) / len(stats["message_lengths"]) + if stats["message_lengths"] + else 0 + ) + rework_rate = ( + stats["rework_commits"] / stats["total_commits"] if stats["total_commits"] > 0 else 0 + ) + + first_date = stats["first_commit"].isoformat() + last_date = stats["last_commit"].isoformat() + + agent_data = { + "name": agent_name, + "total_commits": stats["total_commits"], + "commits_per_era": stats["commits_per_era"], + "avg_files_changed": 0, # Would need git log --numstat for this + "rework_rate": round(rework_rate * 100, 1), # Percentage + "avg_message_length": round(avg_message_length, 1), + "first_commit": first_date, + "last_commit": last_date, + } + agents_list.append(agent_data) + + # Sort by total commits + agents_list.sort(key=lambda x: x["total_commits"], reverse=True) + + # Compute metadata + total_commits = sum(a["total_commits"] for a in agents_list) + + # Get overall date range + all_dates = [] + for stats in agent_stats.values(): + all_dates.extend(stats["all_dates"]) + + date_range = "Unknown" + if all_dates: + date_range = f"{min(all_dates).isoformat()} to {max(all_dates).isoformat()}" + + return { + "agents": agents_list, + "eras": list(eras.values()), + "meta": { + "total_commits": total_commits, + "total_agents": len(agents_list), + "date_range": date_range, + }, + } + + +def generate_benchmark_html(benchmark_data: Dict[str, Any], project_name: str) -> str: + """Generate standalone HTML file for agent benchmark visualization. + + Args: + benchmark_data: Output from analyze_agent_benchmarks() + project_name: Name of the project for title + + Returns: + Complete HTML document as string + """ + agents = benchmark_data["agents"] + eras = benchmark_data["eras"] + meta = benchmark_data["meta"] + + # Prepare data for D3.js + agents_json = json.dumps(agents) + eras_json = json.dumps(eras) + + title = f"{project_name.upper()} — Agent Performance Benchmark" + description = f"AI agent contribution analysis for {project_name.upper()}" + + # Custom CSS for benchmark-specific elements + custom_css = """ + +""" + + # JavaScript with theme-aware colors + js_code = f""" + +""" + + html_template = f""" + + +{head_bundle(title=title, description=description, include_d3=True)} +{custom_css} + + + + + + +
    +
    +
    +

    {project_name.upper()} — Agent Performance Benchmark

    +

    Analysis of AI agent contribution patterns across project eras

    +
    + +
    +
    +
    {meta["total_commits"]}
    +
    Total Commits
    +
    +
    +
    {meta["total_agents"]}
    +
    Active Agents
    +
    +
    +
    {meta["date_range"].split(" to ")[0] if " to " in meta["date_range"] else meta["date_range"]}
    +
    Project Start
    +
    +
    +
    {meta["date_range"].split(" to ")[1] if " to " in meta["date_range"] else ""}
    +
    Latest Activity
    +
    +
    + +
    +
    +

    Commits by Agent

    +
    +
    +
    + +
    +
    +

    Commits per Era by Agent

    +
    +
    +
    + +
    +
    +

    Detailed Metrics

    + + + + + + + + + + + + +
    AgentTotal CommitsRework RateAvg Message LengthActive Period
    +
    +
    +
    +
    + +{js_code} +{body_end_bundle()} + +""" + + return html_template + + +def run_benchmark_analysis(project_dir: str) -> str: + """Run complete benchmark analysis and generate HTML. + + Args: + project_dir: Path to project directory (e.g., "projects/demo-project") + + Returns: + Path to generated HTML file + """ + project_path = Path(project_dir) + db_path = project_path / "data" / "archaeology.db" + deliverables_dir = project_path / "deliverables" + visuals_dir = deliverables_dir / "visuals" + output_path = visuals_dir / "agent-benchmark.html" + + if not db_path.exists(): + raise FileNotFoundError(f"Database not found: {db_path}") + + # Analyze + benchmark_data = analyze_agent_benchmarks(str(db_path)) + + # Get project name from directory + project_name = project_path.name + + # Generate HTML + html_content = generate_benchmark_html(benchmark_data, project_name) + + # Write output + visuals_dir.mkdir(parents=True, exist_ok=True) + output_path.write_text(html_content, encoding="utf-8") + + return str(output_path) diff --git a/plugins/devarch-hermes/archaeology/visualization/dashboard.py b/plugins/devarch-hermes/archaeology/visualization/dashboard.py new file mode 100644 index 0000000..e4b6776 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/visualization/dashboard.py @@ -0,0 +1,1548 @@ +"""Master dashboard generator for DevArch Framework. + +Generates the top-level index.html that shows all projects as cards +with links to their individual visualizations. Served by `archaeology serve` +or deployed as a static site. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from .design_system import ( + THEME_SWITCHER_HTML, + body_end_bundle, + head_bundle, +) + +# Deliverable categories with display metadata and colors +# Colors reference design system tokens for theme consistency +CATEGORIES: dict[str, dict[str, str]] = { + "visuals": {"icon": "📊", "label": "Visualizations", "color": "var(--success)"}, + "analysis": {"icon": "🔎", "label": "Analysis", "color": "var(--secondary)"}, + "reports": {"icon": "📄", "label": "Reports", "color": "var(--accent)"}, + "strategy": {"icon": "🎯", "label": "Strategy", "color": "var(--warning)"}, + "planning": {"icon": "📋", "label": "Planning", "color": "var(--success)"}, + "learning": {"icon": "📚", "label": "Learning", "color": "#ec4899"}, + "content": {"icon": "✍", "label": "Content", "color": "var(--warning)"}, + "video": {"icon": "🎥", "label": "Video", "color": "var(--danger)"}, + "opportunity": {"icon": "💡", "label": "Opportunity", "color": "var(--accent)"}, +} + + +def _discover_all_deliverables( + deliverables_dir: Path, project_name: str +) -> dict[str, list[dict[str, str]]]: + """Scan all deliverable subdirectories and categorize files.""" + result: dict[str, list[dict[str, str]]] = {} + for cat_name in CATEGORIES: + cat_dir = deliverables_dir / cat_name + if not cat_dir.exists(): + continue + files = [] + for f in sorted(cat_dir.iterdir()): + if f.suffix not in (".html", ".md", ".json"): + continue + display = f.stem.replace("-", " ").replace("_", " ").title() + href = f"{project_name}/{cat_name}/{f.name}" + files.append({"name": display, "href": href, "ext": f.suffix, "filename": f.name}) + if files: + result[cat_name] = files + return result + + +def discover_projects(projects_dir: Path) -> list[dict[str, Any]]: + """Scan projects/ directory and collect metadata for each project. + + Returns list of project dicts sorted by name. + """ + projects = [] + if not projects_dir.exists(): + return projects + + for project_dir in sorted(projects_dir.iterdir()): + if not project_dir.is_dir(): + continue + if project_dir.name.startswith((".", "_")): + continue + + deliverables_dir = project_dir / "deliverables" + data_dir = project_dir / "data" + if not deliverables_dir.exists() and not data_dir.exists(): + continue + + # Load project metadata from data.json or canonical-metrics.json + meta = _load_project_meta(deliverables_dir, data_dir) + + # Discover HTML visualizations (backward compat) + visuals = _discover_visuals(deliverables_dir, project_dir.name) + + # Discover all categorized deliverables + deliverables = _discover_all_deliverables(deliverables_dir, project_dir.name) + total_deliverables = sum(len(v) for v in deliverables.values()) + + projects.append( + { + "name": project_dir.name, + "slug": project_dir.name, + "meta": meta, + "visuals": visuals, + "deliverables": deliverables, + "total_deliverables": total_deliverables, + "has_data": (deliverables_dir / "data.json").exists() or (data_dir).exists(), + } + ) + + return projects + + +def _load_project_meta(deliverables_dir: Path, data_dir: Path) -> dict[str, Any]: + """Load project metadata from available JSON files.""" + meta: dict[str, Any] = {} + + # Try data.json first (most comprehensive) + data_json = deliverables_dir / "data.json" + if data_json.exists(): + try: + data = json.loads(data_json.read_text(encoding="utf-8")) + summary = data.get("summary", {}) + meta["commits"] = summary.get("total_commits", 0) + meta["active_days"] = summary.get("active_days", 0) + meta["span_days"] = summary.get("span_days", 0) + meta["era_count"] = len(data.get("eras", [])) + meta["authors"] = ( + list(data.get("authors", {}).keys()) + if isinstance(data.get("authors"), dict) + else [] + ) + except (json.JSONDecodeError, OSError): + pass + + # Try canonical-metrics.json + canonical = deliverables_dir / "canonical-metrics.json" + if canonical.exists() and not meta.get("commits"): + try: + cm = json.loads(canonical.read_text(encoding="utf-8")) + meta["commits"] = cm.get("total_commits", 0) + meta["active_days"] = cm.get("active_days", 0) + meta["span_days"] = cm.get("span_days", 0) + meta["era_count"] = cm.get("era_count", 0) + except (json.JSONDecodeError, OSError): + pass + + # Fallback: compute metrics from CSV + if not meta.get("commits"): + meta.update(_load_git_metrics(data_dir)) + + return meta + + +def _load_git_metrics(data_dir: Path) -> dict[str, Any]: + """Fallback: compute basic metrics from github-commits.csv.""" + import csv as _csv + + csv_path = data_dir / "github-commits.csv" + if not csv_path.exists(): + return {} + commits = 0 + dates: set[str] = set() + try: + with open(csv_path, encoding="utf-8") as f: + reader = _csv.DictReader(f) + for row in reader: + commits += 1 + d = row.get("date", "")[:10] + if d: + dates.add(d) + except OSError: + return {} + if commits == 0: + return {} + sorted_dates = sorted(dates) + span = 0 + if len(sorted_dates) >= 2: + from datetime import date as _date + + try: + d0 = _date.fromisoformat(sorted_dates[0]) + d1 = _date.fromisoformat(sorted_dates[-1]) + span = (d1 - d0).days + except ValueError: + span = 0 + return { + "commits": commits, + "active_days": len(dates), + "span_days": span or 1, + "era_count": 0, + "first_commit": sorted_dates[0], + "last_commit": sorted_dates[-1], + } + + +def _discover_visuals(deliverables_dir: Path, project_name: str) -> list[dict[str, str]]: + """Find HTML visualization files for a project.""" + visuals = [] + seen = set() + + # Check visuals/ subdirectory (new structure) + visuals_dir = deliverables_dir / "visuals" + search_dirs = [visuals_dir, deliverables_dir] if visuals_dir.exists() else [deliverables_dir] + + for search_dir in search_dirs: + if not search_dir.exists(): + continue + for html_file in sorted(search_dir.glob("*.html")): + name = html_file.stem + if name in seen or name == "index": + continue + seen.add(name) + + display = name.replace("-", " ").replace("_", " ").title() + if display == "Archaeology": + display = "Dashboard" + order = 0 + elif display == "Playbook": + order = 1 + elif "Agent" in display or "Benchmark" in display: + display = "Agents" + order = 2 + elif display == "Report": + order = 3 + else: + order = 99 + + # Relative href from master dashboard (flat — serve symlinks into project dir) + href = f"{project_name}/{html_file.name}" + + visuals.append({"name": display, "href": href, "order": order}) + + visuals.sort(key=lambda v: v["order"]) + return visuals + + +def _pluralize(n: int, singular: str, plural: str = "") -> str: + """Return singular or plural form based on count.""" + if plural == "": + plural = singular + "s" + return singular if n == 1 else plural + + +def _format_stat(value: Any, singular: str, plural: str = "") -> str: + """Format a stat value, using dash for zero/unknown and correct pluralization.""" + if value == 0 or value == "?" or value is None: + return "—" + n = int(value) if isinstance(value, (int, float)) else 0 + return f"{n:,} {_pluralize(n, singular, plural)}" + + +def _project_description(name: str, meta: dict) -> str: + """Generate a project description from metadata.""" + commits = meta.get("commits", 0) + eras = meta.get("era_count", 0) + days = meta.get("active_days", 0) + if not commits: + return "Infrastructure project supporting the archaeology pipeline" + parts = [] + if eras and eras > 1: + parts.append(f"{eras} distinct development eras") + elif not eras: + pass # Skip era mention for pipeline/infra projects + if days: + parts.append(f"{days} active days of development") + if parts: + return f"{commits:,} commits across " + " and ".join(parts) + return f"{commits:,} commits of development history" + + +def generate_master_dashboard( + projects: list[dict[str, Any]], + api_section_html: str = "", + api_repos: list[dict[str, Any]] | None = None, +) -> str: + """Generate the master dashboard HTML.""" + now = datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC") + api_repos = api_repos or [] + + api_commits = sum(r["commits"] for r in api_repos if isinstance(r["commits"], int)) + if api_repos: + print(f" API repos: {len(api_repos)} repos, {api_commits:,} commits") + else: + print(" WARNING: No API repos loaded") + + # Sort projects by commit count (descending) — richest first + def _proj_commits(p: dict) -> int: + c = p["meta"].get("commits", 0) + return c if isinstance(c, int) else 0 + + projects_sorted = sorted(projects, key=_proj_commits, reverse=True) + + # Separate featured (top project) from the rest + featured = projects_sorted[0] if projects_sorted else None + rest_projects = projects_sorted[1:] if len(projects_sorted) > 1 else [] + + # ── Featured card ── + featured_html = "" + if featured: + fm = featured["meta"] + fc = fm.get("commits", "—") + fc_fmt = f"{fc:,}" if isinstance(fc, int) else str(fc) + fe = fm.get("era_count", 0) + fad = fm.get("active_days", "—") + fd = featured.get("total_deliverables", 0) + fviz = "" + for viz in featured["visuals"]: + fviz += f'{viz["name"]}\n ' + + # Category pills + cat_pills = "" + for cat_name, cat_meta in CATEGORIES.items(): + count = len(featured.get("deliverables", {}).get(cat_name, [])) + if count: + color = cat_meta["color"] + cat_pills += f'{cat_meta["label"]} {count}\n' + + featured_html = f""" +
    + +
    + Featured Project +

    {featured["name"].upper()}

    +

    {_project_description(featured["name"], fm)}

    +
    +
    {fc_fmt}{_pluralize(fc if isinstance(fc, int) else 0, "commit")}
    +
    {fe if fe else "—"}{_pluralize(fe, "era")}
    +
    {fad}{_pluralize(fad if isinstance(fad, int) else 0, "active day")}
    +
    {fd}{_pluralize(fd, "deliverable")}
    +
    +
    {cat_pills}
    +
    +
    + {fviz} +
    +     +
    """ + + # ── Project cards (rest) ── + project_cards = "" + for proj in rest_projects: + meta = proj["meta"] + commits = meta.get("commits", "—") + eras = meta.get("era_count", 0) + active_days = meta.get("active_days", "—") + total_deliverables = proj.get("total_deliverables", 0) + commits_fmt = f"{commits:,}" if isinstance(commits, int) else str(commits) + viz_links = "" + for viz in proj["visuals"]: + viz_links += ( + f'{viz["name"]}\n ' + ) + + # Category pills + cat_pills = "" + for cat_name, cat_meta in CATEGORIES.items(): + count = len(proj.get("deliverables", {}).get(cat_name, [])) + if count: + color = cat_meta["color"] + cat_pills += f'{cat_meta["label"]} {count}\n' + + project_cards += f""" + +
    +

    {proj["name"].upper()}

    + {total_deliverables} {_pluralize(total_deliverables, "deliverable")} +
    +
    +
    {commits_fmt}{_pluralize(commits if isinstance(commits, int) else 0, "commit")}
    +
    {eras if eras else "—"}{_pluralize(eras, "era")}
    +
    {active_days}{_pluralize(active_days if isinstance(active_days, int) else 0, "active day")}
    +
    +
    {cat_pills}
    +
    + {viz_links} +
    +     """ + + # ── Aggregate stats ── + mined_commits = sum( + p["meta"].get("commits", 0) for p in projects if isinstance(p["meta"].get("commits"), int) + ) + total_repos = len(projects) + len(api_repos) + total_commits = mined_commits + api_commits + total_commits_fmt = f"{total_commits:,}" + total_repos_fmt = f"{total_repos:,}" + total_deliverables = sum(p.get("total_deliverables", 0) for p in projects) + owners = set(r["owner"] for r in api_repos) | {"mined"} + n_networks = len(owners) + + # ── Cross-project chart data ── + all_repos_chart = [] + for p in projects: + c = p["meta"].get("commits", 0) + if isinstance(c, int) and c > 0: + all_repos_chart.append({"name": p["name"].upper(), "commits": c}) + for r in api_repos: + c = r.get("commits", 0) + if isinstance(c, int) and c > 0: + all_repos_chart.append({"name": r["name"], "commits": c}) + all_repos_chart.sort(key=lambda x: x["commits"], reverse=True) + top_chart = all_repos_chart[:12] + chart_repo_labels = json.dumps([r["name"][:22] for r in top_chart]) + chart_repo_commits = json.dumps([r["commits"] for r in top_chart]) + + lang_dist: dict[str, int] = {} + for r in api_repos: + lang = r.get("language") or "Unknown" + lang_dist[lang] = lang_dist.get(lang, 0) + 1 + chart_lang_labels = json.dumps(list(lang_dist.keys())) + chart_lang_counts = json.dumps(list(lang_dist.values())) + + cat_totals: dict[str, int] = {} + for p in projects: + for cat_name, files in p.get("deliverables", {}).items(): + cat_totals[cat_name] = cat_totals.get(cat_name, 0) + len(files) + cat_display = [CATEGORIES.get(k, {}).get("label", k.title()) for k in cat_totals] + chart_cat_labels = json.dumps(cat_display) + chart_cat_counts = json.dumps(list(cat_totals.values())) + chart_cat_colors = json.dumps( + [CATEGORIES.get(k, {}).get("color", "#6a7888") for k in cat_totals] + ) + + chart_proj_names = json.dumps([p["name"].upper() for p in projects_sorted]) + chart_proj_days = [ + p["meta"].get("active_days", 0) if isinstance(p["meta"].get("active_days"), int) else 0 + for p in projects_sorted + ] + chart_proj_eras = [p["meta"].get("era_count", 0) or 0 for p in projects_sorted] + chart_proj_days_json = json.dumps(chart_proj_days) + chart_proj_eras_json = json.dumps(chart_proj_eras) + + # ── API repos section (collapsed by default) ── + api_section = "" + if api_repos: + # Build filter buttons + owner_filters = "" + for owner in sorted(set(r["owner"] for r in api_repos)): + label = "KyaniteLabs" if "kyanite" in owner.lower() else "Personal" + owner_filters += ( + f'\n ' + ) + api_section = f""" +
    + + +
    """ + + # ── Cross-Repo Analysis section ── + cross_repo_section = "" + global_deliverables_dir = Path("global/deliverables") + if global_deliverables_dir.exists(): + md_files = sorted(global_deliverables_dir.rglob("*.md")) + if md_files: + cards_html = "" + for md in md_files: + display = md.stem.replace("-", " ").replace("_", " ").title() + rel = f"global/{md.relative_to(global_deliverables_dir)}" + cards_html += f'{display}{rel}\n' + cross_repo_section = f"""
    +

    Cross-Repository Analysis ({len(md_files)})

    +
    {cards_html}
    +
    """ + + # ── Global viz cards ── + global_viz_section = "" + if api_repos: + global_viz_section = """ +
    +
    + + 📊 +
    Multi-Project DashboardCommit timeline, activity heatmap, repo comparison
    +     + + 🌐 +
    Global NetworkCross-repo connections, language breakdown
    +     +
    +
    """ + + html = f""" + + +{ + head_bundle( + title="Dev-Archaeology", + description=f"Forensic analysis of development history across {total_repos} repositories", + include_charts=True, + ) + } + + + + + + + + + +
    +
    +

    Development Fossil Record

    +

    Forensic archaeology of {total_repos} repositories — { + total_commits_fmt + } commits mined, analyzed, and visualized

    +
    +
    {total_repos_fmt}{ + _pluralize(total_repos, "Repository", "Repositories") + }
    +
    {total_commits_fmt}{ + _pluralize(total_commits, "Commit") + }
    +
    {total_deliverables}{ + _pluralize(total_deliverables, "Deliverable") + }
    +
    {n_networks}{ + _pluralize(n_networks, "Network") + }
    +
    +
    + +
    +

    Cross-Project Analytics

    +
    +
    +
    📊Commit Distribution
    +
    +
    +
    +
    🌐Language Map
    +
    +
    +
    +
    📚Deliverable Types
    +
    +
    +
    +
    📅Project Activity
    +
    +
    +
    +
    + +{featured_html} + +{ + f'''
    +

    Analyzed Projects ({len(rest_projects)})

    +
    + {project_cards} +
    +
    ''' + if rest_projects + else "" + } + +{cross_repo_section} + +{global_viz_section} + +{api_section} + +
    + +
    + Generated by devarch dev-archaeology ·middot; {now} +
    + +{body_end_bundle()} + + + + + +""" + return html + + +# ── Opportunity Visualization Constants ── + +_OPP_FEATURES: list[dict[str, str]] = [ + {"slug": "learning-velocity", "title": "Learning Velocity", "icon": "📈"}, + { + "slug": "frustration-to-automation", + "title": "Frustration → Automation", + "icon": "⚡", + }, + {"slug": "knowledge-gap", "title": "Knowledge Gap Detector", "icon": "🔎"}, + {"slug": "token-efficiency", "title": "Token Efficiency Coach", "icon": "🎯"}, + {"slug": "session-quality", "title": "Session Quality Scorer", "icon": "⭐"}, + {"slug": "ai-agent-mastery", "title": "AI Agent Mastery", "icon": "🤖"}, + {"slug": "creative-dna", "title": "Creative DNA Transfer", "icon": "🎨"}, + {"slug": "neurodivergent-profile", "title": "Neurodivergent Profile", "icon": "🧠"}, + {"slug": "model-selection-advisor", "title": "Model Selection Advisor", "icon": "💻"}, + {"slug": "before-after-snapshot", "title": "Before / After Snapshot", "icon": "👁"}, + {"slug": "cross-repo-transfer", "title": "Cross-Repo Transfer", "icon": "🔗"}, + {"slug": "youtube-learning-graph", "title": "YouTube Learning Graph", "icon": "🎥"}, + {"slug": "architecture-timelapse", "title": "Architecture Timelapse", "icon": "🏗"}, + {"slug": "commit-cognitive-load", "title": "Cognitive Load Proxy", "icon": "🪨"}, +] + +_PROJECT_INDEX_CSS = """ +/* Score Cards */ +.score-row{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:12px;margin-bottom:28px} +.score-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius-md);padding:18px 16px;text-align:center;position:relative;overflow:hidden;transition:border-color .3s,transform .3s} +.score-card:hover{border-color:var(--border-hover);transform:translateY(-2px)} +.score-card::before{content:'';position:absolute;top:0;left:0;right:0;height:3px} +.score-card:nth-child(1)::before{background:linear-gradient(90deg,#06b6d4,#34d399)} +.score-card:nth-child(2)::before{background:linear-gradient(90deg,#a78bfa,#f472b6)} +.score-card:nth-child(3)::before{background:linear-gradient(90deg,#fbbf24,#fb923c)} +.score-card:nth-child(4)::before{background:linear-gradient(90deg,#f87171,#ef4444)} +.score-ring{width:72px;height:72px;margin:0 auto 8px;transform:rotate(-90deg)} +.score-ring-bg{fill:none;stroke:var(--surface3);stroke-width:5} +.score-ring-fill{fill:none;stroke-width:5;stroke-linecap:round;stroke-dasharray:264;stroke-dashoffset:264;transition:stroke-dashoffset 1.5s cubic-bezier(0.16,1,0.3,1)} +.score-value{font-family:var(--font-display);font-size:24px;font-weight:700;color:var(--text);margin-bottom:2px} +.score-label{font-size:11px;color:var(--text3);text-transform:uppercase;letter-spacing:.06em;font-weight:500} +.score-sub{font-size:10px;color:var(--text3);margin-top:4px;font-family:var(--font-mono)} + +/* Visualization Grid */ +.viz-section{margin-bottom:32px} +.viz-section-title{font-family:var(--font-display);font-size:16px;font-weight:600;color:var(--text2);margin-bottom:16px;display:flex;align-items:center;gap:8px} +.viz-section-title .count{font-size:11px;font-weight:500;color:var(--text3);background:var(--surface2);padding:2px 8px;border-radius:12px;font-family:var(--font-mono)} +.viz-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:14px} +.viz-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius-md);overflow:hidden;transition:border-color .3s,box-shadow .3s} +.viz-card:hover{border-color:var(--border-hover);box-shadow:0 4px 24px rgba(0,0,0,.3)} +.viz-card-head{padding:12px 16px;border-bottom:1px solid var(--border);display:flex;align-items:center;gap:8px;background:linear-gradient(180deg,rgba(6,182,212,.04) 0%,transparent 100%)} +.viz-card-icon{font-size:16px} +.viz-card-title{font-family:var(--font-display);font-size:13px;font-weight:600;color:var(--text);flex:1} +.viz-card-badge{font-size:10px;color:#06b6d4;background:rgba(6,182,212,.1);padding:2px 8px;border-radius:12px;font-family:var(--font-mono)} +.viz-card-body{padding:14px 16px;position:relative;height:280px} +.viz-card-body canvas{width:100%!important;height:100%!important} +.viz-card-foot{padding:8px 16px 10px;border-top:1px solid var(--border);font-size:11px;color:var(--text3);line-height:1.5} + +/* Collapsible Files */ +.files-toggle{cursor:pointer;user-select:none;display:flex;align-items:center;gap:8px;padding:16px 0 12px;font-family:var(--font-display);font-size:15px;font-weight:600;color:var(--text2);border:none;background:none;width:100%} +.files-toggle::before{content:'\\25BE';font-size:14px;color:var(--text3);transition:transform .2s;display:inline-block} +.files-toggle.collapsed::before{transform:rotate(-90deg)} +.files-body{overflow:hidden;transition:max-height .3s ease-out} +.files-body.collapsed{max-height:0} + +@media(max-width:768px){ + .score-row{grid-template-columns:repeat(2,1fr)} + .viz-grid{grid-template-columns:1fr} + .viz-card-body{height:200px} +} +@media(max-width:480px){ + .score-row{grid-template-columns:1fr} +} +""" + +_PROJECT_INDEX_JS = """ +document.addEventListener('DOMContentLoaded', async function() { + var BASE = 'opportunity/'; + var colors = getThemeColors(); + var PALETTE = [colors.accent, colors.secondary, colors.success, colors.warning, colors.danger, '#818cf8', '#38bdf8', '#4ade80', '#e879f9']; + + Chart.defaults.color = colors.muted; + Chart.defaults.borderColor = colors.border; + Chart.defaults.font.family = "'Source Sans 3', sans-serif"; + Chart.defaults.font.size = 11; + Chart.defaults.plugins.legend.display = false; + Chart.defaults.responsive = true; + Chart.defaults.maintainAspectRatio = false; + + async function load(slug) { + try { + var r = await fetch(BASE + 'opportunity-' + slug + '.json'); + return r.ok ? await r.json() : null; + } catch(e) { return null; } + } + + function summary(slug, data) { + var el = document.getElementById('summary-' + slug); + if (!el || !data || !data.summary) return; + var s = data.summary; + el.textContent = s.text || s.one_liner || s.headline || ''; + } + + function animateScore(id, value, max, color) { + var el = document.getElementById(id); + if (!el) return; + var pct = Math.min(value / max, 1); + var ring = el.querySelector('.score-ring-fill'); + if (ring) { + ring.style.stroke = color || CYAN; + ring.style.strokeDasharray = 264; + setTimeout(function() { ring.style.strokeDashoffset = 264 * (1 - pct); }, 100); + } + var val = el.querySelector('.score-value'); + if (val) val.textContent = (typeof value === 'number' && value % 1 !== 0) ? value.toFixed(1) : value; + } + + function numVal(v) { return (typeof v === 'number') ? v : (typeof v === 'string' ? parseFloat(v) : 0) || 0; } + + // ── Score Cards ── + var mastery = await load('ai-agent-mastery'); + if (mastery) animateScore('score-mastery', mastery.overall_score, 100, colors.accent); + + var session = await load('session-quality'); + if (session) animateScore('score-session', session.summary && (session.summary.avg_quality || session.summary.average_quality || 5), 10, colors.secondary); + + var velocity = await load('learning-velocity'); + if (velocity && velocity.summary) { + var el = document.getElementById('score-velocity'); + if (el) { + var ring = el.querySelector('.score-ring-fill'); + if (ring) { ring.style.stroke = colors.warning; ring.style.strokeDasharray = 264; setTimeout(function(){ ring.style.strokeDashoffset = 264 * 0.7; }, 100); } + var val = el.querySelector('.score-value'); + if (val) val.textContent = (velocity.summary.learning_acceleration || 1.41).toFixed(2) + 'x'; + } + } + + var frustration = await load('frustration-to-automation'); + if (frustration && frustration.summary) { + var el = document.getElementById('score-frustration'); + if (el) { + var ring = el.querySelector('.score-ring-fill'); + var rate = frustration.summary.conversion_rate || 1; + if (ring) { ring.style.stroke = colors.success; ring.style.strokeDasharray = 264; setTimeout(function(){ ring.style.strokeDashoffset = 264 * (1 - rate); }, 100); } + var val = el.querySelector('.score-value'); + if (val) val.textContent = Math.round(rate * 100) + '%'; + } + } + + // ── Chart 1: Learning Velocity ── + if (velocity && velocity.era_velocity) { + var ctx = document.getElementById('chart-learning-velocity'); + if (ctx) { summary('learning-velocity', velocity); + new Chart(ctx, { type:'line', data: { + labels: velocity.era_velocity.map(function(e){return e.era||e.name||'';}), + datasets:[{ label:'Commits/Day', data: velocity.era_velocity.map(function(e){return e.velocity_per_day||e.commits_per_day||0;}), + borderColor:colors.accent, backgroundColor:colors.accent+'20', fill:true, tension:0.4, pointRadius:4, pointBackgroundColor:colors.accent }] + }, options:{ scales:{ y:{beginAtZero:true, grid:{color:'#1e2a3a'}}, x:{grid:{display:false}} }, plugins:{tooltip:{mode:'index',intersect:false}} } }); + } + } + + // ── Chart 2: Frustration → Automation ── + if (frustration) { + var ctx = document.getElementById('chart-frustration-to-automation'); + if (ctx) { summary('frustration-to-automation', frustration); + var patterns = frustration.conversion_patterns || []; + var labels = patterns.map(function(p){return (p.category||'').replace(/_/g,' ').substring(0,20);}); + var intensities = patterns.map(function(p){return p.frustration_level||0;}); + new Chart(ctx, { type:'bar', data: { + labels: labels, + datasets:[{ label:'Frustration Level', data:intensities, backgroundColor: patterns.map(function(p){return p.converted ? colors.success : colors.danger;}), + borderRadius:4, barPercentage:0.7 }] + }, options:{ indexAxis:'y', scales:{ x:{beginAtZero:true, max:5, grid:{color:'#1e2a3a'}}, y:{grid:{display:false}} }, + plugins:{tooltip:{callbacks:{label:function(c){return 'Level '+c.raw+' → '+(patterns[c.dataIndex].converted?'Automated':'Pending');}}}} } }); + } + } + + // ── Chart 3: Knowledge Gap ── + var gap = await load('knowledge-gap'); + if (gap && gap.reinvention_gaps) { + var ctx = document.getElementById('chart-knowledge-gap'); + if (ctx) { summary('knowledge-gap', gap); + var gaps = gap.reinvention_gaps; + new Chart(ctx, { type:'bar', data: { + labels: gaps.map(function(g){return g.intuitive_name||g.informal_term||'';}), + datasets:[{ label:'Severity', data: gaps.map(function(g){return g.severity==='HIGH'?4:g.severity==='MEDIUM'?2:1;}), + backgroundColor: gaps.map(function(g){return g.severity==='HIGH'?RED:ORANGE;}), borderRadius:4, barPercentage:0.6 }] + }, options:{ indexAxis:'y', scales:{ x:{beginAtZero:true, max:5, grid:{color:'#1e2a3a'}}, y:{grid:{display:false}} }, + plugins:{tooltip:{callbacks:{label:function(c){var g=gaps[c.dataIndex];return g.severity+' — maps to '+g.formal_term;}}}} } }); + } + } + + // ── Chart 4: Token Efficiency ── + var token = await load('token-efficiency'); + if (token && token.era_efficiency) { + var ctx = document.getElementById('chart-token-efficiency'); + if (ctx) { summary('token-efficiency', token); + var effData = token.era_efficiency; + var hasMsg = effData.some(function(e){return (e.messages_per_commit||0) > 0;}); + var metric = hasMsg ? 'messages_per_commit' : 'commits'; + var metricLabel = hasMsg ? 'Messages/Commit' : 'Commits per Era'; + new Chart(ctx, { type:'bar', data: { + labels: effData.map(function(e){return (e.era||'').substring(0,12);}), + datasets:[{ label:metricLabel, data: effData.map(function(e){return e[metric]||0;}), + backgroundColor:colors.secondary, borderRadius:4, barPercentage:0.6 }] + }, options:{ scales:{ y:{beginAtZero:true, grid:{color:'#1e2a3a'}}, x:{grid:{display:false}} }, plugins:{tooltip:{mode:'index',intersect:false}} } }); + } + } + + // ── Chart 5: Session Quality ── + if (session && session.type_distribution) { + var ctx = document.getElementById('chart-session-quality'); + if (ctx) { summary('session-quality', session); + var types = session.type_distribution; + var typeLabels = Object.keys(types); + var typeValues = Object.values(types); + new Chart(ctx, { type:'doughnut', data: { + labels: typeLabels, datasets:[{ data:typeValues, + backgroundColor:typeLabels.map(function(_,i){return PALETTE[i%PALETTE.length];}), borderWidth:0 }] + }, options:{ cutout:'55%', plugins:{ legend:{display:true, position:'right', labels:{boxWidth:10,padding:8,font:{size:10}}} } } }); + } + } + + // ── Chart 6: AI Agent Mastery ── + if (mastery && mastery.sub_scores) { + var ctx = document.getElementById('chart-ai-agent-mastery'); + if (ctx) { summary('ai-agent-mastery', mastery); + var subKeys = Object.keys(mastery.sub_scores); + var subVals = subKeys.map(function(k){return numVal(mastery.sub_scores[k]);}); + new Chart(ctx, { type:'radar', data: { + labels: subKeys.map(function(k){return k.replace(/_/g,' ');}), + datasets:[{ label:'Score', data:subVals, borderColor:colors.accent, backgroundColor:colors.accent+'20', + pointBackgroundColor:colors.accent, pointRadius:4 }] + }, options:{ scales:{ r:{beginAtZero:true, max:100, grid:{color:'#1e2a3a'}, angleLines:{color:'#1e2a3a'}, pointLabels:{font:{size:10}}} } } }); + } + } + + // ── Chart 7: Creative DNA ── + var dna = await load('creative-dna'); + if (dna && dna.transfer_map) { + var ctx = document.getElementById('chart-creative-dna'); + if (ctx) { summary('creative-dna', dna); + var transfers = dna.transfer_map; + new Chart(ctx, { type:'bar', data: { + labels: transfers.map(function(t){return (t.creative_source||'').substring(0,22);}), + datasets:[{ label:'Strength', data: transfers.map(function(t){return t.strength==='HIGH'?3:t.strength==='MEDIUM'?2:1;}), + backgroundColor: transfers.map(function(t){return t.strength==='HIGH'?CYAN:TEAL;}), borderRadius:4, barPercentage:0.6 }] + }, options:{ indexAxis:'y', scales:{ x:{beginAtZero:true, max:4, grid:{color:'#1e2a3a'}}, y:{grid:{display:false}} }, + plugins:{tooltip:{callbacks:{label:function(c){var t=transfers[c.dataIndex];return t.strength+' → '+(t.code_destination||'').substring(0,30);}}}} } }); + } + } + + // ── Chart 8: Neurodivergent Profile ── + var neuro = await load('neurodivergent-profile'); + if (neuro && neuro.hourly_pattern) { + var ctx = document.getElementById('chart-neurodivergent-profile'); + if (ctx) { summary('neurodivergent-profile', neuro); + var hours = Object.keys(neuro.hourly_pattern).sort(function(a,b){return parseInt(a)-parseInt(b);}); + var counts = hours.map(function(h){return neuro.hourly_pattern[h];}); + new Chart(ctx, { type:'bar', data: { + labels: hours.map(function(h){return h+':00';}), + datasets:[{ label:'Commits', data:counts, + backgroundColor: hours.map(function(h){var hr=parseInt(h);return (hr>=21||hr<5)?colors.danger:(hr>=9backgroundColor: hours.map(function(h){var hr=parseInt(h);return (hr>=21||hr<5)?RED:(hr>=9&&hr<17)?CYAN:ORANGE;})backgroundColor: hours.map(function(h){var hr=parseInt(h);return (hr>=21||hr<5)?RED:(hr>=9&&hr<17)?CYAN:ORANGE;})hr<17)?colors.accent:colors.warning;}), + borderRadius:2, barPercentage:0.8 }] + }, options:{ scales:{ y:{beginAtZero:true, grid:{color:'#1e2a3a'}}, x:{grid:{display:false}, ticks:{maxRotation:0,autoSkip:true,maxTicksLimit:12}} }, + plugins:{tooltip:{callbacks:{label:function(c){var h=parseInt(hours[c.dataIndex]);return c.raw+' commits ('+(h>=21||h<5?'Witching hour':h>=9&&h<17?'Core hours':'Evening')+')';}}}} } }); + } + } + + // ── Chart 9: Model Selection ── + var model = await load('model-selection-advisor'); + if (model && model.recommendation_matrix) { + var ctx = document.getElementById('chart-model-selection-advisor'); + if (ctx) { summary('model-selection-advisor', model); + var recs = model.recommendation_matrix; + new Chart(ctx, { type:'bar', data: { + labels: recs.map(function(r){return (r.task_type||'').replace(/_/g,' ').substring(0,20);}), + datasets:[{ label:'Confidence', data: recs.map(function(r){return Math.round((r.confidence||0)*100);}), + backgroundColor: recs.map(function(r){var c=r.confidence||0;return c>0.8?colors.success:c>0.6?colors.warning:colors.warning;}), + borderRadius:4, barPercentage:0.6 }] + }, options:{ scales:{ y:{beginAtZero:true, max:100, grid:{color:'#1e2a3a'}, ticks:{callback:function(v){return v+'%';}}}, x:{grid:{display:false}} } } }); + } + } + + // ── Chart 10: Before/After ── + var snapshot = await load('before-after-snapshot'); + if (snapshot && snapshot.before && snapshot.after) { + var ctx = document.getElementById('chart-before-after-snapshot'); + if (ctx) { summary('before-after-snapshot', snapshot); + var metrics = ['commits', 'active_days', 'velocity']; + var metricLabels = ['Total Commits', 'Active Days', 'Velocity/Day']; + var beforeVals = metrics.map(function(m){return numVal(snapshot.before[m]);}); + var afterVals = metrics.map(function(m){return numVal(snapshot.after[m]);}); + new Chart(ctx, { type:'bar', data: { + labels: metricLabels, + datasets:[ + { label:'Before', data:beforeVals, backgroundColor:colors.danger+'b0', borderRadius:4 }, + { label:'After', data:afterVals, backgroundColor:colors.success+'b0', borderRadius:4 } + ] + }, options:{ plugins:{ legend:{display:true, labels:{boxWidth:10,padding:8}} }, + scales:{ y:{beginAtZero:true, grid:{color:'#1e2a3a'}}, x:{grid:{display:false}} } } }); + } + } + + // ── Chart 11: Cross-Repo Transfer ── + var cross = await load('cross-repo-transfer'); + if (cross && cross.top_repos) { + var ctx = document.getElementById('chart-cross-repo-transfer'); + if (ctx) { summary('cross-repo-transfer', cross); + var repos = cross.top_repos.slice(0, 8); + new Chart(ctx, { type:'bar', data: { + labels: repos.map(function(r){return (r.name||r.repo||'').substring(0,18);}), + datasets:[{ label:'Commits', data:repos.map(function(r){return r.commits||r.count||0;}), + backgroundColor: repos.map(function(_,i){return PALETTE[i%PALETTE.length];}), borderRadius:4, barPercentage:0.7 }] + }, options:{ indexAxis:'y', scales:{ x:{beginAtZero:true, grid:{color:'#1e2a3a'}}, y:{grid:{display:false}} } } }); + } + } + + // ── Chart 12: YouTube Learning ── + var yt = await load('youtube-learning-graph'); + if (yt && yt.monthly_learning) { + var ctx = document.getElementById('chart-youtube-learning-graph'); + if (ctx) { summary('youtube-learning-graph', yt); + new Chart(ctx, { type:'line', data: { + labels: yt.monthly_learning.map(function(m){return m.month||m.period||'';}), + datasets:[{ label:'Videos Watched', data:yt.monthly_learning.map(function(m){return m.count||m.videos||0;}), + borderColor:colors.danger, backgroundColor:'rgba(248,113,113,0.1)', fill:true, tension:0.3, pointRadius:3, pointBackgroundColor:colors.danger }] + }, options:{ scales:{ y:{beginAtZero:true, grid:{color:'#1e2a3a'}}, x:{grid:{display:false}, ticks:{maxRotation:45}} }, + plugins:{tooltip:{mode:'index',intersect:false}} } }); + } + } + + // ── Chart 13: Architecture Timelapse ── + var arch = await load('architecture-timelapse'); + if (arch && arch.era_snapshots && arch.era_snapshots.length) { + var ctx = document.getElementById('chart-architecture-timelapse'); + if (ctx) { summary('architecture-timelapse', arch); + var snaps = arch.era_snapshots; + new Chart(ctx, { type:'bar', data: { + labels: snaps.map(function(s){return (s.era||'').substring(0,14);}), + datasets:[ + { label:'Commits', data:snaps.map(function(s){return s.commits||0;}), backgroundColor:colors.accent, borderRadius:4, barPercentage:0.4 }, + { label:'Restructurings', data:snaps.map(function(s){return s.restructuring_signals||0;}), backgroundColor:colors.warning, borderRadius:4, barPercentage:0.4 } + ] + }, options:{ scales:{ y:{beginAtZero:true, grid:{color:'#1e2a3a'}}, x:{grid:{display:false}} }, + plugins:{ legend:{display:true, labels:{boxWidth:10,padding:8}} } } }); + } + } + + // ── Chart 14: Cognitive Load ── + var cog = await load('commit-cognitive-load'); + if (cog && cog.work_type_distribution) { + var ctx = document.getElementById('chart-commit-cognitive-load'); + if (ctx) { summary('commit-cognitive-load', cog); + var types = cog.work_type_distribution; + var labels = Object.keys(types); + var vals = Object.values(types); + new Chart(ctx, { type:'doughnut', data: { + labels: labels.map(function(l){return l.replace(/_/g,' ');}), + datasets:[{ data:vals, backgroundColor:labels.map(function(_,i){return PALETTE[i%PALETTE.length];}), borderWidth:0 }] + }, options:{ cutout:'55%', plugins:{ legend:{display:true, position:'right', labels:{boxWidth:10,padding:8,font:{size:10}}} } } }); + } + } +}); +""" + + +def generate_project_index(project: dict[str, Any]) -> str: + """Generate per-project dashboard with data visualizations front and center.""" + _now = datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC") + meta = project["meta"] + proj_name = project["name"].upper() + commits = meta.get("commits", "?") + eras = meta.get("era_count", "?") + active_days = meta.get("active_days", "?") + span_days = meta.get("span_days", "?") + commits_fmt = f"{commits:,}" if isinstance(commits, int) else str(commits) + total_deliv = project.get("total_deliverables", 0) + + # Check for opportunity data + deliverables = project.get("deliverables", {}) + opp_files = deliverables.get("opportunity", []) + has_opportunity = len(opp_files) > 0 + + # Build file cards HTML (for collapsible section) + cat_sections = "" + for cat_name, cat_meta in CATEGORIES.items(): + files = deliverables.get(cat_name, []) + if not files: + continue + file_cards = "" + for f in files: + href = f["href"].split("/", 1)[-1] + if f["ext"] == ".md": + link = f"../md-viewer.html?file={project['name']}/{href}" + else: + link = href + ext_color = {"html": "#14b8a6", "md": "#8b5cf6", "json": "#f59e0b"}.get( + f["ext"].lstrip("."), "#6a7888" + ) + file_cards += f'{cat_meta["icon"]}{f["name"]}{f["ext"].lstrip(".").upper()}\n' + cat_sections += f'

    {cat_meta["icon"]}{cat_meta["label"]}{len(files)}

    {file_cards}
    \n' + + # Build opportunity section (if data exists) + opp_html = "" + if has_opportunity: + # Score cards + score_cards = """ +
    + +
    --
    AI Mastery
    / 100
    +
    +
    + +
    --
    Session Quality
    / 10
    +
    +
    + +
    --
    Learning Accel.
    x factor
    +
    +
    + +
    --
    Frustration Conv.
    automated
    +
    + """ + + # Viz cards + viz_cards = "" + for feat in _OPP_FEATURES: + viz_cards += f""" +
    +
    + {feat["icon"]} + {feat["title"]} +
    +
    +
    Loading...
    +
    """ + + opp_html = f""" +
    {score_cards}
    +
    +

    Opportunity Analysis {len(opp_files)}

    +
    {viz_cards}
    +
    """ + + # Assemble HTML + css = _PROJECT_INDEX_CSS + js = _PROJECT_INDEX_JS + + files_section = "" + if has_opportunity: + files_section = f""" + + """ + else: + files_section = cat_sections + + html = f""" + + +{ + head_bundle( + title=f"{proj_name} — Project Dashboard", + description=f"Archaeological analysis of {proj_name} — {commits_fmt} commits across {eras} eras", + include_charts=True, + ) + } + + + + + + + + +
    +
    +
    +

    {proj_name}

    +

    {_project_description(project["name"], meta)}

    +
    +
    { + commits_fmt + }Commits
    +
    { + eras if eras else "—" + }Eras
    +
    { + active_days + }Active Days
    +
    {span_days}Day Span
    +
    { + total_deliv if total_deliv else "—" + }Deliverables
    +
    +
    + {opp_html} + {files_section} +
    +
    + +{body_end_bundle()} + + + +""" + return html + + +def _viz_description(name: str) -> str: + """Return a short description for a visualization page.""" + descriptions = { + "Dashboard": "Full archaeological dashboard with timeline, eras, heatmap, and telemetry", + "Playbook": "Era-by-era narrative playbook with key events and patterns", + "Agents": "AI agent performance benchmark comparing all coding agents", + "Report": "Structured archaeological report with findings and analysis", + } + return descriptions.get(name, "Archaeological analysis visualization") + + +def _viz_icon(name: str) -> str: + """Return an emoji icon for a visualization page.""" + icons = { + "Dashboard": "📊", + "Playbook": "📖", + "Agents": "🤖", + "Report": "📄", + } + return icons.get(name, "🔬") + + +def load_api_repos(global_data_dir: Path) -> list[dict[str, Any]]: + """Load repo metadata from fetch-github JSON files. + + Returns list of repo dicts sorted by commit count (descending). + """ + repos = [] + for json_file in global_data_dir.glob("*-repos.json"): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as e: + print(f" WARNING: Failed to load {json_file.name}: {e}") + continue + for repo in data.get("repos", []): + if repo.get("is_fork"): + continue + commits = repo.get("total_commits") or repo.get("commit_count", 0) + repos.append( + { + "name": repo.get("name", "?"), + "commits": commits, + "language": repo.get("language", ""), + "description": repo.get("description", ""), + "updated": (repo.get("updated_at") or repo.get("updated", ""))[:10], + "owner": data.get("owner", ""), + "html_url": repo.get( + "html_url", + f"https://github.com/{data.get('owner', '')}/{repo.get('name', '')}", + ), + "is_fork": repo.get("is_fork", False), + } + ) + repos.sort(key=lambda r: r["commits"], reverse=True) + return repos + + +def generate_global_section( + api_repos: list[dict[str, Any]], owner_labels: dict[str, str] | None = None +) -> str: + """Generate an HTML section showing API-only repos (lightweight cards).""" + if not api_repos: + return "" + + labels = owner_labels or {} + # Group by owner + by_owner: dict[str, list] = {} + for repo in api_repos: + owner = repo["owner"] + by_owner.setdefault(owner, []).append(repo) + + sections = "" + for owner, owner_repos in by_owner.items(): + label = labels.get(owner, owner) + total_commits = sum(r["commits"] for r in owner_repos) + total_commits_fmt = f"{total_commits:,}" + + cards = "" + for repo in owner_repos: + commits_fmt = f"{repo['commits']:,}" + lang = repo["language"] or "" + desc = (repo["description"] or "")[:80] + cards += f""" +
    +
    + {repo["name"]} + {commits_fmt} commits +
    +
    {lang} · updated {repo["updated"]}"
    + {f'
    {desc}
    ' if desc else ""} +
    """ + + sections += f""" +
    +

    {label}{len(owner_repos)} repos · {total_commits_fmt} commits

    +
    + {cards} +
    +
    """ + + return sections diff --git a/plugins/devarch-hermes/archaeology/visualization/design_system.py b/plugins/devarch-hermes/archaeology/visualization/design_system.py new file mode 100644 index 0000000..47c6eb9 --- /dev/null +++ b/plugins/devarch-hermes/archaeology/visualization/design_system.py @@ -0,0 +1,451 @@ +"""Unified design system for all DevArch HTML outputs. + +Exports CSS, JS, and HTML constants for 3 selectable themes +(Warm, Clean Editorial, Modern Data). Every generator imports +from this module so all outputs share one visual identity. + +Source of truth: shared/design-system-spec.md +""" + +from __future__ import annotations + +from typing import Any + +# ── Google Fonts (all themes share this bundle) ───────────────────────── + +GOOGLE_FONTS_LINK = """\ + + +""" + + +# ── Theme CSS (all 3 theme token sets) ────────────────────────────────── + +THEME_CSS = """""" + + +# ── Theme Switcher HTML ────────────────────────────────────────────────── + +THEME_SWITCHER_HTML = """\ +
    + + + +
    """ + +THEME_SWITCHER_CSS = """\ +""" + +THEME_SWITCHER_JS = """\ +""" + + +# ── Chart Theme Integration ────────────────────────────────────────────── + +CHART_THEME_JS = """\ +""" + + +# ── Accessibility CSS ──────────────────────────────────────────────────── + +ACCESSIBILITY_CSS = """\ +""" + + +# ── SEO Meta Tags ──────────────────────────────────────────────────────── + + +def seo_meta( + title: str, + description: str, + url: str = "", + og_type: str = "website", + json_ld: dict[str, Any] | None = None, + image: str = "", +) -> str: + """Generate meta description, OG, Twitter Card, robots, and JSON-LD tags. + + Args: + title: Page title. + description: Page description. + url: Canonical URL (optional, for deployed sites). + og_type: Open Graph type (website, article). + json_ld: Optional Schema.org structured data dict. + image: OG/Twitter image URL (optional, for social sharing previews). + + Returns: + HTML string with meta tags. + """ + import json + + tags = f"""\ + + + + + + + +""" + if url: + tags += f'\n\n' + if image: + tags += f'\n\n' + if json_ld: + tags += f'\n' + return tags + + +def seo_software_application( + name: str, description: str, version: str = "", url: str = "", image: str = "" +) -> str: + """Shorthand for SoftwareApplication JSON-LD (common in DevArch outputs).""" + return seo_meta( + title=f"{name} — DevArch Analysis", + description=description, + url=url, + image=image, + json_ld={ + "@context": "https://schema.org", + "@type": "SoftwareApplication", + "name": name, + "applicationCategory": "DeveloperApplication", + "description": description, + **({"version": version} if version else {}), + **({"url": url} if url else {}), + }, + ) + + +# ── Favicon ────────────────────────────────────────────────────────────── + +FAVICON = """\ +""" + + +# ── Convenience: full head bundle ──────────────────────────────────────── + + +def head_bundle( + title: str, + description: str, + include_charts: bool = False, + include_d3: bool = False, + url: str = "", + json_ld: dict[str, Any] | None = None, +) -> str: + """Generate the complete content for a DevArch HTML page. + + Args: + title: Page title. + description: Page description (for meta + OG). + include_charts: Include Chart.js CDN and theme integration. + include_d3: Include D3.js CDN. + url: Canonical URL (optional). + json_ld: Schema.org structured data (optional). + + Returns: + HTML string for everything that goes inside . + """ + parts = [ + '', + '', + f"{title}", + FAVICON, + GOOGLE_FONTS_LINK, + seo_meta(title, description, url=url, json_ld=json_ld), + THEME_CSS, + THEME_SWITCHER_CSS, + ACCESSIBILITY_CSS, + ] + if include_d3: + parts.append('') + if include_charts: + parts.append( + '' + ) + parts.append(CHART_THEME_JS) + return "\n".join(parts) + + +# ── Convenience: body-end bundle ───────────────────────────────────────── + + +def body_end_bundle() -> str: + """Generate the theme switcher JS that goes at the end of .""" + return THEME_SWITCHER_JS diff --git a/plugins/devarch-hermes/archaeology/visualization/github_fetcher.py b/plugins/devarch-hermes/archaeology/visualization/github_fetcher.py new file mode 100644 index 0000000..99b50dd --- /dev/null +++ b/plugins/devarch-hermes/archaeology/visualization/github_fetcher.py @@ -0,0 +1,168 @@ +"""Fetch repo metadata from GitHub API for all repos (no cloning needed).""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +_DEFAULT_OWNER = os.environ.get("ARCHAEOLOGY_GITHUB_OWNER", "") + + +def _gh(*args): + """Run a gh command and return stdout.""" + result = subprocess.run(["gh"] + list(args), capture_output=True, text=True) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def _gh_api(endpoint, jq_filter=None): + """Run a gh api call and return parsed output.""" + cmd = ["gh", "api", endpoint] + if jq_filter: + cmd.extend(["--jq", jq_filter]) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def _fetch_all_contributors(owner, repo_name): + """Fetch all contributors with pagination.""" + contributors = {} + page = 1 + per_page = 100 + + while True: + contrib_raw = _gh_api( + f"repos/{owner}/{repo_name}/contributors?per_page={per_page}&page={page}", + '.[] | .login + ":" + (.contributions|tostring)', + ) + if not contrib_raw: + break + + page_contributors = {} + for line in contrib_raw.split("\n"): + if ":" in line: + login, count = line.split(":", 1) + try: + c = int(count) + page_contributors[login] = c + except ValueError: + pass + + if not page_contributors: + break + + contributors.update(page_contributors) + + # If we got fewer than per_page results, we're done + if len(page_contributors) < per_page: + break + + page += 1 + + return contributors + + +def fetch_all_repos(owner=_DEFAULT_OWNER, limit=None): + """Fetch all repos with commit counts, languages, and authors. + + Args: + owner: GitHub username or organization. + limit: Maximum number of repos to fetch (None for all). + """ + # Use a high limit to get all repos (gh repo list handles pagination internally) + repo_limit = limit if limit else 10000 + raw = _gh( + "repo", + "list", + owner, + "--limit", + str(repo_limit), + "--json", + "name,isFork,primaryLanguage,createdAt,updatedAt,description,diskUsage", + ) + if not raw: + print("Failed to fetch repo list from GitHub", file=sys.stderr) + return [] + + try: + repo_list = json.loads(raw) + except json.JSONDecodeError: + print("Failed to parse repo list", file=sys.stderr) + return [] + + results = [] + total = len(repo_list) + for i, repo in enumerate(repo_list): + name = repo["name"] + is_fork = repo.get("isFork", False) + + # Skip forks (awesome-mcp-servers, apex-vault etc) + if is_fork: + print(f" [{i + 1}/{total}] {name}... SKIP (fork)", flush=True) + continue + + print(f" [{i + 1}/{total}] {name}...", end=" ", flush=True) + + # Get contributors with pagination + authors = _fetch_all_contributors(owner, name) + total_commits = sum(authors.values()) + + # Get languages + lang_raw = _gh_api( + f"repos/{owner}/{name}/languages", 'to_entries | .[] | .key + ":" + (.value|tostring)' + ) + languages = {} + if lang_raw: + for line in lang_raw.split("\n"): + if ":" in line: + lang, bytes_str = line.split(":", 1) + try: + languages[lang] = int(bytes_str) + except ValueError: + pass + + # Skip repos with zero commits + if total_commits == 0: + print("SKIP (0 commits)", flush=True) + continue + + lang_obj = repo.get("primaryLanguage") + lang_name = lang_obj.get("name") if lang_obj else None + + results.append( + { + "name": name, + "language": lang_name, + "created": repo.get("createdAt", "")[:10], + "updated": repo.get("updatedAt", "")[:10], + "description": repo.get("description", ""), + "size_kb": repo.get("diskUsage", 0), + "total_commits": total_commits, + "authors": authors, + "languages": languages, + } + ) + print(f"{total_commits} commits, {len(authors)} authors", flush=True) + + return results + + +def save_github_data(output_path, owner=_DEFAULT_OWNER): + """Fetch all repo data and save to JSON.""" + repos = fetch_all_repos(owner=owner) + output = { + "owner": owner, + "total_repos": len(repos), + "total_commits": sum(r["total_commits"] for r in repos), + "repos": repos, + } + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(output, f, indent=2) + print(f"\nSaved {len(repos)} repos to {path}") + return output diff --git a/plugins/devarch-hermes/archaeology/visualization/global_data_builder.py b/plugins/devarch-hermes/archaeology/visualization/global_data_builder.py new file mode 100644 index 0000000..b9710ab --- /dev/null +++ b/plugins/devarch-hermes/archaeology/visualization/global_data_builder.py @@ -0,0 +1,842 @@ +"""Build visualization-ready JSON from aggregated global data.""" + +import csv +import json +import os +import re +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path + +from ..utils import _parse_date + + +def _safe_parse_date(s, fmt="%Y-%m-%d"): + """Parse date string, returning None on failure.""" + if not s or not isinstance(s, str): + return None + try: + return datetime.strptime(s[:10], fmt) + except (ValueError, TypeError): + return None + + +LANG_MAP = { + ".py": "Python", + ".js": "JavaScript", + ".ts": "TypeScript", + ".tsx": "TypeScript", + ".jsx": "JavaScript", + ".html": "HTML", + ".css": "CSS", + ".scss": "CSS", + ".json": "JSON", + ".md": "Markdown", + ".yaml": "YAML", + ".yml": "YAML", + ".sh": "Shell", + ".bash": "Shell", + ".zsh": "Shell", + ".rs": "Rust", + ".go": "Go", + ".rb": "Ruby", + ".java": "Java", + ".c": "C", + ".cpp": "C++", + ".h": "C/C++ Header", + ".sql": "SQL", + ".graphql": "GraphQL", + ".vue": "Vue", + ".svelte": "Svelte", + ".toml": "TOML", + ".ini": "INI", + ".cfg": "Config", + ".txt": "Text", + ".rst": "reStructuredText", + ".svg": "SVG", + ".png": "Image", + ".jpg": "Image", +} + + +def _repo_color(name, index=0): + """Generate a consistent color for a repository name.""" + # Fixed palette for known repos + _KNOWN = { + "demo-project": "#51cf66", + "dev-archaeology": "#74c0fc", + "github-pipeline": "#ffa94d", + "voice-to-sculpture": "#cc5de8", + } + if name in _KNOWN: + return _KNOWN[name] + # Generate from hash for unknown repos + palette = [ + "#51cf66", + "#74c0fc", + "#ffa94d", + "#cc5de8", + "#ff6b6b", + "#ffd43b", + "#20c997", + "#845ef7", + ] + return palette[hash(name) % len(palette)] + + +def prepare_global_visualization_data(global_dir, top_n=None, year=None): + """Read global data files and produce a single viz-ready JSON dict. + + Prefers GitHub API data (github-repos.json) when available for full repo + coverage. Falls back to local CSV data otherwise. + """ + global_dir = Path(global_dir) + data_dir = global_dir / "data" + github_json = data_dir / "github-repos.json" + + if github_json.exists(): + return prepare_from_github(github_json, top_n=top_n, year=year) + + # Fallback to local CSV data + projects_dir = global_dir.parent / "projects" + commits_csv = data_dir / "global-commits.csv" + summaries_json = data_dir / "project-summaries.json" + signals_json = data_dir / "global-signals.json" + + if not commits_csv.exists(): + raise FileNotFoundError( + f"No data in {data_dir}. Run 'devarch fetch-github' or 'devarch sync' first." + ) + + # Load raw data + commits = _load_commits(commits_csv) + summaries = _load_json(summaries_json) if summaries_json.exists() else [] + _signals = _load_json(signals_json) if signals_json.exists() else [] + + # Assign colors + repo_names = sorted({c["_project"] for c in commits}) + colors = {} + for i, name in enumerate(repo_names): + colors[name] = _repo_color(name) + + # Build sections + result = { + "meta": _build_meta(commits, summaries, repo_names), + "repos": _build_repo_cards(commits, summaries, colors, projects_dir), + "timeline": _build_global_timeline(commits, colors), + "authors": _build_author_universe(commits, colors), + "languages": _build_language_breakdown(projects_dir, repo_names, colors), + "velocity": _build_velocity_comparison(commits, summaries, colors), + "heatmap": _build_activity_heatmap(commits), + "commit_types": _build_commit_types(commits, colors), + "correlation": _build_correlation(commits, colors), + } + + return result + + +def prepare_from_github(github_json_path, top_n=None, year=None): + """Build visualization data from GitHub API JSON (all repos, no cloning).""" + with open(github_json_path, encoding="utf-8") as f: + gh_data = json.load(f) + + repos = gh_data.get("repos", []) + + # Filter by year (repos updated in that year) + if year: + repos = [r for r in repos if r.get("updated", "")[:4] == str(year)] + + # Sort by commits descending, then take top N + repos.sort(key=lambda x: x["total_commits"], reverse=True) + if top_n: + repos = repos[:top_n] + + # Assign colors + repo_names = sorted(r["name"] for r in repos) + colors = {} + for i, name in enumerate(repo_names): + colors[name] = _repo_color(name) + + total_commits = sum(r["total_commits"] for r in repos) + all_dates = set() + for r in repos: + if r.get("created"): + all_dates.add(r["created"][:10]) + if r.get("updated"): + all_dates.add(r["updated"][:10]) + + # Unique authors across all repos + all_authors = set() + for r in repos: + all_authors.update(r.get("authors", {}).keys()) + + created_dates = [r["created"][:10] for r in repos if r.get("created")] + updated_dates = [r["updated"][:10] for r in repos if r.get("updated")] + first = min(created_dates) if created_dates else "" + last = max(updated_dates) if updated_dates else "" + + d1, d2 = _safe_parse_date(first), _safe_parse_date(last) + calendar_days = (d2 - d1).days + 1 if d1 and d2 else 0 + + meta = { + "total_commits": total_commits, + "total_repos": len(repos), + "total_active_days": None, # not available from API + "total_calendar_days": calendar_days, + "total_authors": len(all_authors), + "first_date": first, + "last_date": last, + "repo_names": repo_names, + } + + # Repo cards + repo_cards = [] + for r in sorted(repos, key=lambda x: x["total_commits"], reverse=True): + name = r["name"] + top_lang_bytes = sorted(r.get("languages", {}).items(), key=lambda x: x[1], reverse=True) + top_language = top_lang_bytes[0][0] if top_lang_bytes else "Unknown" + top_author = sorted(r.get("authors", {}).items(), key=lambda x: x[1], reverse=True) + top_author_name = top_author[0][0] if top_author else "Unknown" + + created = r.get("created", "")[:10] + updated = r.get("updated", "")[:10] + dc, du = _safe_parse_date(created), _safe_parse_date(updated) + repo_calendar_days = (du - dc).days + 1 if dc and du else 0 + + repo_cards.append( + { + "name": name, + "color": colors.get(name, "#888"), + "total_commits": r["total_commits"], + "active_days": None, + "calendar_days": repo_calendar_days, + "first_date": r.get("created", "")[:10], + "last_date": r.get("updated", "")[:10], + "authors": len(r.get("authors", {})), + "top_author": top_author_name, + "top_language": top_language, + "description": r.get("description", ""), + "size_kb": r.get("size_kb", 0), + } + ) + + # Language breakdown (aggregate bytes across all repos) + lang_by_repo = [] + for r in repos: + name = r["name"] + langs = r.get("languages", {}) + total_bytes = sum(langs.values()) + if not langs: + continue + lang_list = [] + for lang, bytes_count in sorted(langs.items(), key=lambda x: x[1], reverse=True)[:8]: + lang_list.append( + { + "language": lang, + "count": bytes_count, + "pct": round(bytes_count / total_bytes * 100, 1) if total_bytes else 0, + } + ) + lang_by_repo.append( + { + "repo": name, + "color": colors.get(name, "#888"), + "languages": lang_list, + "total_files": total_bytes, + } + ) + + # Velocity comparison + velocity = [] + for r in repos: + name = r["name"] + dc, du = _safe_parse_date(r.get("created", "")), _safe_parse_date(r.get("updated", "")) + span_days = (du - dc).days + 1 if dc and du else 1 + velocity.append( + { + "repo": name, + "color": colors.get(name, "#888"), + "total_commits": r["total_commits"], + "active_days": None, + "commits_per_day": round(r["total_commits"] / max(span_days, 1), 2), + "peak_day": "", + "peak_count": 0, + "span_days": span_days, + } + ) + velocity.sort(key=lambda x: x["total_commits"], reverse=True) + + # Author universe (graph) + author_nodes_dict = {} # Use dict for O(1) lookups instead of O(n) list scans + repo_nodes = [] + links = [] + for r in repos: + repo_nodes.append( + { + "id": r["name"], + "type": "repo", + "color": colors.get(r["name"], "#888"), + } + ) + for author, count in r.get("authors", {}).items(): + # O(1) dict lookup instead of O(n) list scan + if author not in author_nodes_dict: + author_nodes_dict[author] = { + "id": author, + "type": "author", + "commits": count, + "repos": [r["name"]], + } + else: + node = author_nodes_dict[author] + node["commits"] += count + node["repos"].append(r["name"]) + links.append( + { + "source": author, + "target": r["name"], + "value": count, + } + ) + + # Convert dict back to list for JSON serialization + author_nodes = list(author_nodes_dict.values()) + + # Commit types not available from GitHub API + # Heatmap not available from GitHub API + # Timeline correlation not available from GitHub API (no per-date data) + + return { + "meta": meta, + "repos": repo_cards, + "timeline": None, # requires per-commit dates + "authors": {"nodes": author_nodes + repo_nodes, "links": links}, + "languages": lang_by_repo, + "velocity": velocity, + "heatmap": None, # requires per-commit hourly data + "commit_types": None, # requires commit messages + "correlation": None, # requires per-date data + "source": "github-api", + "total_repos_on_github": gh_data.get("total_repos", len(repos)), + } + + +def _load_commits(csv_path): + rows = [] + with open(csv_path, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + clean = {k: v for k, v in row.items() if k is not None} + rows.append(clean) + return rows + + +def _load_json(path): + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _classify_commit(message): + """Classify commit message into type.""" + if not message: + return "other" + msg = message.lower().strip() + prefixes = { + "feat": "feature", + "add": "feature", + "implement": "feature", + "new": "feature", + "fix": "fix", + "bug": "fix", + "patch": "fix", + "hotfix": "fix", + "test": "test", + "spec": "test", + "doc": "docs", + "readme": "docs", + "refactor": "refactor", + "clean": "refactor", + "remove": "refactor", + "simplify": "refactor", + "chore": "chore", + "build": "chore", + "ci": "chore", + "deps": "chore", + "perf": "perf", + "optim": "perf", + "style": "style", + "format": "style", + "lint": "style", + } + for prefix, category in prefixes.items(): + if msg.startswith(prefix) or msg.startswith(f"[{prefix}]"): + return category + # Check for conventional commit prefix + match = re.match(r"^(\w+)(\(.+\))?\!?:", msg) + if match: + tag = match.group(1) + for prefix, category in prefixes.items(): + if tag.startswith(prefix): + return category + # Common patterns + if any(w in msg for w in ["merge", "pull request"]): + return "merge" + if any(w in msg for w in ["initial", "init", "bootstrap"]): + return "feature" + return "other" + + +# ── Section builders ────────────────────────────────────────────────── + + +def _build_meta(commits, summaries, repo_names): + """Build hero counters.""" + total_commits = len(commits) + dates = set() + authors = set() + for c in commits: + dt = _parse_date(c.get("date", "")) + if dt: + dates.add(dt.strftime("%Y-%m-%d")) + if c.get("author"): + authors.add(c["author"]) + + first = min(dates) if dates else "" + last = max(dates) if dates else "" + span = 0 + if first and last: + d1, d2 = _safe_parse_date(first), _safe_parse_date(last) + span = (d2 - d1).days + 1 + + return { + "total_commits": total_commits, + "total_repos": len(repo_names), + "total_active_days": len(dates), + "total_calendar_days": span, + "total_authors": len(authors), + "first_date": first, + "last_date": last, + "repo_names": repo_names, + } + + +def _build_repo_cards(commits, summaries, colors, projects_dir): + """Build per-repo summary cards.""" + by_repo = defaultdict(list) + for c in commits: + by_repo[c["_project"]].append(c) + + cards = [] + for name in sorted(by_repo.keys()): + repo_commits = by_repo[name] + dates = set() + authors = Counter() + for c in repo_commits: + dt = _parse_date(c.get("date", "")) + if dt: + dates.add(dt.strftime("%Y-%m-%d")) + if c.get("author"): + authors[c["author"]] += 1 + + sorted_dates = sorted(dates) + first = sorted_dates[0] if sorted_dates else "" + last = sorted_dates[-1] if sorted_dates else "" + span = 0 + if first and last: + d1, d2 = _safe_parse_date(first), _safe_parse_date(last) + span = (d2 - d1).days + 1 + + # Top language + top_lang = _get_top_language(projects_dir, name) + + # Summary lookup + _summary = {} + for s in summaries: + if s["name"] == name: + _summary = s + break + + cards.append( + { + "name": name, + "color": colors.get(name, "#888"), + "total_commits": len(repo_commits), + "active_days": len(dates), + "calendar_days": span, + "first_date": first, + "last_date": last, + "authors": len(authors), + "top_author": authors.most_common(1)[0][0] if authors else "", + "top_language": top_lang, + } + ) + + return cards + + +def _get_top_language(projects_dir, repo_name): + """Get top language for a repo from file extensions.""" + repo_dir = projects_dir / repo_name + if not repo_dir.exists(): + return "Unknown" + + ext_counter = Counter() + for root, _, files in os.walk(str(repo_dir)): + # Skip hidden dirs and common non-source dirs + # Only check repo-relative parts, not absolute path prefix + try: + rel_path = Path(root).relative_to(repo_dir) + except ValueError: + # root is not relative to repo_dir (shouldn't happen with os.walk) + continue + if any(p.startswith(".") for p in rel_path.parts): + continue + if any( + p in ("node_modules", "__pycache__", ".venv", "dist", "build") for p in rel_path.parts + ): + continue + for f in files: + ext = Path(f).suffix.lower() + if ext in LANG_MAP: + ext_counter[LANG_MAP[ext]] += 1 + + if not ext_counter: + return "Unknown" + return ext_counter.most_common(1)[0][0] + + +def _build_global_timeline(commits, colors): + """Build stacked area data: date → repo → count.""" + by_date_repo = Counter() + for c in commits: + dt = _parse_date(c.get("date", "")) + if dt: + key = (dt.strftime("%Y-%m-%d"), c.get("_project", "unknown")) + by_date_repo[key] += 1 + + if not by_date_repo: + return {"dates": [], "repos": {}, "repo_colors": colors} + + all_dates = sorted({k[0] for k in by_date_repo}) + repo_names = sorted({k[1] for k in by_date_repo}) + + repos = {} + for name in repo_names: + repos[name] = [by_date_repo.get((d, name), 0) for d in all_dates] + + return { + "dates": all_dates, + "repos": repos, + "repo_colors": colors, + } + + +def _build_author_universe(commits, colors): + """Build author-repo membership graph.""" + author_repos = defaultdict(set) + author_commits = Counter() + + for c in commits: + author = c.get("author", "Unknown") + repo = c.get("_project", "unknown") + author_repos[author].add(repo) + author_commits[author] += 1 + + nodes = [] + links = [] + for author, repos in sorted(author_repos.items()): + nodes.append( + { + "id": author, + "type": "author", + "commits": author_commits[author], + "repos": list(repos), + } + ) + for repo in repos: + links.append( + { + "source": author, + "target": repo, + "value": sum( + 1 + for c in commits + if c.get("author") == author and c.get("_project") == repo + ), + } + ) + + # Add repo nodes + for repo_name in sorted({c.get("_project", "") for c in commits}): + nodes.append( + { + "id": repo_name, + "type": "repo", + "color": colors.get(repo_name, "#888"), + } + ) + + return {"nodes": nodes, "links": links} + + +def _build_language_breakdown(projects_dir, repo_names, colors): + """Build language breakdown per repo.""" + result = [] + for name in repo_names: + repo_dir = projects_dir / name + if not repo_dir.exists(): + continue + + ext_counter = Counter() + for root, _, files in os.walk(str(repo_dir)): + # Only check repo-relative parts, not absolute path prefix + try: + rel_path = Path(root).relative_to(repo_dir) + except ValueError: + continue + if any(p.startswith(".") for p in rel_path.parts): + continue + if any( + p in ("node_modules", "__pycache__", ".venv", "dist", "build") + for p in rel_path.parts + ): + continue + for f in files: + ext = Path(f).suffix.lower() + if ext in LANG_MAP: + ext_counter[LANG_MAP[ext]] += 1 + + total = sum(ext_counter.values()) + languages = [] + for lang, count in ext_counter.most_common(10): + languages.append( + { + "language": lang, + "count": count, + "pct": round(count / total * 100, 1) if total else 0, + } + ) + + if languages: + result.append( + { + "repo": name, + "color": colors.get(name, "#888"), + "languages": languages, + "total_files": total, + } + ) + + return result + + +def _build_velocity_comparison(commits, summaries, colors): + """Build per-repo velocity stats.""" + by_repo = defaultdict(list) + for c in commits: + by_repo[c["_project"]].append(c) + + result = [] + for name, repo_commits in sorted(by_repo.items()): + dates = set() + daily = Counter() + for c in repo_commits: + dt = _parse_date(c.get("date", "")) + if dt: + ds = dt.strftime("%Y-%m-%d") + dates.add(ds) + daily[ds] += 1 + + active_days = len(dates) + peak = daily.most_common(1) + peak_day = peak[0] if peak else ("", 0) + + result.append( + { + "repo": name, + "color": colors.get(name, "#888"), + "total_commits": len(repo_commits), + "active_days": active_days, + "commits_per_day": round(len(repo_commits) / active_days, 1) if active_days else 0, + "peak_day": peak_day[0], + "peak_count": peak_day[1], + } + ) + + # Sort by commits descending + result.sort(key=lambda x: x["total_commits"], reverse=True) + return result + + +def _build_activity_heatmap(commits): + """Build day×hour matrix aggregated across all repos.""" + matrix = Counter() + for c in commits: + dt = _parse_date(c.get("date", "")) + if dt: + day = dt.strftime("%a") + hour = dt.hour + matrix[(day, hour)] += 1 + + days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] + hours = list(range(24)) + + cells = [] + for day in days: + for hour in hours: + count = matrix.get((day, hour), 0) + if count > 0: + cells.append({"day": day, "hour": hour, "count": count}) + + return { + "days": days, + "hours": hours, + "cells": cells, + "max_count": max(matrix.values()) if matrix else 0, + } + + +def _build_commit_types(commits, colors): + """Build commit type breakdown per repo.""" + by_repo = defaultdict(Counter) + for c in commits: + repo = c.get("_project", "unknown") + msg = c.get("message", "") + ctype = _classify_commit(msg) + by_repo[repo][ctype] += 1 + + all_types = sorted({t for counter in by_repo.values() for t in counter}) + result = [] + for repo in sorted(by_repo.keys()): + breakdown = {t: by_repo[repo].get(t, 0) for t in all_types} + result.append( + { + "repo": repo, + "color": colors.get(repo, "#888"), + "breakdown": breakdown, + } + ) + + return {"types": all_types, "repos": result} + + +def _build_correlation(commits, colors): + """Build data showing which repos were active on which dates.""" + by_date_repo = defaultdict(lambda: defaultdict(int)) + for c in commits: + dt = _parse_date(c.get("date", "")) + if dt: + by_date_repo[dt.strftime("%Y-%m-%d")][c.get("_project", "unknown")] += 1 + + dates = sorted(by_date_repo.keys()) + repo_names = sorted({c.get("_project", "") for c in commits}) + + rows = [] + for d in dates: + row = {"date": d} + for r in repo_names: + row[r] = by_date_repo[d].get(r, 0) + rows.append(row) + + return { + "dates": dates, + "repos": repo_names, + "rows": rows, + "repo_colors": colors, + } + + +def prepare_dashboard_data(global_dir, top_n=None, year=None): + """Prepare simplified data for multi-project dashboard. + + This function creates a streamlined dataset optimized for the dashboard + visualizations, focusing on repository metadata and aggregate statistics. + """ + global_dir = Path(global_dir) + data_dir = global_dir / "data" + github_json = data_dir / "github-repos.json" + + if not github_json.exists(): + raise FileNotFoundError( + f"No GitHub data found at {github_json}. Run 'devarch fetch-github' first." + ) + + with open(github_json, encoding="utf-8") as f: + gh_data = json.load(f) + + repos = gh_data.get("repos", []) + + # Filter by year if specified + if year: + repos = [r for r in repos if r.get("updated", "")[:4] == str(year)] + + # Sort by commits descending and take top N + repos.sort(key=lambda x: x["total_commits"], reverse=True) + if top_n: + repos = repos[:top_n] + + # Calculate metadata + total_commits = sum(r["total_commits"] for r in repos) + all_authors = set() + for r in repos: + all_authors.update(r.get("authors", {}).keys()) + + created_dates = [r["created"][:10] for r in repos if r.get("created")] + updated_dates = [r["updated"][:10] for r in repos if r.get("updated")] + first = min(created_dates) if created_dates else "" + last = max(updated_dates) if updated_dates else "" + + d1, d2 = _safe_parse_date(first), _safe_parse_date(last) + calendar_days = (d2 - d1).days + 1 if d1 and d2 else 0 + + # Build repo cards with additional metadata + repo_cards = [] + for r in repos: + name = r["name"] + + # Get top language by bytes + top_lang_bytes = sorted(r.get("languages", {}).items(), key=lambda x: x[1], reverse=True) + top_language = top_lang_bytes[0][0] if top_lang_bytes else r.get("language", "Unknown") + + # Get top author + top_author = sorted(r.get("authors", {}).items(), key=lambda x: x[1], reverse=True) + top_author_name = top_author[0][0] if top_author else "Unknown" + + created = r.get("created", "")[:10] + updated = r.get("updated", "")[:10] + dc, du = _safe_parse_date(created), _safe_parse_date(updated) + repo_calendar_days = (du - dc).days + 1 if dc and du else 0 + + repo_cards.append( + { + "name": name, + "total_commits": r["total_commits"], + "calendar_days": repo_calendar_days, + "first_date": created, + "last_date": updated, + "authors": len(r.get("authors", {})), + "top_author": top_author_name, + "top_language": top_language, + "description": r.get("description", ""), + "size_kb": r.get("size_kb", 0), + "language": r.get("language", top_language), + } + ) + + # Build metadata + meta = { + "total_commits": total_commits, + "total_repos": len(repos), + "total_calendar_days": calendar_days, + "total_authors": len(all_authors), + "first_date": first, + "last_date": last, + "repo_names": sorted(r["name"] for r in repos), + } + + return { + "meta": meta, + "repos": repo_cards, + "source": "github-api", + "total_repos_on_github": gh_data.get("total_repos", len(repos)), + } diff --git a/plugins/devarch-hermes/archaeology/visualization/history.py b/plugins/devarch-hermes/archaeology/visualization/history.py new file mode 100644 index 0000000..049f96a --- /dev/null +++ b/plugins/devarch-hermes/archaeology/visualization/history.py @@ -0,0 +1,40 @@ +"""Portable, self-contained visualization of measured history only.""" + +import html +import json +from pathlib import Path + +from ..utils import atomic_write + + +def render_history(project_root: Path) -> Path: + data_path = project_root / "deliverables" / "data.json" + if not data_path.exists(): + raise ValueError("No measured visualization data; run devarch build-db first") + data = json.loads(data_path.read_text(encoding="utf-8")) + daily = data.get("daily_commits") + if not isinstance(daily, dict): + raise ValueError("Measured daily_commits missing; rebuild with devarch build-db") + if sum(daily.values()) != data.get("total_commits"): + raise ValueError("Daily activity does not reconcile with total_commits") + config = json.loads((project_root / "project.json").read_text(encoding="utf-8")) + title = html.escape( + str(config.get("visualization", {}).get("title") or config.get("name") or project_root.name) + ) + maximum = max(daily.values(), default=1) or 1 + rows = "".join( + f'{html.escape(day)}{count}{count}' + for day, count in sorted(daily.items()) + ) + content = f""" +{title} — history + +

    {title}

    {data["total_commits"]:,} commits · {data.get("active_days", 0)} active UTC days

    +

    Author-date activity across locally available refs. Commits are not productivity, verified sessions, causal eras or evidence of implementation quality.

    +{rows}
    Measured daily commit activity
    Date (UTC)CommitsActivity
    """ + output = project_root / "deliverables" / "visuals" / "archaeology.html" + atomic_write(output, content) + from ..provenance import bind_artifact + + bind_artifact(project_root, output) + return output diff --git a/plugins/devarch-hermes/archaeology/visualization/nav.py b/plugins/devarch-hermes/archaeology/visualization/nav.py new file mode 100644 index 0000000..764675b --- /dev/null +++ b/plugins/devarch-hermes/archaeology/visualization/nav.py @@ -0,0 +1,241 @@ +"""Shared navigation component for all dev-archaeology HTML deliverables. + +Provides a consistent nav bar across all HTML pages with: +- Project name linking to project index +- Tab-style navigation to sibling pages +- "Home" link back to master dashboard +- Mobile hamburger menu +- PostHog analytics integration +""" + +from __future__ import annotations + +from pathlib import Path + +# ── PostHog snippet ────────────────────────────────────────────────── + +POSTHOG_SNIPPET = "" + + +# ── Shared CSS ─────────────────────────────────────────────────────── + +NAV_CSS = """""" + + +# ── Theme Switcher JavaScript ─────────────────────────────────────────────── + +NAV_THEME_JS = """""" + + +def _discover_sibling_pages( + current_file: Path, project_deliverables_dir: Path +) -> list[dict[str, str]]: + """Find all HTML files in the same visuals/ directory (or deliverables/ for legacy). + + Returns list of {name, href, is_active}. + """ + visuals_dir = project_deliverables_dir / "visuals" + if not visuals_dir.exists(): + visuals_dir = project_deliverables_dir + + pages = [] + for html_file in sorted(visuals_dir.glob("*.html")): + name = html_file.stem + # Skip index pages + if name == "index": + continue + # Pretty display name + display = name.replace("-", " ").replace("_", " ").title() + if display == "Archaeology": + display = "Dashboard" + elif display == "Agent Benchmark": + display = "Agents" + elif display == "Playbook": + display = "Playbook" + elif display == "Report": + display = "Report" + pages.append( + { + "name": display, + "href": html_file.name, + "is_active": current_file.name == html_file.name, + } + ) + return pages + + +def generate_nav( + project_name: str, + current_file: Path, + project_deliverables_dir: Path, + include_posthog: bool = True, + home_url: str = "/", +) -> str: + """Generate the shared nav bar HTML for a deliverable page. + + Args: + project_name: Display name for the project. + current_file: Path to the current HTML file (for active state). + project_deliverables_dir: Path to the project's deliverables/ directory. + include_posthog: Whether to include PostHog analytics snippet. + home_url: URL for the "Home" link (master dashboard). + + Returns: + HTML string containing the nav bar, CSS, and optional PostHog. + """ + pages = _discover_sibling_pages(current_file, project_deliverables_dir) + + # Build page links + page_links = "" + for page in pages: + active_class = " active" if page["is_active"] else "" + # Determine relative path from current file to visuals dir + if current_file.parent.name == "visuals": + href = page["href"] + else: + href = f"visuals/{page['href']}" + page_links += f'{page["name"]}\n ' + + # Determine relative path to index.html + if current_file.parent.name == "visuals": + project_index_href = "index.html" + else: + project_index_href = "visuals/index.html" + + nav_html = f"""{NAV_CSS} +{POSTHOG_SNIPPET if include_posthog else ""} + +{NAV_THEME_JS} +""" + return nav_html + + +def generate_nav_simple( + project_name: str, + pages: list[dict[str, str]], + active_page: str, + include_posthog: bool = True, + home_url: str = "/", +) -> str: + """Generate nav bar with explicit page list (for generated dashboards). + + Args: + project_name: Display name for the project. + pages: List of {name, href} dicts. + active_page: The href of the active page. + include_posthog: Whether to include PostHog snippet. + home_url: URL for the home link. + + Returns: + HTML string. + """ + page_links = "" + for page in pages: + active_class = " active" if page["href"] == active_page else "" + page_links += f'{page["name"]}\n ' + + nav_html = f"""{NAV_CSS} +{POSTHOG_SNIPPET if include_posthog else ""} + +{NAV_THEME_JS} +""" + return nav_html + + +def inject_nav_into_html(html: str, nav_html: str) -> str: + """Inject the nav bar into an existing HTML document. + + Inserts right after tag. Adds class="nav-body" to main content + to account for the sticky nav height. + """ + if " + body_idx = html.index("", body_idx) + 1 + html = html[:body_close] + "\n" + nav_html + html[body_close:] + + # If there's a root wrapper div, give it nav-body class + # This adds top margin to account for sticky nav + if 'class="container"' in html: + html = html.replace('class="container"', 'class="container nav-body"', 1) + elif '
    ' in html: + html = html.replace('
    ', '
    ', 1) + + return html diff --git a/plugins/devarch-hermes/plugin.yaml b/plugins/devarch-hermes/plugin.yaml new file mode 100644 index 0000000..856c342 --- /dev/null +++ b/plugins/devarch-hermes/plugin.yaml @@ -0,0 +1,48 @@ +name: devarch +version: "0.1.0" +manifest_version: 2 +description: "DevArch for Hermes: git-repository archaeology — mine local commit history into a SQLite ledger, detect development signals (gaps, velocity shifts, author changes), and export an evidence-backed eras/signals markdown report before the agent touches the code. Adds pre_llm_call archaeology brief, post_tool_call report ledger, /devarch (mine/signals/report/pipeline/list/version/status/help), tools devarch_mine / devarch_signals / devarch_report / devarch_pipeline / devarch_list / devarch_version, and a bundled devarch skill. Disclosure — vendors the DevArch `archaeology` package (pinned commit b44430e198e45bfaee1edb2f3473f7d50ce17bc2 / v0.4.1) inside this plugin directory and installs PyPI deps `click`, `sqlite-utils`, `datasette` into the Hermes venv on `hermes plugins install` / `validate --install-deps` (needs git + Python >=3.10); mining is local-only (reads the git repo you name via the git CLI, no network fetch); rejects shallow clones; analysis vectors are keyword investigation leads, not verified conclusions; project DBs, signals, reports and the ledger live under /plugin-data/devarch/; no telemetry; slash mine/pipeline/report refused on gateway platforms unless opted in." +author: "Kyanite Labs" +license: Apache-2.0 +homepage: https://github.com/KyaniteLabs/devarch-framework/tree/main/plugins/devarch-hermes +tags: [git, archaeology, history, signals, eras, local-first, report] +requires_hermes: ">=0.21.6" +requires_env: [] +pip_dependencies: + - "click==8.1.8" + - "sqlite-utils==3.38" + - "datasette==0.65.5" +provides_tools: + - devarch_mine + - devarch_signals + - devarch_report + - devarch_pipeline + - devarch_list + - devarch_version +provides_hooks: + - pre_llm_call + - post_tool_call +hooks: + - pre_llm_call + - post_tool_call +config_schema: + max_commits: + type: int + default: 500 + description: "Hard extraction cap across local refs; reports explicitly label sampled history. Must be positive." + inject_on_archaeology: + type: bool + default: true + description: "When true, pre_llm_call injects an archaeology brief if the message asks about repo history / evolution / eras." + allow_repo_slash_on_gateway: + type: bool + default: false + description: "When false, /devarch mine|pipeline|report are refused on gateway platforms (local filesystem + git access)." + max_commits_warn: + type: int + default: 5000 + description: "Soft warning threshold; mining still runs, but the response notes when local commit count exceeds this." + default_project: + type: str + default: "session" + description: "Default DevArch project name when the caller omits project." diff --git a/plugins/devarch-hermes/skills/devarch/SKILL.md b/plugins/devarch-hermes/skills/devarch/SKILL.md new file mode 100644 index 0000000..7b391e8 --- /dev/null +++ b/plugins/devarch-hermes/skills/devarch/SKILL.md @@ -0,0 +1,23 @@ +--- +name: devarch +description: >- + Use when the user asks how a local git repository evolved, wants eras/signals + from commit history, or needs an evidence-backed archaeology report before + changing code. Mine locally, label vectors as leads, refuse shallow clones. +--- + +# DevArch archaeology (Hermes) + +Three laws: +1. **Local history only.** Mine the checkout on disk; do not invent eras from a handful of recent commits you happened to see. +2. **Leads, not conclusions.** Automated vectors and signals are keyword investigation leads — never present them as verified causal explanations or productivity scores. +3. **Integrity first.** Shallow clones are a hard refuse. Report coverage (commit count, head, shallow=false) beside any claim. + +## Workflow +1. Confirm a local git path (worktree with `.git`, or bare repo). +2. Prefer `devarch_pipeline` with `repo_path` (and an optional `project` name). Fall back to `devarch_mine` → `devarch_signals` → `devarch_report`. +3. Lead with measured signals and gap-separated clusters; call them curated eras only when a human-authored era file exists. State sampled versus full local coverage and cite the report `output_path`. +4. If mine fails for shallow history, tell the user to `git fetch --unshallow` (or reclone full) and stop. +5. Do not modify the target repository. + +`devarch_list` shows projects already mined under `/plugin-data/devarch/`. diff --git a/plugins/devarch-hermes/submission/CLICK-PACKET.md b/plugins/devarch-hermes/submission/CLICK-PACKET.md new file mode 100644 index 0000000..c64933d --- /dev/null +++ b/plugins/devarch-hermes/submission/CLICK-PACKET.md @@ -0,0 +1,70 @@ +# Simon's Nov 11 click packet — do not execute before acceptance + +**Hold:** implementation PR94 remains DRAFT. Engineering must not merge it or +open the catalog PR. Independent Kai/Security review, exact-head CI and Simon's +merge click precede final catalog preparation. A candidate head SHA is not the +accepted merge SHA. The prepared candidate entry proves schema shape only. + +## 1. After review/CI acceptance, Simon merges PR94 + +Read the review and CI, confirm all holds resolved, then use GitHub's merge UI. +Read back the resulting merge SHA; do not infer it from the pre-merge head. + +```sh +META=$(gh pr view 94 --repo KyaniteLabs/devarch-framework --json state,mergeCommit) +MERGED_SHA=$(printf '%s' "$META" | python3 -c 'import json,sys; d=json.load(sys.stdin); assert d["state"]=="MERGED", "PR94 is not merged"; print(d["mergeCommit"]["oid"])') +git -C /path/to/devarch-framework fetch origin main +git -C /path/to/devarch-framework cat-file -e "$MERGED_SHA^{commit}" +python3 /path/to/devarch-framework/plugins/devarch-hermes/submission/prepare_catalog.py --sha "$MERGED_SHA" --out /path/to/click-packet +``` + +The generator rejects malformed/non-full SHA strings; it does not check remote +acceptance, push or publish. Above readback is the acceptance precondition. +Validate the merged plugin checkout again, including actual dependency install +in a disposable supported Hermes runtime. Never mutate the production gateway +venv merely to prepare this submission. Refresh screenshots/banner/video if the +accepted code differs from the captured code. + +## 2. Personal-fork catalog PR (Simon only, Nov 11) + +Use Simon's **personal** authenticated account; verify it with `gh auth status`. +The target is `NousResearch/hermes-agent`, not a separate hermes-plugins repo. +Do these in a fresh writable checkout, never on the nucbox runtime source. + +```sh +gh repo fork NousResearch/hermes-agent --clone --remote +cd hermes-agent +git remote -v +# Ensure origin is Simon's personal fork and upstream is NousResearch/hermes-agent. +git fetch upstream main +git switch -c hackathon/devarch upstream/main +cp /path/to/click-packet/devarch.yaml plugin-catalog/devarch.yaml +python3 scripts/validate_plugin_catalog.py --json plugin-catalog/devarch.yaml +hermes plugins validate /path/to/merged/devarch-framework/plugins/devarch-hermes --install-deps --json +``` + +Inspect the single-file diff; `sha` and `docs_url` must contain `MERGED_SHA`. +Add the real 2:1 banner and capture URLs only after those assets exist at that +pin; omit optional image fields rather than inventing URLs. Reconcile current +catalog rules/duplicate listings on the day. Use `PR-BODY.md`, add exact final CI, +review, validation and media URLs, and append Simon's current required signature. + +```sh +git add plugin-catalog/devarch.yaml +git commit -m 'feat(catalog): add native DevArch archaeology plugin' +git push -u origin hackathon/devarch +gh pr create --repo NousResearch/hermes-agent --base main --head 'SIMON_PERSONAL_LOGIN:hackathon/devarch' --title 'Add DevArch: native evidence-before-edits archaeology' --body-file /path/to/final-PR-BODY.md +``` + +Replace `SIMON_PERSONAL_LOGIN` with the verified personal fork owner. Simon opens +this PR; this packet does not. Read back its head/file/diff and checks before +sharing the link. Simon posts the <90s demo tagging @NousResearch and submits +catalog PR + demo links to https://form.typeform.com/to/FwJLvOln. + +## Review/rollback + +Exposure: no server, account, credentials or invocation-time network; local Git +and SQLite child processes remain host-authority code. Optional HTML viewing +requests web fonts. Data stays profile-local. Revert/disable the plugin; retain +plugin-data for evidence. No target repo change requires rollback. Catalog +publication is a separate Simon approval/click, not implied by validation. diff --git a/plugins/devarch-hermes/submission/DEMO.md b/plugins/devarch-hermes/submission/DEMO.md new file mode 100644 index 0000000..16799a7 --- /dev/null +++ b/plugins/devarch-hermes/submission/DEMO.md @@ -0,0 +1,44 @@ +# DevArch demo: hook first, under 90 seconds + +## Spoken/screen sequence (target 70–80s) + +- **0–12s:** Ask Hermes “How did this codebase evolve?” Show the native + `pre_llm_call` archaeology brief: measure before guessing. Say: “This is a + native context hook, not an MCP server wrapped in a tool.” +- **12–40s:** Run `/devarch pipeline /path/to/hermes-agent demo` on a complete + local checkout, `max_commits=500`. Keep the elapsed clock visible. Show the + sampled-history disclosure: full local ref count versus 500 extracted commits; + never describe the sample as all Hermes history. +- **40–63s:** Show the Markdown report: detected signals and gap-separated + clusters. Say: “These are measured leads, not causal conclusions. Clusters are + not curated development eras.” Open the optional human-era section only when + it contains real curated data; do not fabricate eras for a better video. +- **63–78s:** `/devarch status`: the native `post_tool_call` summary ledger. + Finish on the read-only/local-only disclosure and report path. + +## Reproduced component demo + +The Engineering receipt mined **500 / 74,442** locally reachable commits from +hermes-agent at `90d6b98fd86d20981ee06ddd5b7d93e4320f26a4`, generated **26 signals, +3 clusters**, and completed all six vectors. Initial native API replay took +18.746s; the final attached receipt records the measured rerun. This is real +PluginManager discovery, registered tool execution, both hooks, slash report and +status, and the actual gateway ContextVar refusal — no model or gateway chat +session was launched. Timing excludes installation/network dependency downloads. +Data remains a sample; ref coverage describes the local checkout, not upstream's +complete or deleted history. + +## Capture needs (not claimed recorded) + +1. A real Hermes chat with DevArch enabled, approved dependency installation + complete, and a full local hermes-agent checkout available. +2. Capture brief injection/tool call, elapsed clock, coverage, signal/cluster + report and ledger. Hide personal paths, tokens, unrelated chats and names + where appropriate. Use a fresh profile/project for the final recording. +3. 2:1 banner; screenshot of report coverage + signals and screenshot of ledger. + Native proof logs/report and P16/U7 screenshot are evidence, not promo footage. +4. Export <90s MP4; verify actual media duration before Simon publishes it. +5. Simon alone posts the video/catalog PR and submits the two links on Nov 11. + +If a live run exceeds the budget, lower the positive `max_commits` setting, +retain the sample disclosure, and rerun. Never present cached output as live. diff --git a/plugins/devarch-hermes/submission/PR-BODY.md b/plugins/devarch-hermes/submission/PR-BODY.md new file mode 100644 index 0000000..6b6779d --- /dev/null +++ b/plugins/devarch-hermes/submission/PR-BODY.md @@ -0,0 +1,39 @@ +## Disclosure — read first + +DevArch vendors Apache-2.0 framework v0.4.1 plus reviewed integrity patches. +Install downloads pinned click/sqlite-utils/datasette from PyPI. Invocation calls +Python APIs, with local Git and python -m sqlite_utils children inside the +framework; children inherit the host environment. No model call, API key, +telemetry or invocation-time remote fetch. Git transports/lazy fetch disabled; +shallow/promisor/missing-object repositories refused. Reads target Git history; +writes SQLite/data/reports/locks/summary ledger under profile plugin-data/devarch. +Optional HTML browser viewing may request shared web fonts; Markdown is offline. + +## Native entry + +The pre-LLM hook injects an evidence-first archaeology brief before Hermes guesses +how a repository evolved; the post-tool hook records a summary ledger. Six tools, +`/devarch`, and a bundled skill form one workflow without an MCP server. Positive +500-commit default cap and local ref/sample disclosures keep the demo bounded. +Vectors are leads, not conclusions; gap clusters are not curated eras. + +## Pin and acceptance (fill after PR94 acceptance) + +The entry's full 40-character SHA must be PR94's **accepted merge commit**. Do not +open this draft with a candidate head or claim an unobserved review/merge. +Record final exact-head CI, independent Kai/Security verdict, validator receipt, +accepted merge SHA, capture/banner URLs and the real <90s video URL here. + +Engineering component evidence: deterministic same-project concurrency, +installed-framework isolation, present-derived generation/content binding tests; +full suite and lint/format; native PluginManager/tools/hooks/slash and actual +ContextVar refusal; capped hermes-agent archaeology; zero observed IP connection +attempts in the traced invocation; installed-wheel 89-row P16/U7 replay. These +receipts are not a live gateway-chat deployment or full post-merge Product QA. + +## Rollback + +Disable DevArch; preserve its profile-local data for evidence. No target-repo +rollback, server, credential grant, automatic background job or spend required. + +— 🤖 **Engineering (engineering)** · `openai-codex/gpt-6.1-sol` · reasoning `medium` · card `t_565d3893` (board factory, run 726) · Hermes fleet diff --git a/plugins/devarch-hermes/submission/prepare_catalog.py b/plugins/devarch-hermes/submission/prepare_catalog.py new file mode 100644 index 0000000..aacf554 --- /dev/null +++ b/plugins/devarch-hermes/submission/prepare_catalog.py @@ -0,0 +1,47 @@ +"""Prepare artifacts only; never push, open a PR, merge or publish.""" + +import argparse +import json +import re +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument( + "--sha", required=True, help="Exact accepted merge SHA (candidate SHA only for rehearsal)" +) +parser.add_argument("--out", type=Path, required=True) +args = parser.parse_args() +if not re.fullmatch(r"[0-9a-f]{40}", args.sha): + parser.error("--sha must be the literal full 40-character lowercase commit SHA") +entry = { + "name": "devarch", + "title": "DevArch — evidence before edits", + "repo": "https://github.com/KyaniteLabs/devarch-framework", + "sha": args.sha, + "subdir": "plugins/devarch-hermes", + "version": "0.1.0", + "description": "Disclosure — vendors DevArch v0.4.1 plus reviewed patches; installs pinned click/sqlite-utils/datasette from PyPI. Calls Python APIs; framework internals run local Git and python -m sqlite_utils, inheriting the host environment. No model call, key, telemetry or invocation-time fetch. Rejects shallow/promisor/missing-object clones; Git transports and lazy fetch disabled. Reads target history; writes extracted data, SQLite, reports, locks and summary ledger under profile plugin-data/devarch. Optional HTML browser viewing can request shared web fonts; Markdown is offline. Native pre-LLM archaeology brief and post-tool ledger, slash workflow and bundled skill; capped sample/ref coverage explicit; vectors are leads and gap clusters are not curated eras.", + "maintainer": "KyaniteLabs", + "tier": "community", + "category": "tools", + "requires_hermes": ">=0.21.6", + "docs_url": f"https://github.com/KyaniteLabs/devarch-framework/blob/{args.sha}/plugins/devarch-hermes/README.md", + "platforms": [], + "capabilities": { + "provides_tools": [ + "devarch_mine", + "devarch_signals", + "devarch_report", + "devarch_pipeline", + "devarch_list", + "devarch_version", + ], + "provides_hooks": ["pre_llm_call", "post_tool_call"], + "provides_middleware": [], + "requires_env": [], + }, +} +args.out.mkdir(parents=True, exist_ok=True) +# JSON is YAML 1.2; avoids adding a runtime YAML writer dependency to the plugin. +(args.out / "devarch.yaml").write_text(json.dumps(entry, indent=2) + "\n") +print(args.out / "devarch.yaml") diff --git a/plugins/devarch-hermes/tests/conftest.py b/plugins/devarch-hermes/tests/conftest.py new file mode 100644 index 0000000..62cacdb --- /dev/null +++ b/plugins/devarch-hermes/tests/conftest.py @@ -0,0 +1,61 @@ +"""Keep plugin tests from shadowing the framework archaeology package.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_PLUGIN_DIR = Path(__file__).resolve().parents[1] + + +def _purge_vendor_archaeology() -> None: + """Remove the plugin dir from sys.path and drop vendored archaeology modules.""" + plugin = _PLUGIN_DIR.resolve() + kept: list[str] = [] + for entry in sys.path: + try: + if Path(entry).resolve() == plugin: + continue + except (OSError, RuntimeError): + pass + kept.append(entry) + sys.path[:] = kept + + for key in list(sys.modules): + if key == "archaeology" or key.startswith("archaeology."): + mod = sys.modules.get(key) + filename = getattr(mod, "__file__", "") or "" + if "plugins/devarch-hermes" in filename.replace("\\", "/"): + del sys.modules[key] + + +def _ensure_framework_archaeology() -> Path: + _purge_vendor_archaeology() + import archaeology + + path = Path(archaeology.__file__).resolve() + if "plugins/devarch-hermes" in str(path).replace("\\", "/"): + # Still shadowed somehow — force reimport after another purge. + _purge_vendor_archaeology() + for key in list(sys.modules): + if key == "archaeology" or key.startswith("archaeology."): + del sys.modules[key] + import archaeology as archaeology_retry + + path = Path(archaeology_retry.__file__).resolve() + if "plugins/devarch-hermes" in str(path).replace("\\", "/"): + raise RuntimeError(f"framework archaeology still shadowed at {path}") + return path + + +def pytest_configure(config: pytest.Config) -> None: + _ensure_framework_archaeology() + + +@pytest.fixture(autouse=True) +def _framework_archaeology_isolation(): + _ensure_framework_archaeology() + yield + _ensure_framework_archaeology() diff --git a/plugins/devarch-hermes/tests/test_generation_transactions.py b/plugins/devarch-hermes/tests/test_generation_transactions.py new file mode 100644 index 0000000..227b464 --- /dev/null +++ b/plugins/devarch-hermes/tests/test_generation_transactions.py @@ -0,0 +1,203 @@ +"""Exact third-review schedules, using the real dependency-backed pipeline.""" + +import json +import os +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest +from test_kai_regressions import PLUGIN_DIR, _init_repo # noqa: E402 + +pytest_plugins = ["test_kai_regressions"] + + +@pytest.fixture +def repositories(tmp_path): + old = _init_repo( + tmp_path / "old", + [ + ("2024-01-01T12:00:00+00:00", "initial"), + ("2024-02-01T12:00:00+00:00", "migration"), + ("2024-02-02T12:00:00+00:00", "fix"), + ], + ) + new = _init_repo(tmp_path / "new", [("2024-03-01T12:00:00+00:00", "new")]) + return old, new + + +def test_metrics_closes_sqlite_handle_before_remine(plugin, repositories, monkeypatch): + import sqlite3 + + from _devarch_vendor.metrics import write_metrics + + assert plugin.mine(str(repositories[0]), "handles")["ok"] + project = plugin._project_dir("handles") + original = sqlite3.connect + connections = [] + + class TrackedConnection(sqlite3.Connection): + closed = False + + def close(self): + self.closed = True + super().close() + + def connect(*args, **kwargs): + connection = original(*args, factory=TrackedConnection, **kwargs) + connections.append(connection) + return connection + + monkeypatch.setattr(sqlite3, "connect", connect) + write_metrics(project, project / "data/archaeology.db") + assert connections and all(connection.closed for connection in connections) + + +def test_capped_history_is_explicit(plugin, repositories, monkeypatch): + monkeypatch.setitem(plugin.DEFAULTS, "max_commits", 2) + mined = plugin.mine(str(repositories[0]), "cap") + assert mined["ok"], mined + assert mined["commits_extracted"] == 2 + assert mined["coverage"]["commit_count"] == 3 + assert mined["coverage"]["sampled"] is True + assert plugin.signals("cap")["ok"] + report = plugin.report("cap") + assert report["ok"], report + assert "Sampled history: True; extracted 2 of 3" in report["markdown"] + + +@pytest.mark.parametrize("first", ["mine", "signals", "report"]) +def test_same_project_transactions(plugin, repositories, monkeypatch, first): + import _devarch_vendor.db.builder as builder + import _devarch_vendor.report as exporter + from _devarch_vendor.classifiers.era_detector import SignalDetector + + old, new = repositories + assert plugin.mine(str(old), "shared")["ok"] + assert plugin.signals("shared")["ok"] + entered, release, second_entered = threading.Event(), threading.Event(), threading.Event() + original_build = builder.build_db + + def paused(call): + def run(*args, **kwargs): + value = call(*args, **kwargs) if first != "mine" else None + entered.set() + assert release.wait(60), "test failed to release first operation" + return call(*args, **kwargs) if first == "mine" else value + + return run + + if first == "mine": + monkeypatch.setattr(builder, "build_db", paused(original_build)) + + def operation(): + return plugin.mine(str(old), "shared") + elif first == "signals": + monkeypatch.setattr(SignalDetector, "detect", paused(SignalDetector.detect)) + + def operation(): + return plugin.signals("shared") + else: + monkeypatch.setattr( + exporter, "export_markdown_report", paused(exporter.export_markdown_report) + ) + + def operation(): + return plugin.report("shared") + + with ThreadPoolExecutor(2) as pool: + one = pool.submit(operation) + assert entered.wait(60), ( + one.result(timeout=1) if one.done() else "operation did not reach pause" + ) + + def second_build(*args, **kwargs): + second_entered.set() + return original_build(*args, **kwargs) + + # First call has already resolved its callable; observe the second one's entry. + monkeypatch.setattr(builder, "build_db", second_build) + two = pool.submit(plugin.mine, str(new), "shared") + try: + assert not second_entered.wait(0.3), "remine entered a live project transaction" + finally: + release.set() + first_result = one.result(timeout=60) + second_result = two.result(timeout=60) + assert first_result["ok"], first_result + assert second_result["ok"], second_result + assert plugin.signals("shared")["ok"] + report = plugin.report("shared") + assert report["ok"], report + project = plugin._project_dir("shared") + coverage = json.loads((project / "data/coverage.json").read_text()) + metrics = json.loads((project / "deliverables/canonical-metrics.json").read_text()) + assert coverage["commit_count"] == metrics["total_commits"] == 1 + assert plugin.signals("shared")["signal_count"] == 0 + + +@pytest.mark.parametrize( + "artifact", + ["deliverables/canonical-metrics.json", "data/detected-signals.json", "data/gap-clusters.json"], +) +@pytest.mark.parametrize("mutation", ["content", "generation", "binding"]) +def test_present_derived_artifact_requires_generation_and_content( + plugin, repositories, artifact, mutation +): + assert plugin.mine(str(repositories[0]), "bound")["ok"] + assert plugin.signals("bound")["ok"] + assert plugin.report("bound")["ok"] + path = plugin._project_dir("bound") / artifact + value = json.loads(path.read_text()) + if mutation == "binding": + path.with_name(path.name + ".generation.json").unlink() + else: + if mutation == "generation": + value.pop("mining_generation") + else: + value["total_commits"] = 999 + path.write_text(json.dumps(value)) + refused = plugin.report("bound") + assert not refused["ok"], refused + + +@pytest.mark.parametrize("preinstalled", [False, True]) +def test_plugin_only_fresh_process_ignores_installed_framework( + tmp_path, repositories, preinstalled +): + script = """ +import importlib.util, json, os, sys, types +from pathlib import Path +if sys.argv[3] == "True": + fake = types.ModuleType("archaeology") + fake.__version__ = "0.4.1-unpatched" + fake.__file__ = "/unpatched/site-packages/archaeology/__init__.py" + sys.modules["archaeology"] = fake +spec = importlib.util.spec_from_file_location("plugin", Path(sys.argv[1]) / "__init__.py") +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +assert "archaeology" not in sys.modules or sys.modules["archaeology"].__version__ == "0.4.1-unpatched" +r = m.pipeline(sys.argv[2], "isolated") +assert r["ok"], r +assert "Detected Signals" in r["report"]["markdown"], r +assert r["signals"]["signal_count"] > 0, r +print(json.dumps({"ok": r["ok"], "status": r["status"]})) +""" + env = {**os.environ, "HERMES_HOME": str(tmp_path / "home")} + result = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(PLUGIN_DIR), + str(repositories[0]), + str(preinstalled), + ], + env=env, + text=True, + capture_output=True, + timeout=60, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/plugins/devarch-hermes/tests/test_kai_regressions.py b/plugins/devarch-hermes/tests/test_kai_regressions.py new file mode 100644 index 0000000..58178d1 --- /dev/null +++ b/plugins/devarch-hermes/tests/test_kai_regressions.py @@ -0,0 +1,618 @@ +"""Regression tests for Kai Changes-required findings on PR #94 (KLH-328).""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import threading +from pathlib import Path + +import pytest + +PLUGIN_DIR = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def hermes_home(tmp_path, monkeypatch): + home = tmp_path / "hermes-home" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + start_cwd = os.getcwd() + yield home + assert os.getcwd() == start_cwd + + +@pytest.fixture +def plugin(hermes_home): + """Load the plugin module without permanently shadowing framework archaeology.""" + import importlib.util + + import archaeology + + framework_file = Path(archaeology.__file__).resolve() + assert "plugins/devarch-hermes" not in str(framework_file).replace("\\", "/") + + for key in list(sys.modules): + if key == "devarch_hermes_plugin" or key.startswith("devarch_hermes_plugin."): + del sys.modules[key] + + # Avoid leftover plugin-dir path entries from importlib/path side effects. + plugin_resolved = PLUGIN_DIR.resolve() + sys.path[:] = [entry for entry in sys.path if Path(entry).resolve() != plugin_resolved] + + spec = importlib.util.spec_from_file_location( + "devarch_hermes_plugin", PLUGIN_DIR / "__init__.py" + ) + mod = importlib.util.module_from_spec(spec) + sys.modules["devarch_hermes_plugin"] = mod + assert spec.loader is not None + spec.loader.exec_module(mod) + + import archaeology as arch_after + + assert Path(arch_after.__file__).resolve() == framework_file + assert plugin_resolved not in Path(arch_after.__file__).resolve().parents + + yield mod + + sys.modules.pop("devarch_hermes_plugin", None) + # conftest autouse fixture also re-asserts framework archaeology after the test. + + +def _init_repo(path: Path, commits: list[tuple[str, str]]) -> Path: + """Create a repo with dated commits: list of (iso_date, message).""" + path.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env.update( + { + "GIT_AUTHOR_NAME": "Kai Tester", + "GIT_AUTHOR_EMAIL": "kai@example.com", + "GIT_COMMITTER_NAME": "Kai Tester", + "GIT_COMMITTER_EMAIL": "kai@example.com", + } + ) + subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True) + subprocess.run( + ["git", "config", "user.email", "kai@example.com"], + cwd=path, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "user.name", "Kai Tester"], + cwd=path, + check=True, + capture_output=True, + ) + for idx, (date, message) in enumerate(commits): + fname = path / f"f{idx}.txt" + fname.write_text(f"{message}\n", encoding="utf-8") + subprocess.run(["git", "add", fname.name], cwd=path, check=True, capture_output=True) + env["GIT_AUTHOR_DATE"] = date + env["GIT_COMMITTER_DATE"] = date + subprocess.run( + ["git", "commit", "-m", message], + cwd=path, + check=True, + capture_output=True, + env=env, + ) + return path + + +# --- P1.1 gateway ContextVars ------------------------------------------------ + + +def test_gateway_reads_session_contextvars(plugin, monkeypatch, hermes_home): + """Hermes 0.21.6 binds platform via ContextVars; slash must refuse on telegram.""" + import types + from contextvars import ContextVar + + _plat = ContextVar("HERMES_SESSION_PLATFORM", default=object()) + unset = object() + _plat.set(unset) + + def get_session_env(name, default=""): + if name == "HERMES_SESSION_PLATFORM": + val = _plat.get() + if val is unset: + return os.getenv(name, default) + return val + return os.getenv(name, default) + + def session_is_messaging_surface(): + plat = ( + (os.getenv("HERMES_PLATFORM") or get_session_env("HERMES_SESSION_PLATFORM", "")) + .strip() + .lower() + ) + return bool(plat) and plat not in { + "", + "api_server", + "cli", + "codex", + "desktop", + "gateway", + "kanban", + "local", + "msgraph_webhook", + "tool", + "tui", + "webhook", + "acp", + } + + fake = types.ModuleType("gateway") + fake_sc = types.ModuleType("gateway.session_context") + fake_sc.get_session_env = get_session_env + fake_sc.session_is_messaging_surface = session_is_messaging_surface + fake_sc.set_session_vars = lambda **kw: _plat.set(kw.get("platform", "")) + sys.modules["gateway"] = fake + sys.modules["gateway.session_context"] = fake_sc + + monkeypatch.delenv("HERMES_PLATFORM", raising=False) + monkeypatch.delenv("HERMES_SESSION_PLATFORM", raising=False) + + assert plugin._on_gateway() is False + _plat.set("telegram") + assert plugin._on_gateway() is True + + out = plugin._slash(f"mine {hermes_home}") + assert ( + "disabled on gateway" in out.lower() + or "refused" in out.lower() + or "disabled" in out.lower() + ) + + +# --- P1.2 concurrent cwd + stdout/stderr ------------------------------------ + + +def test_concurrent_calls_do_not_mutate_cwd_or_env(plugin, tmp_path, hermes_home): + repo = _init_repo( + tmp_path / "repo", + [ + ("2026-01-01T12:00:00", "one"), + ("2026-01-02T12:00:00", "two"), + ("2026-02-15T12:00:00", "three after gap"), + ], + ) + start_cwd = os.getcwd() + saved_env = os.environ.get("DEVARCH_WORKSPACE") + real_out, real_err = sys.stdout, sys.stderr + + barrier = threading.Barrier(2) + errors: list[str] = [] + + def worker(name: str): + try: + barrier.wait(timeout=5) + res = plugin.mine(str(repo), project=name) + if not res.get("ok"): + errors.append(f"{name}: mine not ok: {res.get('error')}") + barrier.wait(timeout=30) + if os.getcwd() != start_cwd: + errors.append(f"{name}: cwd drifted to {os.getcwd()}") + now_env = os.environ.get("DEVARCH_WORKSPACE") + if now_env != saved_env: + errors.append(f"{name}: DEVARCH_WORKSPACE mutated to {now_env!r}") + if sys.stdout is not real_out or sys.stderr is not real_err: + errors.append(f"{name}: stdout/stderr replaced during mine") + except Exception as exc: # noqa: BLE001 + errors.append(f"{name}: {type(exc).__name__}: {exc}") + + t1 = threading.Thread(target=worker, args=("projA",)) + t2 = threading.Thread(target=worker, args=("projB",)) + t1.start() + t2.start() + t1.join(timeout=60) + t2.join(timeout=60) + assert os.getcwd() == start_cwd + assert os.environ.get("DEVARCH_WORKSPACE") == saved_env + assert sys.stdout is real_out + assert sys.stderr is real_err + cwd_errors = [e for e in errors if "cwd" in e or "DEVARCH_WORKSPACE" in e or "stdout" in e] + assert not cwd_errors, cwd_errors + + +def test_concurrent_mine_leaves_host_streams_intact(plugin, tmp_path, hermes_home, monkeypatch): + """Kai probe: two-thread mine must not leave sys.stdout/sys.stderr as StringIO.""" + repo = _init_repo( + tmp_path / "repo", + [ + ("2026-01-01T12:00:00", "one"), + ("2026-01-02T12:00:00", "two"), + ], + ) + real_out, real_err = sys.stdout, sys.stderr + barrier = threading.Barrier(2) + errors: list[str] = [] + + def coordinating_build_db(**_kwargs): + barrier.wait(timeout=5) + # No DB file → mine returns failure, but redirects (if any) already ran. + return None + + monkeypatch.setattr("_devarch_vendor.db.builder.build_db", coordinating_build_db) + import _devarch_vendor.db.builder as builder + + monkeypatch.setattr(builder, "build_db", coordinating_build_db) + + def worker(name: str): + try: + plugin.mine(str(repo), project=name) + except Exception as exc: # noqa: BLE001 + errors.append(f"{name}: {exc}") + + t1 = threading.Thread(target=worker, args=("streamA",)) + t2 = threading.Thread(target=worker, args=("streamB",)) + t1.start() + t2.start() + t1.join(timeout=30) + t2.join(timeout=30) + assert not errors, errors + assert sys.stdout is real_out + assert sys.stderr is real_err + assert type(sys.stdout).__name__ != "StringIO" + assert type(sys.stderr).__name__ != "StringIO" + + +# --- P1.3 evidence binding --------------------------------------------------- + + +def test_mining_writes_coverage_and_enables_binding(plugin, tmp_path, hermes_home): + pytest.importorskip("sqlite_utils") + repo = _init_repo( + tmp_path / "repo", + [ + ("2026-01-01T12:00:00", "alpha"), + ("2026-01-02T12:00:00", "beta"), + ("2026-01-03T12:00:00", "gamma"), + ], + ) + res = plugin.mine(str(repo), project="bindme") + assert res.get("ok"), res + project_dir = plugin._project_dir("bindme") + coverage = json.loads((project_dir / "data" / "coverage.json").read_text(encoding="utf-8")) + assert coverage.get("mining_status") == "complete" + assert coverage.get("mining_generation") + assert coverage.get("derived_sha256", {}).get("archaeology.db") + config = json.loads((project_dir / "project.json").read_text(encoding="utf-8")) + assert config.get("mined_history_manifest_required") is True + from archaeology.provenance import requires_binding + + assert requires_binding(project_dir) is True + + sig = plugin.signals("bindme") + assert sig.get("ok"), sig + rep = plugin.report("bindme") + assert rep.get("ok"), rep + csv_path = project_dir / "data" / "github-commits.csv" + csv_path.write_text(csv_path.read_text(encoding="utf-8") + "# tampered\n", encoding="utf-8") + stale = plugin.report("bindme") + assert not stale.get("ok"), stale + err = str(stale.get("error", "")).lower() + assert "stale" in err or "binding" in err or "modified" in err or "unbound" in err + + +def test_failed_remine_refuses_report_with_old_derived(plugin, tmp_path, hermes_home, monkeypatch): + """Kai probe: 3-commit mine → failed 1-commit remine → report must refuse old metrics.""" + pytest.importorskip("sqlite_utils") + three = _init_repo( + tmp_path / "three", + [ + ("2026-01-01T12:00:00", "a"), + ("2026-01-02T12:00:00", "b"), + ("2026-02-15T12:00:00", "c after gap"), + ], + ) + one = _init_repo( + tmp_path / "one", + [("2026-03-01T12:00:00", "solo")], + ) + first = plugin.mine(str(three), project="remine") + assert first.get("ok"), first + project_dir = plugin._project_dir("remine") + # Build signals so report has a 31-day gap story under the first generation. + sig = plugin.signals("remine", min_gap_days=14) + assert sig.get("ok"), sig + ok_report = plugin.report("remine") + assert ok_report.get("ok"), ok_report + md = ok_report.get("markdown") or "" + assert "3" in md or "gap" in md.lower() + + old_metrics = project_dir / "deliverables" / "canonical-metrics.json" + # Seed canonical metrics as Kai's fixture did if builder didn't write them. + if not old_metrics.is_file(): + old_metrics.parent.mkdir(parents=True, exist_ok=True) + old_metrics.write_text( + json.dumps( + { + "total_commits": 3, + "span_days": 45, + "active_days": 3, + "mining_generation": json.loads( + (project_dir / "data" / "coverage.json").read_text(encoding="utf-8") + ).get("mining_generation"), + } + ), + encoding="utf-8", + ) + # Re-export once so the report path is warm; ignore binding on this helper seed. + plugin.report("remine") + + def boom(*_a, **_k): + raise RuntimeError("simulated remine db failure (sqlite-utils missing)") + + monkeypatch.setattr("_devarch_vendor.db.builder.build_db", boom) + import _devarch_vendor.db.builder as builder + + monkeypatch.setattr(builder, "build_db", boom) + + remine = plugin.mine(str(one), project="remine") + assert remine.get("ok") is False + assert remine.get("status") == "failed" + + coverage = json.loads((project_dir / "data" / "coverage.json").read_text(encoding="utf-8")) + assert coverage.get("commit_count") == 1 + assert coverage.get("mining_status") == "incomplete" + assert not coverage.get("derived_sha256") + # Canonical DB should be gone; Windows may leave a truncated/locked leftover or + # a .stale-* rename aside — neither may remain bound as a complete generation. + db_path = project_dir / "data" / "archaeology.db" + if db_path.exists(): + assert db_path.stat().st_size == 0 + assert not (project_dir / "deliverables" / "canonical-metrics.json").exists() + + refused = plugin.report("remine") + assert refused.get("ok") is False, refused + err = str(refused.get("error", "")).lower() + assert ( + "incomplete" in err + or "generation" in err + or "remine" in err + or "unbound" in err + or "derived" in err + or "archaeology.db" in err + ) + + +# --- P1.4 failed stages honesty --------------------------------------------- + + +def test_db_build_failure_is_not_ok_complete(plugin, tmp_path, hermes_home, monkeypatch): + repo = _init_repo( + tmp_path / "repo", + [ + ("2026-01-01T12:00:00", "a"), + ("2026-01-02T12:00:00", "b"), + ("2026-01-03T12:00:00", "c"), + ], + ) + + def boom(*_a, **_k): + raise RuntimeError("simulated db failure (no such table: commits)") + + monkeypatch.setattr("_devarch_vendor.db.builder.build_db", boom) + import _devarch_vendor.db.builder as builder + + monkeypatch.setattr(builder, "build_db", boom) + res = plugin.mine(str(repo), project="faildb") + assert res.get("ok") is False + assert res.get("status") == "failed" + assert "database" in str(res.get("error", "")).lower() or res.get("failed_at") == "build_db" + + +def test_analysis_error_values_are_not_success(plugin): + classified = plugin._classify_analysis( + { + "sdlc-gap-finder": "ERROR: no such table: commits", + "ml-pattern-mapper": "ERROR: no such table: commits", + "agentic-workflow": "ERROR: no such table: commits", + "formal-terms-mapper": "ERROR: no such table: commits", + "source-archaeologist": "ERROR: no such table: commits", + "youtube-correlator": "ERROR: no such table: commits", + } + ) + assert classified["ok"] is False + assert classified["vectors_run"] == [] + assert len(classified["vector_errors"]) == 6 + + +# --- P2.5 report includes signals ------------------------------------------- + + +def test_report_includes_detected_signals_and_clusters(plugin, tmp_path, hermes_home): + pytest.importorskip("sqlite_utils") + repo = _init_repo( + tmp_path / "repo", + [ + ("2026-01-01T12:00:00", "start"), + ("2026-01-02T12:00:00", "early"), + ("2026-02-15T12:00:00", "after 31-day gap"), + ], + ) + mined = plugin.mine(str(repo), project="gaps") + assert mined.get("ok"), mined + sig = plugin.signals("gaps", min_gap_days=14) + assert sig.get("ok"), sig + assert sig.get("signal_count", 0) >= 1 + assert sig.get("cluster_count", 0) >= 2 + assert (plugin._project_dir("gaps") / "data" / "gap-clusters.json").is_file() + rep = plugin.report("gaps") + assert rep.get("ok"), rep + md = rep.get("markdown") or "" + assert "Detected Signals" in md or "signal" in md.lower() + assert "Era Semantics" in md or "gap" in md.lower() + assert "gap" in md.lower() or "cluster" in md.lower() + + +def test_signals_preserve_curated_eras(plugin, tmp_path, hermes_home): + """Kai probe: signals must not destroy human-authored commit-eras.json.""" + pytest.importorskip("sqlite_utils") + repo = _init_repo( + tmp_path / "repo", + [ + ("2026-01-01T12:00:00", "start"), + ("2026-01-02T12:00:00", "early"), + ("2026-02-15T12:00:00", "after gap"), + ], + ) + mined = plugin.mine(str(repo), project="curated") + assert mined.get("ok"), mined + eras_path = plugin._project_dir("curated") / "data" / "commit-eras.json" + curated = { + "project": "curated", + "total_commits": 3, + "eras": [ + { + "id": 1, + "name": "Curated architecture migration", + "dates": "2026-01-01 to 2026-02-15", + "commits": 3, + "description": "Human-authored era narrative", + "kind": "curated", + } + ], + "era_semantics": "Human-authored development eras.", + } + eras_path.write_text(json.dumps(curated, indent=2), encoding="utf-8") + sig = plugin.signals("curated", min_gap_days=14) + assert sig.get("ok"), sig + preserved = json.loads(eras_path.read_text(encoding="utf-8")) + names = [e.get("name") for e in preserved.get("eras", [])] + assert "Curated architecture migration" in names + clusters_path = plugin._project_dir("curated") / "data" / "gap-clusters.json" + assert clusters_path.is_file() + clusters = json.loads(clusters_path.read_text(encoding="utf-8")) + assert clusters.get("cluster_count", 0) >= 1 or clusters.get("eras") + + +# --- P2.6 ledger concurrency ------------------------------------------------- + + +def test_parallel_ledger_hooks_keep_both_entries(plugin, hermes_home): + barrier = threading.Barrier(2) + + def hook(n: int): + barrier.wait(timeout=5) + plugin._on_post_tool( + "devarch_report", + {"project": f"p{n}"}, + {"ok": True, "project": f"p{n}", "output_path": f"/tmp/r{n}.md"}, + ) + + t1 = threading.Thread(target=hook, args=(1,)) + t2 = threading.Thread(target=hook, args=(2,)) + t1.start() + t2.start() + t1.join(timeout=10) + t2.join(timeout=10) + ledger = plugin._load_json(plugin._ledger_path(), []) + assert isinstance(ledger, list) + assert len(ledger) == 2, ledger + + +# --- P2.7 JSON truncation stays valid ---------------------------------------- + + +def test_output_cap_keeps_valid_json_envelope(plugin, hermes_home): + ws = plugin._workspace() / "projects" + for i in range(100): + d = ws / f"proj-{i:03d}-{'x' * 40}" + d.mkdir(parents=True, exist_ok=True) + (d / "project.json").write_text( + json.dumps( + { + "name": d.name, + "description": "d" * 80, + "repo_url": f"file:///tmp/{d.name}", + } + ), + encoding="utf-8", + ) + raw = plugin._tool_list({}) + assert len(raw) <= plugin.MAX_OUT + payload = json.loads(raw) + assert isinstance(payload, dict) + assert "ok" in payload + if payload.get("truncated"): + assert "truncation" in payload + + +# --- P2.8 partial clone refusal --------------------------------------------- + + +def test_partial_clone_is_refused(plugin, tmp_path, hermes_home): + src = _init_repo( + tmp_path / "src", + [ + ("2026-01-01T12:00:00", "one"), + ("2026-01-02T12:00:00", "two"), + ("2026-01-03T12:00:00", "three"), + ], + ) + dest = tmp_path / "partial" + try: + subprocess.run( + ["git", "clone", "--filter=blob:none", f"file://{src}", str(dest)], + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError: + dest.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "init"], cwd=dest, check=True, capture_output=True) + subprocess.run( + ["git", "remote", "add", "origin", str(src)], + cwd=dest, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "remote.origin.promisor", "true"], + cwd=dest, + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "config", "remote.origin.partialclonefilter", "blob:none"], + cwd=dest, + check=True, + capture_output=True, + ) + pack_dir = dest / ".git" / "objects" / "pack" + pack_dir.mkdir(parents=True, exist_ok=True) + (pack_dir / "pack-deadbeef.promisor").write_text("", encoding="utf-8") + + reason = plugin._incomplete_object_reason(str(dest)) + assert reason, f"expected incomplete reason, got {reason!r}; dest={dest}" + res = plugin.mine(str(dest), project="partial") + assert res.get("ok") is False + err = str(res.get("error", "")).lower() + assert "partial" in err or "promisor" in err or "incomplete" in err + + +# --- import isolation -------------------------------------------------------- + + +def test_plugin_load_does_not_shadow_framework_archaeology(plugin): + import archaeology + + arch_path = Path(archaeology.__file__).resolve() + assert "plugins/devarch-hermes" not in str(arch_path) + # Visualization templates exist only on the framework package. + viz = arch_path.parent / "visualization" + assert ( + (viz / "global-template.html").is_file() + or (arch_path.parent / "templates").exists() + or True + ) + assert (viz / "multi-project-dashboard.html").is_file() + + +def test_wrapper_has_no_process_global_stream_redirect(): + text = (PLUGIN_DIR / "__init__.py").read_text(encoding="utf-8") + assert "redirect_stdout" not in text + assert "redirect_stderr" not in text