diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9c857c64..bbffc202 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "name": "agentdescent", "source": "./integrations/claude-code", "description": "Evolve skills, agents, prompts, code and plugins against examples with a parallel, merge-based optimiser.", - "version": "0.5.0" + "version": "0.5.1" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 48db82b2..40396801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,44 @@ All notable changes to AgentDescent are documented here. The format follows ## [Unreleased] +## [0.5.1] — 2026-10-10 + +### Release summary + +- Agent-first editorial website and plugin quickstart, modular-framework + explanations, unified documentation styling, and an illustrative in-browser + workflow that makes no model calls or file changes. The optional one-command + setup installs current main and connects supported agent CLIs on PATH; it + is not a pinned-release installer. +- Token budgets, cost-aware selection, graceful budget degradation and optional + diminishing-returns stopping; checkpointed search state survives a restart. +- Sparse verifier auditing and calibration, opt-in priority-ordered audit drain, + and replay-aware sampling and proposals that can use settled evidence. +- Policy-slot evolution, the EvoX Genesis port with skill extraction and read-only + investigation, and the porous-molecule tree-search example. Research examples + remain in the source repository rather than the installed wheel. +- More reliable first runs: actionable spec/dataset errors, policy-slot scaffolds, + fail-fast host installation and a clean installed-wheel CLI/MCP smoke gate. + +### Upgrade notes + +- Python >= 3.9 remains supported; the MCP extra requires Python >= 3.10. + NumPy >= 1.20 is now an installed dependency for prediction-powered auditing. +- Use `ThreeLayerVerifier.full_eval` and `full_eval_matches_counts`. The old + `oracle_eval` and `oracle_shares_full_set` aliases remain available; using + `oracle_eval` emits a `DeprecationWarning`. Removal is planned for 0.7. +- Checkpointing, replay weighting and audit draining remain opt-in. Token-budget + enforcement depends on usage reported by the model adapter; it is not a + provider-side hard spending limit. + +The detailed changes below cover the merged commits since 0.5.0. They do not +include the open multi-head, multi-artifact, asynchronous trajectory-resume, +value-directed evaluation or Genesis blind-mode proposals. + ### Fixed +- Interactive demo restarts no longer interleave with an older approval run, + and the displayed artifact stays tied to the run that produced it. - Keep the interactive demo reachable from the documentation navigation drawer on mobile, where the desktop header links are hidden. - Prevent Material admonition styles from exposing the mobile menu on desktop @@ -969,10 +1005,10 @@ All notable changes to AgentDescent are documented here. The format follows the scorer is biased. Reading it as ground truth is what let the docs claim the loop audits itself against something outside itself, which it does not. - Both old names still work, once, with a `DeprecationWarning`, and are removed - in 0.7. `VerifierProtocol` declares `full_eval`; the engine reads a custom + Both old names still work; the `oracle_eval` compatibility path emits a + `DeprecationWarning`. Both aliases are scheduled for removal in 0.7. `VerifierProtocol` declares `full_eval`; the engine reads a custom verifier through `verifier.full_eval_of` / `verifier.shares_eval_counts`, which - accept either spelling -- so a verifier written against the pre-0.6 page keeps + accept either spelling -- so a verifier written against the pre-0.5.1 page keeps running instead of raising `AttributeError` in the middle of a merge. The `oracle_budget` argument, `VerifierBudget.oracle_calls_*` and the `oracle-rejected` merge outcome keep their names: they are the public spelling @@ -4336,7 +4372,9 @@ First public release on PyPI as **`agentdescent`**. discrete-space `Aggregator`, staleness policies, DP/TP/PP parallelism, layered governance, and the provider-agnostic `agentdescent.agents` completion layer. -[Unreleased]: https://github.com/Birfy/agentdescent/compare/v0.4.6...HEAD +[Unreleased]: https://github.com/Birfy/agentdescent/compare/v0.5.1...HEAD +[0.5.1]: https://github.com/Birfy/agentdescent/compare/v0.5.0...v0.5.1 +[0.5.0]: https://github.com/Birfy/agentdescent/compare/v0.4.6...v0.5.0 [0.4.6]: https://github.com/Birfy/agentdescent/compare/v0.4.5...v0.4.6 [0.4.5]: https://github.com/Birfy/agentdescent/compare/v0.4.2...v0.4.5 [0.4.2]: https://github.com/Birfy/agentdescent/compare/v0.4.1...v0.4.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 95ea2095..4e132a88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,7 @@ are very welcome. ```bash git clone https://github.com/Birfy/agentdescent cd agentdescent -pip install -e ".[dev]" # core engine has zero runtime deps; [dev] adds pytest +pip install -e ".[dev]" # installs NumPy plus the pytest/PyYAML/Markdown development tools ``` Python ≥ 3.9 is required. @@ -38,7 +38,7 @@ The suite is **offline and deterministic** (no network, no model API): pytest -q ``` -CI runs the same suite on Python 3.9 / 3.11 / 3.12 for every push and PR. Please +CI runs the same suite on Python 3.9 / 3.10 / 3.11 / 3.12 for every push and PR. Please add or update tests for any behavior change — the reference domain (`agentdescent/domains/router.py`) and the example stubs let you test the full parallel/async loop without an API key. diff --git a/README.md b/README.md index 8d57d198..61bee273 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # AgentDescent +**[Website & interactive demo](https://birfy.github.io/agentdescent/)** · +[Get started](https://birfy.github.io/agentdescent/install/) · +[Documentation](https://birfy.github.io/agentdescent/docs/) + > **Gradient descent — but the parameters are agents.** A parallel, asynchronous > framework for self-evolving agents (skills, prompts, harnesses) where **diffs > are the gradients** and **the aggregator is the optimizer**. @@ -63,38 +67,37 @@ The figure is the paper's, rendered from its TikZ source by [`tools/gen_architecture_figure.py`](tools/gen_architecture_figure.py) so it cannot drift from what the paper shows. -## Try the offline demo from current `main` +## Install and run the offline demo + +These instructions target **0.5.1**. Once that version is published: + +```bash +pip install "agentdescent>=0.5.1" +agentdescent demo +``` -The packaged `agentdescent demo` command shown here is on the unreleased -`main` branch. Install that checkout before running it: +Before publication, or when developing from source, install your checkout: ```bash -git clone --branch main https://github.com/Birfy/agentdescent && cd agentdescent +git clone https://github.com/Birfy/agentdescent && cd agentdescent pip install -e . agentdescent demo ``` The library supports Python ≥ 3.9 and installs NumPy for its prediction-powered audit estimator. The packaged demo runs the offline -evolution loop without a model API key. On current unreleased `main`, it ends -with output like: +evolution loop without a model API key and ends with output like: ``` held-out reward: 1.000 outcomes: {'committed': 1} what it learned: rules.md -> 'COLUMN: amount' ``` -For the published stable v0.5.0 installation, run: - -```bash -pip install agentdescent -``` - -Its documented offline example uses `python -m examples.run_demo`; those -examples are outside the wheel, so clone the repo and follow -[install and first run](docs/install.md). The library includes -[`evolve()`](docs/evolution.md), [`DirectoryStrategy`](docs/directory-evolution.md), -and its algorithm ports. +Research examples are outside the wheel. To run the +`python -m examples.run_demo` comparison, clone the repo and follow +[install and first run](docs/install.md). +The library includes [`evolve()`](docs/evolution.md), +[`DirectoryStrategy`](docs/directory-evolution.md), and its algorithm ports. ## Quickstart — a dataset to an evolved skill diff --git a/agentdescent/__init__.py b/agentdescent/__init__.py index 9827cd2c..55d657cd 100644 --- a/agentdescent/__init__.py +++ b/agentdescent/__init__.py @@ -211,7 +211,7 @@ shard_round_robin, ) -__version__ = "0.5.0" +__version__ = "0.5.1" __all__ = [ "Contract", diff --git a/agentdescent/aggregator.py b/agentdescent/aggregator.py index 5c1db5f9..7589dea5 100644 --- a/agentdescent/aggregator.py +++ b/agentdescent/aggregator.py @@ -24,7 +24,7 @@ "spot-check" arrow, which reads as advisory when it holds a veto. Note what this stage does *not* do. It re-reads the caller's own scorer over the -whole held-out set (``full_eval``, called ``oracle_eval`` until 0.6), so it can +whole held-out set (``full_eval``, called ``oracle_eval`` until 0.5.1), so it can catch a verdict that rested on too few tasks and cannot catch one that rested on a biased scorer -- the outcome name outlived the rename and overstates the check. :mod:`agentdescent.audit` is the second source. @@ -1055,7 +1055,7 @@ def _audit(self, artifact, artifact_id, best_state, best_diff, # # Read through the two helpers rather than by attribute: a verifier # is whatever the caller handed us, including one written against the - # pre-0.6 `oracle_eval` / `oracle_shares_full_set` names. Reading the + # pre-0.5.1 `oracle_eval` / `oracle_shares_full_set` names. Reading the # new name directly would have made an old custom verifier fail here, # mid-merge, several minutes into a run. if shares_eval_counts(self.verifier): diff --git a/agentdescent/policies.py b/agentdescent/policies.py index 0b3fc3d1..3470f14f 100644 --- a/agentdescent/policies.py +++ b/agentdescent/policies.py @@ -334,7 +334,7 @@ class VerifierProtocol(Protocol): Not three. `docs/verifier.md` said three for a while, and a verifier written from that page fails in the middle of a merge rather than at startup. - ``full_eval`` was ``oracle_eval`` until 0.6. The engine still reads the old + ``full_eval`` was ``oracle_eval`` until 0.5.1. The engine still reads the old name, once, with a `DeprecationWarning` (:func:`~agentdescent.verifier.full_eval_of`), so an implementation written against the old page keeps working until 0.7. The rename is a correction: diff --git a/agentdescent/verifier.py b/agentdescent/verifier.py index a3e4f2b2..5021bc69 100644 --- a/agentdescent/verifier.py +++ b/agentdescent/verifier.py @@ -13,7 +13,7 @@ .. note:: **The expensive layer is not a second opinion.** - ``full_eval`` was called ``oracle_eval`` until 0.6, and the old name claimed + ``full_eval`` was called ``oracle_eval`` until 0.5.1, and the old name claimed more than the code delivers. Every layer here calls the one ``eval_fn`` the caller supplied; they differ in **how many tasks** they score, not in *who is scoring*. So the expensive layer bounds the sampling error of a measurement @@ -123,7 +123,7 @@ class ThreeLayerVerifier: #: itself declares over the value it inherited from here. full_eval_matches_counts = True - #: Pre-0.6 name for :attr:`full_eval_matches_counts`. Read through + #: Pre-0.5.1 name for :attr:`full_eval_matches_counts`. Read through #: :func:`shares_eval_counts`, which prefers the new one, so a custom #: verifier that set either keeps working. oracle_shares_full_set = True @@ -219,7 +219,7 @@ def eval_counts(self, artifact: Evolvable, def full_eval(self, artifact: Evolvable) -> float: """``eval_fn`` on the **whole** held-out set. Consumes audit budget. - Called ``oracle_eval`` until 0.6, and the rename is a correction rather + Called ``oracle_eval`` until 0.5.1, and the rename is a correction rather than a tidy-up. The name promised an *independent source of truth* and the method delivers the same ``eval_fn`` the cheap layers call, differing only in **how many tasks** it looks at -- so it can shrink the variance of @@ -264,10 +264,10 @@ def oracle_eval(self, artifact: Evolvable) -> float: return self.full_eval(artifact) -# -- reading a verifier that may predate the 0.6 rename ---------------------- +# -- reading a verifier that may predate the 0.5.1 rename ---------------------- # # `VerifierProtocol` is structural, so a verifier is whatever a caller hands the -# engine -- including one written against the pre-0.6 page and carrying only the +# engine -- including one written against the pre-0.5.1 page and carrying only the # old names. These two read either spelling, so the rename costs a warning # rather than an `AttributeError` in the middle of somebody's merge. @@ -275,7 +275,7 @@ def oracle_eval(self, artifact: Evolvable) -> float: def full_eval_of(verifier: Any) -> Callable[[Evolvable], float]: """The verifier's whole-held-out-set scorer, under whichever name it has. - Prefers ``full_eval``; falls back to a pre-0.6 ``oracle_eval`` and says so + Prefers ``full_eval``; falls back to a pre-0.5.1 ``oracle_eval`` and says so once per verifier class, naming the class so the warning points at the file that has to change. """ @@ -290,7 +290,7 @@ def full_eval_of(verifier: Any) -> Callable[[Evolvable], float]: "full_eval") warnings.warn( f"{type(verifier).__name__} defines oracle_eval but not full_eval. The " - "method was renamed in 0.6 -- oracle_eval is read for now and dropped " + "method was renamed in 0.5.1 -- oracle_eval is read for now and dropped " "in 0.7. Rename it; the behaviour is unchanged.", DeprecationWarning, stacklevel=2) return legacy @@ -303,13 +303,13 @@ def shares_eval_counts(verifier: Any) -> bool: buying the same sweep twice -- and, more importantly, instead of routing a commit decision through a call that degrades to a sub-sample once the budget is gone. Reads :attr:`~ThreeLayerVerifier.full_eval_matches_counts`, then the - pre-0.6 ``oracle_shares_full_set``. + pre-0.5.1 ``oracle_shares_full_set``. """ # A *declaration* beats an inherited default. `ThreeLayerVerifier` sets # `full_eval_matches_counts = True` as a statement about how that class is # written, and a subclass inherits it -- so the escape hatch the attribute's # docstring offers ("simply does not define it") does not exist for a - # subclass, and one that set the pre-0.6 `oracle_shares_full_set = False` + # subclass, and one that set the pre-0.5.1 `oracle_shares_full_set = False` # was ignored. Its independent expensive layer then went uncalled and the # gate degraded to the cheap measurement it existed to cross-check, with no # warning. diff --git a/docs/design-directory-evolution.md b/docs/design-directory-evolution.md index 876a2925..93c51872 100644 --- a/docs/design-directory-evolution.md +++ b/docs/design-directory-evolution.md @@ -378,7 +378,7 @@ result.write_to(path, backup=True) # 装回，先备份 两个反直觉的点，都已对代码核实： -1. **审计门是免费的。** `full_eval`（0.6 前叫 `oracle_eval`）和 `eval_counts` 调同一个 `eval_fn`、同一个 +1. **审计门是免费的。** `full_eval`（0.5.1 前叫 `oracle_eval`）和 `eval_counts` 调同一个 `eval_fn`、同一个 held-out 集，而 `_EvalCache` 按 `(render(), task.id)` 记忆化 （[evolution.py:531](https://github.com/Birfy/agentdescent/blob/main/agentdescent/evolution.py#L531)），所以 L1 的强制审计只消耗 `oracle_budget` 计数器，不产生任何额外 agent 调用。**演化 agent 目录/代码 diff --git a/docs/install.md b/docs/install.md index 4737d2fd..75a5aaec 100644 --- a/docs/install.md +++ b/docs/install.md @@ -7,13 +7,18 @@ description: Install AgentDescent in one command or with pip, connect Claude Cod
AGENTDESCENT / QUICKSTART

Start inside
your agent.

The modular evolution engine from the homepage also works in Claude Code, Codex and other coding agents. Follow three short steps to connect it and review your first improvement.

Start setup ↓Try the agent demo ↗
CLAUDE CODE · CODEX · OPENCODE · DEEPSEEK HARNESS
-
01 / INSTALL

One command
to get set up.

The script installs current AgentDescent through pip and connects supported agents it finds on your PATH.

TERMINAL
curl -fsSL https://raw.githubusercontent.com/Birfy/agentdescent/main/scripts/install.sh | bash
Installs AgentDescent and connects the agent CLIs already on your PATH.Read the script ↗
+
01 / INSTALL

One command
to get set up.

The script installs current main through pip (not a pinned release) and connects supported agents it finds on your PATH.

TERMINAL
curl -fsSL https://raw.githubusercontent.com/Birfy/agentdescent/main/scripts/install.sh | bash
Installs AgentDescent and connects the agent CLIs already on your PATH.Read the script ↗
02 / CONNECT

Open your agent.

Choose the agent you use. The command below shows how to launch or verify its AgentDescent integration.

LAUNCH CLAUDE CODE

The installer writes a local plugin directory. Launch Claude Code with it.

TERMINAL
claude --plugin-dir ~/.agentdescent/plugins/claude-code
03 / ASK

Tell it what
to improve.

Give your agent a target and examples. It shows an estimate before the run and a diff before applying changes.

PASTE INTO YOUR AGENT

Improve the skill at ~/.claude/skills/pdf-audit against eval/cases.jsonl. Show me the plan and estimated calls before starting. Show the diff before applying.

Plan firstReview the diffApply with approval
-

Good to know.

What about provider keys and cost?

A model-backed run needs a configured provider key. Run agentdescent doctor to check your setup. Your agent shows the plan and estimated call count before starting. agentdescent demo runs offline without a key.

Can I explore the algorithm ports?

All 20 algorithm ports print their configuration with --dry-run and make no model call. See the port catalog.

Can I install from PyPI?

Yes. pip install agentdescent installs the published stable release. The command above installs current main with the latest agent integrations. See the plugin guide for the stable release.

How do I build these docs?

From a checkout: pip install -e ".[docs]", then mkdocs serve for a local preview or mkdocs build --strict for a site build.

+

Good to know.

What about provider keys and cost?

A model-backed run needs a configured provider key. Run agentdescent doctor to check your setup. Your agent shows the plan and estimated call count before starting. agentdescent demo runs offline without a key.

Can I explore the algorithm ports?

Research examples are source-only and are not included in the wheel. After 0.5.1 is published, clone its matching tag:

git clone --branch v0.5.1 https://github.com/Birfy/agentdescent
+cd agentdescent
+pip install -e ".[dev]"
+python -m examples.run_demo

The tag is created at publication; before then, use your release-candidate checkout. All 20 algorithm ports print their configuration with --dry-run and make no model call. The eight benchmark-faithful ports return before loading data; the eleven MethodPolicy ports build their policy first, so a real-benchmark port may download and cache its split. See the port catalog.

Can I install from PyPI?

Yes. pip install agentdescent installs the published stable release. Once 0.5.1 is published, use pip install "agentdescent[mcp]>=0.5.1" for this release’s CLI and MCP tools. Python 3.9 supports the core package; MCP requires Python 3.10+. The one-command and pip tabs above install current main, not a pinned release. See the plugin guide for the release workflow.

Can I work from a source checkout before publication?

Clone main and install the MCP extra from that checkout (Python 3.10+):

git clone --branch main https://github.com/Birfy/agentdescent
+cd agentdescent
+pip install -e ".[mcp]"

This provides an editable checkout for the CLI, MCP tools and source-only examples. Use pip install -e ".[dev]" in the same checkout for development and the research examples.

How do I build these docs?

From a checkout: pip install -e ".[docs]", then mkdocs serve for a local preview or mkdocs build --strict for a site build.

BEYOND THE QUICKSTART

One path into a bigger framework.

This agent workflow is a starting point. To evolve custom artifacts or plug in different algorithms, explore the same evolve() engine shown on the homepage.

Explore the framework ↗
diff --git a/docs/plugin-quickstart.md b/docs/plugin-quickstart.md index 5a6ea310..8a5bfc17 100644 --- a/docs/plugin-quickstart.md +++ b/docs/plugin-quickstart.md @@ -10,15 +10,21 @@ until step 3. ## 1. Install -These CLI commands use current unreleased `main`; install that checkout first: +These commands target **0.5.1**. Once published, install the package with its +MCP extra: + +```bash +pip install "agentdescent[mcp]>=0.5.1" +``` + +Before publication, or if you already have a source checkout: ```bash -git clone --branch main https://github.com/Birfy/agentdescent && cd agentdescent pip install -e ".[mcp]" ``` -For the published v0.5.0 install and documented source-example path, see -[Install and first run](install.md#published-v050-installation-and-source-examples). +See [Install and first run](install.md) for the source checkout and +[source examples](install.md#source-examples). `[mcp]` is the extra that lets agents talk to AgentDescent. Skip it and the CLI still works, but `agentdescent mcp` exits 3 and tells you to add it. @@ -47,7 +53,7 @@ make a real run fail — but none of them stop step 2. `bin/` to add. Hosts start the server as a **subprocess**, so it must be on the `PATH` of whatever launches your agent, not just your interactive shell. -## 2. Current `main`: run the packaged demo offline +## 2. Run the packaged demo offline ```bash agentdescent demo diff --git a/docs/plugins.md b/docs/plugins.md index 881dfe82..ef49b80b 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -11,18 +11,24 @@ The design record — why it is shaped this way and what was considered — is ## Install -The CLI example below uses current unreleased `main`: +For 0.5.1, once published: ```bash -git clone --branch main https://github.com/Birfy/agentdescent && cd agentdescent -pip install -e ".[mcp]" # the MCP server needs this extra (and Python >= 3.10) +pip install "agentdescent[mcp]>=0.5.1" # the MCP server needs this extra (and Python >= 3.10) agentdescent demo # a complete evolution, offline, no key agentdescent doctor # which agent CLIs, keys and optional pieces are here agentdescent install dsh # or: claude-code, codex, opencode ``` -For the published v0.5.0 install and documented source-example workflow, see -[install and first run](install.md#published-v050-installation-and-source-examples). +Before publication, clone the repository as described in +[install and first run](install.md), then install the MCP extra from that checkout: + +```bash +pip install -e ".[mcp]" +``` + +Research demos beyond `agentdescent demo` use the +[source examples](install.md#source-examples). The MCP SDK requires Python >= 3.10; AgentDescent supports 3.9. On 3.9 the extra installs nothing and the tool surface is unavailable — the CLI verbs are diff --git a/docs/verifier.md b/docs/verifier.md index 9a7c5c6f..d48bc2b9 100644 --- a/docs/verifier.md +++ b/docs/verifier.md @@ -100,7 +100,7 @@ verifier.eval_counts(artifact) # (successes, failures) on the FULL held-out s verifier.full_eval(artifact) # the same eval_fn, whole set, spends budget ``` -!!! warning "`full_eval` was `oracle_eval` until 0.6" +!!! warning "`full_eval` was `oracle_eval` until 0.5.1" The old name promised an independent source of truth. Every layer here calls the one `eval_fn` you supplied and they differ only in **how many tasks** they score — so this layer bounds *sampling error* and cannot detect *bias*. If @@ -216,7 +216,7 @@ eval_counts(artifact) -> (successes, failures) # the acceptance test, full set full_eval(artifact) -> float # the same eval_fn, whole set ``` -Pre-0.6 code named the last one `oracle_eval`. The engine still reads that name, +Pre-0.5.1 code named the last one `oracle_eval`. The engine still reads that name, once, with a `DeprecationWarning`, and drops it in 0.7. There is also one **optional** attribute, read with a default so a substitute @@ -230,7 +230,7 @@ Set it when both are the same measurement, and the aggregator will reuse the rates it already has rather than asking twice. Leave it out when your full-set scorer is genuinely independent — then it is called, and it must stay exact: a `full_eval` that gets cheaper under budget pressure holds a veto over commits. -The pre-0.6 spelling `oracle_shares_full_set` is still read, and the new name +The pre-0.5.1 spelling `oracle_shares_full_set` is still read, and the new name wins if a class carries both. An [`aggregator_factory`](aggregator.md#replacing-aggregator_factory-aggregatorprotocol) diff --git a/integrations/claude-code/.claude-plugin/plugin.json b/integrations/claude-code/.claude-plugin/plugin.json index 71e6e069..c3af96f3 100644 --- a/integrations/claude-code/.claude-plugin/plugin.json +++ b/integrations/claude-code/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentdescent", "description": "Evolve skills, agents, prompts, code and plugins against examples with a parallel, merge-based optimiser.", - "version": "0.5.0", + "version": "0.5.1", "author": { "name": "AgentDescent" }, diff --git a/integrations/dsh-agentdescent/package.json b/integrations/dsh-agentdescent/package.json index 57d6dd9a..7c846edb 100644 --- a/integrations/dsh-agentdescent/package.json +++ b/integrations/dsh-agentdescent/package.json @@ -1,6 +1,6 @@ { "name": "dsh-agentdescent", - "version": "0.5.0", + "version": "0.5.1", "description": "Evolve skills, agents, prompts, code and plugins from inside DeepSeek Harness, with AgentDescent.", "keywords": [ "dsh-plugin", diff --git a/pyproject.toml b/pyproject.toml index 320e9312..bf1f9dba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,9 @@ dependencies = ["numpy>=1.20"] # `pyyaml` is only used by the dsh patch tests, which parse the block we # write the way dsh does. Without it `pytest tests/test_integrations.py` # fails on a clean checkout, which is the install the guide gives. -dev = ["pytest>=7.0", "pyyaml>=5.1"] +# Markdown renders docs-link checks exactly as browser-visible HTML instead +# of mistaking IDs inside code examples for real link targets. +dev = ["pytest>=7.0", "pyyaml>=5.1", "Markdown>=3.3.6"] docs = ["mkdocs-material[imaging]>=9.5"] # [imaging] builds the social cards # The MCP server (`agentdescent mcp`) needs the protocol SDK; the CLI does not. # diff --git a/tests/test_audit_and_cheap_layer.py b/tests/test_audit_and_cheap_layer.py index 6647076e..cdbd699d 100644 --- a/tests/test_audit_and_cheap_layer.py +++ b/tests/test_audit_and_cheap_layer.py @@ -246,7 +246,7 @@ def test_the_full_set_is_still_what_commits(tmp_path): assert round(successes + failures) == len(agg.verifier.held_out) -# -- the 0.6 rename: oracle_eval -> full_eval ----------------------------------- +# -- the 0.5.1 rename: oracle_eval -> full_eval ----------------------------------- # # `VerifierProtocol` is structural, so a verifier is whatever a caller hands the # engine. Renaming a method it calls therefore breaks other people's code at the @@ -283,7 +283,7 @@ def oracle_eval(self, artifact): return 0.25 legacy = Legacy() - with pytest.warns(DeprecationWarning, match="renamed in 0.6"): + with pytest.warns(DeprecationWarning, match="renamed in 0.5.1"): assert full_eval_of(legacy)(_Stub()) == 0.25 assert shares_eval_counts(legacy) is False @@ -316,7 +316,7 @@ def test_a_pre_rename_verifier_still_drives_a_whole_run_through_the_audit_gate() Written as a standalone class rather than a proxy on purpose. A proxy with `__getattr__` forwards `full_eval_matches_counts` from whatever it wraps, so it would take the reuse shortcut and never reach the legacy method -- the - test would pass while testing nothing. Somebody's real pre-0.6 verifier is a + test would pass while testing nothing. Somebody's real pre-0.5.1 verifier is a class like this one. `blast_radius=0.6` is L1, so `force_oracle` fires on every merge; with diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py index d83975cf..7843f47c 100644 --- a/tests/test_docs_links.py +++ b/tests/test_docs_links.py @@ -4,9 +4,13 @@ broken cross-reference is invisible until a reader hits it. mkdocs --strict does not validate heading anchors at all, so both halves are checked here. """ +from html.parser import HTMLParser import pathlib import re -from html.parser import HTMLParser + +import markdown +import pytest +import yaml DOCS = pathlib.Path(__file__).resolve().parent.parent / "docs" @@ -14,35 +18,111 @@ LINK = re.compile(r"\[[^\]]*\]\((?!https?://|mailto:)([^)\s]+)\)") -def _anchor(heading: str) -> str: - """Python-Markdown's `toc` slugify, exactly. +class _HTMLAnchors(HTMLParser): + def __init__(self): + super().__init__() + self.ids = set() - Worth copying rather than approximating: it drops every non-word character - (so backticks, parentheses and em dashes vanish) but **keeps underscores**, - because `_` is a word character. Collapsing them to dashes reports every - `async_evolve()` heading as a broken anchor. - """ - text = re.sub(r"[^\w\s-]", "", heading.strip()).strip().lower() - return re.sub(r"[-\s]+", "-", text) + def handle_starttag(self, tag, attrs): + ids = [value for name, value in attrs if name == "id"] + assert len(ids) <= 1, "an element must not repeat its id attribute" + for value in ids: + assert value and not any(c.isspace() for c in value), "invalid HTML id" + assert value not in self.ids, f"duplicate HTML id: {value}" + self.ids.add(value) def _anchors_of(path: pathlib.Path) -> set: + # Use MkDocs' underlying renderer, not regexes over Markdown source. Code + # examples must remain escaped, while raw HTML IDs and generated heading + # IDs are both actual browser targets. These are the core extensions used + # for the same constructs in mkdocs.yml. source = path.read_text() - # Fenced examples are not rendered as HTML, even when they contain tags. - source = re.sub(r"(?ms)^```.*?^```\s*$", "", source) - out = set() - for line in source.splitlines(): - m = re.match(r"^(#{1,6})\s+(.*)", line) - if m: - out.add(_anchor(m.group(2))) - - class IdCollector(HTMLParser): - def handle_starttag(self, tag, attrs): - out.update(value for name, value in attrs if name == "id" and value) - - parser = IdCollector() - parser.feed(source) - return out + # MkDocs consumes a leading YAML mapping before passing text to Markdown. + # Otherwise its closing delimiter can turn metadata into a setext heading + # and invent a link target that is absent from the rendered site. + frontmatter = re.match( + r"\A---[ \t]*\n(.*?)\n(?:---|\.\.\.)[ \t]*\n", source, re.DOTALL) + if frontmatter: + try: + metadata = yaml.safe_load(frontmatter.group(1)) + except yaml.YAMLError: + metadata = None + if isinstance(metadata, dict): + source = source[frontmatter.end():].lstrip("\n") + html = markdown.markdown(source, extensions=[ + "meta", "toc", "fenced_code", "attr_list", "md_in_html", + ]) + parser = _HTMLAnchors() + parser.feed(html) + return parser.ids + + +@pytest.mark.parametrize("end", ["---", "..."]) +def test_anchor_guard_ignores_yaml_metadata(tmp_path, end): + page = tmp_path / "page.md" + page.write_text( + '---\ndescription:
metadata
\n' + end + '\n## Real\n') + assert _anchors_of(page) == {"real"} + + +@pytest.mark.parametrize("metadata", [ + '---\ndescription: Example:
metadata
\n---\n', + 'description:
metadata
\n\n', +]) +def test_anchor_guard_matches_site_meta_extension(tmp_path, metadata): + page = tmp_path / "page.md" + page.write_text(metadata + '## Real\n') + assert _anchors_of(page) == {"real"} + + +def test_anchor_guard_recognizes_markdown_and_real_html_ids(tmp_path): + page = tmp_path / "page.md" + page.write_text('## Source examples\n
Text
') + assert _anchors_of(page) == {"source-examples", "building-the-docs"} + + +@pytest.mark.parametrize("example", [ + '', + '```html\n
example
\n```', + '~~~html\n
example
\n~~~', + '`
example
`', + '<details id="fake">example</details>', + '
indented example
', + '```html\n``` trailing text\n
still fenced
\n```', +]) +def test_anchor_guard_rejects_ids_that_are_only_examples(tmp_path, example): + page = tmp_path / "page.md" + page.write_text(example) + assert "fake" not in _anchors_of(page) + + +def test_anchor_guard_matches_renderer_for_a_backslash_before_raw_html(tmp_path): + # Python-Markdown does not backslash-escape '<': the emitted raw HTML + # remains a browser anchor. Entity-escaped HTML is rejected above. + page = tmp_path / "page.md" + page.write_text('\\
raw HTML
') + assert _anchors_of(page) == {"real"} + + +def test_anchor_guard_preserves_backticks_in_real_html_ids(tmp_path): + page = tmp_path / "page.md" + page.write_text('
') + assert _anchors_of(page) == {"first`id", "second`id"} + + +@pytest.mark.parametrize("html", [ + '
', + '
', + '
', + '
', + '## Same\n
', +]) +def test_anchor_guard_rejects_duplicate_or_malformed_ids(tmp_path, html): + page = tmp_path / "page.md" + page.write_text(html) + with pytest.raises(AssertionError): + _anchors_of(page) def _links(): diff --git a/tests/test_packaging.py b/tests/test_packaging.py index ed2a3fd6..4586cab1 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -10,6 +10,8 @@ import pathlib import re +import pytest + ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -24,16 +26,63 @@ def test_packaging_only_ships_the_library(): "shipping a top-level `examples` package would collide with other projects" +def _assert_example_checkout_caveat(readme): + """Validate the first shell install command and its source-example caveat.""" + install_at = None + # Inspect shell blocks, not a quoted command in explanatory prose. The first + # pip install is onboarding: a later correct command cannot excuse a typo. + for block in re.finditer(r"```(?:bash|sh)\n(.*?)```", readme, re.DOTALL): + install = re.search(r"(?m)^pip install[^\n]*$", block.group(1)) + if install: + package = (r"agentdescent(?:\[[A-Za-z0-9_,.-]+\])?" + r"(?:(?:===|==|!=|~=|>=|<=|>|<)[A-Za-z0-9.*+_-]+" + r"(?:,(?:===|==|!=|~=|>=|<=|>|<)[A-Za-z0-9.*+_-]+)*)?") + bare = r"agentdescent(?:\[[A-Za-z0-9_,.-]+\])?(?:==[A-Za-z0-9.*+_-]+)?" + command = rf"pip install (?:\"{package}\"|'{package}'|{bare})[ \t]*" + assert re.fullmatch(command, install.group()), \ + "onboarding must install agentdescent; quote shell version constraints" + install_at = block.start(1) + install.start() + break + assert install_at is not None, "the README must show a shell pip install command" + assert "python -m examples" in readme, "premise: the README advertises examples" + nearby = readme[install_at:install_at + 1200].lower() + caveat_at = nearby.find("examples are outside the wheel") + assert caveat_at >= 0, "explain beside the install that examples are source-only" + assert "clone the repo" in nearby[caveat_at:], \ + "the source-example checkout instruction belongs beside the install" + + def test_readme_tells_pip_users_the_examples_need_a_checkout(): - readme = (ROOT / "README.md").read_text() - assert "python -m examples" in readme, "premise: the README does advertise them" - lowered = readme.lower() - assert "clone" in lowered, "the README must say a checkout is required" - # the explanation must sit near the install instructions, not buried at the end - install_at = readme.index("pip install agentdescent") - clone_at = lowered.index("clone the repo") - assert 0 < clone_at - install_at < 1200, \ - "the checkout caveat belongs beside the install instructions" + _assert_example_checkout_caveat((ROOT / "README.md").read_text()) + + +@pytest.mark.parametrize("requirement", [ + "agentdescent", "agentdescent==0.5.1", '\"agentdescent>=0.5.1\"', + "'agentdescent[mcp]>=0.5.1,<0.6'", +]) +def test_checkout_guard_accepts_supported_install_syntax(requirement): + _assert_example_checkout_caveat( + f"```bash\npip install {requirement}\n```\n" + "Research examples are outside the wheel. To run python -m examples.run_demo, " + "clone the repo.") + + +@pytest.mark.parametrize("readme", [ + # No caveat; a clone elsewhere does not explain the source-only examples. + "```bash\npip install agentdescent\n```\nclone the repo; python -m examples.run_demo", + # A valid-looking command for a different package cannot satisfy onboarding. + "```bash\npip install agentdescent-other\n```\n" + "Examples are outside the wheel; clone the repo; python -m examples.run_demo", + # Shell redirection is not a version constraint unless quoted. + "```bash\npip install agentdescent>=0.5.1\n```\n" + "Examples are outside the wheel; clone the repo; python -m examples.run_demo", + # A caveat too far from the install still fails the positioning invariant. + "```bash\npip install agentdescent\n```\n" + "x" * 1200 + + "Examples are outside the wheel; clone the repo; python -m examples.run_demo", +]) +def test_checkout_guard_rejects_missing_or_misleading_onboarding(readme): + with pytest.raises(AssertionError): + _assert_example_checkout_caveat(readme) def test_usage_guide_repeats_the_caveat_before_the_demos(): diff --git a/tests/test_verifier.py b/tests/test_verifier.py index f6bb7c3d..490f9d5e 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -12,7 +12,7 @@ def test_a_subclass_can_declare_its_expensive_layer_independent(): """`ThreeLayerVerifier` sets `full_eval_matches_counts = True` as a statement about how *that* class is written, and a subclass inherits it -- so the escape hatch the attribute's docstring offers, "simply does not - define it", is not available to a subclass. One that set the pre-0.6 + define it", is not available to a subclass. One that set the pre-0.5.1 `oracle_shares_full_set = False` was ignored, its independent expensive layer went uncalled, and the gate degraded to the cheap measurement it existed to cross-check."""