From 874c470a2f5b0c184b4a86b77d22fbb49f0ba282 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 21 Oct 2026 00:18:36 +0800 Subject: [PATCH 1/3] chore(release): prepare OpenCollab-Eval 0.9.5 --- .github/workflows/ci.yml | 4 ++-- CHANGELOG.md | 16 +++++++++++++++- MIGRATION.md | 2 +- MIGRATION.zh-CN.md | 2 +- README.md | 4 ++-- RELEASING.md | 2 +- docs/README.md | 2 +- docs/evaluation-suite.md | 2 +- docs/getting-started.md | 8 ++++---- docs/swe-prolite-operations.md | 2 +- docs/zh-CN/README.md | 2 +- docs/zh-CN/evaluation-suite.md | 2 +- docs/zh-CN/getting-started.md | 8 ++++---- docs/zh-CN/swe-prolite-operations.md | 2 +- pyproject.toml | 2 +- src/opencollab_eval/__init__.py | 2 +- tests/packaging/test_release_metadata.py | 2 +- 17 files changed, 39 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 400baeb..0f92d59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: RISE-X-Lab/OpenCollab - ref: d4780297a2835d713f727500876d1276ff4d734d + ref: df4ccb279567761482df2d5f838fdcfc741821c1 token: ${{ secrets.OPENCOLLAB_READ_TOKEN || github.token }} persist-credentials: false path: opencollab-source @@ -138,7 +138,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: RISE-X-Lab/OpenCollab - ref: d4780297a2835d713f727500876d1276ff4d734d + ref: df4ccb279567761482df2d5f838fdcfc741821c1 token: ${{ secrets.OPENCOLLAB_READ_TOKEN || github.token }} persist-credentials: false path: opencollab-source diff --git a/CHANGELOG.md b/CHANGELOG.md index 5820ad4..390b63c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to OpenCollab-Eval are recorded in this file. ## [Unreleased] +## [0.9.5] - 2026-10-21 + +Paired with the signed OpenCollab 0.9.5 release. The supported runtime range +remains `opencollab>=0.9,<0.10`, and CI verifies the immutable 0.9.5 runtime +commit. + +Evaluation and experiment inputs now reside in this repository. Shared setup +for generation, evaluation, orchestration, reports, and tests has been +consolidated while retaining the existing execution and scoring behavior. +Contributor guidance and development documentation reflect the current +repository layout. + ## [0.9.2] - 2026-10-06 Paired with OpenCollab 0.9.2. The supported runtime dependency remains @@ -203,4 +215,6 @@ truncated, or mismatched execution output. [0.9.2]: https://github.com/RISE-X-Lab/OpenCollab-Eval/compare/v0.9.1...v0.9.2 -[Unreleased]: https://github.com/RISE-X-Lab/OpenCollab-Eval/compare/v0.9.2...HEAD +[0.9.5]: https://github.com/RISE-X-Lab/OpenCollab-Eval/compare/v0.9.2...v0.9.5 + +[Unreleased]: https://github.com/RISE-X-Lab/OpenCollab-Eval/compare/v0.9.5...HEAD diff --git a/MIGRATION.md b/MIGRATION.md index 7718b18..2b9d94c 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -21,7 +21,7 @@ The evaluator uses a `src` package layout. Installed entry points are `oc-eval` ## Choose compatible versions -The current evaluator release is 0.9.2. Its dependency is `opencollab>=0.9,<0.10`, and CI tests it with OpenCollab 0.9.2. The exact tested framework commit is recorded in `.github/workflows/ci.yml`. +The current evaluator release is 0.9.5. Its dependency is `opencollab>=0.9,<0.10`, and CI tests it with OpenCollab 0.9.5. The exact tested framework commit is recorded in `.github/workflows/ci.yml`. Follow [Getting started](docs/getting-started.md) for release-wheel or source installation. Confirm what the active environment actually imports before starting a run. diff --git a/MIGRATION.zh-CN.md b/MIGRATION.zh-CN.md index d52c024..d7e047b 100644 --- a/MIGRATION.zh-CN.md +++ b/MIGRATION.zh-CN.md @@ -21,7 +21,7 @@ OpenCollab 负责让智能体做题。OpenCollab-Eval 负责准备题目、保 ## 选择配套版本 -当前评测器版本为 0.9.2，依赖范围为 `opencollab>=0.9,<0.10`，CI 使用 OpenCollab 0.9.2 验证。实际验证的框架提交写在 `.github/workflows/ci.yml`。 +当前评测器版本为 0.9.5，依赖范围为 `opencollab>=0.9,<0.10`，CI 使用 OpenCollab 0.9.5 验证。实际验证的框架提交写在 `.github/workflows/ci.yml`。 按照[入门指南](docs/zh-CN/getting-started.md)安装发布的 wheel 或源码。开始运行前，先确认当前环境实际加载的两个包版本。 diff --git a/README.md b/README.md index 192a44b..69fcc3f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@

Install · Choose a command · Run Duo

-OpenCollab-Eval prepares tasks, runs [OpenCollab](https://github.com/RISE-X-Lab/OpenCollab) solvers, and evaluates their patches in isolated official test workspaces. The current version is **0.9.2**, paired with **OpenCollab >=0.9,<0.10**. +OpenCollab-Eval prepares tasks, runs [OpenCollab](https://github.com/RISE-X-Lab/OpenCollab) solvers, and evaluates their patches in isolated official test workspaces. The current version is **0.9.5**, paired with **OpenCollab >=0.9,<0.10**. @@ -118,7 +118,7 @@ releases. The license is [MulanPSL-2.0](LICENSE), with dependency notices in

安装 · 选择命令 · 运行 Duo

-OpenCollab-Eval 负责准备任务、调用 [OpenCollab](https://github.com/RISE-X-Lab/OpenCollab) 求解器，并在独立的正式测试工作区中评测补丁。当前版本为 **0.9.2**，配套依赖为 **OpenCollab >=0.9,<0.10**。 +OpenCollab-Eval 负责准备任务、调用 [OpenCollab](https://github.com/RISE-X-Lab/OpenCollab) 求解器，并在独立的正式测试工作区中评测补丁。当前版本为 **0.9.5**，配套依赖为 **OpenCollab >=0.9,<0.10**。 diff --git a/RELEASING.md b/RELEASING.md index f68140d..3dd496b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,6 +1,6 @@ # Releasing OpenCollab-Eval -OpenCollab-Eval 0.9.2 is paired with OpenCollab 0.9.2. Releases use the verified compatible OpenCollab release. PyPI publishing is outside the current release process. +OpenCollab-Eval 0.9.5 is paired with OpenCollab 0.9.5. Releases use the verified compatible OpenCollab release. PyPI publishing is outside the current release process. ## Release invariants diff --git a/docs/README.md b/docs/README.md index 9489d35..747b0c5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ **English** | [简体中文](zh-CN/README.md) -Choose a guide by the task you want to complete. The repository [README](../README.md) introduces version 0.9.2 and the main commands. [Getting started](getting-started.md) walks through installation, task input, a first run, and the result file. +Choose a guide by the task you want to complete. The repository [README](../README.md) introduces version 0.9.5 and the main commands. [Getting started](getting-started.md) walks through installation, task input, a first run, and the result file. ## Prepare and run tasks diff --git a/docs/evaluation-suite.md b/docs/evaluation-suite.md index eaccad5..2f80121 100644 --- a/docs/evaluation-suite.md +++ b/docs/evaluation-suite.md @@ -2,7 +2,7 @@ **English** | [简体中文](zh-CN/evaluation-suite.md) -This guide covers worker packaging, provider capacity, research workflow settings, and saved candidate recovery for OpenCollab-Eval 0.9.2 with OpenCollab >=0.9,<0.10. Begin a new official evaluation with the [Duo tutorial](swe-prolite-operations.md#run-duo-on-one-linux-worker). The settings here support existing operator deployments and controlled experiments. +This guide covers worker packaging, provider capacity, research workflow settings, and saved candidate recovery for OpenCollab-Eval 0.9.5 with OpenCollab >=0.9,<0.10. Begin a new official evaluation with the [Duo tutorial](swe-prolite-operations.md#run-duo-on-one-linux-worker). The settings here support existing operator deployments and controlled experiments. OC runs the agents and workflows. OCE prepares the public task and dependencies, captures a trusted candidate, executes official tests, and records the outcome. diff --git a/docs/getting-started.md b/docs/getting-started.md index 0fdb063..dec61fd 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,7 +6,7 @@ Start by choosing what you want to run. A task for your own repository uses gene ## Install the package -The current evaluator is 0.9.2 and requires OpenCollab 0.9.x (`opencollab>=0.9,<0.10`). Use Python 3.10 through 3.12 on Linux or macOS. The commands below use Python 3.12 and keep the environment beside the two source checkouts. +The current evaluator is 0.9.5 and requires OpenCollab 0.9.x (`opencollab>=0.9,<0.10`). Use Python 3.10 through 3.12 on Linux or macOS. The commands below use Python 3.12 and keep the environment beside the two source checkouts. ```bash mkdir -p "$HOME/oc-evaluation" @@ -21,11 +21,11 @@ oc-eval --version oc-eval --help ``` -The version command should print 0.9.2 for the evaluator. For installation from built wheels, activate an environment and install both compatible packages. +The version command should print 0.9.5 for the evaluator. For installation from built wheels, activate an environment and install both compatible packages. ```bash -python -m pip install /path/to/opencollab-0.9.2-py3-none-any.whl -python -m pip install /path/to/opencollab_eval-0.9.2-py3-none-any.whl +python -m pip install /path/to/opencollab-0.9.5-py3-none-any.whl +python -m pip install /path/to/opencollab_eval-0.9.5-py3-none-any.whl ``` Official SWE-bench support requires the corresponding extra and Docker on the Linux worker. OpenHands support uses its own extra in Python 3.12. From the source installation directory, install the extra needed by your run. diff --git a/docs/swe-prolite-operations.md b/docs/swe-prolite-operations.md index 2777998..90b414f 100644 --- a/docs/swe-prolite-operations.md +++ b/docs/swe-prolite-operations.md @@ -6,7 +6,7 @@ Use Duo for a new OpenCollab evaluation. The first tutorial runs the controller, ## Run Duo on one Linux worker -Install matching OpenCollab 0.9.x and OpenCollab-Eval 0.9.2 packages with the SWE-bench extra following [Getting started](getting-started.md). Activate that Python environment on the Linux worker and allow the evaluation account to access Docker Engine. Use an OpenAI-compatible Responses endpoint with streamed responses, function tools, and the reasoning settings selected below. Keep the environment active throughout this tutorial. +Install matching OpenCollab 0.9.x and OpenCollab-Eval 0.9.5 packages with the SWE-bench extra following [Getting started](getting-started.md). Activate that Python environment on the Linux worker and allow the evaluation account to access Docker Engine. Use an OpenAI-compatible Responses endpoint with streamed responses, function tools, and the reasoning settings selected below. Keep the environment active throughout this tutorial. ### Prepare the dataset and task images diff --git a/docs/zh-CN/README.md b/docs/zh-CN/README.md index 263fd1a..6b3b9d5 100644 --- a/docs/zh-CN/README.md +++ b/docs/zh-CN/README.md @@ -2,7 +2,7 @@ [English](../README.md) | **简体中文** -按要完成的任务选择指南。仓库 [README](../../README.md#simplified-chinese)介绍 0.9.2 和常用命令。[快速入门](getting-started.md)从安装开始，带你准备输入、运行任务并打开结果文件。 +按要完成的任务选择指南。仓库 [README](../../README.md#simplified-chinese)介绍 0.9.5 和常用命令。[快速入门](getting-started.md)从安装开始，带你准备输入、运行任务并打开结果文件。 ## 准备与运行任务 diff --git a/docs/zh-CN/evaluation-suite.md b/docs/zh-CN/evaluation-suite.md index 970ebfe..6a68df1 100644 --- a/docs/zh-CN/evaluation-suite.md +++ b/docs/zh-CN/evaluation-suite.md @@ -2,7 +2,7 @@ [English](../evaluation-suite.md) | **简体中文** -本指南介绍 OpenCollab-Eval 0.9.2 与 OpenCollab >=0.9,<0.10 配套使用时的 worker 打包、API 请求容量、研究工作流设置和已保存候选恢复。新的正式评测从 [Duo 教程](swe-prolite-operations.md#在一台-linux-工作机上运行-duo)开始。这里的设置用于已有部署和受控实验。 +本指南介绍 OpenCollab-Eval 0.9.5 与 OpenCollab >=0.9,<0.10 配套使用时的 worker 打包、API 请求容量、研究工作流设置和已保存候选恢复。新的正式评测从 [Duo 教程](swe-prolite-operations.md#在一台-linux-工作机上运行-duo)开始。这里的设置用于已有部署和受控实验。 OC 执行 Agent 与工作流。OCE 准备公开题面和依赖，提取可信候选，执行正式测试，并记录结果。 diff --git a/docs/zh-CN/getting-started.md b/docs/zh-CN/getting-started.md index f948f58..dbf076e 100644 --- a/docs/zh-CN/getting-started.md +++ b/docs/zh-CN/getting-started.md @@ -6,7 +6,7 @@ ## 安装软件包 -当前评测器版本为 0.9.2，依赖 OpenCollab 0.9.x（`opencollab>=0.9,<0.10`）。Linux 和 macOS 支持 Python 3.10 至 3.12。下面使用 Python 3.12，把环境放在两个源码目录旁边。 +当前评测器版本为 0.9.5，依赖 OpenCollab 0.9.x（`opencollab>=0.9,<0.10`）。Linux 和 macOS 支持 Python 3.10 至 3.12。下面使用 Python 3.12，把环境放在两个源码目录旁边。 ```bash mkdir -p "$HOME/oc-evaluation" @@ -21,11 +21,11 @@ oc-eval --version oc-eval --help ``` -评测器版本命令应输出 0.9.2。使用已构建的 wheel 时，激活环境后安装两个兼容软件包。 +评测器版本命令应输出 0.9.5。使用已构建的 wheel 时，激活环境后安装两个兼容软件包。 ```bash -python -m pip install /path/to/opencollab-0.9.2-py3-none-any.whl -python -m pip install /path/to/opencollab_eval-0.9.2-py3-none-any.whl +python -m pip install /path/to/opencollab-0.9.5-py3-none-any.whl +python -m pip install /path/to/opencollab_eval-0.9.5-py3-none-any.whl ``` 正式 SWE-bench 评测需要对应的可选依赖，以及 Linux 工作机上的 Docker。OpenHands 支持使用单独的可选依赖，需要 Python 3.12。在源码安装目录中，安装本次运行所需的依赖。 diff --git a/docs/zh-CN/swe-prolite-operations.md b/docs/zh-CN/swe-prolite-operations.md index db93987..3b9bab4 100644 --- a/docs/zh-CN/swe-prolite-operations.md +++ b/docs/zh-CN/swe-prolite-operations.md @@ -6,7 +6,7 @@ ## 在一台 Linux 工作机上运行 Duo -按[快速开始](getting-started.md)安装匹配的 OpenCollab 0.9.x 与带 SWE-bench extra 的 OpenCollab-Eval 0.9.2。在 Linux worker 上激活该 Python 环境，并允许评测账号访问 Docker Engine。配置支持流式 Responses、function tools 和所选推理设置的 OpenAI 兼容入口。后续命令始终在这个已激活环境中执行。 +按[快速开始](getting-started.md)安装匹配的 OpenCollab 0.9.x 与带 SWE-bench extra 的 OpenCollab-Eval 0.9.5。在 Linux worker 上激活该 Python 环境，并允许评测账号访问 Docker Engine。配置支持流式 Responses、function tools 和所选推理设置的 OpenAI 兼容入口。后续命令始终在这个已激活环境中执行。 ### 准备数据与题目镜像 diff --git a/pyproject.toml b/pyproject.toml index 240c2de..87bd63c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "opencollab-eval" -version = "0.9.2" +version = "0.9.5" description = "Independent evaluation layer for OpenCollab solvers." readme = "README.md" requires-python = ">=3.10" diff --git a/src/opencollab_eval/__init__.py b/src/opencollab_eval/__init__.py index 1a222dc..10cf86c 100644 --- a/src/opencollab_eval/__init__.py +++ b/src/opencollab_eval/__init__.py @@ -1,3 +1,3 @@ """Independent evaluation layer for OpenCollab-based solvers.""" -__version__ = "0.9.2" +__version__ = "0.9.5" diff --git a/tests/packaging/test_release_metadata.py b/tests/packaging/test_release_metadata.py index 2f9de6f..3950913 100644 --- a/tests/packaging/test_release_metadata.py +++ b/tests/packaging/test_release_metadata.py @@ -19,7 +19,7 @@ def test_release_metadata_keeps_versions_aligned_and_includes_license() -> None: license_bytes = (_REPO_ROOT / "LICENSE").read_bytes() assert opencollab_eval.__version__ - assert opencollab_eval.__version__ == "0.9.2" + assert opencollab_eval.__version__ == "0.9.5" assert 'requires = ["hatchling==1.31.0"]' in pyproject assert f'version = "{opencollab_eval.__version__}"' in pyproject assert 'dependencies = ["opencollab>=0.9,<0.10", "httpx>=0.27"]' in pyproject From 6d23025ae388cb08b2d85cce83a1cf7b961c6762 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 21 Oct 2026 00:23:00 +0800 Subject: [PATCH 2/3] fix(experiment): select owned inputs with the paired runtime --- CHANGELOG.md | 2 ++ src/opencollab_eval/experiment/arm_audit.py | 12 +++++++----- tests/experiment/test_arm_alignment_gate.py | 17 +++++++++++++++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 390b63c..e85e5e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ commit. Evaluation and experiment inputs now reside in this repository. Shared setup for generation, evaluation, orchestration, reports, and tests has been consolidated while retaining the existing execution and scoring behavior. +Arm audits select the owned evaluation inputs when both source checkouts are +provided, as they are in CI. Contributor guidance and development documentation reflect the current repository layout. diff --git a/src/opencollab_eval/experiment/arm_audit.py b/src/opencollab_eval/experiment/arm_audit.py index fe8eb63..35d61cf 100644 --- a/src/opencollab_eval/experiment/arm_audit.py +++ b/src/opencollab_eval/experiment/arm_audit.py @@ -71,15 +71,17 @@ def default_team_config() -> str: """Resolve current owned inputs, or an explicitly selected historical checkout. A wheel needs an evaluation source checkout named by - ``OPENCOLLAB_EVAL_SOURCE_ROOT``. An explicit ``OPENCOLLAB_SOURCE_ROOT`` - retains the historical framework layout. Missing selected inputs remain - missing; never substitute another revision's prompt silently. + ``OPENCOLLAB_EVAL_SOURCE_ROOT``. Without it, an explicit + ``OPENCOLLAB_SOURCE_ROOT`` retains the historical framework layout. + Missing selected inputs remain missing. """ + source_root = os.environ.get("OPENCOLLAB_EVAL_SOURCE_ROOT", "").strip() + if source_root: + return str(Path(source_root) / "experiment/framework" / EXPERIMENT_TEAM_CONFIG) legacy_root = os.environ.get("OPENCOLLAB_SOURCE_ROOT", "").strip() if legacy_root: return str(Path(legacy_root) / EXPERIMENT_TEAM_CONFIG) - source_root = os.environ.get("OPENCOLLAB_EVAL_SOURCE_ROOT", "").strip() - root = Path(source_root) if source_root else Path(__file__).resolve().parents[3] + root = Path(__file__).resolve().parents[3] return str(root / "experiment/framework" / EXPERIMENT_TEAM_CONFIG) diff --git a/tests/experiment/test_arm_alignment_gate.py b/tests/experiment/test_arm_alignment_gate.py index 70a5a2b..2f0e865 100644 --- a/tests/experiment/test_arm_alignment_gate.py +++ b/tests/experiment/test_arm_alignment_gate.py @@ -77,13 +77,13 @@ def test_the_team_arm_s_configuration_is_reachable_from_here() -> None: def test_the_config_lookup_prefers_the_checkout_the_environment_names() -> None: - # CI installs OpenCollab from a wheel, which carries no ``configs/``, and - # names the checkout it was built from. That checkout has to win. + # Historical runs can select a framework checkout without an evaluator root. import opencollab from _pytest.monkeypatch import MonkeyPatch patch = MonkeyPatch() try: + patch.delenv("OPENCOLLAB_EVAL_SOURCE_ROOT", raising=False) root = Path(tempfile.mkdtemp()) planted = root / arm_audit.EXPERIMENT_TEAM_CONFIG planted.parent.mkdir(parents=True, exist_ok=True) @@ -103,6 +103,19 @@ def test_the_config_lookup_prefers_the_checkout_the_environment_names() -> None: patch.undo() +def test_current_evaluation_inputs_take_precedence_over_runtime_checkout( + monkeypatch, tmp_path, +) -> None: + evaluation = tmp_path / "evaluation" + runtime = tmp_path / "runtime" + monkeypatch.setenv("OPENCOLLAB_EVAL_SOURCE_ROOT", str(evaluation)) + monkeypatch.setenv("OPENCOLLAB_SOURCE_ROOT", str(runtime)) + + assert arm_audit.default_team_config() == str( + evaluation / "experiment/framework" / arm_audit.EXPERIMENT_TEAM_CONFIG + ) + + def test_the_registry_covers_exactly_the_arms_the_batch_driver_runs() -> None: # A fifth arm added to the driver and not to the registry would otherwise # be compared against the other four with nothing checking its inputs. From 79e63d44e981684e42f56e65eaaa00d408532f6e Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 21 Oct 2026 00:27:11 +0800 Subject: [PATCH 3/3] test(release): align paired OpenCollab 0.9.5 checks --- tests/orchestration/test_analyst_solve_target_tests.py | 4 +++- tests/orchestration/test_duo_adapter.py | 2 +- tests/orchestration/test_phase1_toolset_whitelist.py | 4 +++- tests/orchestration/test_phase2_warts.py | 4 +++- tests/packaging/test_publication_workflows.py | 2 +- tests/packaging/test_wheel_contract.py | 4 +++- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/orchestration/test_analyst_solve_target_tests.py b/tests/orchestration/test_analyst_solve_target_tests.py index e6b966b..500c1bf 100644 --- a/tests/orchestration/test_analyst_solve_target_tests.py +++ b/tests/orchestration/test_analyst_solve_target_tests.py @@ -25,7 +25,9 @@ async def agent(self, prompt, *, schema=None, label=None, tools=None, **kw): if isinstance(reply, dict) and tools: for tool in tools: if getattr(tool, "name", "") == "bash": - tool._verified_targets.update(reply.get("tests_run") or ()) + tool._verified_targets.update( + (False, target) for target in (reply.get("tests_run") or ()) + ) return reply async def parallel(self, thunks): diff --git a/tests/orchestration/test_duo_adapter.py b/tests/orchestration/test_duo_adapter.py index ba67107..9d6630f 100644 --- a/tests/orchestration/test_duo_adapter.py +++ b/tests/orchestration/test_duo_adapter.py @@ -36,7 +36,7 @@ def test_registry_uses_oc_implementation_and_preserves_each_workflow_identity(na def test_only_one_duo_workflow_is_exposed(): assert workflows.duo is duo - assert [spec.name for spec in get_builtin_workflows().list_specs()] == ["duo"] + assert [spec.name for spec in get_builtin_workflows().list_specs()].count("duo") == 1 registry = _bundled_workflow_registry() assert "duo-v3" not in registry assert "validation-council-dual-coder-selection-v2" not in registry diff --git a/tests/orchestration/test_phase1_toolset_whitelist.py b/tests/orchestration/test_phase1_toolset_whitelist.py index b22a1c1..4d894e3 100644 --- a/tests/orchestration/test_phase1_toolset_whitelist.py +++ b/tests/orchestration/test_phase1_toolset_whitelist.py @@ -91,7 +91,9 @@ async def agent(self, prompt, *, schema=None, label=None, tools=None, **kw): if isinstance(reply, dict) and reply.get("verdict") == "PASS": for tool in tools or (): if getattr(tool, "name", "") == "bash": - tool._verified_targets.update(reply.get("tests_run") or ()) + tool._verified_targets.update( + (False, target) for target in (reply.get("tests_run") or ()) + ) return reply async def parallel(self, thunks): diff --git a/tests/orchestration/test_phase2_warts.py b/tests/orchestration/test_phase2_warts.py index aa87b1b..ea5fc73 100644 --- a/tests/orchestration/test_phase2_warts.py +++ b/tests/orchestration/test_phase2_warts.py @@ -37,7 +37,9 @@ async def agent(self, prompt, *, schema=None, label=None, tools=None, **kw): if isinstance(reply, dict) and tools: for tool in tools: if getattr(tool, "name", "") == "bash": - tool._verified_targets.update(reply.get("tests_run") or ()) + tool._verified_targets.update( + (False, target) for target in (reply.get("tests_run") or ()) + ) return reply async def parallel(self, thunks): diff --git a/tests/packaging/test_publication_workflows.py b/tests/packaging/test_publication_workflows.py index f75b3c7..561b9aa 100644 --- a/tests/packaging/test_publication_workflows.py +++ b/tests/packaging/test_publication_workflows.py @@ -190,7 +190,7 @@ def test_push_gates_reject_an_unrelated_but_reachable_before_commit( def test_ci_uses_verified_action_release_commits() -> None: workflow = _workflow("ci.yml") - opencollab_ref = "d4780297a2835d713f727500876d1276ff4d734d" + opencollab_ref = "df4ccb279567761482df2d5f838fdcfc741821c1" assert "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" in workflow assert "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97" in workflow diff --git a/tests/packaging/test_wheel_contract.py b/tests/packaging/test_wheel_contract.py index 9c97d2f..204834e 100644 --- a/tests/packaging/test_wheel_contract.py +++ b/tests/packaging/test_wheel_contract.py @@ -31,7 +31,9 @@ def test_opencollab_sdk_can_come_from_the_built_wheel() -> None: sdk_version = Version(distribution_version("opencollab")).release assert (0, 9, 0) <= sdk_version < (0, 10) assert callable(opencollab.builtin_workflows.duo) - assert [spec.name for spec in opencollab.builtin_workflows.get_builtin_workflows().list_specs()] == ["duo"] + assert [ + spec.name for spec in opencollab.builtin_workflows.get_builtin_workflows().list_specs() + ].count("duo") == 1 assert opencollab.profiles.resolve_profile_name("base") == "single2" assert callable(opencollab.patches.patch_paths) assert callable(opencollab.tools.evidence_tools)