From 0df170a8f2687ab8f5201e896081dded1270a626 Mon Sep 17 00:00:00 2001 From: Majd Abdallah Date: Sat, 12 Sep 2026 15:44:01 +0200 Subject: [PATCH 1/3] Establish production verification, artifact integrity, and CLI e2e --- .github/dependabot.yml | 9 + .github/workflows/ci.yml | 132 +-- .github/workflows/release.yml | 62 +- .github/workflows/verify.yml | 208 +++++ .pre-commit-config.yaml | 5 + CHANGELOG.md | 15 + MANIFEST.in | 2 + Makefile | 41 +- README.md | 17 + docs/audit_2026_09_12_inventory.json | 995 +++++++++++++++++++++ docs/audit_2026_09_12_observations.json | 132 +++ docs/audit_2026_09_12_reproduce.py | 276 ++++++ docs/audit_2026_09_12_validation.md | 85 ++ docs/codebase-review-2026-09-12.md | 176 ++++ docs/production-roadmap.md | 97 ++ docs/production-validation.md | 37 + docs/release.md | 152 +++- mkdocs.yml | 2 + pyproject.toml | 1 + pytest.ini | 1 + scripts/installed_smoke.py | 76 ++ src/trialmatchai/cli/artifacts.py | 40 + src/trialmatchai/cli/bootstrap_data.py | 105 ++- src/trialmatchai/cli/demo.py | 135 +++ src/trialmatchai/cli/main.py | 12 +- src/trialmatchai/interop/importers/fhir.py | 3 + src/trialmatchai/services/preflight.py | 12 +- src/trialmatchai/utils/integrity.py | 117 +++ tests/test_artifact_integrity.py | 67 ++ tests/test_bootstrap_data.py | 44 + tests/test_cli_e2e.py | 35 + tests/test_deployment_readiness.py | 11 + tests/test_fhir_robustness.py | 27 + 33 files changed, 2895 insertions(+), 234 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/verify.yml create mode 100644 MANIFEST.in create mode 100644 docs/audit_2026_09_12_inventory.json create mode 100644 docs/audit_2026_09_12_observations.json create mode 100644 docs/audit_2026_09_12_reproduce.py create mode 100644 docs/audit_2026_09_12_validation.md create mode 100644 docs/codebase-review-2026-09-12.md create mode 100644 docs/production-roadmap.md create mode 100644 docs/production-validation.md create mode 100644 scripts/installed_smoke.py create mode 100644 src/trialmatchai/cli/artifacts.py create mode 100644 src/trialmatchai/cli/demo.py create mode 100644 src/trialmatchai/utils/integrity.py create mode 100644 tests/test_artifact_integrity.py create mode 100644 tests/test_cli_e2e.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..fecd44db --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecca6936..1af2f1a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,131 +2,19 @@ name: CI on: push: - branches: [main, dev, deployment-readiness-audit] + branches: [main, dev, 'production/**'] pull_request: branches: [main, dev] + merge_group: workflow_dispatch: -jobs: - deployment-readiness: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install dependencies - run: uv sync --frozen - - - name: Validate lockfile - run: uv lock --check - - - name: Lint - run: uv run ruff check . - - - name: Run tests - run: uv run pytest -v - - - name: Build package - run: uv build - - - name: Installed package smoke - run: | - WHEEL="$(ls "$PWD"/dist/trialmatchai-*.whl | head -n 1)" - mkdir -p /tmp/trialmatchai-installed-smoke - cd /tmp/trialmatchai-installed-smoke - uv run --python 3.11 --isolated --with "$WHEEL" python - <<'PY' - from pathlib import Path - from trialmatchai.config.config_loader import load_config - - cfg = load_config() - assert Path(cfg["entity_extraction"]["schema_path"]).exists() - assert cfg["paths"]["output_dir"].endswith("results") - PY +permissions: + contents: read - - name: CLI help smoke - run: | - WHEEL="$(ls dist/trialmatchai-*.whl | head -n 1)" - uv run --python 3.11 --isolated --with "$WHEEL" trialmatchai --help - for cmd in pipeline healthcheck bootstrap-data index build-concepts update-registry \ - import-patient build run e2e trec finetune; do - uv run --python 3.11 --isolated --with "$WHEEL" trialmatchai "$cmd" --help - done +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true - - name: Secret scan - run: uv run pre-commit run gitleaks --all-files - - - name: Dependency audit - # vllm==0.23.0 (pins torch 2.11.0) and its transitive diskcache 5.6.3 are hard-pinned - # for the inference stack. The advisories below were newly published against those exact - # pinned versions and list NO fixed version, so there is nothing to upgrade to. Revisit - # and drop these ignores once vllm / diskcache ship patched releases. - run: >- - uv run pip-audit --progress-spinner off - --ignore-vuln CVE-2025-3000 - --ignore-vuln CVE-2025-69872 - --ignore-vuln CVE-2026-53923 - --ignore-vuln CVE-2026-54236 - --ignore-vuln CVE-2026-12491 - --ignore-vuln CVE-2026-54235 - --ignore-vuln CVE-2026-54233 - - ml-extras-smoke: - # Installs the optional ML extras (CPU-only) and imports the local-model - # surface that the default job never touches, so a broken import or wrong - # transformers/gliner API call is caught instead of shipping silently. - # Non-blocking: it exercises heavy third-party deps (torch/gliner) whose - # availability we don't gate releases on; failures show up as a signal - # without failing the workflow. - continue-on-error: true - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install with entity extra (CPU) - run: uv sync --extra entity - - - name: Import ML surface - run: | - uv run python - <<'PY' - import importlib - - # The entity extra's heavy libs must import cleanly in the resolved - # environment (catches a bad transformers API or a torch pin conflict). - for lib in ("torch", "transformers", "gliner", "gliner2"): - importlib.import_module(lib) - print(f"imported {lib}") - - # The package's local-model modules must import with extras present. - modules = [ - "trialmatchai.entities.recognizers", - "trialmatchai.models.embedding.text_embedder", - "trialmatchai.models.llm.llm_reranker", - "trialmatchai.models.llm.vllm_loader", - "trialmatchai.matching.eligibility_reasoning_vllm", - "trialmatchai.finetuning.cot", - "trialmatchai.finetuning.reranker", - "trialmatchai.finetuning.ner", - ] - for name in modules: - importlib.import_module(name) - print(f"imported {name}") - PY +jobs: + verification: + uses: ./.github/workflows/verify.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c225b296..bdbea6fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,44 +1,54 @@ name: Release -# Publishes trialmatchai to PyPI when a GitHub Release is published. -# Uses PyPI Trusted Publishing (OIDC) — no API token is stored in the repo. -# One-time setup on PyPI: add a Trusted Publisher for this repo + workflow + -# the `pypi` environment (https://docs.pypi.org/trusted-publishers/). - on: release: types: [published] - workflow_dispatch: {} permissions: contents: read +concurrency: + group: pypi-release + cancel-in-progress: false + jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - - name: Build sdist + wheel - run: uv build - - name: Check the built metadata - run: uvx twine check dist/* - - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/ + verification: + uses: ./.github/workflows/verify.yml + with: + release_tag: ${{ github.event.release.tag_name }} pypi-publish: - needs: build + needs: verification runs-on: ubuntu-latest + timeout-minutes: 15 environment: pypi permissions: - id-token: write # required for PyPI Trusted Publishing (OIDC) + contents: read + id-token: write + attestations: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - name: dist + name: trialmatchai-dist-${{ github.sha }} path: dist/ - - uses: astral-sh/setup-uv@v5 - - name: Publish to PyPI - run: uv publish --trusted-publishing always + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + with: + version: "0.11.24" + - name: Verify the tested artifacts + env: + PYTHONPATH: src + run: python -m trialmatchai artifacts verify dist/SHA256SUMS --require-exact --json + - uses: actions/attest-build-provenance@43d14bc2b83dec42d39ecae14e916627a18bb661 # v3 + with: + subject-path: | + dist/*.whl + dist/*.tar.gz + dist/SHA256SUMS + - name: Publish the tested distributions with OIDC + run: uv publish dist/*.whl dist/*.tar.gz --trusted-publishing always diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 00000000..1728e2a6 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,208 @@ +name: Production verification + +on: + workflow_call: + inputs: + release_tag: + description: Optional release tag, which must match v + required: false + type: string + default: "" + +permissions: + contents: read + +jobs: + quality: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + with: + version: "0.11.24" + enable-cache: true + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - run: uv sync --frozen + - run: uv lock --check + - run: uv run --frozen ruff check . + - run: uv run --frozen pre-commit run actionlint --all-files + - run: uv run --frozen pytest -v -m 'not e2e' --junitxml=test-results/unit.xml + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: unit-results + path: test-results/unit.xml + retention-days: 14 + + security: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + with: + version: "0.11.24" + enable-cache: true + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - run: uv sync --frozen + - run: uv run --frozen pre-commit run gitleaks --all-files + - run: make audit + + model-imports: + # Import compatibility is required; GPU inference is a separate qualification. + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + with: + version: "0.11.24" + enable-cache: true + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - run: uv sync --frozen --extra entity + - name: Import optional model surface + run: | + uv run --frozen --extra entity python - <<'PY' + import importlib + for module in ( + "torch", "transformers", "gliner2", + "trialmatchai.entities.recognizers", + "trialmatchai.models.embedding.text_embedder", + "trialmatchai.models.llm.llm_reranker", + "trialmatchai.models.llm.vllm_loader", + "trialmatchai.matching.eligibility_reasoning_vllm", + "trialmatchai.finetuning.cot", + "trialmatchai.finetuning.reranker", + "trialmatchai.finetuning.ner", + ): + importlib.import_module(module) + print(f"Imported {module}") + PY + + package: + needs: quality + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + with: + version: "0.11.24" + enable-cache: true + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - run: uv sync --frozen + - name: Validate release identity and record build inputs + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + uv run --frozen python - <<'PY' + import hashlib + import json + import os + import platform + import subprocess + import tomllib + from pathlib import Path + from trialmatchai import __version__ + version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"] + assert version == __version__, "Package and module versions differ" + tag = os.environ["RELEASE_TAG"] + assert not tag or tag == f"v{version}", f"Tag {tag!r} does not match v{version}" + Path("dist").mkdir(exist_ok=True) + info = { + "schema_version": 1, "version": version, + "source_commit": os.environ["GITHUB_SHA"], + "python": platform.python_version(), + "uv": subprocess.check_output(["uv", "--version"], text=True).strip(), + "lock_sha256": hashlib.sha256(Path("uv.lock").read_bytes()).hexdigest(), + } + Path("dist/build-info.json").write_text(json.dumps(info, indent=2) + "\n") + PY + - run: uv build + - run: uv run --frozen trialmatchai artifacts manifest dist + - run: uv run --frozen trialmatchai artifacts verify dist/SHA256SUMS --require-exact --json + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: trialmatchai-dist-${{ github.sha }} + path: dist/ + include-hidden-files: true # uv build writes .gitignore, which SHA256SUMS includes + if-no-files-found: error + retention-days: 14 + + installed-cli: + needs: package + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + with: + version: "0.11.24" + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: trialmatchai-dist-${{ github.sha }} + path: dist/ + - name: Verify artifacts before installation + env: + PYTHONPATH: src + run: python -m trialmatchai artifacts verify dist/SHA256SUMS --require-exact --json + - name: Run installed wheel through the public CLI + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + run: | + WHEEL="$(realpath dist/trialmatchai-*.whl)" + SMOKE_SCRIPT="$PWD/scripts/installed_smoke.py" + mkdir -p "$RUNNER_TEMP/trialmatchai-smoke" + cd "$RUNNER_TEMP/trialmatchai-smoke" + uv run --no-project --isolated --with "$WHEEL" python -I "$SMOKE_SCRIPT" \ + --expect-installed --workspace "$RUNNER_TEMP/trialmatchai-smoke/workspace" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: failure() + with: + name: installed-cli-failure + path: ${{ runner.temp }}/trialmatchai-smoke/workspace/ + retention-days: 7 + + required: + if: always() + needs: [quality, security, model-imports, package, installed-cli] + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Require every verification job to pass + env: + NEEDS: ${{ toJSON(needs) }} + run: | + python - <<'PY' + import json + import os + outcomes = {name: job["result"] for name, job in json.loads(os.environ["NEEDS"]).items()} + print(outcomes) + assert all(result == "success" for result in outcomes.values()), "Verification failed or skipped" + PY diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index acd7ccbc..3caa9a0f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,11 @@ # Install once: uv run pre-commit install # Run on demand: uv run pre-commit run --all-files repos: + - repo: https://github.com/rhysd/actionlint + rev: v1.7.12 + hooks: + - id: actionlint + - repo: https://github.com/gitleaks/gitleaks rev: v8.21.2 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index fae865d8..346a4f03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,20 @@ All notable changes to TrialMatchAI are documented here. The format follows ## [Unreleased] +### Added +- A production roadmap covering all 38 audit findings, clinical workflow, retrieval, + evaluation, bounded agents, CLI automation, and deployment qualification. +- `trialmatchai demo` runs synthetic FHIR import, real CPU indexing, retrieval, + report generation, and resume; `--version` exposes the package version. +- `trialmatchai artifacts manifest/verify` manages portable SHA-256 manifests. + Bootstrap supports a trusted manifest and a strict mode requiring all digests. +- CI and releases share required verification, including an isolated installed-wheel + CLI run. Releases verify the tested artifacts and attach provenance before OIDC + publication. Repository protections still require administrator configuration. + ### Security +- Explicit unmatched FHIR subject references no longer attach records to the only + patient in a bundle. Strict import rejects them; lenient import quarantines them. - **Cleared 13 Dependabot advisories (9 of them high) in leaf dependencies**, without disturbing the pinned inference stack: `pillow` 12.2.0 → 12.3.0 (decompression-bomb and heap out-of-bounds issues in image decoding), `aiohttp` 3.14.1 → 3.14.3 (request smuggling and a @@ -24,6 +37,8 @@ All notable changes to TrialMatchAI are documented here. The format follows so they are a deliberate re-baselining exercise rather than a patch. ### Fixed +- Embedder catalog resources are included in built wheels; Make targets use the + unified CLI. Disabled model stages no longer trigger matching Hub access checks. - **`pip` upgraded to 26.2.1** so CI's dependency audit passes (PYSEC-2026-3721). `pip` is dev-only here, arriving transitively via `pip-api` ← `pip-audit`, and is never shipped in the wheel. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..2532ad92 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include pytest.ini +include scripts/installed_smoke.py diff --git a/Makefile b/Makefile index 8ffb301a..b0581051 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,8 @@ -.PHONY: audit bootstrap build clean healthcheck index lint lock release-check sync sync-model test update-registry +.PHONY: audit bootstrap build clean demo e2e healthcheck index lint lock release-check sync sync-model test update-registry -# Advisories with no fixed version, published against the hard-pinned inference stack -# (vllm and its transitive diskcache). Kept in sync with the same list in -# .github/workflows/ci.yml -- if you change one, change the other. Revisit and drop these -# once vllm / diskcache ship patched releases. +# Legacy advisory exceptions for the pinned inference stack. CI calls this target +# as the single source. These do not imply that current advisories lack fixes; +# audit and requalify the optional stack separately (roadmap P09/P14). PIP_AUDIT_IGNORES := --ignore-vuln CVE-2025-3000 --ignore-vuln CVE-2025-69872 --ignore-vuln CVE-2026-53923 --ignore-vuln CVE-2026-54236 --ignore-vuln CVE-2026-12491 --ignore-vuln CVE-2026-54235 --ignore-vuln CVE-2026-54233 sync: @@ -16,36 +15,46 @@ lock: uv lock --check lint: - uv run ruff check . + uv run --frozen ruff check . test: - uv run pytest + uv run --frozen pytest audit: - uv run pip-audit --progress-spinner off $(PIP_AUDIT_IGNORES) + uv run --frozen pip-audit --progress-spinner off $(PIP_AUDIT_IGNORES) build: uv build healthcheck: - uv run trialmatchai-healthcheck --registry + uv run --frozen trialmatchai healthcheck --registry bootstrap: - uv run trialmatchai-bootstrap-data + uv run --frozen trialmatchai bootstrap-data update-registry: - uv run trialmatchai-update-registry + uv run --frozen trialmatchai update-registry index: - uv run trialmatchai-index --prepare + uv run --frozen trialmatchai index --prepare + +demo: + uv run --frozen trialmatchai demo + +e2e: + uv run --frozen pytest -m e2e release-check: + uv run --frozen pre-commit run actionlint --all-files uv lock --check - uv run ruff check . - uv run pytest + uv run --frozen ruff check . + uv run --frozen pytest uv build - uv run pre-commit run gitleaks --all-files - uv run pip-audit --progress-spinner off $(PIP_AUDIT_IGNORES) + uv run --frozen trialmatchai artifacts manifest dist + uv run --frozen trialmatchai artifacts verify dist/SHA256SUMS --require-exact --json + uv run --no-project --isolated --with "$$(realpath dist/trialmatchai-*.whl)" python -I scripts/installed_smoke.py --expect-installed --workspace "$$(mktemp -d /tmp/trialmatchai-wheel-smoke.XXXXXX)" + uv run --frozen pre-commit run gitleaks --all-files + uv run --frozen pip-audit --progress-spinner off $(PIP_AUDIT_IGNORES) clean: rm -rf build dist src/*.egg-info .pytest_cache .ruff_cache diff --git a/README.md b/README.md index c76f17e1..658deb01 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,23 @@ below produces. ## Quickstart +Try the full software path on synthetic data with Python 3.11, without downloading +models or a trial corpus (available from this development branch): + +```bash +uv sync --frozen +uv run --frozen trialmatchai demo --workdir /tmp/trialmatchai-demo +uv run --frozen trialmatchai demo --workdir /tmp/trialmatchai-demo --resume +``` + +The command creates a real CPU search index, imports a synthetic FHIR patient, +ranks trials, and prints the HTML report path. This demonstrates retrieval and +reporting; it does not validate model accuracy or clinical eligibility. Choose an +empty directory for a new run. See the [delivery runbook](docs/release.md) for +checksums, installed-wheel checks, and release gates, and the +[production roadmap](docs/production-roadmap.md) for remaining qualification work. + + ### 1. Configure the runtime ```bash diff --git a/docs/audit_2026_09_12_inventory.json b/docs/audit_2026_09_12_inventory.json new file mode 100644 index 00000000..3a2b2748 --- /dev/null +++ b/docs/audit_2026_09_12_inventory.json @@ -0,0 +1,995 @@ +{ + "revision": "508db336ee79fdf6cc076fc3da9731fe9cba6751", + "files": [ + { + "path": ".github/workflows/ci.yml", + "lines": 132, + "sha256": "012dc43bb5d2b7de6ad33ac6a3af5bd3f5c4ef6f3289603a9edcd59c71d1f15b" + }, + { + "path": ".github/workflows/docs.yml", + "lines": 53, + "sha256": "e7f8290c520dcee0798291ac2ffcb7d5edc1fd2a8944b611c99dabd0b01193f9" + }, + { + "path": ".github/workflows/release.yml", + "lines": 44, + "sha256": "7d0a6d7b7dabb3bbb2b0aa2afce4360e58a8aa9674816cedca3f3e5843fbb855" + }, + { + "path": ".pre-commit-config.yaml", + "lines": 25, + "sha256": "db74ede544cf5aea5c40a94828b2818442523d24226d9da0d7ee5c7f4bcc31b8" + }, + { + "path": "CHANGELOG.md", + "lines": 425, + "sha256": "6dec9915dae2c4a409dab1d0f9434dccfa0b515e631baeae130198b4cd26a2f7" + }, + { + "path": "Makefile", + "lines": 52, + "sha256": "6a3c1c10cc437b993d8449b1b780bd2ac215972b13e26386e34e74eb23f15237" + }, + { + "path": "README.md", + "lines": 470, + "sha256": "d0d2fbf19f93c14e5d16996ec6df481b0f229db2283b870a04e48d3757c8882e" + }, + { + "path": "SECURITY.md", + "lines": 36, + "sha256": "215b5ad71e3eb378e71a4ac50c7d56a891c4394429a0ce2d63dfa2504b61bdd5" + }, + { + "path": "benchmarks/embedders/bge-m3-vw0.6.json", + "lines": 55, + "sha256": "cd409d59c6f6aafaac6bba7d78aeb83409d0660103abe46067ca874ca71cb459" + }, + { + "path": "benchmarks/embedders/medcpt-vw0.6.json", + "lines": 55, + "sha256": "e5b1a93aedfe875a96f16dc6be7a2514fecb2e56a4e11308a160bd160d209602" + }, + { + "path": "benchmarks/embedders/pubmedbert-neuml-vw0.6.json", + "lines": 55, + "sha256": "6b01d9a00782f1d5e825f2ae7eca7ae896148bd76e6e433de553e223ff23d455" + }, + { + "path": "benchmarks/embedders/qwen3-0.6b-vw0.6.json", + "lines": 55, + "sha256": "6f90d8acd8af76eb59ec8399f8886b6e75d44b8753707c4310429fa72d267c75" + }, + { + "path": "benchmarks/embedders/registry.json", + "lines": 59, + "sha256": "14252644ff93fbe28df3f7312db9113e04a074bde870a18a26eec90e1eefdfc9" + }, + { + "path": "docs/api.md", + "lines": 64, + "sha256": "19d66b8a453bac8ee321639e27b8e7fb506ea706a13c4c93cbcac58453a286da" + }, + { + "path": "docs/architecture.md", + "lines": 37, + "sha256": "897e9b5691d88a503b23b1775f77e2bf658133c8e81cc4267d45aa80ed392c9b" + }, + { + "path": "docs/finetuning.md", + "lines": 165, + "sha256": "88e9e6f895f7503a87c6d89104fd32e049e3d81d901a72d85a1031f8d8446ebb" + }, + { + "path": "docs/index.md", + "lines": 64, + "sha256": "efe6729d7b4807e1c3d3c4f9d703c69cb8254dce45dc7be8a74fb349ad79540c" + }, + { + "path": "docs/interoperability.md", + "lines": 105, + "sha256": "f19977e146b558deb1ed6215e82890afe71a11389f57bc8a0af2e5be6085119a" + }, + { + "path": "docs/pipeline.md", + "lines": 101, + "sha256": "363d55cb693ffa647a02b5f3bf5b54f3b532a85559eaf0f3b3cff84bf324e00b" + }, + { + "path": "docs/registry-updater.md", + "lines": 89, + "sha256": "f3840baabf2af68f848e536ad523f71e625800cf5dd856fb71656e8048534fe3" + }, + { + "path": "docs/release.md", + "lines": 47, + "sha256": "70738ae03fd50b43d2733b738168a89ff54c93419c0b6f2b7eea2a2c2556e2e5" + }, + { + "path": "mkdocs.yml", + "lines": 65, + "sha256": "1195264ad9d111e0ebe3c44af1658916e62b4456044ce22b5bfcc29422ea45e7" + }, + { + "path": "pyproject.toml", + "lines": 122, + "sha256": "cc1703a6c9eea0025093823bed8b26e64a25f69f0bbf1a037e4b2d9583aadba7" + }, + { + "path": "pytest.ini", + "lines": 5, + "sha256": "52269999411bd0ce6ca690c4abbae494632a9adb6f2c0e25e65555896ec5dfb6" + }, + { + "path": "scripts/benchmark_embedder.py", + "lines": 248, + "sha256": "c4150db7eccfff6e3a4897bce0b0a1d5ab1ee87f9035b178bd7c062b48ef1053" + }, + { + "path": "scripts/plot_embedder_recall.py", + "lines": 92, + "sha256": "b4afa8692085d2c00ed19968ef8ba4b3c05bc7305b07378e981f8799edd53b1b" + }, + { + "path": "src/trialmatchai/__init__.py", + "lines": 3, + "sha256": "09ee229de422abc4fbde00da2cfcf925f92f8e836d3e4544997376cab809fb27" + }, + { + "path": "src/trialmatchai/__main__.py", + "lines": 4, + "sha256": "e59d2ecba5d8143c73ce8864b458bee783866c480d478784692b2d7c5161bcfe" + }, + { + "path": "src/trialmatchai/cli/__init__.py", + "lines": 1, + "sha256": "75ad4f4c2943325f88064ad2beae890b59674be2bf0404b30afc5584dc4de4bb" + }, + { + "path": "src/trialmatchai/cli/bootstrap_data.py", + "lines": 285, + "sha256": "6a384dbf2f574cb17e500f9a29254049dd76f1b9d3f5d47fd4217cc6660acf4e" + }, + { + "path": "src/trialmatchai/cli/build.py", + "lines": 118, + "sha256": "6eddcf0855d565a4dd2f12a3aab7548bea7031452b1ec8e4ea9dc25c58ace484" + }, + { + "path": "src/trialmatchai/cli/build_concepts.py", + "lines": 263, + "sha256": "766556993e668f33f064a1202f2c0863c8b58743712b1a58b01a199ac833cada" + }, + { + "path": "src/trialmatchai/cli/e2e.py", + "lines": 69, + "sha256": "691247ab3d8a0de36b394967ca53f555048c7d2cef64234b65819e2770bf5b61" + }, + { + "path": "src/trialmatchai/cli/healthcheck.py", + "lines": 106, + "sha256": "a09d3635718af27bf6f683428613fba6f155de0e35787b3e4f7b51797657027a" + }, + { + "path": "src/trialmatchai/cli/import_patient.py", + "lines": 98, + "sha256": "8143a9f16f60c97e8f090294a65865503fe1f43d0abdbe52eb5b46bec0b9656a" + }, + { + "path": "src/trialmatchai/cli/index_data.py", + "lines": 91, + "sha256": "3fb5bad2006dc7743a4ee14c17da4f3cc9a455552888b2bf075548e707212406" + }, + { + "path": "src/trialmatchai/cli/main.py", + "lines": 107, + "sha256": "9ba58169466fa71e93b936e75ca6264216c0355bbd7acedadf694311f0f5bfc3" + }, + { + "path": "src/trialmatchai/cli/pipeline.py", + "lines": 106, + "sha256": "5e52d3397e6ea0e2e52f90c55fc9a0de3ca23d78a10f4e13e0e8dc33eed6efad" + }, + { + "path": "src/trialmatchai/cli/report.py", + "lines": 91, + "sha256": "3ac6c7f09f5d0cb915c991b0e38f9d96b39d08dc90713fc23e78fcb6e468b81f" + }, + { + "path": "src/trialmatchai/cli/run.py", + "lines": 29, + "sha256": "c109c596b566591b092ae895697bbc3441ba7474f47d5f9d01de7e31b3b5892b" + }, + { + "path": "src/trialmatchai/cli/trec.py", + "lines": 75, + "sha256": "b67da51fa0519104ac853e1cff4730fbfd96177ea90e0ea3a29e86ecbf1d7674" + }, + { + "path": "src/trialmatchai/cli/update_registry.py", + "lines": 204, + "sha256": "2f535be85fabf7f324ea0c6ffb344e5a784fde05495ed3b3f0b08a734dc36333" + }, + { + "path": "src/trialmatchai/config/__init__.py", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/trialmatchai/config/catalog/embedders.json", + "lines": 20, + "sha256": "d78cecfd2b2c7ae184494e7dd2c978febf78e59f8b2ab25bf8517c4a578752d8" + }, + { + "path": "src/trialmatchai/config/config.json", + "lines": 145, + "sha256": "364bfb43d97758da9f19b5750701da09e78f0ba080515b318d8ff92a0dbd218f" + }, + { + "path": "src/trialmatchai/config/config_loader.py", + "lines": 272, + "sha256": "41d0b6c59c7890f150dd898cf02e4540e0a74e000780a4a28205f0d71ccd361c" + }, + { + "path": "src/trialmatchai/config/settings.py", + "lines": 535, + "sha256": "9acb9b3732beafd577869064fd340f70e144e70926a07e96378889d804692b83" + }, + { + "path": "src/trialmatchai/constraints/__init__.py", + "lines": 30, + "sha256": "66b5cf608c3585553f346788011598bf5d6ffde1d07035fbe8b1f27f1528681a" + }, + { + "path": "src/trialmatchai/constraints/evaluation.py", + "lines": 466, + "sha256": "e5f7756e18b3015413eba020cf00e49c744d0baf31162fb7e47480d8d9a4caa5" + }, + { + "path": "src/trialmatchai/constraints/extraction.py", + "lines": 500, + "sha256": "d324b4e1aadbb3a2432156619ea6f51ec5aec65d9368dd7674e108a8d7ee8dd6" + }, + { + "path": "src/trialmatchai/constraints/models.py", + "lines": 121, + "sha256": "936cb46943b6909a75463c0d7e132817ca0ebe82a79160af8cc9a0eabd963867" + }, + { + "path": "src/trialmatchai/constraints/reports.py", + "lines": 119, + "sha256": "c7ca01ef82f8cce78b3b5937985425d3050a45fa0d14b1c1a57decd8493eb371" + }, + { + "path": "src/trialmatchai/entities/__init__.py", + "lines": 13, + "sha256": "fb3d3700e7a10055bd1b14d6980e575312c0ece29ae65caac72c1db6f591c647" + }, + { + "path": "src/trialmatchai/entities/annotator.py", + "lines": 121, + "sha256": "e27b6e20c310b0fd381c9d221a86bea5acfc2fbe63b85c0d90bc4b2d1e7c50d9" + }, + { + "path": "src/trialmatchai/entities/builder.py", + "lines": 203, + "sha256": "85abdadbb1fe2ccd2603b425ae1c715397c39aed9723d958ab07c64a52a0a12d" + }, + { + "path": "src/trialmatchai/entities/concept_sources.py", + "lines": 241, + "sha256": "bb5562e5b7c30c5a560bbdd80c2b48c3596e04c5202483a7465c3139ff0173bf" + }, + { + "path": "src/trialmatchai/entities/linker.py", + "lines": 419, + "sha256": "ee23e861a86e2642e3b4b8254c6a78cac3b384afcb50bb448b1eafb8a9d61dfd" + }, + { + "path": "src/trialmatchai/entities/linker_eval.py", + "lines": 173, + "sha256": "e81bc3088730054fd18ec9765e1c071f7bd78d51b568d8a3d56fb533c95c8958" + }, + { + "path": "src/trialmatchai/entities/recognizers.py", + "lines": 375, + "sha256": "d660a5707d4bc9453d67bb2a1d14d61688e029ff066d7e2ca550911e1b2c3176" + }, + { + "path": "src/trialmatchai/entities/resources/variant_patterns.tsv", + "lines": 52, + "sha256": "38fed7e6f386ba205d12b04679360f59ffbb603e21987003e4792215ddbafc3d" + }, + { + "path": "src/trialmatchai/entities/schemas.py", + "lines": 111, + "sha256": "9a329ab6c409bdb5c9d713dd5a9fceca8ac5057e25a568818ae59c650bede27f" + }, + { + "path": "src/trialmatchai/entities/types.py", + "lines": 135, + "sha256": "9b5210af513a28e52b9a3b0b61e48e4b4c1f2e9377d3ff9783736b07de633f43" + }, + { + "path": "src/trialmatchai/entity_schemas/trialmatchai.yaml", + "lines": 134, + "sha256": "c3699ba0780071ba00878e29de5d35330d69dbbae011e4b3645315705596c814" + }, + { + "path": "src/trialmatchai/finetuning/__init__.py", + "lines": 16, + "sha256": "ed98b6b09b68e03c0bb965b5e49e6bb1f970215f21c268faea85a3f7687b3589" + }, + { + "path": "src/trialmatchai/finetuning/_sft.py", + "lines": 230, + "sha256": "ef3c2836d11454812b34d19b04d7422edd04b651307aca2607b02c907bfd53a4" + }, + { + "path": "src/trialmatchai/finetuning/cli.py", + "lines": 208, + "sha256": "709a3e32f17d5a338fa2cefb5d1c8967482dbcd9954740adfe4333b0e26d7883" + }, + { + "path": "src/trialmatchai/finetuning/config.py", + "lines": 94, + "sha256": "61b0d6ded19ace0bd632e3af46e88c450211cb05129f25eed9d8b965e847d74a" + }, + { + "path": "src/trialmatchai/finetuning/cot.py", + "lines": 25, + "sha256": "0350b126d29ef691da99a08fe2b80d11d38ca5dbbd3defc9aab4a92277f55dec" + }, + { + "path": "src/trialmatchai/finetuning/data.py", + "lines": 177, + "sha256": "197cb9f2d64935b56ebb82dc4a2d3be16337dd85efc14d5454a6cb65dbfeee33" + }, + { + "path": "src/trialmatchai/finetuning/merge.py", + "lines": 45, + "sha256": "19bf9d3cbc4ac7b0d83e03d01cd180fa45787cec4f8dbe1d6e2d83cc3db05ab4" + }, + { + "path": "src/trialmatchai/finetuning/ner.py", + "lines": 196, + "sha256": "a432944e059bf93cb4a842dafb9a8df34999d52b2ab0a3751c60cce6a7aa7c13" + }, + { + "path": "src/trialmatchai/finetuning/reranker.py", + "lines": 30, + "sha256": "759a03f89fb9b3561c24bfc3849708882fb34de057b12e91a754490f94fb5ea8" + }, + { + "path": "src/trialmatchai/interop/__init__.py", + "lines": 25, + "sha256": "74de28c7ca584c1aa89abccabadf0d247aaa5de2efc30a546423eb7bc7c316b8" + }, + { + "path": "src/trialmatchai/interop/detect.py", + "lines": 57, + "sha256": "ed93ee08b0f6b8900202556d52244731a2a46303a05de29d963faa5b7bc2ad6a" + }, + { + "path": "src/trialmatchai/interop/exporters/__init__.py", + "lines": 19, + "sha256": "c3d0e2795b967f143fa21a45fcfb13ee5ca85b5fef48ca503998c9cc36285c6d" + }, + { + "path": "src/trialmatchai/interop/exporters/fhir.py", + "lines": 83, + "sha256": "80b643f11e33164fbd3ecdc0df59c33c9bddef35e87b7d5ac574c2f2c25cc219" + }, + { + "path": "src/trialmatchai/interop/exporters/html_report.py", + "lines": 310, + "sha256": "1109493a3969bca785b59905819f506678bd825a835111f560759aeeea9d359d" + }, + { + "path": "src/trialmatchai/interop/exporters/matching_summary.py", + "lines": 27, + "sha256": "7c794505b5cbffeff0872939eb72f945a755945364c93466470125e0aeffcadf" + }, + { + "path": "src/trialmatchai/interop/exporters/phenopacket.py", + "lines": 87, + "sha256": "078e5c3889d4fdba6f68a6ebdd301f5332515a5af6c407813d0d654893139316" + }, + { + "path": "src/trialmatchai/interop/exporters/templates/report.html", + "lines": 553, + "sha256": "d8534ddd7d1499f4b0593d278f2b924196f418cc41737b2c616523818a391e4d" + }, + { + "path": "src/trialmatchai/interop/importers/__init__.py", + "lines": 35, + "sha256": "499a73f9dd917f8239d1819b9eb7784cf5be0274653a574b5a6eeccad727eb87" + }, + { + "path": "src/trialmatchai/interop/importers/fhir.py", + "lines": 791, + "sha256": "78fd9658c4571f1c904d3f1865526b9940ed65cdcc59f77e88c48cfeb75c1d60" + }, + { + "path": "src/trialmatchai/interop/importers/omop.py", + "lines": 485, + "sha256": "36d1fa9d8bad58a749db5fcf903976d157c693dd8415cc6fee47d962795aac01" + }, + { + "path": "src/trialmatchai/interop/importers/phenopacket.py", + "lines": 317, + "sha256": "fe175ec41c222cba8bbeb9c4d72baf68be6d3d52b92624c93cfda66d60359cde" + }, + { + "path": "src/trialmatchai/interop/importers/text.py", + "lines": 118, + "sha256": "3a6db05082ece0ab6ae3c09776c86f9362db84efb4ffd2ae037b87c8220c7c58" + }, + { + "path": "src/trialmatchai/interop/models.py", + "lines": 148, + "sha256": "dcc1816bc20965c4fbd6dcafc6cbcf5eb7b46ed369d5fe1c359c335011595e86" + }, + { + "path": "src/trialmatchai/interop/narrative.py", + "lines": 111, + "sha256": "838132954d3c828113ebbf6f27523da7a476973e3cd3b42c26d5e4965c82adf6" + }, + { + "path": "src/trialmatchai/interop/utils.py", + "lines": 255, + "sha256": "b576809f403b78f42e786e68a26d1d5204bb45260532b53440bd80af89a8c081" + }, + { + "path": "src/trialmatchai/linking.py", + "lines": 287, + "sha256": "91fa14cff5613b98e85a17705ddabe3737310793451dd955aca38e01fe2b875b" + }, + { + "path": "src/trialmatchai/main.py", + "lines": 797, + "sha256": "a9757840ed7ab1c33490f6bfbf47ffa3fedff759babc41ef3fcfecd38db04a80" + }, + { + "path": "src/trialmatchai/matching/__init__.py", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/trialmatchai/matching/eligibility_base.py", + "lines": 276, + "sha256": "cba5450c58cfbaf0e184084966f14d3595b24425196d09da0b9abe0343b92f4d" + }, + { + "path": "src/trialmatchai/matching/eligibility_reasoning_transformers.py", + "lines": 142, + "sha256": "c1df37cc2bca68289cbdf6955cbfbf501a21830f0639968b2b674eac02d7e33f" + }, + { + "path": "src/trialmatchai/matching/eligibility_reasoning_vllm.py", + "lines": 417, + "sha256": "119165d91442e25e5554748c46369c65da73a2af2c396fb7267a1d1d219d49b5" + }, + { + "path": "src/trialmatchai/matching/query_expansion.py", + "lines": 259, + "sha256": "af0fc7eeb0a72d9b57a6851265b3aded6df31958f37013d477aae8d88caab176" + }, + { + "path": "src/trialmatchai/matching/retrieval/__init__.py", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/trialmatchai/matching/retrieval/criteria_retrieval.py", + "lines": 330, + "sha256": "8a5b1d55fb4f2532fff6c86c92e8f815b57b8050b471bc1d87fe980f859c2722" + }, + { + "path": "src/trialmatchai/matching/retrieval/first_level_planner.py", + "lines": 417, + "sha256": "42d6432df0bbf81cb93cb1d07451bffe84524bd25751acf0b864d0bb10ac1cd5" + }, + { + "path": "src/trialmatchai/matching/retrieval/location.py", + "lines": 58, + "sha256": "dda0d8eef4eefbcf297b319a4f51fce8dc36395f125da82499b3e3f94ac5230d" + }, + { + "path": "src/trialmatchai/matching/retrieval/synonyms.py", + "lines": 32, + "sha256": "4404adba03afdabbc786c642ef012da879b706805f9ec14ad6a5b63077173057" + }, + { + "path": "src/trialmatchai/matching/retrieval/trial_retrieval.py", + "lines": 323, + "sha256": "95a2c789d23c7d1c8ae8797aeffd59d34368d8352c80c5f08571f64024278341" + }, + { + "path": "src/trialmatchai/matching/trial_ranker.py", + "lines": 201, + "sha256": "0975e8bcdebfaf7f4a7435bdf1bc9139a2f827a6916e4d073b2243d5010d429d" + }, + { + "path": "src/trialmatchai/models/__init__.py", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/trialmatchai/models/embedding/__init__.py", + "lines": 15, + "sha256": "f93c999838c1dfb4ad5d2ea3a287550ad19d163cf2b071b411e70679c0d9f08b" + }, + { + "path": "src/trialmatchai/models/embedding/text_embedder.py", + "lines": 415, + "sha256": "0f10502b3a170175b6ea7c2e267ebf14d21af38260e43a42ab3ba192a19480af" + }, + { + "path": "src/trialmatchai/models/llm/__init__.py", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/trialmatchai/models/llm/llm_reranker.py", + "lines": 140, + "sha256": "8516228ee9119583879c78f9345333e15ba06a3b06a25655e08b1456a2673cf9" + }, + { + "path": "src/trialmatchai/models/llm/transformers_reranker.py", + "lines": 103, + "sha256": "a576e0a24c433cb99806b013d42eb942ad51e19b7b5a352147343a6bbf0e42ab" + }, + { + "path": "src/trialmatchai/models/llm/vllm_loader.py", + "lines": 326, + "sha256": "a6940d1588b5ac2aa9abd86b081b4b11c77b9da12e69783bc49eebdb8057bccc" + }, + { + "path": "src/trialmatchai/orchestration.py", + "lines": 873, + "sha256": "204d8a0854176d0f4faea5ba713174981b338788ae1e9f5310c83b899740db9c" + }, + { + "path": "src/trialmatchai/pipeline.py", + "lines": 222, + "sha256": "db7790db5d3cfa3d5cbcbb0e416b989c2303f010e591ec197410769acf53a937" + }, + { + "path": "src/trialmatchai/plugins.py", + "lines": 42, + "sha256": "63ae8985aa89bcccefdb0708d293828872ae99823d706207cd0866fe05058651" + }, + { + "path": "src/trialmatchai/registry/__init__.py", + "lines": 13, + "sha256": "44377b620bee032f71f20b39b076104804569c27483f7be5dd9f382298e88370" + }, + { + "path": "src/trialmatchai/registry/clinicaltrials_gov.py", + "lines": 235, + "sha256": "c43fa1a928d4c325683d7d6e51a6ab941dcebd99db9d9426fdb50b20bf6bf03f" + }, + { + "path": "src/trialmatchai/registry/criteria_chunking.py", + "lines": 260, + "sha256": "85ef4ad97711c2149a040fdcf771c736e1d7920fa1ddb54f6109abc80a20da7a" + }, + { + "path": "src/trialmatchai/registry/defaults.py", + "lines": 20, + "sha256": "d6be9e3ea6002479f32c001bb1b68ecc9c80e478c9c779cce62489579e2532be" + }, + { + "path": "src/trialmatchai/registry/manifest.py", + "lines": 55, + "sha256": "44231404398b2191cbad48603ece5a015681af1ad852b098251f521e54e00a7d" + }, + { + "path": "src/trialmatchai/registry/normalization.py", + "lines": 140, + "sha256": "1e7341bed0fbd379cb0e85202209772b3532f6692fa32921415fc63f4ccfbd51" + }, + { + "path": "src/trialmatchai/registry/preparation.py", + "lines": 310, + "sha256": "29c2b94ec5745eb240905a234c0b7e03f024244caabd3de7cc27a213889dbf69" + }, + { + "path": "src/trialmatchai/registry/updater.py", + "lines": 255, + "sha256": "48db9469c62a19f7da2977429b87b4b008f50dadbf7adf1b586644a2ba312d47" + }, + { + "path": "src/trialmatchai/schemas/__init__.py", + "lines": 3, + "sha256": "7657090c48403b04166876fa2fb69df3fa1b605ca908cedf45a87315a872f688" + }, + { + "path": "src/trialmatchai/schemas/phenopacket.py", + "lines": 22, + "sha256": "e3ceca20db8a285f7fac582c164a96ef45480bcd42388a6752b2cd12c53a1348" + }, + { + "path": "src/trialmatchai/search/__init__.py", + "lines": 33, + "sha256": "a992d23c4b408f68a132dab54f3199f2f8fe1bbb6afef21598083f5ba7f9593f" + }, + { + "path": "src/trialmatchai/search/lancedb_backend.py", + "lines": 1079, + "sha256": "cfbd02036a77e3cf85720005594dc55835f3fa190b86fc7197956127d5b7ca3d" + }, + { + "path": "src/trialmatchai/services/__init__.py", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/trialmatchai/services/preflight.py", + "lines": 292, + "sha256": "21a0ea1a0c93cfeafeabb3b58469f0a7e9a518c71e8a177a0bf569b51b63960b" + }, + { + "path": "src/trialmatchai/trec/__init__.py", + "lines": 14, + "sha256": "ec0adbf7118690f9bca834d903f4af22396f40a0269c4a1b160f891e3a0f05bc" + }, + { + "path": "src/trialmatchai/trec/backfill.py", + "lines": 137, + "sha256": "ea7e9d8055713be7e122fdbd5262446c8a2fd29c4740a3f3ef3181b5f661d00f" + }, + { + "path": "src/trialmatchai/trec/corpus.py", + "lines": 75, + "sha256": "1c0931af81e1a44b2def42b2ad223a13ca20f8c88cdad7d888276fe1b601df66" + }, + { + "path": "src/trialmatchai/trec/metrics.py", + "lines": 123, + "sha256": "62ec1692fd0906cbcac04051e083c81523012bc734794890370a2d0845b74ef5" + }, + { + "path": "src/trialmatchai/trec/qrels.py", + "lines": 243, + "sha256": "7c84ae4d7ee6ac4da793d080615dd2eed4d2c1e4c8325fa5eb7c05dbdaba86f0" + }, + { + "path": "src/trialmatchai/trec/runner.py", + "lines": 207, + "sha256": "86bc8d0cc97dc714de701649c7c84ff2f6123d998da0cf6c79aa937765929a2a" + }, + { + "path": "src/trialmatchai/trec/topics.py", + "lines": 326, + "sha256": "7bcc4647a83af846c8e4497fda9576a185ad2b5e973d66c5eef467a6a92341dd" + }, + { + "path": "src/trialmatchai/utils/__init__.py", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "src/trialmatchai/utils/file_utils.py", + "lines": 85, + "sha256": "18cc7f18dfb347685be79b303beefdcf8cced429390bb45f4e54b2c38c83483c" + }, + { + "path": "src/trialmatchai/utils/json_utils.py", + "lines": 35, + "sha256": "858460e470dc18c163fd8eff0e0a9a44bffd7d6e30b23fd2d0f822ede3a6fe36" + }, + { + "path": "src/trialmatchai/utils/logging_config.py", + "lines": 74, + "sha256": "976cda1d5503bfbf7ea970e11c8d882c0e96ae022b86cc8b3eb899ac6d5f286d" + }, + { + "path": "src/trialmatchai/utils/pipeline_state.py", + "lines": 91, + "sha256": "c5f6e72f80e409af25c0cff90d367cd9a66b61e7e657ac4b01e6ee3430bc3a22" + }, + { + "path": "src/trialmatchai/utils/temporal_utils.py", + "lines": 43, + "sha256": "22995a0e046220aa6002b17a373dd503fda4ea1d440379fe9cbaf2850f04f305" + }, + { + "path": "src/trialmatchai/utils/text.py", + "lines": 19, + "sha256": "44a189f0e7c8ba58de3dba4539d5a00e08338716d33d75fc3527ffd0d353095c" + }, + { + "path": "src/trialmatchai/utils/timing.py", + "lines": 15, + "sha256": "aee03d8ca8e919b1abe7593379ced30658dd43532bd7fb1a5181aeda26aad166" + }, + { + "path": "tests/conftest.py", + "lines": 5, + "sha256": "b596eee5c0ed59e9b0ce755f8fcc5d536f49c4139cb0fc57917fa9f91ac891a9" + }, + { + "path": "tests/test_audit_fixes.py", + "lines": 427, + "sha256": "ee6a3192764998cc92d312907519bcae89924a31f52c94604b5e214b6cc10b2e" + }, + { + "path": "tests/test_audit_fixes_batch8.py", + "lines": 630, + "sha256": "0c018689ad210681fae0795a536b2c8a4a1c43ea5f53c8721364454a93a034cb" + }, + { + "path": "tests/test_audit_fixes_reaudit.py", + "lines": 814, + "sha256": "7c5e1f70d5179075feac25ac26b4b2c6c00def73791e987e802fd26b42b4e24f" + }, + { + "path": "tests/test_bm25_fusion.py", + "lines": 154, + "sha256": "b278d6c6b0b33b3eafb2f4ca57baae6c014eba2849ee329bc90cb29c99eea44c" + }, + { + "path": "tests/test_bootstrap_data.py", + "lines": 126, + "sha256": "7941d648aeabab17d11fd10ed0920aa464dc95647abb4a8c9b2dd6c18ea7d83e" + }, + { + "path": "tests/test_concept_sources.py", + "lines": 51, + "sha256": "c6c0fde09d533dfc3ffb6748456826c048f4cd168fed3c166ae21e43152c9b6a" + }, + { + "path": "tests/test_concept_store.py", + "lines": 83, + "sha256": "99d70bad6eb2acb5b32fce16f55178d12bd39dc753bca415acbe98b190ed7dbb" + }, + { + "path": "tests/test_config_pytest.py", + "lines": 86, + "sha256": "8791616f9dab2a10cbf9fb521a0c3ca0349589ec93de94111fa9798987087aca" + }, + { + "path": "tests/test_constraint_exclusion_aggregation.py", + "lines": 46, + "sha256": "4631cbbac4c0fcba2a86b0fa7e8fe1033b8ede061d4bd20c900a91410e8affe8" + }, + { + "path": "tests/test_constraints.py", + "lines": 317, + "sha256": "cf24aaacdec75c422fd6d2e9ba4b319ceb2ce889aa966d7f3e716e87fefb2422" + }, + { + "path": "tests/test_criteria_chunking.py", + "lines": 156, + "sha256": "00ab22dbb96aed86fd5702847c7f023e06f2dbb64be1257ebfb38050eadfe2d8" + }, + { + "path": "tests/test_deployment_readiness.py", + "lines": 112, + "sha256": "98aa83a653d362b6eb62542c06bfabbfcfd7680e88a20424c81384d465ba1f29" + }, + { + "path": "tests/test_eligibility_vllm.py", + "lines": 58, + "sha256": "14ac26c435e6324637f1b6f1be0d7311763aeecaf8a79133a0c05fec2d8d0222" + }, + { + "path": "tests/test_embedder_contract.py", + "lines": 37, + "sha256": "5c1dba00320d7ea684a390ca59b2ed76176f074377d0a44abd92899461fe2100" + }, + { + "path": "tests/test_embedding.py", + "lines": 22, + "sha256": "47aa544c206cc65e74e195ec4c56a4a75b3df3433f2a5da8be572a1c986934bc" + }, + { + "path": "tests/test_entities.py", + "lines": 285, + "sha256": "34f924ac65f14777d140f1ee3d5434651e0c83c4afac9464cb0be7967d92cf27" + }, + { + "path": "tests/test_fhir_robustness.py", + "lines": 229, + "sha256": "9fde0cd58bb2bd21f47a9929972ce36a9e5c7e1f8a59f8a04c8220d29fa71dde" + }, + { + "path": "tests/test_file_utils_pytest.py", + "lines": 15, + "sha256": "28aab218c22b00791555cd3c81bdf8a32ef815482c43eab5d00f6491189fb55d" + }, + { + "path": "tests/test_finetuning.py", + "lines": 220, + "sha256": "48057eea0f0feb200acceb2226ebd7662cc4ec6de9485834e4941eea2f3c001d" + }, + { + "path": "tests/test_first_level_planner.py", + "lines": 367, + "sha256": "01a8c8fc0e77631f14c5541aa8308319967998702450a2b903348836ea3a95f6" + }, + { + "path": "tests/test_first_level_search_pytest.py", + "lines": 42, + "sha256": "3de080e1257e106bcfbef0586887af32a9b99d9530fb8931d8ce9219006e220f" + }, + { + "path": "tests/test_html_report.py", + "lines": 268, + "sha256": "5d3357948e66f010077d99dadf0c65d9dbc60b2d43de574cf1a8cdda73b23969" + }, + { + "path": "tests/test_import_patient_cli.py", + "lines": 37, + "sha256": "bc7e9a65c1a1bce0609b98c7481563db6545e3456ec0949d5447d8ba591a36b9" + }, + { + "path": "tests/test_index_provenance.py", + "lines": 85, + "sha256": "1ec6a0515b99e790765148adced3c8c4aea79f5d8a4b0ba93278f2979b4bd424" + }, + { + "path": "tests/test_interop_fixtures.py", + "lines": 16, + "sha256": "34463d2dec452836a6133697403a3bfcfc949a2d1166f6b00580ae1b020addee" + }, + { + "path": "tests/test_lancedb_search_backend.py", + "lines": 245, + "sha256": "d71d594b1c6b30c92f2a98456ac1ce02630c8100489322ace48c35d1632bdbcb" + }, + { + "path": "tests/test_linking.py", + "lines": 293, + "sha256": "6ac7c2312fd6d486d25a145b5e3cc1189976a80227370dea4751ed9abd8ebe1d" + }, + { + "path": "tests/test_location_filter.py", + "lines": 165, + "sha256": "93fa867e346b00c5490fcbaed04cd8ca427e497249ae39da281fa175b45d1f62" + }, + { + "path": "tests/test_logging.py", + "lines": 34, + "sha256": "b9e5d6fdfe493c31e98bdcde12a0f54ba2e9de5f6b9c649dbd79551ad84e3d67" + }, + { + "path": "tests/test_logging_pytest.py", + "lines": 27, + "sha256": "09bcbcbc993795054a80ef1ec09fccd137706531583b7a9ed8d42cf962fe1d5f" + }, + { + "path": "tests/test_medium_bug_fixes.py", + "lines": 72, + "sha256": "0d1169436bbe6bcab79e82789227c81d6025edea575374c3cdb3920c31c4fbac" + }, + { + "path": "tests/test_metrics.py", + "lines": 70, + "sha256": "0c668d9e14fef5f2c3ca432e941e8ea939594014f53f42fc34e93019bd3efa4c" + }, + { + "path": "tests/test_model_catalog.py", + "lines": 57, + "sha256": "3ba1b46337abfeb783470781337c96b0d6ad1bf753e2e86d16fcdf985ab191ef" + }, + { + "path": "tests/test_model_helpers.py", + "lines": 45, + "sha256": "bb9825a86999ef9fa3323539b8679c6c2ac7c329b2862a3313fbba99adf08c74" + }, + { + "path": "tests/test_package_imports.py", + "lines": 52, + "sha256": "fce485952c5643829c530121a7cb049a78d2c6dfc1b557d94deb9870763a54cd" + }, + { + "path": "tests/test_patient_interop.py", + "lines": 435, + "sha256": "f53e3f3958cbe2c1cc141b7f5421f9d1091c7389ea60ad11feebfe532eaf43fd" + }, + { + "path": "tests/test_patient_runtime_loading.py", + "lines": 108, + "sha256": "9bdc14ba7a9edca34b67fdf50146795a3d953c1b0a658456c9a830dd07708a3b" + }, + { + "path": "tests/test_pipeline.py", + "lines": 128, + "sha256": "289a6b37358fcd3f235cdbfa8fddc837fb5e8e5b930895390f0e07fee745c21d" + }, + { + "path": "tests/test_pipeline_wrappers.py", + "lines": 48, + "sha256": "c04c4de2033e49b54a1d829361b018e72bd8582b2c6caa9b2a2002ed798d2427" + }, + { + "path": "tests/test_plugins.py", + "lines": 39, + "sha256": "8045ee14cc425e738e0e0e70068ff327cfe9d6a64b2fb7749d031f2ed67937a1" + }, + { + "path": "tests/test_preflight_and_indexer.py", + "lines": 222, + "sha256": "13a36fb742970695696e88e1a6c3e8c8698cede4bf1b0a040d28ef8a5850e8eb" + }, + { + "path": "tests/test_qrels_eval.py", + "lines": 95, + "sha256": "82054f9a5e4b938af33e93aa6bc18038576e9923d869b481c6b55542b4b7da24" + }, + { + "path": "tests/test_registry_updater.py", + "lines": 347, + "sha256": "6d68fdde9e5dd0b6740ce11e5e20b8505aa0ce08186e706deee65c8be7b7762c" + }, + { + "path": "tests/test_reranker_engine.py", + "lines": 76, + "sha256": "3a91b197ebf8c974cf8e2d606be419d0c9edc1cfc14d869750722f278fdb1592" + }, + { + "path": "tests/test_resume_and_atomicity.py", + "lines": 404, + "sha256": "3c470cca14512b36ff4cbf1fa4c089af0ebcfabf63251a7358a1ec951732b61b" + }, + { + "path": "tests/test_schemas.py", + "lines": 30, + "sha256": "7f9eec208c7e30638000c489eacd24e27a2e8dba4ca266c93d8a3aaa4a83d1fd" + }, + { + "path": "tests/test_search_queries_pytest.py", + "lines": 55, + "sha256": "aca6ac1a57296d505fdff93479e48fd53a2b49d5e9207215a4b2ef388188b5f8" + }, + { + "path": "tests/test_second_level_search_pytest.py", + "lines": 299, + "sha256": "fdf8bc57f40bf53903df70c5f1355ab2d3715860665a9aca5770eddbd7e93339" + }, + { + "path": "tests/test_settings.py", + "lines": 102, + "sha256": "31a924659d966c08f779cc0fd8c7b114d6f53e9fcbba7da15162a78c939fd03f" + }, + { + "path": "tests/test_settings_validators.py", + "lines": 29, + "sha256": "e4d818d32f1c7295b17891315c9023a27ad8f6878339f88f82f8e52c08f5f827" + }, + { + "path": "tests/test_trec_2023.py", + "lines": 183, + "sha256": "4fdcb0f9a28a0ffddbe30e948e21a4316d33913bdac07e3142f06df21136d98e" + }, + { + "path": "tests/test_trec_reproduction.py", + "lines": 46, + "sha256": "ff7bf523f5e53336bc34d3ea17b7d3895f0027e17c77a51e545c9e83028ccc3d" + }, + { + "path": "tests/test_trial_ranker_pytest.py", + "lines": 68, + "sha256": "a41a85602ba2ac4b6759cd6cdd15d70ebe6bafc5105746f7045007cbd666c977" + }, + { + "path": "tests/test_trial_ranker_scoring_contract.py", + "lines": 144, + "sha256": "5ba35fefb101dc59a0969b5dcdc40e9a698a0d0a343140e118ebdb700c3e4e45" + }, + { + "path": "tests/test_trial_ranker_tiebreak.py", + "lines": 77, + "sha256": "d12234f9659e00f18408b35478a0bd52d7c05a4a8595bea29d65314f872594e2" + }, + { + "path": "tests/test_variant_recognizer.py", + "lines": 30, + "sha256": "58cffcca94455f3f9f92bf9e8d572dff20d01f62d759e5c4d8219d6820fa7108" + } + ] +} diff --git a/docs/audit_2026_09_12_observations.json b/docs/audit_2026_09_12_observations.json new file mode 100644 index 00000000..ea19fe5c --- /dev/null +++ b/docs/audit_2026_09_12_observations.json @@ -0,0 +1,132 @@ +{ + "A01_remote_adapter_paths": { + "all_absolute": true, + "any_exists": false, + "normalized": { + "cot_adapter_path": "/nfs/scratch/mabdallah/TrialMatchAI/majdabd33/trialmatchai-phi4-reasoning-lora", + "reranker_adapter_path": "/nfs/scratch/mabdallah/TrialMatchAI/majdabd33/trialmatchai-gemma2-reranker-lora" + }, + "status": "observed" + }, + "A02_exclusion_threshold": { + "raw_score": 1.0, + "retained_trials": [], + "status": "observed", + "weighted_score": 0.25 + }, + "A03_cross_patient_fhir": { + "conditions": [ + "Synthetic disease belonging to B" + ], + "patient": "synthetic-A", + "status": "observed", + "strict": true + }, + "A04_stale_eligibility_worklist": { + "new_batches": 0, + "reused": { + "Final Decision": "Eligible" + }, + "status": "observed" + }, + "A05_incomplete_match_signature": { + "signature_unchanged": true, + "status": "observed" + }, + "A06_invalid_eligibility_accepted": { + "scheduled_for_retry": false, + "status": "observed", + "stored": { + "unrelated": true + } + }, + "A07_failed_inclusion_score": { + "all_exclusions_unknown": 1.0, + "exclusion_violation": -1.0, + "generation_error": -1.0, + "insufficient_information": 0.5, + "mandatory_inclusion_failed": 0.9, + "status": "observed" + }, + "A08_missing_fact_becomes_absence": { + "exclusion_present": { + "evidence": null, + "signal": 0.25, + "status": "matched" + }, + "inclusion_absent": { + "evidence": null, + "signal": 1.0, + "status": "matched" + }, + "status": "observed" + }, + "A09_gene_mention_becomes_mutation": { + "clinical_status": "matched", + "signal": 1.0, + "status": "observed" + }, + "A10_empty_candidate_scope": { + "requested_nct_ids": [], + "returned_nct_ids": [ + "NCT00000001" + ], + "status": "observed" + }, + "A11_vector_dimension_mismatch": { + "cosine": 1.0, + "left": [ + 1.0 + ], + "right": [ + 1.0, + 99.0 + ], + "status": "observed" + }, + "A12_rejected_concept_indexed": { + "indexed_synonyms": "unrelated disease", + "status": "observed", + "text": "synthetic mention" + }, + "A13_missing_topics_omitted": { + "expected_topics": 3, + "mean_ndcg10": 1.0, + "ranked_topics": 1, + "scored_topics": 2, + "status": "observed" + }, + "A14_condensed_ndcg_one_hit": { + "eligible_found": 1, + "eligible_total": 10, + "ndcg10": 1.0, + "ndcg_full10": 0.22009176629808017, + "status": "observed" + }, + "A15_failure_report_state": { + "final_decision": null, + "reasoning_available": true, + "status": "observed" + }, + "A16_fhir_export_required_fields": { + "has_required_medication": false, + "has_required_status": false, + "resource_keys": [ + "code", + "id", + "note", + "resourceType", + "subject" + ], + "status": "observed" + }, + "A17_empty_prepared_criteria_stale": { + "old_criterion_remains": true, + "status": "observed" + }, + "A18_unvalidated_canonical_patient_id": { + "id_accepted": "../synthetic-escape", + "status": "observed", + "target_outside_output_root": true + } +} diff --git a/docs/audit_2026_09_12_reproduce.py b/docs/audit_2026_09_12_reproduce.py new file mode 100644 index 00000000..11589a8e --- /dev/null +++ b/docs/audit_2026_09_12_reproduce.py @@ -0,0 +1,276 @@ +"""Synthetic audit probes for revision 508db33; no models, network, or real patients. + +Run from the repository root: + .venv/bin/python docs/audit_2026_09_12_reproduce.py + +These observations deliberately describe current defects, rather than asserting +desired behavior. They are review evidence, not additions to the regression suite. +All temporary patient and trial artifacts live under a TemporaryDirectory. +""" + +from __future__ import annotations + +import copy +import json +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +RESULTS: dict[str, dict] = {} + + +def probe(name): + def decorate(fn): + try: + RESULTS[name] = {"status": "observed", **fn()} + except Exception as exc: + RESULTS[name] = {"status": "probe_error", "error": repr(exc)} + print(json.dumps({name: RESULTS[name]}, sort_keys=True), flush=True) + return fn + return decorate + + +@probe("A01_remote_adapter_paths") +def adapters(): + from trialmatchai.config.config_loader import load_config + + config = load_config(str(ROOT / "src/trialmatchai/config/config.json")) + paths = {key: config["model"][key] for key in ("cot_adapter_path", "reranker_adapter_path")} + return {"normalized": paths, "all_absolute": all(Path(p).is_absolute() for p in paths.values()), + "any_exists": any(Path(p).exists() for p in paths.values())} + + +@probe("A02_exclusion_threshold") +def exclusions(): + from trialmatchai.matching.retrieval.criteria_retrieval import SecondStageRetriever + + class PerfectReranker: + def rank_pairs(self, pairs): + return [1.0] * len(pairs) + + retriever = SecondStageRetriever(None, PerfectReranker(), None) + rows = [{"query": "synthetic cancer", "_source": { + "criterion": "Excluded if synthetic cancer", "eligibility_type": "exclusion", + "criteria_id": "c1", "nct_id": "NCT00000001"}}] + scored = retriever.rerank_criteria(rows) + return {"raw_score": 1.0, "weighted_score": scored[0]["llm_score"], + "retained_trials": retriever.aggregate_to_trials(scored)} + + +@probe("A03_cross_patient_fhir") +def cross_patient(): + from trialmatchai.interop.importers.fhir import import_fhir + + bundle = {"resourceType": "Bundle", "type": "collection", "entry": [ + {"resource": {"resourceType": "Patient", "id": "synthetic-A"}}, + {"resource": {"resourceType": "Condition", "id": "condition-B", + "subject": {"reference": "Patient/synthetic-B"}, + "code": {"text": "Synthetic disease belonging to B"}}}]} + with tempfile.TemporaryDirectory(prefix="tmai-audit-") as tmp: + path = Path(tmp) / "bundle.json" + path.write_text(json.dumps(bundle)) + profiles = import_fhir(path, strict=True) + return {"patient": profiles[0].patient_id, "strict": True, + "conditions": [f.label for f in profiles[0].conditions]} + + +@probe("A04_stale_eligibility_worklist") +def stale_eligibility(): + from trialmatchai.matching.eligibility_base import BaseTrialProcessor + + class Recorder(BaseTrialProcessor): + def __init__(self): + self.calls = [] + + def _process_batch(self, batch, output_folder): + self.calls.extend(batch) + + with tempfile.TemporaryDirectory(prefix="tmai-audit-") as tmp: + root = Path(tmp) + trials, outputs = root / "trials", root / "outputs" + trials.mkdir() + outputs.mkdir() + (trials / "NCT00000001.json").write_text(json.dumps({"eligibility_criteria": "NEW criteria"})) + (outputs / "NCT00000001.json").write_text(json.dumps({"Final Decision": "Eligible"})) + processor = Recorder() + processor.process_trials(["NCT00000001"], str(trials), str(outputs), ["CHANGED patient facts"]) + return {"new_batches": len(processor.calls), "reused": json.loads((outputs / "NCT00000001.json").read_text())} + + +@probe("A05_incomplete_match_signature") +def signatures(): + from trialmatchai.orchestration import _match_signature + + base = {"model": {"base_model": "base", "cot_adapter_path": "adapter-A"}, + "search": {"mode": "hybrid", "first_level": {"max_trials": 100}}, + "constraints": {"score_weight": 0.25}} + changed = copy.deepcopy(base) + changed["model"]["cot_adapter_path"] = "adapter-B" + changed["search"]["first_level"]["max_trials"] = 900 + changed["constraints"]["score_weight"] = 0.9 + return {"signature_unchanged": _match_signature(base) == _match_signature(changed)} + + +@probe("A06_invalid_eligibility_accepted") +def invalid_eligibility(): + from trialmatchai.matching.eligibility_base import BaseTrialProcessor, _is_error_output + + with tempfile.TemporaryDirectory(prefix="tmai-audit-") as tmp: + BaseTrialProcessor()._save_outputs("NCT00000001", '{"unrelated": true}', tmp) + path = Path(tmp) / "NCT00000001.json" + return {"stored": json.loads(path.read_text()), "scheduled_for_retry": _is_error_output(str(path))} + + +@probe("A07_failed_inclusion_score") +def inclusion_score(): + from trialmatchai.matching.trial_ranker import score_trial + + def score(inclusions, exclusions=()): + return score_trial({"Inclusion_Criteria_Evaluation": [{"Classification": c} for c in inclusions], + "Exclusion_Criteria_Evaluation": [{"Classification": c} for c in exclusions]}) + return {"mandatory_inclusion_failed": score(["Met"] * 9 + ["Not Met"]), + "insufficient_information": score(["Unclear"]), + "all_exclusions_unknown": score(["Met"], ["Unclear"] * 10), + "exclusion_violation": score(["Met"], ["Violated"]), + "generation_error": score_trial({"error": "processing_failed"})} + + +@probe("A08_missing_fact_becomes_absence") +def absence(): + from trialmatchai.constraints.evaluation import evaluate_constraint_set + from trialmatchai.constraints.models import Constraint, ConstraintSet, PatientConstraintContext + + context = PatientConstraintContext(patient_id="synthetic") + out = {} + for polarity, comparator in [("inclusion", "absent"), ("exclusion", "present")]: + criteria = ConstraintSet(nct_id="NCT00000001", criteria_id="c1", polarity=polarity, + source_text="Synthetic criterion", constraints=[Constraint( + kind="condition", label="diabetes", comparator=comparator)]) + result = evaluate_constraint_set(criteria, context) + out[f"{polarity}_{comparator}"] = {"status": result.evaluations[0].status, + "signal": result.constraint_signal, + "evidence": result.evaluations[0].patient_evidence} + return out + + +@probe("A09_gene_mention_becomes_mutation") +def biomarker(): + from trialmatchai.constraints.evaluation import evaluate_constraint_set + from trialmatchai.constraints.models import Constraint, ConstraintSet, PatientConstraintContext, PatientConstraintFact + + criteria = ConstraintSet(nct_id="NCT00000001", criteria_id="c1", polarity="inclusion", + source_text="EGFR mutation required", constraints=[Constraint( + kind="biomarker", label="EGFR", comparator="mutated")]) + context = PatientConstraintContext(patient_id="synthetic", facts=[PatientConstraintFact( + kind="biomarker", label="EGFR", evidence_text="EGFR testing ordered; result pending")]) + result = evaluate_constraint_set(criteria, context) + return {"clinical_status": result.evaluations[0].status, "signal": result.constraint_signal} + + +@probe("A10_empty_candidate_scope") +def empty_scope(): + from trialmatchai.search.lancedb_backend import InMemorySearchBackend + + backend = InMemorySearchBackend(criteria=[{"nct_id": "NCT00000001", "criteria_id": "c1", + "criterion": "synthetic cancer", "eligibility_type": "inclusion"}]) + hits = backend.search_criteria(query="synthetic cancer", nct_ids=[], search_mode="bm25") + return {"requested_nct_ids": [], "returned_nct_ids": [h["_source"]["nct_id"] for h in hits]} + + +@probe("A11_vector_dimension_mismatch") +def mismatched_vectors(): + from trialmatchai.search.lancedb_backend import _cosine + + return {"left": [1.0], "right": [1.0, 99.0], "cosine": _cosine([1.0], [1.0, 99.0])} + + +@probe("A12_rejected_concept_indexed") +def rejected_concepts(): + from trialmatchai.search.lancedb_backend import _flatten_entities + + texts, synonyms = _flatten_entities([{"text": "synthetic mention", "linker_status": "rejected", + "concept_candidates": [{"concept_name": "unrelated disease", "score": 1.0}]}]) + return {"text": texts, "indexed_synonyms": synonyms} + + +@probe("A13_missing_topics_omitted") +def missing_topics(): + from trialmatchai.trec.qrels import evaluate + + with tempfile.TemporaryDirectory(prefix="tmai-audit-") as tmp: + root = Path(tmp) + p = root / "synthetic-A" + p.mkdir() + (p / "ranked_trials.json").write_text(json.dumps({"RankedTrials": [{"TrialID": "NCT00000001", "Score": 1}]})) + (root / "synthetic-C").mkdir() + qrels = {pid: {"NCT00000001": 2} for pid in ["synthetic-A", "synthetic-B", "synthetic-C"]} + result = evaluate(qrels, root) + return {"expected_topics": 3, "scored_topics": result["num_queries_scored"], + "ranked_topics": result["num_queries_ranked"], "mean_ndcg10": result["mean"]["ndcg@10"]} + + +@probe("A14_condensed_ndcg_one_hit") +def one_hit(): + from trialmatchai.trec.metrics import condensed_ndcg + + judgments = {f"NCT{i:08d}": 2 for i in range(1, 11)} + ranked, scores = ["NCT00000001"], {"NCT00000001": 1.0} + return {"eligible_found": 1, "eligible_total": 10, + "ndcg10": condensed_ndcg(ranked, scores, judgments, [10])[10], + "ndcg_full10": condensed_ndcg(ranked, scores, judgments, [10], full_ideal=True)[10]} + + +@probe("A15_failure_report_state") +def failure_report(): + from trialmatchai.interop.exporters.html_report import build_report_model + + model = build_report_model(patient_summary={"patient_id": "synthetic"}, + ranked=[{"TrialID": "NCT00000001", "Score": -1}], + eligibility_by_id={"NCT00000001": {"error": "processing_failed"}}, + meta_by_id={}, generated_at="synthetic") + return {"reasoning_available": model["trials"][0]["reasoning_available"], + "final_decision": model["trials"][0]["final_decision"]} + + +@probe("A16_fhir_export_required_fields") +def fhir_export(): + from trialmatchai.interop.exporters.fhir import profile_to_fhir_bundle + from trialmatchai.interop.models import PatientProfile, Provenance + from trialmatchai.interop.utils import make_fact + + profile = PatientProfile(patient_id="synthetic", medications=[make_fact( + category="medication", label="synthetic drug", provenance=Provenance(source_format="text"))]) + bundle = profile_to_fhir_bundle(profile) + medication = next(e["resource"] for e in bundle["entry"] if e["resource"]["resourceType"] == "MedicationStatement") + return {"resource_keys": sorted(medication), "has_required_status": "status" in medication, + "has_required_medication": any(k in medication for k in ["medicationCodeableConcept", "medicationReference"])} + + +@probe("A17_empty_prepared_criteria_stale") +def empty_criteria(): + from trialmatchai.registry.preparation import write_prepared_criteria + + with tempfile.TemporaryDirectory(prefix="tmai-audit-") as tmp: + write_prepared_criteria([{"nct_id": "NCT00000001", "criteria_id": "old", "criterion": "Old criterion"}], tmp) + write_prepared_criteria([], tmp) + return {"old_criterion_remains": (Path(tmp) / "NCT00000001/old.json").exists()} + + +@probe("A18_unvalidated_canonical_patient_id") +def patient_id(): + from trialmatchai.interop.models import PatientProfile + + profile = PatientProfile(patient_id="../synthetic-escape") + with tempfile.TemporaryDirectory(prefix="tmai-audit-") as tmp: + output = Path(tmp) / "outputs" + target = (output / profile.patient_id).resolve() + return {"id_accepted": profile.patient_id, "target_outside_output_root": not target.is_relative_to(output.resolve())} + + +if __name__ == "__main__": + output = Path(__file__).with_name("audit_2026_09_12_observations.json") + output.write_text(json.dumps(RESULTS, indent=2, sort_keys=True) + "\n") + print(f"Wrote {output}") + sys.exit(1 if any(v["status"] == "probe_error" for v in RESULTS.values()) else 0) diff --git a/docs/audit_2026_09_12_validation.md b/docs/audit_2026_09_12_validation.md new file mode 100644 index 00000000..accaef11 --- /dev/null +++ b/docs/audit_2026_09_12_validation.md @@ -0,0 +1,85 @@ +**Validation record — TrialMatchAI review, 12 September 2026** + +Source revision: `508db33`, package version `0.8.2`. Application code was not changed. Review files and synthetic evidence were added under `docs/`. + +| Check | Result | Limit | +|---|---|---| +| Repository Ruff check | Passed | Lint does not establish functional correctness | +| AST parse of tracked Python source/tests/scripts | Passed | Inventory contains file hashes and line counts | +| Focused existing test selection | **181 passed, 4 deselected in 6.29s** | 29 selected test modules; four tests requiring an actual LanceDB connection were explicitly excluded | +| Full pytest attempt | **Incomplete; timed out with exit 124** | Stalled in `lancedb.connect` during `test_health_reports_never_built_search_db`; no full-suite pass is claimed | +| Synthetic audit probes | **18 observations reproduced** | Uses fakes/pure functions and invented patient/trial data; no GPU inference or actual LanceDB search | +| Wheel build in a temporary source copy | Passed | Local setuptools 81.0.0; built wheel lacks `trialmatchai/config/catalog/embedders.json` | +| GPU quality, end-to-end latency, clinical validation | Not run | Recommendations are not measured score improvements | +| Live registry reconciliation, dependency vulnerability scan, browser interaction test | Not run | Registry/UI findings are source-level findings; this review makes no new security-advisory status claims | + +The full run used offline model settings and a 240-second process timeout. A faulthandler dump identified the blocked call: + +```text +tests/test_audit_fixes.py:424 test_health_reports_never_built_search_db +src/trialmatchai/search/lancedb_backend.py:223 __init__ +lancedb/__init__.py:121 connect +lancedb/db.py:493 __init__ +lancedb/background_loop.py:25 run +concurrent/futures/_base.py:451 result +threading.py:327 wait +``` + +This identifies the observed blocking location, not its root cause. It is not classified as a proven application defect. Intermediate narrowed runs encountered the same connection wait in catalog/backend tests. The final selection excludes all four known connection-dependent tests below and completes successfully. + +Reproduce the focused existing tests from the repository root: + +```bash +HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 .venv/bin/python -m pytest \ + tests/test_constraints.py \ + tests/test_constraint_exclusion_aggregation.py \ + tests/test_criteria_chunking.py \ + tests/test_fhir_robustness.py \ + tests/test_patient_interop.py \ + tests/test_interop_fixtures.py \ + tests/test_metrics.py \ + tests/test_qrels_eval.py \ + tests/test_trial_ranker_pytest.py \ + tests/test_trial_ranker_scoring_contract.py \ + tests/test_trial_ranker_tiebreak.py \ + tests/test_first_level_planner.py \ + tests/test_first_level_search_pytest.py \ + tests/test_second_level_search_pytest.py \ + tests/test_search_queries_pytest.py \ + tests/test_settings.py \ + tests/test_settings_validators.py \ + tests/test_config_pytest.py \ + tests/test_model_catalog.py \ + tests/test_html_report.py \ + tests/test_entities.py \ + tests/test_finetuning.py \ + tests/test_eligibility_vllm.py \ + tests/test_embedder_contract.py \ + tests/test_variant_recognizer.py \ + tests/test_location_filter.py \ + tests/test_concept_sources.py \ + tests/test_schemas.py \ + tests/test_patient_runtime_loading.py \ + -k 'not catalog_swap_makes_metric_follow and not explicit_metric_still_pins_over_embedder and not concept_table_no_recreate_appends_rows and not search_backend_metric_follows_embedder' +``` + +Reproduce the audit probes: + +```bash +.venv/bin/python docs/audit_2026_09_12_reproduce.py +``` + +The probe command refreshes `docs/audit_2026_09_12_observations.json`. A zero exit means all probes executed; it does **not** mean the observed application behavior is correct. The probes are deliberately separate from the normal regression suite. Convert each relevant example into a desired-behavior regression test when implementing its fix. + +Wheel contents were checked by copying `src/trialmatchai`, `pyproject.toml`, `README.md`, and `LICENSE` into a new temporary directory and invoking `setuptools.build_meta.build_wheel` there, without dependency installation. Inspection of the resulting ZIP returned: + +```json +{ + "setuptools": "81.0.0", + "wheel": "trialmatchai-0.8.2-py3-none-any.whl", + "catalog_present": false, + "config_present": true +} +``` + +The review's evidence labels distinguish executable observations, source-level findings, and proposed experiments. No predicted retrieval/clinical improvement is presented as a measured result. diff --git a/docs/codebase-review-2026-09-12.md b/docs/codebase-review-2026-09-12.md new file mode 100644 index 00000000..4de398ff --- /dev/null +++ b/docs/codebase-review-2026-09-12.md @@ -0,0 +1,176 @@ +**TrialMatchAI repository review — 12 September 2026** + +Reviewed revision: `508db33` (version `0.8.2`). Priority: balanced ranking quality and a usable clinical workflow. + +**Assessment:** TrialMatchAI has a substantial research implementation, useful component boundaries, and a meaningful regression suite. Its main obstacle is inconsistent contracts between patient import, retrieval, constraints, model assessment, cached results, evaluation, and reporting. Several of those inconsistencies can produce confidently presented but stale or unsupported results. Correct these before expanding model autonomy or using headline ranking metrics to select a production configuration. + +This is a repository-wide static review with targeted execution, not a certification of clinical accuracy or a completed GPU benchmark. The tracked inventory includes **110 package Python files / 19,252 lines**, **57 test files / 8,713 lines**, two benchmark scripts, configuration, schemas, templates, documentation, and CI/release workflows. Source inspection followed the import → prepare/link/index → first retrieval → criterion reranking → eligibility → ranking → report flows, plus registry updates, training, resume, and evaluation. Binary model weights, private patient records, and large runtime corpora were not audited. No application fixes were applied. + +Review evidence is in [synthetic probes](audit_2026_09_12_reproduce.py), [recorded observations](audit_2026_09_12_observations.json), [file inventory with hashes](audit_2026_09_12_inventory.json), and [validation record](audit_2026_09_12_validation.md). The probes use invented patients and trials, local temporary directories, and no inference or network calls. They reproduce current behavior; they deliberately are not regression tests asserting desired behavior. + +Severity below means: **P0** blocks trustworthy clinical workflow; **P1** materially affects correctness, ranking, installation, or repeatability; **P2** affects maintainability, performance, or usability. “Reproduced” means an executable synthetic observation exists. “Static” means the behavior follows from source and call flow; it was not exercised with production models/data. Proposed improvements and their expected benefits remain hypotheses until evaluated. + +**Release and clinical correctness findings** + +1. **P0 — FHIR can attach another patient's resource to the only available patient, even in strict mode. Reproduced A03.** `_profile_for_resource` falls back to the sole profile after failing to resolve an explicit subject reference. A bundle with Patient A and a Condition whose subject is Patient B imports B's condition into A. Bundles without Patient resources also collapse onto one fallback profile. Resolve explicit references strictly; quarantine or reject unresolved subjects, and distinguish a genuinely absent subject from a contradictory subject. Add mixed-patient, missing-patient, and versioned-reference fixtures. Evidence: [src/trialmatchai/interop/importers/fhir.py:203](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/importers/fhir.py#L203). + +2. **P0 — Rematching can reuse eligibility decisions from an earlier patient/trial/model version. Reproduced A04/A05/A06.** The outer runner disables patient-level resume on force or a changed corpus, but `BaseTrialProcessor.process_trials` independently skips existing trial JSON whenever it has no `error` key. The inner cache has no patient, trial, prompt, adapter, or generation fingerprint. Even `{"unrelated": true}` becomes a completed result. Separately, `_match_signature` omits the CoT adapter, its revision, many search/constraint settings, and assessment limits; ingest skips existing patient IDs, and runtime loading trusts existing summaries. Use immutable run IDs, schema-validated outputs, and per-artifact dependency hashes; propagate force to every dependent stage. A changed fact must invalidate affected criterion assessments and ranking. Evidence: [src/trialmatchai/matching/eligibility_base.py:249](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/eligibility_base.py#L249), [src/trialmatchai/orchestration.py:381](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L381), [src/trialmatchai/orchestration.py:65](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L65), [src/trialmatchai/main.py:716](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L716). + +3. **P0 — Missing information becomes affirmative clinical evidence in deterministic constraints. Reproduced A08/A09.** No recorded condition satisfies an “absence required” inclusion with signal `+1`; no recorded exclusion condition produces “matched” with `+0.25`, without evidence. An “EGFR testing ordered; result pending” fact satisfies a mutation requirement because any matching non-negative gene mention is accepted. Missing, explicitly absent, pending, historical, and present need separate states. Require a scoped negative assertion for absence and a gene–alteration–result relationship for biomarkers. Evidence: [src/trialmatchai/constraints/evaluation.py:273](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/constraints/evaluation.py#L273), [src/trialmatchai/constraints/evaluation.py:306](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/constraints/evaluation.py#L306). + +4. **P0 — Generated query expansion overwrites the narrative subsequently used as clinical evidence. Static; active when expansion is enabled.** `enrich_summary` replaces `patient_narrative` with generated `expanded_sentences`; the eligibility stage consumes that narrative. An omission or hallucination in expansion can therefore alter the evidentiary basis of the final decision, while deterministic constraints still consume the canonical profile. Keep generated retrieval terms in a separate derived object. Clinical assessment should use source facts and verbatim evidence, with generated summaries traceable to them. Evidence: [src/trialmatchai/matching/query_expansion.py:237](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/query_expansion.py#L237), [src/trialmatchai/main.py:365](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L365). + +5. **P1 — Default adapter identifiers are converted into nonexistent local paths. Reproduced A01.** Both shipped Hugging Face adapter IDs become absolute paths beneath the runtime root. The reranker preflight requires that local path to exist, so the documented default first-use download path cannot succeed on a clean setup. The CoT loader likewise receives the transformed path. Introduce an explicit model reference contract distinguishing local directories from Hub repositories, resolve adapters to pinned snapshots, and validate the resulting adapter before loading it. Also make “merged checkpoint, no adapter” a supported reranker configuration: current preflight requires a reranker adapter despite the merge documentation. Evidence: [src/trialmatchai/config/config_loader.py:213](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/config/config_loader.py#L213), [src/trialmatchai/services/preflight.py:83](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/services/preflight.py#L83), [src/trialmatchai/models/llm/vllm_loader.py:260](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/models/llm/vllm_loader.py#L260). + +6. **P1 — Ranking does not consistently encode eligibility. Reproduced A07.** Nine met inclusions plus one explicitly failed inclusion score `0.9`; one met inclusion plus ten unclear exclusions scores `1.0`; a generation error and a violated exclusion both score `-1`. The model's `Final Decision` is ignored by scoring but displayed by the report. These are different clinical and operational states. Derive an explicit decision from the complete logical criterion assessment; keep retrieval relevance, eligibility state, evidence completeness, and operational failure separate. Rank within appropriate states without presenting the numeric score as a probability. Evidence: [src/trialmatchai/matching/trial_ranker.py:58](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/trial_ranker.py#L58), [src/trialmatchai/interop/exporters/html_report.py:125](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/html_report.py#L125). + +7. **P1 — Criteria are flattened without preserving logical groups. Static.** Constraint sets contain a flat list, and aggregation takes the minimum nonzero signal. “A OR B” inclusions cannot be represented faithfully; “A AND B” exclusions can be penalized when only A holds. Chunking also loses parent clauses, and criterion IDs hash trial ID plus text, omitting polarity and position, so identical inclusion/exclusion text collides. Preserve sections, parent-child relationships, AND/OR/NOT, exceptions, time anchors, and source spans. Only apply a hard decision when the relevant logical group is complete. Evidence: [src/trialmatchai/constraints/models.py:64](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/constraints/models.py#L64), [src/trialmatchai/constraints/evaluation.py:46](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/constraints/evaluation.py#L46), [src/trialmatchai/registry/criteria_chunking.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/criteria_chunking.py#L1), [src/trialmatchai/registry/preparation.py:205](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/preparation.py#L205). + +8. **P1 — Clinical numeric and temporal semantics are too weak for definitive decisions. Static.** Measurements are parsed from the first number in a description, losing inequality/range semantics. Missing units are treated as compatible; matching measurements are chosen by lexical/code similarity and input order, without selecting by date. Temporal constraints return unknown, while prior/current concept constraints effectively use presence. Implement typed quantities with comparators, normalized units, measurement dates, encounter context, and explicit temporal intervals. Missing units or unresolved timing should cause abstention. Evidence: [src/trialmatchai/constraints/evaluation.py:228](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/constraints/evaluation.py#L228), [src/trialmatchai/constraints/evaluation.py:236](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/constraints/evaluation.py#L236), [src/trialmatchai/constraints/evaluation.py:413](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/constraints/evaluation.py#L413). + +9. **P1 — Import preserves some source content but does not preserve enough assertion semantics for matching. Static.** Text NER turns entity mentions into positive patient facts without negation, experiencer, or temporal scope. A disease in “no diabetes” or a relative's history can become a patient condition. FHIR resolved/remission/inactive conditions are represented as negated, obscuring historical disease; medication orders and actual exposure are not distinguished by downstream constraints. OMOP observation values are dropped; measurement operators and some dates are not carried into decision logic. Phenopacket disease exclusion and staging information are not consistently propagated into the narrative/assessment path. Add assertions and typed observations to the canonical model, then test clinically meaningful round trips and source-to-criterion evidence. Evidence: [src/trialmatchai/interop/importers/text.py:89](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/importers/text.py#L89), [src/trialmatchai/interop/importers/fhir.py:285](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/importers/fhir.py#L285), [src/trialmatchai/interop/importers/omop.py:258](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/importers/omop.py#L258), [src/trialmatchai/interop/importers/phenopacket.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/importers/phenopacket.py#L1). + +10. **P1 — Standard exports are lossy and can be structurally invalid. Reproduced A16; schema comparison.** The FHIR exporter uses one generic `code`-based resource constructor. MedicationStatement has no required `status` or `medication[x]`; Observation similarly lacks required status, and actual measurement values are placed in notes rather than typed value fields. Several fact categories are omitted without a FHIR conversion report. Phenopacket export also approximates values and dates, and its loss report only covers selected omitted categories. Use resource-specific mappers and declare/export the supported standard version. Validate against official schemas and report every meaningful loss, including assertion/timing loss. Evidence: [src/trialmatchai/interop/exporters/fhir.py:53](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/fhir.py#L53), [src/trialmatchai/interop/exporters/phenopacket.py:6](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/phenopacket.py#L6). The required FHIR fields are documented in [HL7 MedicationStatement R4](https://hl7.org/fhir/R4/medicationstatement.html) and [HL7 Observation R4](https://hl7.org/fhir/R4/observation.html). + +**Retrieval and ranking findings** + +11. **P1 — Default second-stage scoring removes exclusion criteria from trial aggregation. Reproduced A02.** A maximum reranker probability of `1.0` becomes `0.25` after exclusion weighting, below the default aggregation threshold `0.5`. Even the default favorable exclusion constraint adjustment does not bridge that gap. Exclusion evidence therefore cannot influence aggregation as intended under these defaults. Threshold criterion relevance before polarity weighting, and carry contradictory evidence separately instead of discarding it. Test both an exclusion-only hit and a trial with strong inclusion relevance plus a disqualifying exclusion. Evidence: [src/trialmatchai/matching/retrieval/criteria_retrieval.py:30](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/criteria_retrieval.py#L30), [src/trialmatchai/matching/retrieval/criteria_retrieval.py:112](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/criteria_retrieval.py#L112), [src/trialmatchai/matching/retrieval/criteria_retrieval.py:203](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/criteria_retrieval.py#L203). + +12. **P1 — The reranker often receives a query fragment in place of patient evidence. Static.** Its prompt asks whether the patient text contains enough information to assess a criterion, including satisfaction or violation. However, the supplied “patient text” is the retrieval query. The caller takes the first ten deduplicated main conditions, other conditions, and narrative fragments, in that order; ten condition terms can remove the narrative entirely. A high score means assessability/relevance, not eligibility, yet it drives candidate selection and score refinement. Supply a stable clinical context plus criterion-specific evidence to reranking, preserve a relevance score, and train/evaluate a separate support/contradiction/unknown classifier if needed. Evidence: [src/trialmatchai/main.py:288](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L288), [src/trialmatchai/matching/retrieval/criteria_retrieval.py:112](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/criteria_retrieval.py#L112), [src/trialmatchai/models/llm/llm_reranker.py:82](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/models/llm/llm_reranker.py#L82). + +13. **P1 — Embedding changes can leave old vectors in place and label them as current. Static.** Build-level fingerprints detect some configuration changes, but per-trial preparation skips based only on source/output mtime and parseable JSON. `build_system` then calls `build_index(force=True)`, bypassing the latter's auto-detection of an embedder change. Existing prepared vectors may be reused, after which the new model identity is written. Direct index builds can skip on existing tables despite a changed corpus/filter or an explicit re-embed request. The identity omits pooling, some lengths/instructions, and revision. Use the embedder's existing full fingerprint everywhere, attach it to every prepared artifact, and never publish an index whose document vectors disagree with the query encoder contract. Evidence: [src/trialmatchai/orchestration.py:240](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L240), [src/trialmatchai/orchestration.py:303](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L303), [src/trialmatchai/orchestration.py:606](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L606), [src/trialmatchai/orchestration.py:757](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L757), [src/trialmatchai/orchestration.py:849](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L849). + +14. **P1 — Vector corruption is hidden by truncation/padding. Reproduced A11; static index paths.** Cosine/dot helpers crop unequal vectors to the shorter length; index preparation pads/truncates to a majority dimension. Cosine between `[1]` and `[1,99]` becomes `1.0`. Mixed models of the same dimension are also undetectable by shape alone. Fail on any fingerprint/dimension mismatch; validate finite values and expected normalization before index publication. Evidence: [src/trialmatchai/search/lancedb_backend.py:849](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L849), [src/trialmatchai/search/lancedb_backend.py:965](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L965). + +15. **P1 — BGE-M3 has conflicting pooling contracts in the repository. Static plus upstream verification.** The shipped explicit config and benchmark registry use mean pooling, while the internal model catalog uses CLS. Thus selecting the BGE catalog name changes representation even for the same model name, and the benchmark does not test its published native pooling configuration. Standardize the contract and rerun a controlled CLS-versus-mean ablation with freshly embedded documents and a compatible concept store. No numerical gain is established by this review. Evidence: [src/trialmatchai/config/config.json:87](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/config/config.json#L87), [src/trialmatchai/config/catalog/embedders.json:2](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/config/catalog/embedders.json#L2), [benchmarks/embedders/registry.json:3](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/benchmarks/embedders/registry.json#L3), [scripts/benchmark_embedder.py:43](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/scripts/benchmark_embedder.py#L43). [BGE-M3's published pooling configuration](https://huggingface.co/BAAI/bge-m3/blob/main/1_Pooling/config.json) enables CLS and disables mean pooling. + +16. **P1 — Filtering after candidate retrieval can destroy recall without refill. Static.** Age/sex/status filtering happens after a bounded ANN/FTS candidate set is acquired. If that set is dominated by incompatible trials, valid matches just beyond the limit disappear. Location filtering is also applied after fusion. Additionally, the runtime sets `overall_status = "All"`, making the configured status hard-filter channel ineffective for a clinical “open trials” workflow. Push suitable filters into candidate acquisition, overfetch/refill with bounds, retain unknown demographics, and make recruitment preference explicit. Measure how many eligible candidates each filter removes. Evidence: [src/trialmatchai/search/lancedb_backend.py:355](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L355), [src/trialmatchai/search/lancedb_backend.py:613](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L613), [src/trialmatchai/main.py:122](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L122), [src/trialmatchai/matching/retrieval/trial_retrieval.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/trial_retrieval.py#L1). + +17. **P1 — Rejected concept candidates re-enter the search index as synonyms. Reproduced A12.** `_flatten_entities` appends candidate names based on candidate score without checking the final linker acceptance state. The linker's RRF normalization can give its best candidate score `1.0` even when the absolute acceptance gate rejects it. This bypasses the NIL-aware linker and introduces unrelated search terms. Index accepted concepts/synonyms only; retain rejected candidates as diagnostics. Evidence: [src/trialmatchai/search/lancedb_backend.py:941](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L941), [src/trialmatchai/entities/linker.py:246](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/entities/linker.py#L246), [src/trialmatchai/entities/linker.py:409](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/entities/linker.py#L409). + +18. **P2 — Empty candidate scopes broaden criterion search. Reproduced A10.** `nct_ids=[]` behaves like an unrestricted search because the backend treats an empty set as no filter. The current pipeline ultimately slices to zero trials, but still performs global retrieval and potentially reranking; direct callers receive unrelated trials. Reserve `None` for unrestricted search and return immediately for an empty allowed set. Evidence: [src/trialmatchai/search/lancedb_backend.py:680](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L680), [src/trialmatchai/search/lancedb_backend.py:1037](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L1037), [src/trialmatchai/main.py:295](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L295). + +19. **P2 — Candidate aggregation favors criterion volume and lacks coverage guarantees. Static/design risk.** The default aggregate combines `sum(scores)/sqrt(count)` with a maximum, rewarding trials with more retrieved high-scoring criteria. Each query retrieves a shared pool of 250 criteria, so some candidate trials can receive no assessment evidence. Concurrent query completion also influences insertion order and tied trial order. Add per-trial evidence quotas, deterministic ID tie-breaks, criterion-group deduplication, and learned/calibrated aggregation evaluated against simple coverage-normalized baselines. Evidence: [src/trialmatchai/matching/retrieval/criteria_retrieval.py:98](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/criteria_retrieval.py#L98), [src/trialmatchai/matching/retrieval/criteria_retrieval.py:229](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/criteria_retrieval.py#L229). + +20. **P2 — Search fallbacks and metric handling can mask a degraded retrieval path. Static.** Backend search exceptions can fall back to a bounded table scan; a successful-looking result may reflect an arbitrary prefix of the corpus. Criterion vector-index creation uses the default cosine argument rather than consistently passing the selected backend metric. Report channel health, fallback reasons, and candidate counts in the run; require metric parity when building and querying both tables. Treat a failed retrieval channel as degraded execution with an explicit recovery policy. Evidence: [src/trialmatchai/search/lancedb_backend.py:297](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L297), [src/trialmatchai/search/lancedb_backend.py:329](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L329), [src/trialmatchai/search/lancedb_backend.py:441](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L441). + +**Eligibility generation, state, and registry findings** + +21. **P1 — Context budgeting can omit criteria and produce incomplete assessments. Static.** The vLLM processor truncates criterion text from the tail; exclusions commonly appear there. Input budgeting reserves only a fraction of requested output tokens, so the input allowance and requested output can exceed the context window together. The Transformers processor left-truncates the whole prompt, potentially removing instructions and early criteria. Optional guided JSON controls shape but does not ensure every criterion was assessed. Chunk logical criterion groups, allocate input/output against the actual loaded model window, require every expected criterion ID to receive a state, and explicitly mark incomplete assessments. Evidence: [src/trialmatchai/matching/eligibility_reasoning_vllm.py:78](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/eligibility_reasoning_vllm.py#L78), [src/trialmatchai/matching/eligibility_reasoning_vllm.py:115](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/eligibility_reasoning_vllm.py#L115), [src/trialmatchai/matching/eligibility_reasoning_transformers.py:49](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/eligibility_reasoning_transformers.py#L49), [src/trialmatchai/matching/eligibility_reasoning_transformers.py:127](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/eligibility_reasoning_transformers.py#L127). + +22. **P1 — Failure handling and completion markers disagree. Static; A06/A15 support the output-state issue.** Output persistence accepts any parsed object and can swallow write failures. Partial eligibility output can omit trials from the final ranking. Mixed patient success/failure returns success, and the match fingerprint is recorded regardless of the matching return code. An old ranked file plus a failed rerun can therefore satisfy future resume checks. Give every trial and stage an explicit pending/running/complete/partial/failed state; propagate failed persistence; publish a completed run only after validating expected outputs. Evidence: [src/trialmatchai/matching/eligibility_base.py:196](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/eligibility_base.py#L196), [src/trialmatchai/main.py:443](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L443), [src/trialmatchai/main.py:697](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L697), [src/trialmatchai/orchestration.py:471](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L471). + +23. **P1 — Registry refresh does not reliably remove stale recruitment status. Static.** Queries fetch only configured active statuses. A previously indexed recruiting trial that transitions to completed/withdrawn is then absent from those responses and can retain its old status locally. Reconcile all tracked IDs independently of the “discover open trials” query; maintain a durable update watermark with overlap, retries, and tombstones. Separate recruitment status from clinical eligibility. Evidence: [src/trialmatchai/registry/defaults.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/defaults.py#L1), [src/trialmatchai/registry/clinicaltrials_gov.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/clinicaltrials_gov.py#L1), [src/trialmatchai/registry/updater.py:89](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/updater.py#L89). + +24. **P1 — Registry updates can create different trial versions across retrieval and reasoning. Static.** The updater writes raw/normalized data and directly updates index tables, but does not refresh the prepared trial/criterion files. Eligibility loading prefers prepared trials before normalized source data, so retrieval may use new criteria while CoT uses old criteria. A previous `fetched` manifest record is also considered complete even when a later run enables reindexing, leaving that trial unindexed until its source changes. Publish a versioned trial snapshot used by both retrieval and assessment, with separate fetched/prepared/indexed milestones. Evidence: [src/trialmatchai/registry/updater.py:159](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/updater.py#L159), [src/trialmatchai/registry/updater.py:175](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/updater.py#L175), [src/trialmatchai/main.py:350](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L350). + +25. **P1 — Corpus publication is not atomic across artifacts. Static; stale-empty case reproduced A17.** Criteria replacement deletes the old directory before writing the new files; an empty replacement leaves the old directory intact. Temporary prepared JSON files also end in `.json`, unlike the safer utility writers, so crash leftovers can match reader globs. Trial and criterion tables are replaced separately; updates delete then add rows. A crash or concurrent reader can observe a mixed snapshot. Write a complete immutable version, validate counts and fingerprints, then switch one manifest pointer; use batch upserts and delayed cleanup. Evidence: [src/trialmatchai/registry/preparation.py:161](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/preparation.py#L161), [src/trialmatchai/registry/preparation.py:187](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/preparation.py#L187), [src/trialmatchai/orchestration.py:343](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L343), [src/trialmatchai/search/lancedb_backend.py:301](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L301). + +26. **P1 — Backend and fine-tuning interchangeability is incomplete. Static.** Transformers reranker/eligibility constructors never apply the configured adapters. Switching to that backend alone also passes the default numeric device as string `"0"`, which is not a valid Torch device specification. NER fine-tuning defaults to adapter-only output and recommends using that directory as `model_name`, but the recognizer calls only `GLiNER2.from_pretrained`; the installed loader explicitly requires loading the base and then `load_adapter`. Provide a single resolved model specification with backend capability validation, normalized devices, and a train → save → reload → infer smoke test. Evidence: [src/trialmatchai/main.py:390](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L390), [src/trialmatchai/main.py:536](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L536), [src/trialmatchai/models/llm/transformers_reranker.py:17](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/models/llm/transformers_reranker.py#L17), [src/trialmatchai/entities/recognizers.py:196](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/entities/recognizers.py#L196), [src/trialmatchai/finetuning/ner.py:173](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/finetuning/ner.py#L173). + +**Evaluation and user experience findings** + +27. **P1 — Missing patients and empty rankings are omitted from reported averages. Reproduced A13.** Evaluation skips absent patient directories entirely, and skips ranking metrics for empty ranked lists. With three expected topics, one successful ranking, one missing patient, and one empty directory, only two topics are “scored,” one is “ranked,” and mean nDCG is `1.0`. Evaluate the complete declared topic set; failed/empty retrieval or ranking contributes zero where the metric is defined. Report both service completion and quality, and distinguish topics without relevant judgments from execution failures. Evidence: [src/trialmatchai/trec/qrels.py:194](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/trec/qrels.py#L194), [src/trialmatchai/trec/qrels.py:218](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/trec/qrels.py#L218). + +28. **P1 — Default nDCG measures ordering within the returned judged set, not end-to-end retrieval quality. Reproduced A14.** Returning one of ten equally eligible trials yields default nDCG@10 `1.0`; the existing full-ideal variant yields approximately `0.2201`. That conditional metric is useful when labeled precisely, but is not a substitute for full-task nDCG. Both variants condense the numerator by removing unjudged results, so `ndcg_full` is still not simply standard uncondensed `trec_eval` nDCG. Keep conditional, condensed, and standard metrics under explicit names and verify reference parity on miniature runs. Evidence: [src/trialmatchai/trec/metrics.py:61](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/trec/metrics.py#L61), [src/trialmatchai/trec/qrels.py:219](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/trec/qrels.py#L219). Use the [NIST trec_eval implementation](https://github.com/usnistgov/trec_eval/blob/main/m_ndcg_cut.c) as a reference for standard nDCG rather than relying on a “trec_eval-style” label. + +29. **P1 — Benchmark corpus/provenance limits prevent strong conclusions from the headline plots alone. Static.** The TREC runner derives its corpus filter from the union of qrels trial IDs, creating a restricted candidate task. Backfill can use live registry records in a historical evaluation. The tracked benchmark JSONs support first-level embedder curves, but the repository does not contain equivalent full per-topic ranking bundles/configurations for every reasoner in the headline performance figure. The README already acknowledges the TrialGPT cohort mismatch and the restricted 2023 task; retain those qualifications and avoid interpreting the chart as a matched full-corpus comparison. Archive frozen corpus IDs/versions, exact configs/adapters, run files, evaluator version, all topic outcomes, and figure-generation code for each reported result. Evidence: [src/trialmatchai/trec/runner.py:132](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/trec/runner.py#L132), [src/trialmatchai/trec/backfill.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/trec/backfill.py#L1), [scripts/benchmark_embedder.py:168](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/scripts/benchmark_embedder.py#L168), [README.md:62](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/README.md#L62). Official track data are indexed by [NIST](https://trec.nist.gov/data/trials.html). + +30. **P1 — Report data can mix versions, and operational failure is not a first-class UI state. Reproduced A15; static joins.** Reports prefer current summaries and prepared metadata while reading existing rankings. `processing_failed` is treated as `reasoning_available=true` because only one error sentinel is excluded. The report can therefore show a new patient description next to an old assessment, or an empty assessment without a useful retry state. Render one immutable run snapshot; include patient/trial update times, assessed/total criterion counts, evidence links, missing facts, and a specific failed/partial state. The displayed decision must come from the same validated decision object used for ranking. Evidence: [src/trialmatchai/interop/exporters/html_report.py:125](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/html_report.py#L125), [src/trialmatchai/interop/exporters/html_report.py:220](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/html_report.py#L220). + +31. **P1 for an interactive deployment — There is no complete clinical work queue or feedback loop. Product gap.** The current product is a batch CLI plus a portable HTML report. It has useful search/filter/print behavior, but no persistent fact correction, pending questions, job cancellation/retry, reviewer overrides, trial comparison, case assignment, or referral tracking. The report also does not present site distance/contact details needed to act on a match. Build a thin application over a versioned job API, retain the CLI, and make the primary task “review evidence and decide the next action.” Add user/account access controls and export handling when introducing shared hosting; the existing self-contained report embeds patient data and is not access-controlled merely because only one patient is visible. Evidence: [src/trialmatchai/cli/report.py:26](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/cli/report.py#L26), [src/trialmatchai/interop/exporters/templates/report.html:327](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/templates/report.html#L327), [src/trialmatchai/interop/exporters/templates/report.html:365](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/templates/report.html#L365), [src/trialmatchai/registry/preparation.py:86](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/preparation.py#L86). + +32. **P1 — Canonical patient IDs are not constrained at the filesystem boundary. Reproduced A18 without writing outside a temporary output root.** Importers sanitize IDs, but a directly supplied `PatientProfile` accepts `../synthetic-escape`; runtime output paths join that ID without containment validation. This is a local input-boundary defect, not evidence of an exposed remote endpoint. Validate canonical IDs, enforce resolved-path containment, and namespace imported identities by source organization/dataset to avoid collisions. Evidence: [src/trialmatchai/interop/models.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/models.py#L1), [src/trialmatchai/main.py:588](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L588). + +**Duplication, dead paths, and maintenance findings** + +33. **P1 — The built wheel omits the embedder catalog. Build reproduced.** Building this revision with local setuptools `81.0.0` produced `trialmatchai-0.8.2-py3-none-any.whl` containing `config/config.json` but no `config/catalog/embedders.json`. The package-data patterns do not include the nested catalog. Missing catalogs silently become `{}`, so `model: "medcpt"` resolves differently outside the checkout. Include the resource and test every catalog entry from an installed wheel outside the repository. Evidence: [pyproject.toml:122](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/pyproject.toml#L122), [src/trialmatchai/config/config_loader.py:52](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/config/config_loader.py#L52). + +34. **P2 — Several exposed settings are inert or have competing sources of truth. Static.** `constraints.llm_extraction_enabled` has no runtime consumer; first-level LLM expansion is not wired to an expander by the main caller; the global NER threshold is not used for schema-level filtering; GLiNER batch size is stored but recognition loops per text; runtime linker construction ignores configured margin. CoT and reranker batching use different configuration sections depending on backend, including `rag.batch_size * 2` for vLLM reranking. The old model quantization/tokenizer sections do not drive the main loaders, and allowing arbitrary vLLM extra keys does not forward them automatically. Publish effective resolved settings and reject unsupported knobs, then consolidate each behavior under one contract. Evidence: [src/trialmatchai/config/settings.py:237](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/config/settings.py#L237), [src/trialmatchai/registry/preparation.py:144](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/preparation.py#L144), [src/trialmatchai/entities/annotator.py:85](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/entities/annotator.py#L85), [src/trialmatchai/entities/recognizers.py:199](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/entities/recognizers.py#L199), [src/trialmatchai/main.py:558](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/main.py#L558). + +35. **P2 — Multiple pipelines and fingerprint systems duplicate policy and drift. Static.** `pipeline.py`, `orchestration.py`, `main.py`, and the TREC runner have overlapping lifecycle decisions; build/match/link/index/concept-store caches use different identities. Runtime query expansion and planner expansion are separate mechanisms. Multiple JSON/atomic writers have different temporary-file guarantees. `--skip expand` does not remove an already-expanded summary, so it is not a clean ablation by itself. Share one stage executor, artifact store, model contract, expansion interface, and atomic writer; preserve thin CLI/TREC presets. Evidence: [src/trialmatchai/pipeline.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/pipeline.py#L1), [src/trialmatchai/orchestration.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L1), [src/trialmatchai/utils/file_utils.py:20](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/utils/file_utils.py#L20), [src/trialmatchai/utils/pipeline_state.py:70](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/utils/pipeline_state.py#L70). + +36. **P2 — Dead-code candidates should be removed or explicitly retained as public compatibility. Static reference search.** `_has_entries` in bootstrap has no caller. `utils/temporal_utils.py` has no package/test caller; newer temporal helpers live in interop utilities. `_INDEX_TEMPLATE`/`render_index_html` are unused by the active reporting flow but remain exported and tested, so deprecate before removal if external callers matter. The embedder `fingerprint()` contract is implemented and tested but unused by runtime index provenance: wire it in, rather than delete it. Evidence: [src/trialmatchai/cli/bootstrap_data.py:248](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/cli/bootstrap_data.py#L248), [src/trialmatchai/utils/temporal_utils.py:1](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/utils/temporal_utils.py#L1), [src/trialmatchai/interop/exporters/html_report.py:269](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/interop/exporters/html_report.py#L269), [src/trialmatchai/models/embedding/text_embedder.py:101](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/models/embedding/text_embedder.py#L101). + +37. **P2 — Build/docs/CI do not fully exercise the supported product. Static.** Four Make targets invoke removed executables instead of unified subcommands. Fine-tuning docs say vLLM is the only backend despite the Transformers branch. Installed-wheel smoke checks do not resolve catalog models; the ML smoke job is non-blocking and mostly imports modules, so it cannot validate adapter loading or inference/training API compatibility. Release workflow builds/publishes without an explicit dependency on tests in that workflow. Add installed CLI scenario tests, a required tiny-model adapter round trip, and a separate GPU qualification job for supported stacks. Make release artifacts depend on the applicable verification results. Evidence: [Makefile:33](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/Makefile#L33), [docs/finetuning.md:42](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/docs/finetuning.md#L42), [.github/workflows/ci.yml:41](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/.github/workflows/ci.yml#L41), [.github/workflows/ci.yml:91](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/.github/workflows/ci.yml#L91), [.github/workflows/release.yml:19](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/.github/workflows/release.yml#L19). + +38. **P2 — Bulk operations have avoidable memory and indexing costs. Static/design risk.** Index creation materializes both prepared corpora in lists; registry updates rebuild search structures per changed trial; second-stage queries invoke the same embedder concurrently; normal expansion/matching paths can retain multiple model engines while TREC explicitly frees the expansion model. Add bounded batch ingestion, batch index maintenance, one GPU scheduler with microbatching, and explicit model lifetime budgets. Measure throughput and memory under the real corpus before changing concurrency. Evidence: [src/trialmatchai/orchestration.py:324](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/orchestration.py#L324), [src/trialmatchai/registry/updater.py:175](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/registry/updater.py#L175), [src/trialmatchai/search/lancedb_backend.py:301](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/search/lancedb_backend.py#L301), [src/trialmatchai/matching/retrieval/criteria_retrieval.py:98](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/matching/retrieval/criteria_retrieval.py#L98), [src/trialmatchai/trec/runner.py:145](https://github.com/cbib/TrialMatchAI/blob/508db336ee79fdf6cc076fc3da9731fe9cba6751/src/trialmatchai/trec/runner.py#L145). + +There are good foundations to preserve: canonical patient profiles with provenance fields; explicit query/document embedding APIs; RRF candidate fusion; separation of retrieval and model wrappers; a shared eligibility base class; constraint diagnostics; incremental registry manifests; numerous synthetic tests; safe HTML text insertion/JSON escaping; and atomic utility writers. The review found substantive gaps around those foundations, rather than a need to rewrite everything. + +**A practical agentic design** + +Use a durable, bounded state machine with explicit tool contracts. “Agentic” should mean the system can diagnose missing evidence, choose a useful next retrieval or question, recover from failure, and resume coherently. A longer CoT prompt alone does not provide those capabilities. + +```mermaid +flowchart TD + A[Import and validate patient identity] --> B[Review canonical facts and provenance] + B --> C[Plan bounded retrieval queries] + C --> D[First level: lexical, dense, accepted concepts] + D --> E{Candidate coverage adequate?} + E -->|No; budget remains| C + E -->|Yes or budget reached| F[Second level: retrieve evidence and rerank] + F --> G[Assess logical criterion groups] + G --> H[Validate evidence, coverage, and contradictions] + H -->|Repairable failure| G + H -->|Important missing fact| I[Ask one targeted question or retrieve authorized evidence] + I --> B + H -->|Validated or explicitly partial| J[Rank with eligibility state and uncertainty] + J --> K[Clinical review, compare trials, choose next action] +``` + +Start with these versioned contracts: + +| Object | Required content | +|---|---| +| Patient snapshot | Identity namespace, source hash, facts, assertion/experiencer, quantities/units, dates, evidence spans, correction history | +| Trial snapshot | Registry ID/version, retrieval timestamp, recruitment/site status, complete criterion tree and source spans | +| Retrieval plan | Source fact IDs, proposed terms, allowed filters, candidate budget, why a refinement was triggered | +| Criterion assessment | Criterion ID/group, met/contradicted/unknown/not applicable, evidence fact IDs, concise justification, rule/model version, coverage state | +| Trial decision | Explicit clinical state, unresolved decisive questions, coverage, relevance score, decision-policy version | +| Run record | Input/config/model/corpus hashes, stage states, tool events, latency/cost/token budgets, retries, artifacts, cancellation state | + +Tools should have narrow typed arguments: `retrieve_trials`, `retrieve_criteria(allowed_trial_ids, query)`, `lookup_patient_evidence`, `evaluate_constraints`, `get_trial_snapshot`, `validate_assessment`, `propose_question`, and `assemble_report`. Patient and trial text are evidence, not instructions for the agent. Tool results need source IDs and freshness. External retrieval and data sharing should be explicit deployment capabilities, with a separate user action for outreach/referrals. + +At **first retrieval**, seed queries with the clinical problem, stage, biomarker, therapy history, and relevant alternatives. Preserve negation and provenance; generated synonyms must not become facts. Run lexical and model-native dense channels, with accepted ontology concepts as a separate channel. Retry only when a measurable symptom warrants it: empty results, excessive hard-filter loss, low channel diversity, or missing coverage. Allow a small number of refinements and cap newly added candidates. Do not relax confirmed clinical facts to make results appear. + +At **second retrieval**, build a compact evidence packet for each candidate rather than scoring isolated query fragments. Retrieve both supportive and contradictory criterion groups. Keep relevance, evidence availability, and eligibility classification separate. Allocate a minimum criterion/evidence budget per trial, then spend extra assessment on promising or uncertain candidates. Log the eligible-candidate retention curve so this stage cannot silently erase first-stage recall. + +At **eligibility/CoT**, a reasoning-capable model can assess ambiguous groups internally, but the durable/user-facing output should be structured conclusions, source citations, uncertainty, and short justifications. Do not use a raw thinking transcript as proof. Deterministic rules should handle well-specified numeric/date conditions; uncertain extraction or contradictory sources should trigger abstention or a targeted review. Validate that every expected criterion group has an assessment, every evidence reference exists, and the final decision follows the same policy used by ranking. Use a bounded repair retry for malformed/incomplete output and selective second-model review for consequential disagreements, rather than always doubling inference. + +At **follow-up**, prioritize questions by whether their answers could change the top actionable matches. For example, an unresolved biomarker result shared by several otherwise promising trials is more useful than asking about every unknown criterion. Present the source gap, why the question matters, and an “unknown/unavailable” option. Save the response as a sourced correction, invalidate affected decisions, and rerun only dependent work. Batch runs should return a pending-question artifact rather than block indefinitely. + +**How to improve the scores credibly** + +First establish the correct baseline. Correcting failed-topic accounting and using full-task metrics may initially lower reported numbers. That is an evaluation fix, not evidence that retrieval became worse. Keep a conditional ranking metric for diagnosis, but optimize against complete-task quality and clinical error rates together. + +| Layer | Measure | First experiment | Acceptance evidence | +|---|---|---|---| +| First retrieval | Grade-2 and grade-1+2 recall@100/500/1000; per-filter loss; channel contribution | Native BGE CLS vs current mean; MedCPT native query/document contract; candidate refill | Paired per-topic improvement on a frozen holdout, with freshly built compatible indexes | +| Second retrieval | Relevant/eligible retention@20/50/100; oracle ceiling from first stage; criterion coverage | Full patient evidence vs query fragments; exclusion threshold fix; coverage-normalized aggregate | Retains more eligible candidates at the same assessment budget, with no hidden missing-trial omission | +| Eligibility | Per-class precision/recall/F1; false eligibility/exclusion; unknown calibration; evidence correctness | Criterion-group chunks + output validation; typed quantities/dates; targeted disagreement review | Clinician-adjudicated set, including negation, temporal, biomarker, and missing-data cases | +| Final ranking | Standard nDCG@10/20, eligible P@10, eligible recall@20; conditional metrics separately | Explicit state + relevance tie-break vs current fractional inclusion score | All expected topics included, evaluator parity, paired confidence intervals | +| Workflow | Time to first shortlist; time to reviewed shortlist; questions per useful match; correction burden | Evidence review + prioritized follow-up + actionable site information | Observed clinician tasks with recorded completion/error rates | +| Operations | Successful run fraction, stale-result rate, schema/coverage failure rate, p50/p95 latency, GPU memory, cost/patient | Versioned artifacts, partial states, bounded retries, batching | Crash/restart and patient-update tests; load measurements on stated hardware | + +Use exact trial IDs and source snapshots for historical TREC evaluations. Run both the restricted-qrels task and a full-corpus task, clearly labeled. Split training, development, and test by patient and trial where feasible, and include temporal or site holdouts. Fine-tune only after establishing semantic labels: the existing reranker “enough information” target cannot be assumed to train eligibility. Mine hard negatives from actual retrieval errors, especially same disease/wrong stage, wrong mutation, prohibited therapy, outdated measurement, and relative-history cases. Preserve a held-out clinical error set and evaluate calibration/abstention, not only training loss. + +The separation of retrieval, criterion matching, and trial ranking has a primary-source precedent in [TrialGPT's implementation](https://github.com/ncbi-nlp/TrialGPT). The bounded question/recovery state machine proposed here is a design recommendation for this codebase; it is not a claim that adopting an agent framework automatically improves clinical accuracy. + +**Implementation order and acceptance gates** + +These are work packages, not calendar promises; model experiments and clinical validation require data, hardware, and reviewers beyond this static review. + +| Order | Work package | Scope | Done when | +|---|---|---|---| +| 1 | Correctness and installation | Findings 1–6, 10–11, 27–28, 32–33; shared decision/output schema | Cross-patient import rejected; changed facts cannot reuse stale decisions; missing facts stay unknown; malformed outputs cannot complete; installed default/catalog resolution works; all evaluation topics count | +| 2 | Reproducible corpus and execution | Findings 13–14, 21–25, 29–30 | Every result identifies one patient/trial/model/config snapshot; forced and interrupted runs behave correctly; index/criteria versions are consistent; long criteria are fully covered or explicitly incomplete | +| 3 | Retrieval/ranking experiments | Findings 12, 15–20; ablation harness | Frozen full-task baseline and paired comparison bundles establish which changes help at equal latency/cost | +| 4 | Clinical evidence model | Findings 7–9, 17, 26 | Negation, experiencer, units, dates, logical groups, and biomarkers survive import → assessment; adapter round trips work on supported backends | +| 5 | Usable application | Finding 31 plus reporting improvements | A reviewer can import/correct a case, monitor/cancel/retry matching, inspect evidence, compare trials, answer key questions, and record the next action without editing files | +| 6 | Bounded agent loop | Retrieval refinement, evidence lookup, follow-up, selective repair | Every tool action is traceable, budgets stop loops, missing answers remain pending, and agentic mode outperforms the fixed pipeline on predefined quality/workflow measures | +| 7 | Simplification and sustained validation | Findings 34–38 | One source of truth per setting/stage, obsolete paths removed/deprecated, effective config visible, installed and model integration checks qualify releases | + +Small improvements can accompany those packages: fix Make targets; consolidate vocabulary names and field aliases; normalize UI classifications with the same function as ranking; show original rank after filtering; add trial-site links and last-updated dates; distinguish “not run,” “failed,” and “needs information”; persist shortlist/comparison choices; support keyboard/focus navigation after patient changes; redact query/patient text from routine shared logs; make reports atomic; record all omitted/conversion fields; add source checksums and timestamps to downloaded artifacts; and turn unsupported configuration options into actionable errors. + +The first useful milestone is a trustworthy fixed pipeline with stable evidence and repeatable metrics. That provides the baseline needed to determine whether retrieval refinement, targeted questions, or additional reasoning actually improves the clinical workflow. diff --git a/docs/production-roadmap.md b/docs/production-roadmap.md new file mode 100644 index 00000000..ae471211 --- /dev/null +++ b/docs/production-roadmap.md @@ -0,0 +1,97 @@ +**TrialMatchAI production roadmap** + +Owner: TrialMatchAI maintainers. Started 12 September 2026. Baseline: `508db33` / 0.8.2. Initial implementation branch: `production/reliability-foundation`. + +This plan incorporates all 38 findings in [the codebase review](codebase-review-2026-09-12.md), plus production checksums, automated checks, CI/CD, executable end-to-end workflows, and a CLI suitable for both people and automation. The objective is balanced clinical matching quality and practical usability. Completion of the first PR does not imply that the entire system is production-qualified. + +**Delivery model** + +Use small, reviewed pull requests with one operational objective and an explicit acceptance gate. Keep `main` releasable, use short-lived feature branches, and ship immutable versioned artifacts. A PR must identify the behavior changed, migration implications, tests, remaining limits, and linked review findings. Start with a reproducible fixed pipeline, then add bounded agent actions against that baseline. + +The first PR establishes the verification/release foundation, a runnable synthetic CLI example, checksum tooling, installed-package validation, and the immediately reproducible patient-attribution and packaging defects. Later work packages remain separately reviewable. The historical audit/probes remain baseline evidence; do not reinterpret their old observations as the behavior of a fixed release. + +**Work packages, dependencies, and acceptance gates** + +| Package | Scope and review findings | Depends on | Acceptance gate | +|---|---|---|---| +| P01 — Release and CLI foundation | Reusable required verification; installed-wheel CLI e2e; artifact checksum creation/verification; strict bootstrap integrity mode; CLI version/demo; package catalog; Make targets; FHIR subject attribution. Findings 1, 33, 37; checksum/CI/CLI requests | None | A clean installed wheel runs the synthetic example, verifies artifacts, rejects corruption and contradictory patient references, and produces a report; publishing consumes the exact tested artifacts and waits for verification | +| P02 — Evidence identity and result lifecycle | Canonical ID validation/containment; source namespaces; patient/trial/config/model/prompt fingerprints; schema-valid completion; force propagation; atomic run publication; immutable report inputs. Findings 2, 22, 30, 32 | P01 | Changed patient, trial, adapter, prompt, or settings cannot reuse incompatible output; crash/resume and concurrent jobs cannot publish mixed results; partial failure has a nonzero automation outcome | +| P03 — Shared decision semantics | Separate relevance, eligibility state, completeness, and operational failure; unknown vs absent; pending biomarkers; inclusion/exclusion policy; consistent report labels. Findings 3, 6, 11 | P02 | Contract tests cover required inclusion failure, exclusion violation, unknown evidence, pending tests, malformed outputs, and identical decisions across ranking/reporting | +| P04 — Clinical fact/criterion model | Negation, experiencer, dates, typed quantities/units, assertions, logical criterion groups, source spans, relation-aware biomarkers; importer/exporter validation and conversion-loss reports. Findings 7–10 | P02, P03 | Clinically meaningful FHIR/OMOP/Phenopacket/text fixtures survive import → evidence → assessment; unsupported semantics abstain; supported FHIR resources pass standard validation | +| P05 — Corpus/model integrity | Shared embedder fingerprint; per-artifact model identity; strict dimension/finite-value checks; versioned trial snapshots; registry status reconciliation; fetched/prepared/indexed states; atomic snapshot publication; linked-concept provenance. Findings 13, 14, 23–25 | P02 | Model swaps re-embed correctly; changed/closed trials reach all stages; rebuilds cannot relabel old vectors; corrupt/mixed snapshots are rejected | +| P06 — Honest evaluation and experiment records | All-topic denominators; standard/condensed/conditional metric names; reference evaluator parity; frozen corpora; per-topic run files/configs; figure reproduction and confidence intervals. Findings 27–29 | P01; P02 for artifact lineage | Hand-computed miniature runs match the reference; failed/empty topics count; each published number has immutable inputs and a repeatable command | +| P07 — First retrieval quality | Native pooling/metrics, field channels, accepted-only concepts, filter pushdown/refill, status/location preferences, bounded expansion kept separate from source facts. Findings 4, 15–17, 20 | P04–P06 | Held-out recall improves or is preserved at a declared cost; filter losses/channel failures are visible; no generated query becomes clinical evidence | +| P08 — Second retrieval and assessment | Real patient evidence packets; supportive/contradictory criteria; per-trial coverage; deterministic ties; empty-scope behavior; grouped/chunked reasoning; complete criterion IDs; bounded schema repair. Findings 12, 18, 19, 21 | P03, P04, P06, P07 | Eligible-candidate retention and criterion coverage are measured; long inputs cannot silently lose criteria; malformed or incomplete assessment cannot become complete | +| P09 — Model/training qualification | One model reference contract for Hub/local/merged/adapters; pinned revisions; device normalization; adapter round trips; label validation; train/dev/test isolation; calibration and error mining. Findings 5, 26 | P01, P03, P06 | Every supported backend loads its declared adapter and completes tiny train/save/reload/infer tests; GPU stack passes actual inference and memory qualification before release | +| P10 — CLI and operational interface | Structured errors/exit codes; `init`, `config validate/show`, `doctor`, plan/dry-run/status/resume/cancel; patient/run selection; JSON stdout and logs on stderr; deterministic noninteractive behavior | P02, P05 | Scripted and interactive onboarding require no source edits; failures have actionable messages; machine output parses; Ctrl-C cancels safely and resumes from valid state | +| P11 — Clinical workspace | Fact review/correction; evidence citations; questions; persistent shortlists/comparison; reviewer overrides with rationale; current site/contact/distance data; referral preparation; accessibility. Finding 31 | P03, P04, P08, P10 | A clinician completes representative screening tasks without editing files; time, errors, questions, and correction burden are recorded; results identify their source snapshot | +| P12 — Bounded agent workflow | Typed retrieval/evidence/validation tools; explicit state graph; bounded refinement; useful missing-information questions; selective disagreement review; tool traces/budgets and safe data boundaries | P06–P11 | Agentic mode improves predefined quality/workflow measures against the fixed pipeline; no unbounded retries; unanswered questions remain pending; external actions require the appropriate explicit user action | +| P13 — Simplification and scale | Consolidate stage executor/config/atomic writers/expansion APIs; deprecate dead exports; wire effective knobs; batch index maintenance and GPU work; model lifetime budgets. Findings 34–36, 38 | Incremental throughout | One source of truth per behavior; no exposed inert knobs; compatibility migrations documented; throughput and p95 latency measured on a representative corpus | +| P14 — Deployment qualification | Immutable container/package/model/data versions; staging promotion, migrations, canary, rollback, backup/restore, observability, access control, retention, disaster recovery | P01–P13 as applicable to supported deployment | Staging e2e and recovery drills pass; SLOs and support boundaries published; production promotion/rollback uses previously qualified immutable artifacts | + +Every review finding maps to at least one package. Package owners should convert each finding into an issue and a desired-behavior regression test when implementing its fix. Do not merge broad scoring/model changes into the release-infrastructure PR merely to make the roadmap look complete. + +**Checksums and artifact trust** + +Use SHA-256 for transport/content integrity and content-addressed artifact identities. A checksum alone does not authenticate a publisher: its expected value must come from a trusted manifest or release provenance. Release artifacts should have a checksum manifest, source commit, package/version metadata, dependency inventory/SBOM, and build provenance. Verify checksums after artifact transfer and before publication/deployment. Promote exactly the artifact that passed installed-package tests; do not rebuild it in the publish job. + +For downloaded trial/model/training artifacts, validate expected digest syntax and require all requested checksums before starting a strict bootstrap. Hash cached downloads too, reject mismatches before extraction, use atomic downloads/extraction publication, and retain verification provenance. Legacy archives without published SHA-256 values need a documented one-time acquisition/verification process; do not invent trusted digests or silently claim that old caches are verified. Later corpus manifests must record individual file hashes, source versions, schema/model fingerprints, row counts, and completeness. Legacy extraction sentinels cannot establish verified provenance. + +**CI/CD contract** + +| Trigger | Required checks and behavior | +|---|---| +| Every PR / main push | Frozen lockfile; Ruff; complete CPU unit/integration suite; bounded job timeouts; JUnit artifacts; package build; checksum verification; installed-wheel CLI e2e from outside the checkout; secret/dependency checks | +| Merge queue, if enabled | The same reusable verification against the merge-group commit; one stable aggregate check for branch protection | +| Scheduled / model-stack change | Supported model/backend adapter load/infer round trips; GPU e2e on trusted workers; model revision availability; representative memory/latency checks; vulnerability exception review | +| Benchmark change | Frozen-data smoke evaluator parity on PRs; full held-out experiments on trusted scheduled/manual jobs; archive exact configs, per-topic metrics, run files, and provenance | +| Release | Tag/version consistency; same reusable verification at the release commit; install/test exact wheel; checksums/provenance; OIDC publishing; deployment uses the validated artifact | +| Staging / production | Environment-scoped identity, qualified artifact digest, readiness/liveness checks, synthetic e2e, migration compatibility, canary SLO comparison, automated rollback criteria | + +Pin third-party Actions to reviewed commit SHAs and update them with Dependabot. Keep `contents: read` as the default, and grant publishing/attestation permissions only to the relevant jobs. Never execute untrusted PR code on a privileged self-hosted GPU worker or expose deployment secrets to it. Use job timeouts and cancellation for superseded CI runs; releases should not cancel one another mid-publish. + +Repository branch protection, required reviewers, protected deployment environments, OIDC trust configuration, runners, registry credentials, and production infrastructure are account-level settings. Record the exact settings in a deployment runbook and apply them only to the intended repository/environments; YAML alone does not activate them. No production environment is assumed or deployed by the first PR. + +These choices follow [GitHub reusable-workflow semantics](https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows), [artifact provenance guidance](https://docs.github.com/en/actions/concepts/security/artifact-attestations), [uv's GitHub integration](https://docs.astral.sh/uv/guides/integration/github/), and [PyPI trusted publishing](https://docs.pypi.org/trusted-publishers/). + +**End-to-end validation pyramid** + +1. Fast deterministic tests exercise clinical/data invariants and failure paths with synthetic records. Assert desired outcomes, not implementation details. +2. Installed-package CPU e2e invokes the public CLI in a clean working directory with a real temporary LanceDB store, synthetic trial/patient files, deterministic embeddings, retrieval, ranking, report generation, and resume. It requires no model download and is explicitly a software smoke test, not clinical/model validation. +3. Tiny-model integration covers tokenizer/prompt budgets, adapters, structured output, save/reload, and backend parity. Use small pinned fixtures and fail supported configurations when these checks fail. +4. GPU e2e covers the actual supported NER/retriever/reranker/reasoner stack, long criteria, malformed generation, cancellation/recovery, model memory, and artifact lineage. Run on isolated trusted workers. +5. Full evaluation uses frozen TREC and clinician-adjudicated holdouts with failed topics included, confidence intervals, and model/data/config provenance. Include temporal/site subgroups and error categories. +6. Staging synthetic transactions validate the deployed endpoint, worker, storage, search snapshot, report, access boundaries, and rollback—not only that a process starts. + +Required negative cases include corrupt archives; unsafe archive/manifest paths; missing/wrong checksums; expired or unavailable model revisions; explicit foreign patient references; absent vs negated facts; pending tests; incompatible units; stale measurements; logical OR/AND; criteria beyond context; partial output; disk/write failure; interrupted download/build/run; stale cache after edits; empty query/candidate/result; registry closure; and mixed model/index identity. + +**CLI user experience contract** + +Provide progressive disclosure: a one-command synthetic demo, a validated starter configuration, then reproducible commands for real data. Show the active workspace/run, artifact locations, progress by stage, skipped work and its reason, and a concrete next command. Common failures should identify the field/path/model at fault without exposing patient notes, tokens, or stack traces by default. Keep verbose diagnostics opt-in. + +Support explicit patient/run selection and keep different datasets/organizations isolated. `--json` should emit only a versioned JSON object/event stream on stdout; normal diagnostics go to stderr. Document exit codes for success, invalid input/config, execution failure, and partial completion. Noninteractive jobs must never wait for prompts. Cancellation must save a resumable state and return the appropriate termination code. Dry-run must be genuinely read-only, showing planned work and estimated resources without downloading models or creating completion markers. + +An operator should be able to answer: “What will run?”, “Which artifacts/models are used?”, “What changed since the previous run?”, “Why did this trial rank here?”, “What failed?”, and “What exact command resumes it?” + +**Agent and clinical quality gates** + +Maintain separate states for clinical eligibility, evidence completeness, and execution status. Model explanations should be concise conclusions grounded in source facts and criterion IDs. Generated query text must not replace clinical evidence. Unknown information remains unknown. Let an agent perform bounded retrieval refinement, evidence lookup, question prioritization, and repair; record every tool input/output/version and stop at a declared budget. Additional reasoning is justified by measured improvement and consequential uncertainty, not by a default demand for longer CoT. + +Before claiming quality improvements, establish corrected baselines for all-topic recall/nDCG/eligible precision, per-criterion accuracy, false eligibility/exclusion, evidence fidelity, calibration, successful-run fraction, review time, and cost/latency. Thresholds are to be agreed from measured baselines and clinical error tolerance; no arbitrary production score is invented here. + +**Initial implementation status** + +P01 is implemented on the foundation branch with recorded local validation in +[the implementation checks](production-validation.md). Hosted checks and repository +protection configuration must be verified on the draft PR. No release or deployment +has been performed. P02–P14 remain planned work, including production GPU security +and actual model qualification. The local full model environment's dependency audit +reported 23 advisory rows across six packages on 12 September 2026, after the legacy +exceptions; some rows are repeated aliases. Do not describe that stack as security +qualified. A clean base/dev audit is a separate, narrower check. + +**Operational definition of done** + +A release is qualified only for its declared deployment profile. The maintainers need a supported OS/Python/GPU/model matrix, reproducible installation, tested data migrations, stable CLI/API contracts, release notes, observability dashboards/alerts, retention and access policy, encrypted storage/transport where applicable, backup/restore and rollback drills, capacity measurements, and a documented incident owner. For a shared clinical application, conduct usability/accessibility and clinical error review using representative users and data. + +The first implementation PR supplies a foundation and makes remaining gates visible. Production readiness remains an earned result of the later work packages and recorded acceptance evidence. diff --git a/docs/production-validation.md b/docs/production-validation.md new file mode 100644 index 00000000..70a654a4 --- /dev/null +++ b/docs/production-validation.md @@ -0,0 +1,37 @@ +# Foundation implementation validation + +Branch: `production/reliability-foundation`, based on `508db33`. Date: 12 September +2026. This records checks for the implementation, separately from the historical +[baseline review evidence](audit_2026_09_12_validation.md). + +- Full local pytest suite: **442 passed**, 41.56 seconds. Includes real CLI e2e. + Hugging Face and Transformers downloads were disabled. +- Fresh frozen base/dev environment: **440 passed, 1 skipped, 1 e2e deselected**; + its dependency audit found **no known vulnerabilities without exceptions**. + The skipped test requires an optional dependency covered by the separate import job. +- Frozen lockfile validation: passed with uv 0.11.24. +- Wheel and sdist build: passed. Packaged MedCPT and bge-m3 catalogs resolve. +- Isolated installed-wheel CLI smoke from `/tmp`, with checkout imports forbidden: + passed. Covers synthetic FHIR import, real LanceDB indexing, adult/pediatric + filtering, ranking, report content, resume preserving ranking output, and + public checksum verification. No model downloads or clinical data. +- Ruff, actionlint 1.7.12, and strict MkDocs build: passed. +- CPU imports of torch, Transformers, GLiNER2, embedding, reranking, reasoning, + and fine-tuning modules: passed in the existing local environment. This is not + actual GPU inference or an isolated optional-dependency qualification. +- Gitleaks scan: passed; repeated on staged files before committing. +- Existing full model environment dependency audit: **failed** with 23 advisory + rows across accelerate, mcp, mkdocs-material, setuptools, transformers, and vllm + after legacy exceptions (13 rows ignored). Some entries are repeated aliases. + GPU stack migration/requalification and docs/toolchain updates remain follow-up + work; no additional exceptions were added to hide this result. + +LanceDB connections hang in this session's restricted sandbox but succeed outside +it. Full pytest and installed-wheel e2e therefore ran outside that sandbox. The +historical audit's timeout is an environment limitation, not evidence of a LanceDB +product defect. + +The synthetic e2e demonstrates software integration, not retrieval benchmark +improvement or clinical validity. Published ranking metrics have not been rerun. +Hosted CI and release execution are separate from these local checks; no production +release, model migration, or application deployment was performed. diff --git a/docs/release.md b/docs/release.md index cd97fd01..b0495b54 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,47 +1,131 @@ -# Release Checklist +# Development and release runbook -TrialMatchAI uses a `src/` package layout and exposes the public package name `trialmatchai`. +The [production roadmap](production-roadmap.md) tracks the remaining clinical, +ranking, agent, and deployment work. These delivery gates establish the first +foundation; they do not qualify the GPU models or a clinical deployment. -## Local Checks +## Develop and exercise the CLI + +Use Python 3.11 and uv 0.11.24, matching CI. Work on a short-lived branch and keep +`uv.lock` committed. `uv sync --frozen` installs the declared development environment. ```bash -uv lock --check -uv run ruff check . -uv run pytest -uv build -uv run pre-commit run --all-files -uv run pip-audit --progress-spinner off \ - --ignore-vuln CVE-2025-3000 \ - --ignore-vuln CVE-2025-69872 \ - --ignore-vuln CVE-2026-53923 \ - --ignore-vuln CVE-2026-54236 \ - --ignore-vuln CVE-2026-12491 \ - --ignore-vuln CVE-2026-54235 \ - --ignore-vuln CVE-2026-54233 +uv sync --frozen +make lint +make test +make demo +make e2e ``` -Smoke test console commands: +`demo` creates a temporary workspace with synthetic trials and a FHIR patient, +then runs preparation, a real LanceDB index, import, matching, and HTML reporting. +It uses deterministic CPU embeddings and disables model stages. The output states +that eligibility reasoning is unavailable, identifies the report, and prints a +resume command. Use `trialmatchai demo --workdir /path/to/empty-directory` to choose +the location. `--resume` requires the original, unmodified fixture configuration; +create a new workspace to try a different configuration. Runtime state is retained +for inspection. No real patient data is used by this example or by CI. + +`trialmatchai --version` prints the package version. The new artifact commands +support `--json`: one JSON object on stdout, exit 0 for success or 1 for verification +failure. Argument errors return 2. Demo interruption returns 130. Existing commands +are being standardized separately in roadmap P10. + +## Verify transferred artifacts ```bash -uv run trialmatchai --help -uv run trialmatchai healthcheck --help -uv run trialmatchai bootstrap-data --help -uv run trialmatchai build --help -uv run trialmatchai index --help -uv run trialmatchai build-concepts --help -uv run trialmatchai update-registry --help -uv run trialmatchai import-patient --help -uv run trialmatchai run --help -uv run trialmatchai e2e --help -uv run trialmatchai trec --help -uv run trialmatchai finetune --help +trialmatchai artifacts manifest /path/to/artifacts --json +trialmatchai artifacts verify /path/to/artifacts/SHA256SUMS --require-exact --json ``` -Wheel install smoke: +The manifest uses GNU `sha256sum` text syntax. Paths are relative to its directory, +with nested directories allowed. Verification rejects missing or corrupt files, +unsafe paths, duplicate entries, and symlink artifacts. `--require-exact` also +rejects extra files. `--directory` selects the artifact root when the manifest is +stored elsewhere. A manifest hashes every regular file, including hidden files, +except itself; keep the directory limited to the intended deliverables. + +A checksum verifies content against an expected value. Obtain that value from a +trusted release or controlled artifact source; hashing an untrusted download does +not establish its publisher. Preserve build provenance alongside release digests. + +For bootstrap, obtain a trusted manifest listing `processed_trials.tar.gz` and +`criteria_part_0.zip` through `criteria_part_5.zip` (adjust to `--criteria-chunks`). +Include `models.tar.gz` with `--with-models` and `finetuning_datasets.zip` with +`--finetune-data`. ```bash -uv build -python -m venv /tmp/trialmatchai-wheel-smoke -/tmp/trialmatchai-wheel-smoke/bin/pip install dist/trialmatchai-*.whl -/tmp/trialmatchai-wheel-smoke/bin/python -c "import trialmatchai; print(trialmatchai.__version__)" +trialmatchai bootstrap-data --root /path/to/workspace \ + --checksum-manifest /path/to/trusted/SHA256SUMS ``` + +The manifest implies `--require-checksums`. Alternatively, strict mode accepts +`TRIALMATCHAI_PROCESSED_TRIALS_SHA256`, `TRIALMATCHAI_CRITERIA_PART__SHA256`, +`TRIALMATCHAI_MODELS_SHA256`, and `TRIALMATCHAI_FINETUNE_DATA_SHA256`. Every requested +digest must exist and contain exactly 64 hexadecimal characters before work starts. +Cached archives are hashed before extraction; mismatches fail. Successful extraction +records archive hashes in its completion marker. Strict mode re-extracts legacy +markers without this provenance. Resume trusts a matching extraction marker; it +does not rehash every extracted file. Atomic publication of complete corpus trees +and per-file corpus verification remain roadmap P05 work. Default bootstrap remains +compatible with legacy archives whose digests have not been supplied. + +## Local release rehearsal + +From a clean checkout with no distributions from older versions: + +```bash +make release-check +``` + +This validates workflows with actionlint and the lockfile, lints, runs tests, builds distributions, creates and +checks their manifest, installs the wheel in an isolated environment, exercises the +public CLI, scans for secrets, and audits installed dependencies. The installed +smoke verifies packaged embedder catalogs, synthetic patient import, a real index, +adult/pediatric filtering, ranked output, HTML content, resume, and checksum CLI. +Model downloads are disabled. The smoke is also callable via +`scripts/installed_smoke.py --workspace /empty/path --expect-installed` using an +installed environment's Python. + +The dependency advisory exceptions currently live in `Makefile`; CI calls `make +audit` so the lists cannot drift. The default audit covers installed base and dev +dependencies. Optional GPU dependency auditing, dated exception reviews, SBOMs, +and actual model load/infer checks still need P09/P14 qualification. CPU import +compatibility for the entity extra is required in CI. + +## CI and publishing + +`ci.yml` calls `verify.yml` on pull requests, supported branch pushes, merge groups, +and manual runs. Required verification includes frozen dependencies, tests, lint, +secret/dependency checks, model imports, a single package build, and the installed +wheel e2e. Jobs have timeouts; test reports and synthetic failure artifacts have +limited retention. The `verification / required` aggregate succeeds only when all +its dependencies succeed. Use the exact check name displayed by the first run when +configuring required status checks. + +Publishing a GitHub release calls the same reusable verification at that commit. +The tag must equal `v` and `pyproject.toml` must agree with `__version__`. +The build records the source commit, Python/uv versions, and lockfile hash. The +publisher downloads the previously tested artifacts, verifies the manifest, creates +[GitHub provenance](https://docs.github.com/en/actions/concepts/security/artifact-attestations), +and publishes the wheel/sdist using [PyPI OIDC](https://docs.pypi.org/trusted-publishers/). +It does not rebuild. Only the publish job receives write-capable identity permissions. +A failed verification blocks publication. Diagnose a partial publish before retrying; +do not overwrite an existing PyPI version. + +Repository administrators must configure these settings; workflow files do not +activate them automatically: + +- Protect `main` with reviewed pull requests and the aggregate verification check; + include merge-group checks if using a merge queue. Restrict bypass and force pushes. +- Protect the `pypi` environment, select permitted release refs, and configure its + trusted publisher for this repository and `release.yml` on PyPI. +- Review pinned Action updates from Dependabot. Use trusted isolated runners for + GPU jobs and keep them inaccessible to untrusted pull-request code. +- Define the staging/production target, immutable model/data versions, health + probes, access and retention policy, backup/restore, and rollback criteria before + enabling deployment. Package publication is not application deployment. + +Staging and production automation will be implemented for the chosen deployment +profile in P14. No deployment credentials, infrastructure, or clinical users are +assumed by the initial foundation PR. diff --git a/mkdocs.yml b/mkdocs.yml index 7a566fec..d15fa17d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -63,3 +63,5 @@ nav: - Fine-tuning: finetuning.md - API reference: api.md - Release process: release.md + - Production roadmap: production-roadmap.md + - Codebase review: codebase-review-2026-09-12.md diff --git a/pyproject.toml b/pyproject.toml index 3e8b0646..5f589524 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ where = ["src"] [tool.setuptools.package-data] trialmatchai = [ "config/*.json", + "config/catalog/*.json", "entity_schemas/*.yaml", "entities/resources/*.tsv", "interop/exporters/templates/*.html", diff --git a/pytest.ini b/pytest.ini index 175d379b..e3eecc2f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,4 @@ testpaths = tests addopts = -q markers = integration: tests that require external services or large models + e2e: synthetic public CLI end-to-end tests with a real local search database diff --git a/scripts/installed_smoke.py b/scripts/installed_smoke.py new file mode 100644 index 00000000..3c3e0dc0 --- /dev/null +++ b/scripts/installed_smoke.py @@ -0,0 +1,76 @@ +"""Exercise public CLI commands from an installed wheel, outside the checkout. + +Run with the installed environment's Python and --expect-installed in CI. +This checks the software path with synthetic CPU retrieval, not model accuracy. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from importlib import metadata, resources +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--expect-installed", action="store_true") + args = parser.parse_args() + root = args.workspace.resolve() + root.mkdir(parents=True, exist_ok=True) + + import trialmatchai + from trialmatchai.config.config_loader import load_config + + if args.expect_installed: + source = Path(__file__).resolve().parents[1] / "src" + if Path(trialmatchai.__file__).resolve().is_relative_to(source): + raise RuntimeError("Smoke test imported the checkout, not the installed wheel") + + command = Path(sys.executable).parent / "trialmatchai" + env = {key: value for key, value in os.environ.items() if not key.startswith("TRIALMATCHAI_")} + env.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1") + + def run(*arguments): + result = subprocess.run([str(command), *arguments], cwd=root, env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120) + print(result.stdout, end="", flush=True) + if result.returncode: + raise RuntimeError(f"CLI {arguments} returned {result.returncode}") + return result.stdout + + assert run("--version").strip() == f"trialmatchai {metadata.version('trialmatchai')}" + package = resources.files("trialmatchai") + for name, expected_model in (("medcpt", "ncbi/MedCPT-Article-Encoder"), ("bge-m3", "BAAI/bge-m3")): + config = json.loads(package.joinpath("config/config.json").read_text()) + config["embedder"] = {"model": name} + path = root / f"{name}.json" + path.write_text(json.dumps(config)) + resolved = load_config(path) + assert resolved["embedder"]["model_name"] == expected_model + assert Path(resolved["entity_extraction"]["schema_path"]).is_file() + + demo = root / "demo" + run("demo", "--workdir", str(demo)) + result = json.loads((demo / "demo-result.json").read_text()) + assert result["mode"] == "synthetic_retrieval_only" + assert result["ranked_trial_ids"][0] == "NCT00000001" + assert "NCT00000002" not in result["ranked_trial_ids"] # Pediatric age filter. + report = Path(result["report"]).read_text() + assert "demo-patient" in report and "NCT00000001" in report + assert '"reasoning_available": false' in report + ranked = demo / "results/demo-patient/ranked_trials.json" + original = (ranked.read_bytes(), ranked.stat().st_mtime_ns) + run("demo", "--workdir", str(demo), "--resume") + assert (ranked.read_bytes(), ranked.stat().st_mtime_ns) == original + assert json.loads((demo / "demo-result.json").read_text())["resumed"] is True + run("artifacts", "verify", str(demo / "SHA256SUMS"), "--json") + print("Installed CLI smoke passed: catalog, import, real index, retrieval, report, resume, checksums.") + + +if __name__ == "__main__": + main() diff --git a/src/trialmatchai/cli/artifacts.py b/src/trialmatchai/cli/artifacts.py new file mode 100644 index 00000000..3b0161e1 --- /dev/null +++ b/src/trialmatchai/cli/artifacts.py @@ -0,0 +1,40 @@ +"""Create and verify portable SHA-256 manifests for artifact promotion.""" + +from __future__ import annotations + +import argparse +import json +import sys + +from trialmatchai.utils.integrity import verify_manifest, write_manifest + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog="trialmatchai artifacts") + commands = parser.add_subparsers(dest="action", required=True) + create = commands.add_parser("manifest", help="Write SHA256SUMS for an artifact directory") + create.add_argument("directory") + create.add_argument("--json", action="store_true", help="Emit machine-readable output") + verify = commands.add_parser("verify", help="Verify every artifact in a SHA-256 manifest") + verify.add_argument("manifest") + verify.add_argument("--directory", default=None, help="Artifact root (default: manifest directory)") + verify.add_argument("--require-exact", action="store_true", help="Reject unlisted files too") + verify.add_argument("--json", action="store_true", help="Emit machine-readable output") + args = parser.parse_args(argv) + try: + if args.action == "manifest": + path = write_manifest(args.directory) + result = {"ok": True, "manifest": str(path)} + message = f"Wrote {path}" + else: + names = verify_manifest(args.manifest, directory=args.directory, require_exact=args.require_exact) + result = {"ok": True, "verified": names, "count": len(names)} + message = f"Verified {len(names)} artifact(s)." + except (OSError, ValueError) as exc: + if args.json: + print(json.dumps({"ok": False, "error": str(exc)})) + else: + print(f"Artifact verification failed: {exc}", file=sys.stderr) + return 1 + print(json.dumps(result) if args.json else message) + return 0 diff --git a/src/trialmatchai/cli/bootstrap_data.py b/src/trialmatchai/cli/bootstrap_data.py index 71a1531a..de6afa07 100644 --- a/src/trialmatchai/cli/bootstrap_data.py +++ b/src/trialmatchai/cli/bootstrap_data.py @@ -1,7 +1,7 @@ from __future__ import annotations import argparse -import hashlib +import json import os import stat import sys @@ -12,6 +12,9 @@ import requests +from trialmatchai.utils.file_utils import write_json_file +from trialmatchai.utils.integrity import normalize_sha256, read_manifest, verify_sha256 + DATA_URL = "https://zenodo.org/records/15516900/files/processed_trials.tar.gz?download=1" MODELS_URL = "https://zenodo.org/records/15516900/files/models.tar.gz?download=1" CRITERIA_ZIP_BASE_URL = "https://zenodo.org/records/15516900/files" @@ -78,6 +81,15 @@ def main(argv: Sequence[str] | None = None) -> int: action="store_true", help="Re-extract archives even when target directories already exist", ) + parser.add_argument( + "--checksum-manifest", type=Path, + help="Trusted SHA256SUMS file for requested archives; implies --require-checksums.", + ) + parser.add_argument( + "--require-checksums", action="store_true", + help="Require SHA-256 for every requested archive before any download/extraction. " + "Use --checksum-manifest or TRIALMATCHAI_*_SHA256 environment variables.", + ) args = parser.parse_args(argv) root = (args.root or _runtime_root()).resolve() @@ -91,6 +103,8 @@ def main(argv: Sequence[str] | None = None) -> int: finetune_data=args.finetune_data, finetune_data_url=args.finetune_data_url, force=args.force, + checksum_manifest=args.checksum_manifest, + require_checksums=args.require_checksums, ) return 0 @@ -106,14 +120,44 @@ def bootstrap_data( finetune_data: bool = False, finetune_data_url: str = FINETUNE_DATA_URL, force: bool = False, + checksum_manifest: str | Path | None = None, + require_checksums: bool = False, ) -> None: + if criteria_chunks < 1: + raise ValueError("criteria_chunks must be at least 1") + require_checksums = require_checksums or checksum_manifest is not None + supplied = read_manifest(checksum_manifest) if checksum_manifest is not None else {} + env_names = { + PROCESSED_TRIALS_ARCHIVE: "TRIALMATCHAI_PROCESSED_TRIALS_SHA256", + **{f"{CHUNK_PREFIX}_{i}.zip": f"TRIALMATCHAI_CRITERIA_PART_{i}_SHA256" for i in range(criteria_chunks)}, + } + if with_models: + env_names[MODELS_ARCHIVE] = "TRIALMATCHAI_MODELS_SHA256" + if finetune_data: + env_names[FINETUNE_ARCHIVE] = "TRIALMATCHAI_FINETUNE_DATA_SHA256" + checksums: dict[str, str | None] = {} + for name, env in env_names.items(): + value = supplied.get(name) or os.getenv(env) + checksums[name] = normalize_sha256(value) if value else None + missing = [name for name, value in checksums.items() if value is None] + if require_checksums and missing: + raise ValueError(f"Missing required SHA-256 checksums: {', '.join(missing)}") + + def stage_checksums(names): + return {name: checksums[name] for name in names if checksums[name] is not None} + + def complete(path, expected): + return _extract_complete(path, expected_checksums=expected if require_checksums else None) + data_dir = root / "data" models_dir = root / "models" data_dir.mkdir(parents=True, exist_ok=True) criteria_dir = data_dir / "processed_criteria" - if force or not _extract_complete(criteria_dir): + criteria_sums = stage_checksums(f"{CHUNK_PREFIX}_{i}.zip" for i in range(criteria_chunks)) + if force or not complete(criteria_dir, criteria_sums): criteria_dir.mkdir(parents=True, exist_ok=True) + (criteria_dir / _EXTRACT_MARKER).unlink(missing_ok=True) for index in range(criteria_chunks): chunk_name = f"{CHUNK_PREFIX}_{index}.zip" chunk_path = data_dir / chunk_name @@ -123,41 +167,47 @@ def bootstrap_data( ) _verify_sha256( chunk_path, - os.getenv(f"TRIALMATCHAI_CRITERIA_PART_{index}_SHA256"), + checksums[chunk_name], ) _safe_extract_zip(chunk_path, criteria_dir) - _mark_extract_complete(criteria_dir) + _mark_extract_complete(criteria_dir, checksums=criteria_sums) processed_trials_dir = data_dir / "processed_trials" - if force or not _extract_complete(processed_trials_dir): + trial_sums = stage_checksums([PROCESSED_TRIALS_ARCHIVE]) + if force or not complete(processed_trials_dir, trial_sums): + (processed_trials_dir / _EXTRACT_MARKER).unlink(missing_ok=True) processed_archive = data_dir / PROCESSED_TRIALS_ARCHIVE _download_if_missing(data_url, processed_archive) _verify_sha256( - processed_archive, os.getenv("TRIALMATCHAI_PROCESSED_TRIALS_SHA256") + processed_archive, checksums[PROCESSED_TRIALS_ARCHIVE] ) _safe_extract_tar_gz(processed_archive, data_dir) - _mark_extract_complete(processed_trials_dir) + _mark_extract_complete(processed_trials_dir, checksums=trial_sums) if with_models: models_dir.mkdir(parents=True, exist_ok=True) - if force or not _extract_complete(models_dir): + model_sums = stage_checksums([MODELS_ARCHIVE]) + if force or not complete(models_dir, model_sums): + (models_dir / _EXTRACT_MARKER).unlink(missing_ok=True) models_archive = data_dir / MODELS_ARCHIVE _download_if_missing(models_url, models_archive) - _verify_sha256(models_archive, os.getenv("TRIALMATCHAI_MODELS_SHA256")) + _verify_sha256(models_archive, checksums[MODELS_ARCHIVE]) _safe_extract_tar_gz(models_archive, models_dir) - _mark_extract_complete(models_dir) + _mark_extract_complete(models_dir, checksums=model_sums) if finetune_data: finetune_dir = data_dir / "finetune" - if force or not _extract_complete(finetune_dir): + finetune_sums = stage_checksums([FINETUNE_ARCHIVE]) + if force or not complete(finetune_dir, finetune_sums): finetune_dir.mkdir(parents=True, exist_ok=True) + (finetune_dir / _EXTRACT_MARKER).unlink(missing_ok=True) finetune_archive = data_dir / FINETUNE_ARCHIVE _download_if_missing(finetune_data_url, finetune_archive) _verify_sha256( - finetune_archive, os.getenv("TRIALMATCHAI_FINETUNE_DATA_SHA256") + finetune_archive, checksums[FINETUNE_ARCHIVE] ) _safe_extract_zip(finetune_archive, finetune_dir) - _mark_extract_complete(finetune_dir) + _mark_extract_complete(finetune_dir, checksums=finetune_sums) _cleanup_archives(data_dir, criteria_chunks) @@ -186,15 +236,7 @@ def _verify_sha256(path: Path, expected: str | None) -> None: _warn(f"No SHA-256 checksum configured for {path.name}; skipping verification.") return - digest = hashlib.sha256() - with path.open("rb") as file: - for chunk in iter(lambda: file.read(1024 * 1024), b""): - digest.update(chunk) - actual = digest.hexdigest() - if actual != expected: - raise ValueError( - f"Checksum mismatch for {path}: expected {expected}, got {actual}" - ) + verify_sha256(path, expected) def _safe_extract_tar_gz(archive: Path, target: Path) -> None: @@ -245,24 +287,27 @@ def _cleanup_archives(data_dir: Path, criteria_chunks: int) -> None: (data_dir / f"{CHUNK_PREFIX}_{index}.zip").unlink(missing_ok=True) -def _has_entries(path: Path) -> bool: - return path.exists() and any(path.iterdir()) - - _EXTRACT_MARKER = ".bootstrap_complete" -def _extract_complete(path: Path) -> bool: +def _extract_complete(path: Path, *, expected_checksums: dict[str, str] | None = None) -> bool: """True only when a prior extract wrote its completion sentinel. Presence of *some* entries is not proof: a killed extract leaves a partial tree that would otherwise bake a truncated corpus into every later run. """ - return (path / _EXTRACT_MARKER).exists() + marker = path / _EXTRACT_MARKER + if expected_checksums is None: + return marker.is_file() + try: + data = json.loads(marker.read_text(encoding="utf-8")) + return isinstance(data, dict) and data.get("schema") == 1 and data.get("sha256") == expected_checksums + except (OSError, ValueError): + return False -def _mark_extract_complete(path: Path) -> None: - (path / _EXTRACT_MARKER).write_text("ok\n", encoding="utf-8") +def _mark_extract_complete(path: Path, *, checksums: dict[str, str] | None = None) -> None: + write_json_file({"schema": 1, "sha256": checksums or {}}, str(path / _EXTRACT_MARKER)) def _runtime_root() -> Path: diff --git a/src/trialmatchai/cli/demo.py b/src/trialmatchai/cli/demo.py new file mode 100644 index 00000000..b337f2e3 --- /dev/null +++ b/src/trialmatchai/cli/demo.py @@ -0,0 +1,135 @@ +"""A repeatable synthetic CLI walkthrough using the real CPU search pipeline.""" + +from __future__ import annotations + +import argparse +import json +import shlex +import sys +import tempfile +from importlib import resources +from pathlib import Path + +from trialmatchai.utils.file_utils import write_json_file +from trialmatchai.utils.integrity import verify_manifest, write_manifest + +_MARKER = ".trialmatchai-demo.json" +_FIXTURE_VERSION = 1 + + +def _create_workspace(root: Path) -> dict: + from trialmatchai.config.settings import TrialMatchSettings + + package = resources.files("trialmatchai") + config = json.loads(package.joinpath("config/config.json").read_text(encoding="utf-8")) + data = root / "data" + config["paths"] = {"output_dir": str(root / "results"), "trials_json_folder": str(data / "trials_jsons")} + config["patient_inputs"].update({ + "raw_dir": str(data / "patients/raw"), "profile_dir": str(data / "patients/profiles"), + "summary_dir": str(data / "patients/summaries"), "strict_validation": True, + }) + config["entity_extraction"].update({ + "backend": "disabled", "model_name": "demo-unused", "device": "cpu", + "schema_path": str(package.joinpath("entity_schemas/trialmatchai.yaml")), + }) + config["concept_linker"].update({"enabled": False, "db_path": str(data / "concepts")}) + config["search_backend"].update({"db_path": str(data / "search"), "candidate_limit": 50}) + config["embedder"] = {"backend": "hashing", "model_name": "hashing", "use_gpu": False, + "hashing_dimensions": 64, "normalize": True} + config["model"].update({"base_model": "demo-unused", "reranker_model_path": "demo-unused", + "cot_adapter_path": None, "reranker_adapter_path": ""}) + config["global"]["device"] = "cpu" + config["LLM_reranker"]["enabled"] = False + config["rag"]["enabled"] = False + config["use_cot_reasoning"] = False + config["query_expansion"]["enabled"] = False + config["search"].update({"mode": "bm25", "second_level_keep_divisor": 1}) + config["registry"].update({"raw_dir": str(data / "registry/raw"), + "reports_dir": str(data / "registry/reports"), + "manifest_path": str(data / "registry/manifest.jsonl")}) + # The demo deliberately ignores deployment env overrides and .env files. + # Its files and database must stay inside this synthetic workspace. + config = TrialMatchSettings.model_validate(config).to_dict() + write_json_file(config, str(root / "config.json")) + for number, condition, title, minimum, maximum in ( + (1, "lung cancer", "SYNTHETIC adult lung cancer study", "18 Years", "120 Years"), + (2, "lung cancer", "SYNTHETIC pediatric lung cancer study", "0 Years", "12 Years"), + (3, "diabetes", "SYNTHETIC adult diabetes study", "18 Years", "120 Years"), + ): + nct_id = f"NCT{number:08d}" + write_json_file({ + "nct_id": nct_id, "brief_title": title, "brief_summary": f"Synthetic research on {condition}.", + "condition": condition, "overall_status": "RECRUITING", "gender": "All", + "minimum_age": minimum, "maximum_age": maximum, + "eligibility_criteria": f"Inclusion Criteria:\n- Diagnosis of {condition}.\n- Age within the study limits.", + }, str(data / "trials_jsons" / f"{nct_id}.json")) + write_json_file({"resourceType": "Bundle", "type": "collection", "entry": [ + {"resource": {"resourceType": "Patient", "id": "demo-patient", "gender": "female", "birthDate": "1980-01-01"}}, + {"resource": {"resourceType": "Condition", "id": "demo-condition", + "subject": {"reference": "Patient/demo-patient"}, "code": {"text": "lung cancer"}}}, + ]}, str(root / "patient.fhir.json")) + write_json_file({"fixture_version": _FIXTURE_VERSION}, str(root / _MARKER)) + # Covers the initial fixture/config inputs, not the runtime outputs added later. + write_manifest(root) + return config + + +def run_demo(workdir: str | Path | None = None, *, resume: bool = False) -> dict: + root = Path(workdir).resolve() if workdir is not None else Path(tempfile.mkdtemp(prefix="trialmatchai-demo-")) + if resume: + marker = root / _MARKER + if not marker.is_file() or json.loads(marker.read_text()) != {"fixture_version": _FIXTURE_VERSION}: + raise ValueError("--resume requires an existing TrialMatchAI demo workspace of this version") + verify_manifest(root / "SHA256SUMS") + config = json.loads((root / "config.json").read_text(encoding="utf-8")) + # A copied workspace's absolute config paths must not write back to its origin. + if config.get("paths", {}).get("output_dir") != str(root / "results"): + raise ValueError("Demo workspace moved; create a new demo in an empty directory") + else: + if root.exists() and any(root.iterdir()): + raise ValueError(f"Directory is not empty: {root}. Choose an empty directory or use --resume for an existing demo.") + root.mkdir(parents=True, exist_ok=True) + config = _create_workspace(root) + + from trialmatchai.pipeline import StageContext, run_pipeline + + print(f"Synthetic CPU demo: {root}\nThis run tests retrieval and reporting; no eligibility model is used.", flush=True) + ctx = StageContext( + config=config, processed_trials_folder=root / "data/processed_trials", + processed_criteria_folder=root / "data/processed_criteria", + inputs=[str(root / "patient.fhir.json")], input_format="fhir", with_entities=False, + ) + rc = run_pipeline(ctx, only=["prepare", "index", "ingest", "match"]) + if rc: + raise RuntimeError(f"Demo pipeline failed with exit code {rc}") + patient_dir = root / "results/demo-patient" + ranked = json.loads((patient_dir / "ranked_trials.json").read_text())["RankedTrials"] + result = { + "schema_version": 1, "mode": "synthetic_retrieval_only", "workspace": str(root), + "config": str(root / "config.json"), "report": str(patient_dir / "report.html"), + "ranked_trial_ids": [row["TrialID"] for row in ranked], "resumed": resume, + } + if not ranked or not Path(result["report"]).is_file(): + raise RuntimeError("Demo did not produce a non-empty ranking and HTML report") + write_json_file(result, str(root / "demo-result.json")) + return result + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog="trialmatchai demo", description=__doc__) + parser.add_argument("--workdir", type=Path, help="Empty workspace directory (default: a new temporary directory)") + parser.add_argument("--resume", action="store_true", help="Resume an existing, unmodified demo workspace") + args = parser.parse_args(argv) + if args.resume and args.workdir is None: + parser.error("--resume requires --workdir") + try: + result = run_demo(args.workdir, resume=args.resume) + except KeyboardInterrupt: + print("Demo interrupted. Resume with the same --workdir and --resume.", file=sys.stderr) + return 130 + except (OSError, ValueError, RuntimeError) as exc: + print(f"Demo failed: {exc}", file=sys.stderr) + return 1 + print(f"Report: {result['report']}\nRun details: {Path(result['workspace']) / 'demo-result.json'}") + print(f"To resume: trialmatchai demo --workdir {shlex.quote(result['workspace'])} --resume") + return 0 diff --git a/src/trialmatchai/cli/main.py b/src/trialmatchai/cli/main.py index 8280256f..0ab75287 100644 --- a/src/trialmatchai/cli/main.py +++ b/src/trialmatchai/cli/main.py @@ -9,8 +9,14 @@ def main() -> int: prog="trialmatchai", description="TrialMatchAI command group.", ) + from trialmatchai import __version__ + + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("artifacts", help="Create or verify artifact checksums", add_help=False) + subparsers.add_parser("demo", help="Run a synthetic CPU example in an isolated workspace", add_help=False) + subparsers.add_parser( "pipeline", help="Run the unified e2e pipeline, or any slice (--only/--from/--to/--skip/--force)", @@ -70,7 +76,11 @@ def main() -> int: ) args, remainder = parser.parse_known_args() - if args.command == "pipeline": + if args.command == "artifacts": + from trialmatchai.cli.artifacts import main as command + elif args.command == "demo": + from trialmatchai.cli.demo import main as command + elif args.command == "pipeline": from trialmatchai.cli.pipeline import main as command elif args.command == "healthcheck": from trialmatchai.cli.healthcheck import main as command diff --git a/src/trialmatchai/interop/importers/fhir.py b/src/trialmatchai/interop/importers/fhir.py index f8b30323..58ae28e8 100644 --- a/src/trialmatchai/interop/importers/fhir.py +++ b/src/trialmatchai/interop/importers/fhir.py @@ -210,6 +210,9 @@ def _profile_for_resource( for candidate in _reference_candidates(reference): if candidate in profiles_by_reference: return profiles_by_reference[candidate] + # An explicit foreign/unresolved subject must never be attributed to the + # sole patient. Only a genuinely absent subject permits that fallback. + return None if len(profiles) == 1: return profiles[0] return None diff --git a/src/trialmatchai/services/preflight.py b/src/trialmatchai/services/preflight.py index c66ca1c5..f17e2bec 100644 --- a/src/trialmatchai/services/preflight.py +++ b/src/trialmatchai/services/preflight.py @@ -122,11 +122,13 @@ def run_preflight_checks( # Check gated base models up front so an HF auth failure surfaces here, not # after first-level search. - issues.extend( - check_hf_access( - [model_cfg.get("base_model"), model_cfg.get("reranker_model_path")] - ) - ) + requested_models = [] + if rag_enabled: + requested_models.append(model_cfg.get("base_model")) + if reranker_enabled: + requested_models.append(model_cfg.get("reranker_model_path")) + if requested_models: + issues.extend(check_hf_access(requested_models)) search_cfg = config.get("search_backend", {}) if search_cfg: diff --git a/src/trialmatchai/utils/integrity.py b/src/trialmatchai/utils/integrity.py new file mode 100644 index 00000000..6b5efb07 --- /dev/null +++ b/src/trialmatchai/utils/integrity.py @@ -0,0 +1,117 @@ +"""SHA-256 manifests for transferred artifacts (not publisher authentication).""" + +from __future__ import annotations + +import hashlib +import hmac +import re +from pathlib import Path, PurePosixPath + +from trialmatchai.utils.file_utils import write_text_file + + +def normalize_sha256(value: str) -> str: + if not re.fullmatch(r"[0-9a-fA-F]{64}", value): + raise ValueError("Expected a SHA-256 digest containing exactly 64 hexadecimal characters") + return value.lower() + + +def sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_sha256(path: str | Path, expected: str) -> str: + expected = normalize_sha256(expected) + actual = sha256_file(path) + if not hmac.compare_digest(actual, expected): + raise ValueError(f"Checksum mismatch for {Path(path).name}: expected {expected}, got {actual}") + return actual + + +def _validate_name(name: str) -> str: + path = PurePosixPath(name) + if ( + not name or path.is_absolute() or "\\" in name or ":" in name + or any(part in {"", ".", ".."} for part in name.split("/")) + or any(ord(char) < 32 for char in name) + ): + raise ValueError(f"Unsafe artifact path in checksum manifest: {name!r}") + return name + + +def read_manifest(path: str | Path) -> dict[str, str]: + """Read the portable GNU sha256sum text format; reject ambiguity/traversal.""" + entries: dict[str, str] = {} + for number, line in enumerate(Path(path).read_text(encoding="utf-8").splitlines(), 1): + if not line.strip() or line.startswith("#"): + continue + match = re.fullmatch(r"([0-9a-fA-F]{64}) [ *](.+)", line) + if match is None: + raise ValueError(f"Invalid SHA-256 manifest line {number}") + name = _validate_name(match.group(2)) + if name in entries: + raise ValueError(f"Duplicate artifact in checksum manifest: {name}") + entries[name] = normalize_sha256(match.group(1)) + if not entries: + raise ValueError("Checksum manifest contains no artifacts") + return entries + + +def _artifact_path(root: Path, name: str) -> Path: + candidate = root + for part in PurePosixPath(_validate_name(name)).parts: + candidate = candidate / part + if candidate.is_symlink(): + raise ValueError(f"Artifact must not be a symlink: {name}") + if not candidate.resolve().is_relative_to(root): + raise ValueError(f"Artifact escapes its directory: {name}") + if not candidate.is_file(): + raise ValueError(f"Artifact is missing or is not a regular file: {name}") + return candidate + + +def write_manifest(directory: str | Path, *, filename: str = "SHA256SUMS") -> Path: + root = Path(directory).resolve() + if not root.is_dir(): + raise ValueError(f"Artifact directory does not exist: {root}") + if len(PurePosixPath(_validate_name(filename)).parts) != 1: + raise ValueError("Manifest filename must be a single filename") + manifest = root / filename + if manifest.is_symlink(): + raise ValueError("Checksum manifest must not be a symlink") + lines = [] + for path in sorted(root.rglob("*")): + if path == manifest: + continue + if path.is_symlink(): + raise ValueError(f"Artifact must not be a symlink: {path.relative_to(root)}") + if path.is_file(): + name = _validate_name(path.relative_to(root).as_posix()) + lines.append(f"{sha256_file(path)} {name}") + if not lines: + raise ValueError("No artifacts to checksum") + write_text_file(lines, str(manifest)) + return manifest + + +def verify_manifest( + manifest: str | Path, *, directory: str | Path | None = None, require_exact: bool = False, +) -> list[str]: + manifest = Path(manifest).resolve() + root = Path(directory).resolve() if directory is not None else manifest.parent + entries = read_manifest(manifest) + for name, expected in entries.items(): + verify_sha256(_artifact_path(root, name), expected) + if require_exact: + found = { + path.relative_to(root).as_posix() + for path in root.rglob("*") + if (path.is_file() or path.is_symlink()) and path != manifest + } + if found != set(entries): + raise ValueError(f"Unlisted artifacts: {sorted(found - set(entries))}") + return sorted(entries) diff --git a/tests/test_artifact_integrity.py b/tests/test_artifact_integrity.py new file mode 100644 index 00000000..b4bf191b --- /dev/null +++ b/tests/test_artifact_integrity.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import json + +import pytest + +from trialmatchai.cli.artifacts import main +from trialmatchai.utils.integrity import read_manifest, verify_manifest, write_manifest + + +def test_manifest_round_trip_and_corruption(tmp_path): + (tmp_path / "model.bin").write_bytes(b"synthetic weights\x00\xff") + (tmp_path / "nested").mkdir() + (tmp_path / "nested/data file.json").write_text('{"synthetic": true}') + manifest = write_manifest(tmp_path) + assert verify_manifest(manifest, require_exact=True) == ["model.bin", "nested/data file.json"] + (tmp_path / "model.bin").write_bytes(b"corrupt") + with pytest.raises(ValueError, match="Checksum mismatch"): + verify_manifest(manifest) + + +@pytest.mark.parametrize("name", ["../escape", "/absolute", "a/../../escape", "a/./b", "a//b", r"a\b", "C:/secret"]) +def test_manifest_rejects_unsafe_paths(tmp_path, name): + path = tmp_path / "SHA256SUMS" + path.write_text(f"{'0' * 64} {name}\n") + with pytest.raises(ValueError, match="Unsafe artifact path"): + read_manifest(path) + + +def test_manifest_rejects_duplicate_and_invalid_entries(tmp_path): + path = tmp_path / "SHA256SUMS" + path.write_text(f"{'0' * 64} weights.bin\n{'1' * 64} weights.bin\n") + with pytest.raises(ValueError, match="Duplicate"): + read_manifest(path) + path.write_text("not-a-checksum weights.bin\n") + with pytest.raises(ValueError, match="line 1"): + read_manifest(path) + + +def test_exact_manifest_rejects_unlisted_files(tmp_path): + (tmp_path / "package.whl").write_text("wheel") + manifest = write_manifest(tmp_path) + (tmp_path / "unexpected.whl").write_text("other wheel") + with pytest.raises(ValueError, match="Unlisted artifacts"): + verify_manifest(manifest, require_exact=True) + + +def test_verification_rejects_symlink_substitution(tmp_path): + weights = tmp_path / "weights.bin" + weights.write_text("weights") + manifest = write_manifest(tmp_path) + weights.rename(tmp_path / "renamed.bin") + weights.symlink_to(tmp_path / "renamed.bin") + with pytest.raises(ValueError, match="symlink"): + verify_manifest(manifest) + + +def test_artifact_cli_json_exit_status(tmp_path, capsys): + artifact = tmp_path / "package.whl" + artifact.write_text("wheel") + assert main(["manifest", str(tmp_path), "--json"]) == 0 + result = json.loads(capsys.readouterr().out) + assert main(["verify", result["manifest"], "--require-exact", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["count"] == 1 + artifact.write_text("bad wheel") + assert main(["verify", result["manifest"], "--json"]) == 1 + assert json.loads(capsys.readouterr().out)["ok"] is False diff --git a/tests/test_bootstrap_data.py b/tests/test_bootstrap_data.py index 532d293b..84018b06 100644 --- a/tests/test_bootstrap_data.py +++ b/tests/test_bootstrap_data.py @@ -73,6 +73,50 @@ def test_verify_sha256_rejects_mismatches(tmp_path): _verify_sha256(path, "0" * 64) +def test_strict_bootstrap_requires_all_checksums_before_writing(tmp_path, monkeypatch): + monkeypatch.delenv("TRIALMATCHAI_PROCESSED_TRIALS_SHA256", raising=False) + monkeypatch.delenv("TRIALMATCHAI_CRITERIA_PART_0_SHA256", raising=False) + with pytest.raises(ValueError, match="Missing required SHA-256"): + bootstrap_data(root=tmp_path, criteria_chunks=1, require_checksums=True) + assert not (tmp_path / "data").exists() + + +def test_strict_bootstrap_verifies_cached_archives_and_resume_provenance(tmp_path, monkeypatch): + from trialmatchai.cli.bootstrap_data import _extract_complete + from trialmatchai.utils.integrity import write_manifest + + data = tmp_path / "data" + data.mkdir() + _write_tar_gz(data / PROCESSED_TRIALS_ARCHIVE, {"processed_trials/NCT00000001.json": b"{}"}) + _write_zip(data / "criteria_part_0.zip", {"c1.json": "{}"}) + manifest = write_manifest(data) + sums = {"criteria_part_0.zip": _sha256(data / "criteria_part_0.zip")} + bootstrap_data(root=tmp_path, criteria_chunks=1, checksum_manifest=manifest) + assert _extract_complete(data / "processed_criteria", expected_checksums=sums) + monkeypatch.setattr("requests.get", lambda *a, **kw: pytest.fail("Verified resume must not download")) + bootstrap_data(root=tmp_path, criteria_chunks=1, checksum_manifest=manifest) + assert not _extract_complete(data / "processed_criteria", expected_checksums={"criteria_part_0.zip": "0" * 64}) + + +def test_legacy_marker_does_not_count_as_verified(tmp_path): + from trialmatchai.cli.bootstrap_data import _extract_complete + + (tmp_path / ".bootstrap_complete").write_text("ok\n") + assert not _extract_complete(tmp_path, expected_checksums={"archive.zip": "0" * 64}) + + +def test_strict_bootstrap_corruption_prevents_extraction(tmp_path, monkeypatch): + data = tmp_path / "data" + data.mkdir() + (data / "criteria_part_0.zip").write_bytes(b"corrupt") + monkeypatch.setenv("TRIALMATCHAI_PROCESSED_TRIALS_SHA256", "0" * 64) + monkeypatch.setenv("TRIALMATCHAI_CRITERIA_PART_0_SHA256", "0" * 64) + monkeypatch.setattr("trialmatchai.cli.bootstrap_data._safe_extract_zip", lambda *a: pytest.fail("Must not extract corrupt data")) + with pytest.raises(ValueError, match="Checksum mismatch"): + bootstrap_data(root=tmp_path, criteria_chunks=1, require_checksums=True) + assert not (data / "processed_criteria/.bootstrap_complete").exists() + + def test_tar_extraction_rejects_path_traversal(tmp_path): archive = tmp_path / "unsafe.tar.gz" with tarfile.open(archive, "w:gz") as tar: diff --git a/tests/test_cli_e2e.py b/tests/test_cli_e2e.py new file mode 100644 index 00000000..4e5933d4 --- /dev/null +++ b/tests/test_cli_e2e.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +@pytest.mark.e2e +def test_public_cli_synthetic_workflow(tmp_path): + script = Path(__file__).resolve().parents[1] / "scripts/installed_smoke.py" + env = dict(os.environ, HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1") + result = subprocess.run([sys.executable, str(script), "--workspace", str(tmp_path)], + env=env, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=180) + assert result.returncode == 0, result.stdout + + +def test_demo_refuses_to_overwrite_existing_data(tmp_path): + from trialmatchai.cli.demo import run_demo + + original = tmp_path / "patient.txt" + original.write_text("do not change") + with pytest.raises(ValueError, match="not empty"): + run_demo(tmp_path) + assert original.read_text() == "do not change" + + +def test_demo_resume_requires_its_own_workspace(tmp_path): + from trialmatchai.cli.demo import run_demo + + with pytest.raises(ValueError, match="existing TrialMatchAI demo"): + run_demo(tmp_path, resume=True) diff --git a/tests/test_deployment_readiness.py b/tests/test_deployment_readiness.py index 898abfa9..aa253281 100644 --- a/tests/test_deployment_readiness.py +++ b/tests/test_deployment_readiness.py @@ -110,3 +110,14 @@ def test_json_extraction_rejects_malformed_output(): assert "Unbalanced JSON object" in str(exc) else: raise AssertionError("Malformed output should fail JSON extraction") + + +def test_disabled_llms_do_not_check_model_access(tmp_path, monkeypatch): + from trialmatchai.services import preflight + + monkeypatch.setattr(preflight, "check_hf_access", lambda *a: (_ for _ in ()).throw(AssertionError("Disabled models must not contact the Hub"))) + assert preflight.run_preflight_checks({ + "paths": {"output_dir": str(tmp_path / "results")}, + "LLM_reranker": {"enabled": False}, "rag": {"enabled": False}, + "model": {"base_model": "org/private-model", "reranker_model_path": "org/private-reranker"}, + }, require_models=True) == [] diff --git a/tests/test_fhir_robustness.py b/tests/test_fhir_robustness.py index 7b0b8ac2..84fb7263 100644 --- a/tests/test_fhir_robustness.py +++ b/tests/test_fhir_robustness.py @@ -4,6 +4,8 @@ import json +import pytest + from trialmatchai.interop.importers.fhir import ( _medication_codes_label, _resource_disposition, @@ -227,3 +229,28 @@ def test_fhir_ndjson_strict_raises_on_malformed_line(tmp_path): assert len(profiles) >= 1 with pytest.raises(Exception): import_fhir(path, input_format="fhir-ndjson", strict=True) + + +@pytest.mark.parametrize("strict", [False, True]) +def test_foreign_subject_never_falls_back_to_only_patient(tmp_path, strict): + path = tmp_path / "mixed.json" + path.write_text(json.dumps({"resourceType": "Bundle", "entry": [ + {"resource": {"resourceType": "Patient", "id": "patient-A"}}, + {"resource": {"resourceType": "Condition", "id": "foreign-condition", + "subject": {"reference": "Patient/patient-B"}, "code": {"text": "diabetes"}}}, + ]})) + if strict: + with pytest.raises(ValueError, match="no resolvable patient reference"): + import_fhir(path, strict=True) + else: + profile = import_fhir(path)[0] + assert profile.conditions == [] + assert profile.unsupported[0]["id"] == "foreign-condition" + + +def test_absent_subject_retains_single_patient_fallback(tmp_path): + profile = _import_bundle(tmp_path, [ + {"resourceType": "Patient", "id": "patient-A"}, + {"resourceType": "Condition", "code": {"text": "asthma"}}, + ])[0] + assert [fact.label for fact in profile.conditions] == ["asthma"] From e94d2ccdf6e633e595d3765bc9862e6652f94dce Mon Sep 17 00:00:00 2001 From: Majd Abdallah Date: Sat, 12 Sep 2026 16:31:52 +0200 Subject: [PATCH 2/3] fix: prevent checksum redirection and make demo initialization atomic --- CHANGELOG.md | 3 ++ docs/production-validation.md | 10 ++-- docs/release.md | 6 ++- src/trialmatchai/cli/demo.py | 31 +++++++++--- src/trialmatchai/utils/integrity.py | 8 ++- tests/test_artifact_integrity.py | 29 +++++++++++ tests/test_cli_e2e.py | 77 +++++++++++++++++++++++++++++ 7 files changed, 150 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 346a4f03..7564e3c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,9 @@ All notable changes to TrialMatchAI are documented here. The format follows so they are a deliberate re-baselining exercise rather than a patch. ### Fixed +- Checksum verification rejects manifest symlinks that could redirect verification + to another artifact directory. Interrupted demo initialization leaves its chosen + directory ready for a normal retry; completed initialization supports `--resume`. - Embedder catalog resources are included in built wheels; Make targets use the unified CLI. Disabled model stages no longer trigger matching Hub access checks. - **`pip` upgraded to 26.2.1** so CI's dependency audit passes (PYSEC-2026-3721). `pip` is diff --git a/docs/production-validation.md b/docs/production-validation.md index 70a654a4..a8cd43f8 100644 --- a/docs/production-validation.md +++ b/docs/production-validation.md @@ -4,11 +4,15 @@ Branch: `production/reliability-foundation`, based on `508db33`. Date: 12 Septem 2026. This records checks for the implementation, separately from the historical [baseline review evidence](audit_2026_09_12_validation.md). -- Full local pytest suite: **442 passed**, 41.56 seconds. Includes real CLI e2e. +- Full local pytest suite after review fixes: **452 passed**, 47.34 seconds. + Includes real CLI e2e and 10 regression cases for manifest redirection, + interrupted initialization, safe publication, and retry instructions. Hugging Face and Transformers downloads were disabled. -- Fresh frozen base/dev environment: **440 passed, 1 skipped, 1 e2e deselected**; +- Initial frozen base/dev check, before review fixes: **440 passed, 1 skipped, 1 e2e deselected**; its dependency audit found **no known vulnerabilities without exceptions**. - The skipped test requires an optional dependency covered by the separate import job. + The skipped test compares the package version with an adjacent `pyproject.toml`, + which is absent in this non-editable install. The package job checks version + consistency from the source checkout separately. - Frozen lockfile validation: passed with uv 0.11.24. - Wheel and sdist build: passed. Packaged MedCPT and bge-m3 catalogs resolve. - Isolated installed-wheel CLI smoke from `/tmp`, with checkout imports forbidden: diff --git a/docs/release.md b/docs/release.md index b0495b54..7769876f 100644 --- a/docs/release.md +++ b/docs/release.md @@ -22,7 +22,9 @@ then runs preparation, a real LanceDB index, import, matching, and HTML reportin It uses deterministic CPU embeddings and disables model stages. The output states that eligibility reasoning is unavailable, identifies the report, and prints a resume command. Use `trialmatchai demo --workdir /path/to/empty-directory` to choose -the location. `--resume` requires the original, unmodified fixture configuration; +the location. Initialization publishes its fixtures atomically; if it is interrupted +before completion, retry the same command without `--resume`. Once initialized, +`--resume` requires the original, unmodified fixture configuration; create a new workspace to try a different configuration. Runtime state is retained for inspection. No real patient data is used by this example or by CI. @@ -39,7 +41,7 @@ trialmatchai artifacts verify /path/to/artifacts/SHA256SUMS --require-exact --js ``` The manifest uses GNU `sha256sum` text syntax. Paths are relative to its directory, -with nested directories allowed. Verification rejects missing or corrupt files, +with nested directories allowed. Verification rejects symlinked manifests, missing or corrupt files, unsafe paths, duplicate entries, and symlink artifacts. `--require-exact` also rejects extra files. `--directory` selects the artifact root when the manifest is stored elsewhere. A manifest hashes every regular file, including hidden files, diff --git a/src/trialmatchai/cli/demo.py b/src/trialmatchai/cli/demo.py index b337f2e3..f7a234cc 100644 --- a/src/trialmatchai/cli/demo.py +++ b/src/trialmatchai/cli/demo.py @@ -18,6 +18,17 @@ def _create_workspace(root: Path) -> dict: + # Build beside the destination so publishing is one same-filesystem rename. + # A failed or interrupted setup leaves the requested directory empty; retry + # can start normally without trusting partially written ownership markers. + with tempfile.TemporaryDirectory(prefix=f".{root.name}-init-", dir=root.parent) as temp: + staging = Path(temp) + config = _write_workspace(root, destination=staging) + staging.replace(root) + return config + + +def _write_workspace(root: Path, *, destination: Path) -> dict: from trialmatchai.config.settings import TrialMatchSettings package = resources.files("trialmatchai") @@ -50,7 +61,7 @@ def _create_workspace(root: Path) -> dict: # The demo deliberately ignores deployment env overrides and .env files. # Its files and database must stay inside this synthetic workspace. config = TrialMatchSettings.model_validate(config).to_dict() - write_json_file(config, str(root / "config.json")) + write_json_file(config, str(destination / "config.json")) for number, condition, title, minimum, maximum in ( (1, "lung cancer", "SYNTHETIC adult lung cancer study", "18 Years", "120 Years"), (2, "lung cancer", "SYNTHETIC pediatric lung cancer study", "0 Years", "12 Years"), @@ -62,15 +73,15 @@ def _create_workspace(root: Path) -> dict: "condition": condition, "overall_status": "RECRUITING", "gender": "All", "minimum_age": minimum, "maximum_age": maximum, "eligibility_criteria": f"Inclusion Criteria:\n- Diagnosis of {condition}.\n- Age within the study limits.", - }, str(data / "trials_jsons" / f"{nct_id}.json")) + }, str(destination / "data/trials_jsons" / f"{nct_id}.json")) write_json_file({"resourceType": "Bundle", "type": "collection", "entry": [ {"resource": {"resourceType": "Patient", "id": "demo-patient", "gender": "female", "birthDate": "1980-01-01"}}, {"resource": {"resourceType": "Condition", "id": "demo-condition", "subject": {"reference": "Patient/demo-patient"}, "code": {"text": "lung cancer"}}}, - ]}, str(root / "patient.fhir.json")) - write_json_file({"fixture_version": _FIXTURE_VERSION}, str(root / _MARKER)) + ]}, str(destination / "patient.fhir.json")) + write_json_file({"fixture_version": _FIXTURE_VERSION}, str(destination / _MARKER)) # Covers the initial fixture/config inputs, not the runtime outputs added later. - write_manifest(root) + write_manifest(destination) return config @@ -122,10 +133,16 @@ def main(argv=None) -> int: args = parser.parse_args(argv) if args.resume and args.workdir is None: parser.error("--resume requires --workdir") + workdir = args.workdir try: - result = run_demo(args.workdir, resume=args.resume) + if workdir is None: + workdir = Path(tempfile.mkdtemp(prefix="trialmatchai-demo-")) + result = run_demo(workdir, resume=args.resume) except KeyboardInterrupt: - print("Demo interrupted. Resume with the same --workdir and --resume.", file=sys.stderr) + print("Demo interrupted.", file=sys.stderr) + if workdir is not None: + resume_flag = " --resume" if (workdir / "SHA256SUMS").is_file() else "" + print(f"To retry: trialmatchai demo --workdir {shlex.quote(str(workdir))}{resume_flag}", file=sys.stderr) return 130 except (OSError, ValueError, RuntimeError) as exc: print(f"Demo failed: {exc}", file=sys.stderr) diff --git a/src/trialmatchai/utils/integrity.py b/src/trialmatchai/utils/integrity.py index 6b5efb07..95ebe7b0 100644 --- a/src/trialmatchai/utils/integrity.py +++ b/src/trialmatchai/utils/integrity.py @@ -101,8 +101,12 @@ def write_manifest(directory: str | Path, *, filename: str = "SHA256SUMS") -> Pa def verify_manifest( manifest: str | Path, *, directory: str | Path | None = None, require_exact: bool = False, ) -> list[str]: - manifest = Path(manifest).resolve() - root = Path(directory).resolve() if directory is not None else manifest.parent + manifest = Path(manifest) + if manifest.is_symlink(): + raise ValueError("Checksum manifest must not be a symlink") + # Bind the artifact root to the requested location before resolving the file. + root = Path(directory).resolve() if directory is not None else manifest.parent.resolve() + manifest = manifest.resolve() entries = read_manifest(manifest) for name, expected in entries.items(): verify_sha256(_artifact_path(root, name), expected) diff --git a/tests/test_artifact_integrity.py b/tests/test_artifact_integrity.py index b4bf191b..da912192 100644 --- a/tests/test_artifact_integrity.py +++ b/tests/test_artifact_integrity.py @@ -65,3 +65,32 @@ def test_artifact_cli_json_exit_status(tmp_path, capsys): artifact.write_text("bad wheel") assert main(["verify", result["manifest"], "--json"]) == 1 assert json.loads(capsys.readouterr().out)["ok"] is False + + +@pytest.mark.parametrize("explicit_directory", [False, True]) +def test_manifest_symlink_cannot_redirect_verification(tmp_path, capsys, explicit_directory): + trusted = tmp_path / "trusted" + incoming = tmp_path / "incoming" + trusted.mkdir() + incoming.mkdir() + (trusted / "package.whl").write_bytes(b"known good") + (incoming / "package.whl").write_bytes(b"corrupted incoming wheel") + manifest = write_manifest(trusted) + link = incoming / "SHA256SUMS" + link.symlink_to(manifest) + + arguments = ["verify", str(link), "--require-exact", "--json"] + if explicit_directory: + arguments += ["--directory", str(incoming)] + assert main(arguments) == 1 + result = json.loads(capsys.readouterr().out) + assert result["ok"] is False + assert "manifest must not be a symlink" in result["error"] + + +def test_external_manifest_verifies_the_explicit_artifact_directory(tmp_path): + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + (artifacts / "package.whl").write_bytes(b"wheel") + manifest = write_manifest(artifacts).rename(tmp_path / "SHA256SUMS") + assert verify_manifest(manifest, directory=artifacts, require_exact=True) == ["package.whl"] diff --git a/tests/test_cli_e2e.py b/tests/test_cli_e2e.py index 4e5933d4..4d4d2c4a 100644 --- a/tests/test_cli_e2e.py +++ b/tests/test_cli_e2e.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shlex import subprocess import sys from pathlib import Path @@ -33,3 +34,79 @@ def test_demo_resume_requires_its_own_workspace(tmp_path): with pytest.raises(ValueError, match="existing TrialMatchAI demo"): run_demo(tmp_path, resume=True) + + +@pytest.mark.parametrize("phase", ["config", "manifest"]) +@pytest.mark.parametrize("failure", [OSError, KeyboardInterrupt]) +def test_demo_can_restart_after_interrupted_initialization(tmp_path, monkeypatch, phase, failure): + from trialmatchai.cli import demo + from trialmatchai.utils.integrity import verify_manifest + + workspace = tmp_path / "demo" + hook = "write_json_file" if phase == "config" else "write_manifest" + original = getattr(demo, hook) + + def interrupt_after_write(*args, **kwargs): + original(*args, **kwargs) + raise failure("simulated initialization failure") + + with monkeypatch.context() as interrupted: + interrupted.setattr(demo, hook, interrupt_after_write) + with pytest.raises(failure, match="simulated initialization failure"): + demo.run_demo(workspace) + + assert list(workspace.iterdir()) == [] + + # Stop at the real pipeline boundary; this test exercises startup recovery, + # while the installed CLI e2e covers execution after successful startup. + class PipelineStarted(Exception): + pass + + def pipeline_started(ctx, **kwargs): + assert ctx.config["paths"]["output_dir"] == str(workspace / "results") + assert "config.json" in verify_manifest(workspace / "SHA256SUMS", require_exact=True) + raise PipelineStarted + + monkeypatch.setattr("trialmatchai.pipeline.run_pipeline", pipeline_started) + with pytest.raises(PipelineStarted): + demo.run_demo(workspace) + + +def test_demo_publish_preserves_files_added_during_initialization(tmp_path, monkeypatch): + from trialmatchai.cli import demo + + workspace = tmp_path / "demo" + original = demo.write_manifest + + def add_user_file_before_publish(*args, **kwargs): + result = original(*args, **kwargs) + (workspace / "patient.txt").write_text("preserve this file") + return result + + monkeypatch.setattr(demo, "write_manifest", add_user_file_before_publish) + with pytest.raises(OSError): + demo.run_demo(workspace) + assert (workspace / "patient.txt").read_text() == "preserve this file" + assert not (workspace / "SHA256SUMS").exists() + + +@pytest.mark.parametrize("phase", ["initialization", "pipeline"]) +def test_demo_interrupt_prints_the_correct_retry(tmp_path, monkeypatch, capsys, phase): + from trialmatchai.cli import demo + + workspace = tmp_path / "demo with spaces" + + def interrupt(*args, **kwargs): + raise KeyboardInterrupt + + if phase == "initialization": + monkeypatch.setattr(demo, "write_manifest", interrupt) + else: + monkeypatch.setattr("trialmatchai.pipeline.run_pipeline", interrupt) + assert demo.main(["--workdir", str(workspace)]) == 130 + error = capsys.readouterr().err + command = error.split("To retry: ", 1)[1].strip() + expected = ["trialmatchai", "demo", "--workdir", str(workspace)] + if phase == "pipeline": + expected.append("--resume") + assert shlex.split(command) == expected From 7da1cf7242be4e5f5e8dafc133a770c5c2d7b855 Mon Sep 17 00:00:00 2001 From: Majd Abdallah Date: Sat, 12 Sep 2026 17:09:01 +0200 Subject: [PATCH 3/3] Fix review recovery gaps and prepare the 0.9.0 foundation release --- .github/workflows/docs.yml | 18 +- .github/workflows/verify.yml | 3 +- CHANGELOG.md | 79 +-- README.md | 660 ++++++++------------- docs/assets/readme-overview.svg | 47 ++ docs/index.md | 97 ++- docs/production-roadmap.md | 25 +- docs/production-validation.md | 18 +- docs/release.md | 43 +- pyproject.toml | 2 +- scripts/installed_smoke.py | 13 +- src/trialmatchai/__init__.py | 2 +- src/trialmatchai/cli/bootstrap_data.py | 277 ++++++--- src/trialmatchai/cli/demo.py | 20 +- src/trialmatchai/interop/importers/fhir.py | 4 + src/trialmatchai/main.py | 2 +- src/trialmatchai/orchestration.py | 4 + src/trialmatchai/services/preflight.py | 13 + tests/test_audit_fixes.py | 2 +- tests/test_bootstrap_data.py | 156 +++++ tests/test_deployment_readiness.py | 29 + tests/test_fhir_robustness.py | 9 +- uv.lock | 2 +- 23 files changed, 893 insertions(+), 632 deletions(-) create mode 100644 docs/assets/readme-overview.svg diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3dc3926e..7c42b070 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,18 +21,23 @@ concurrency: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + with: + version: "0.11.24" - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" - name: Build docs (strict) - run: uv run --group docs mkdocs build --strict + run: uv run --frozen --group docs mkdocs build --strict - name: Upload Pages artifact if: github.ref == 'refs/heads/main' - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 with: path: site @@ -40,6 +45,7 @@ jobs: # PRs and other branches only build (validate) above; main publishes. if: github.ref == 'refs/heads/main' needs: build + timeout-minutes: 10 runs-on: ubuntu-latest permissions: pages: write # deploy to Pages @@ -50,4 +56,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 1728e2a6..46fe8493 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -138,14 +138,13 @@ jobs: } Path("dist/build-info.json").write_text(json.dumps(info, indent=2) + "\n") PY - - run: uv build + - run: uv build --no-create-gitignore - run: uv run --frozen trialmatchai artifacts manifest dist - run: uv run --frozen trialmatchai artifacts verify dist/SHA256SUMS --require-exact --json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: trialmatchai-dist-${{ github.sha }} path: dist/ - include-hidden-files: true # uv build writes .gitignore, which SHA256SUMS includes if-no-files-found: error retention-days: 14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7564e3c7..6c367b28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,47 +4,52 @@ All notable changes to TrialMatchAI are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/) and the project adheres to [Semantic Versioning](https://semver.org/). -## [Unreleased] +## [0.9.0] — 2026-09-12 ### Added -- A production roadmap covering all 38 audit findings, clinical workflow, retrieval, - evaluation, bounded agents, CLI automation, and deployment qualification. -- `trialmatchai demo` runs synthetic FHIR import, real CPU indexing, retrieval, - report generation, and resume; `--version` exposes the package version. -- `trialmatchai artifacts manifest/verify` manages portable SHA-256 manifests. - Bootstrap supports a trusted manifest and a strict mode requiring all digests. -- CI and releases share required verification, including an isolated installed-wheel - CLI run. Releases verify the tested artifacts and attach provenance before OIDC - publication. Repository protections still require administrator configuration. - -### Security -- Explicit unmatched FHIR subject references no longer attach records to the only - patient in a bundle. Strict import rejects them; lenient import quarantines them. -- **Cleared 13 Dependabot advisories (9 of them high) in leaf dependencies**, without disturbing - the pinned inference stack: `pillow` 12.2.0 → 12.3.0 (decompression-bomb and heap - out-of-bounds issues in image decoding), `aiohttp` 3.14.1 → 3.14.3 (request smuggling and a - malformed-response parser read), `cryptography` 49.0.0 → 50.0.1 (Bleichenbacher oracle in - PKCS#7 `EnvelopedData` decryption) and `pymdown-extensions` 11.0 → 11.0.2 (ReDoS in the - markdown extensions used to build these docs). `vllm`, `torch`, `torchvision`, `torchaudio` - and `mcp` are unchanged. - - None of these were reachable in normal use — this project processes clinical text rather than - untrusted images or HTTP, and never calls PKCS#7 decryption — but all four had fixes available - and none is load-bearing, so upgrading is cheaper than carrying the advisories. - - The remaining open advisories are against `vllm` and `torch`. Those require moving off the - hard-pinned 0.23.0 / 2.11.0 inference stack that every published benchmark was measured on, - so they are a deliberate re-baselining exercise rather than a patch. +- Synthetic CPU demo with real LanceDB indexing, FHIR import, ranking, HTML reports, + repeatable resume, and a CLI version command. +- Portable SHA-256 manifest creation/verification, including exact file-set checks, + safe path handling, and strict bootstrap digests from a trusted manifest. +- Shared CI/release verification with frozen dependencies, installed-wheel CLI e2e, + build metadata, artifact checksums, provenance, and PyPI trusted publishing. +- A codebase audit and sequenced production roadmap covering ranking quality, + clinical workflow, bounded agents, CLI automation, and deployment qualification. ### Fixed -- Checksum verification rejects manifest symlinks that could redirect verification - to another artifact directory. Interrupted demo initialization leaves its chosen - directory ready for a normal retry; completed initialization supports `--resume`. -- Embedder catalog resources are included in built wheels; Make targets use the - unified CLI. Disabled model stages no longer trigger matching Hub access checks. -- **`pip` upgraded to 26.2.1** so CI's dependency audit passes (PYSEC-2026-3721). `pip` is - dev-only here, arriving transitively via `pip-api` ← `pip-audit`, and is never shipped in - the wheel. +- Changed bootstrap archives replace managed extraction trees without retaining old + files. Publication stages complete trees, serializes writers, recovers interrupted + replacement, and retains previous trees and unrelated model directories. +- Corrupt cached archives are quarantined and retried with a bounded fresh download; + completion checks reject symlinks and non-regular marker files. +- Explicit unresolved FHIR patient references, including identifier-only, display-only, + empty, and null references, cannot fall back to the sole patient in a bundle. +- Interrupted demo initialization can be retried; resume repairs missing/truncated + patient reports without repeating completed ranking. Automatic patient reports + are written atomically. +- Query-expansion preflight checks the actual explicit or fallback model before + expansion; disabled reranking/reasoning stages do not request unused model access. +- Embedder catalogs ship in wheels, Make targets use the current CLI, and checksum + verification rejects manifest redirection through symlinks. + +### Changed +- Rewritten README with an SVG overview, pipeline diagram, executable onboarding, + and explicit boundaries between available features and planned qualification. +- Documentation builds use frozen dependencies and pinned Actions. +- Updated locked leaf dependencies: Pillow, aiohttp, cryptography, + pymdown-extensions, and the development pip toolchain. + +### Known limitations +- This release validates the CPU software path, not GPU inference or clinical + eligibility. Ranking metrics were not rerun or claimed to improve. +- The optional inference environment still has unresolved dependency advisories; + the clean base/dev audit is a narrower check. See `docs/production-validation.md`. +- Bootstrap replacement is recoverable between two renames, not an online snapshot + switch. Stop readers during updates; strict resume does not hash extracted files. + Legacy model entries without ownership metadata are preserved conservatively. +- Full evidence identity, clinical decision semantics, evaluator corrections, + model/adapter qualification, review workflows, bounded agents, and application + deployment remain in `docs/production-roadmap.md`. ## [0.8.2] — 2026-08-31 diff --git a/README.md b/README.md index 658deb01..a0bba563 100644 --- a/README.md +++ b/README.md @@ -1,487 +1,313 @@
-TrialMatchAI - -

TrialMatchAI matches patients to the clinical trials they're eligible for. Give it a patient as clinical notes, FHIR, a Phenopacket, or OMOP data, and it returns a ranked shortlist of trials, each one explained criterion by criterion so you can see why the patient does or doesn't qualify. Everything runs on your own infrastructure on a single GPU server, so patient data never leaves your environment.

- -

- CI - Docs - PyPI - Python versions - License: MIT - Linted with Ruff -

- -

- Install · - Quickstart · - How it works · - Configuration · - CLI -

+TrialMatchAI — patient profiles, trial retrieval, criterion review, and reports -
- -> **⚕️ For research and informational use only.** TrialMatchAI is not medical -> advice, not a medical device, and must not replace review by qualified -> healthcare professionals. - -## Overview - -TrialMatchAI works in two halves. You build the matching system once, then you -match patients against it as often as you like. Both halves only do work that -isn't already done, and both resume cleanly after an interruption. - -- **Build** is the heavy one-time setup, and it needs a GPU. It prepares a - searchable corpus of clinical trials (embeddings, biomedical entity - annotations, and parsed eligibility constraints) and builds a local search - index. -- **Match** ingests a patient, retrieves candidate trials, and reasons over each - trial's eligibility criteria to produce a ranked, explained shortlist. It writes - that out as a self-contained HTML report you can open in any browser. - -Trials, models, indexes, and results all live on your own machine. - -## Performance - -This release ships two TrialMatchAI configurations: a clinical one that pairs the -MedCPT retriever with MedGemma, and a general one that pairs bge-m3 with phi-4. The -bars below also include three experimental reasoners on MedCPT retrieval: the -thinking-model Baichuan-M2-32B, a 2-bit Qwen3.6-35B-A3B MoE, and a compact -II-Medical-8B. On the official **TREC Clinical Trials** benchmark (2021 and 2022, -125 topics pooled), all five rank eligible trials more accurately than the published -TrialMatchAI system (the “paper” bars) and than **TrialGPT** (Jin et al., *Nature -Communications* 2024): - - - - Clinical-trial ranking performance on TREC CT 2021+2022 (pooled): nDCG@10 and graded P@10 for five TrialMatchAI reasoners (MedCPT+Qwen3.6-35B-A3B, MedCPT+Baichuan-M2-32B, MedCPT+MedGemma, bge-m3+phi-4, MedCPT+II-Medical-8B) vs the paper vs TrialGPT - - -nDCG@10 and graded P@10 pooled over TREC Clinical Trials 2021 and 2022 (125 -topics), computed on judged trials. The five TrialMatchAI configurations and the -paper use the identical ranking metric on the same topics, so those comparisons are -exact. TrialGPT shows the value reported in Jin et al. (2024); their evaluation also -includes the SIGIR 2016 cohort, so treat it as an indicative reference rather than a -matched run. The 2-bit Qwen3.6-35B-A3B MoE (about 3B active parameters) leads on both -metrics, narrowly ahead of Baichuan-M2-32B; both run with grammar-constrained JSON so -nearly all of their verdicts parse, while MedGemma and phi-4 had roughly 8% parse -failures that can drop otherwise-eligible trials, so part of that gap is a formatting -effect. The margins among the top reasoners are small (about 0.01 to 0.04). At the -small-model end, II-Medical-8B is the most compact reasoner here yet still matches -TrialGPT on nDCG@10 and beats it on graded P@10, so even an 8B model clears the -reference bar. Reproduce ours with `trialmatchai trec --tracks "21 22"`. - -**Retrieval recall by embedder.** This is the share of eligible trials (qrels -grade 2) that the first-level search surfaces among its top *k* candidates, across -four embedders: two general-purpose ones (bge-m3, Qwen3-Embedding) and two -clinical ones (MedCPT, PubMedBERT). The clinical embedders lead on both tracks: - - - - First-level retrieval recall@k (10 to 2000) of eligible trials, per track (TREC CT 2021 and 2022), for four embedders: bge-m3 and Qwen3-Embedding (general) vs MedCPT and PubMedBERT (clinical) - - -TREC-standard recall@k over eligible trials (qrels grade 2), first-level -retrieval only, per track (TREC CT 2021 and 2022; a candidate pool of roughly 26k -trials) with hybrid retrieval at vector weight 0.6. Each embedder indexes the same -corpus with concept linking held fixed, so the retrieval embedder is the only -thing that changes. The two clinical embedders, MedCPT (native dot product) and -PubMedBERT, lead the two general ones (bge-m3 and the strong general-purpose -Qwen3-Embedding) at every depth. Domain specialization beats a larger general -model here. Reproduce with `scripts/benchmark_embedder.py`. - -**TREC Clinical Trials 2023.** The 2023 track replaces the free-text patient -narratives of 2021/2022 with structured questionnaire templates — 40 topics across -eight disorders, each 5 to 12 optional fields — over a May-2023 registry snapshot. -TrialMatchAI parses those templates natively; the numbers below are the bge-m3 + -phi-4 configuration over the 37 judged topics: - -| Metric | rel ≥ 1 | eligible (grade 2) | -| --- | --- | --- | -| nDCG@5 | 0.8871 | — | -| nDCG@10 | 0.8806 | — | -| nDCG@20 | 0.8718 | — | -| graded P@10 | 0.8811 | — | -| Precision@10 | 0.9351 | 0.8270 | -| Recall@1000 (first level) | 0.6442 | — | - -Condensed metrics (unjudged trials dropped) over the 37 topics NIST judged of -the 40 released; grade 2 = eligible, 1 = relevant but excluded, 0 = not relevant. -2023 topics average 604 relevant trials each — four times the 2021/2022 average of -about 150 — which is why recall@1000 sits at 0.64: covering a 604-trial relevant set -from a 1000-candidate budget requires 60% of the pool to be relevant. Per-disorder -nDCG@10 ranges from 0.79 (sickle cell anemia) to 0.99 (rheumatoid arthritis). -Scored against a corpus restricted to the judged pool -(17,103 of 17,106 judged NCTs; three are no longer in the registry), so these are -not directly comparable to full-corpus TREC 2023 submissions. To reproduce, first -fetch the judged trials the bootstrap corpus does not carry — `python -m -trialmatchai.trec.backfill --tracks 23`, which pulls them from the live -ClinicalTrials.gov API — then `trialmatchai build` and `trialmatchai trec --tracks -"23"`. Skipping the backfill leaves the index short of the judged pool and the run -warns rather than silently understating recall. - -## Requirements - -- Python 3.11 (`pyproject.toml` requires `>=3.11,<3.12`) -- `pip` or `uv`, to install from PyPI or from a source checkout for development -- An NVIDIA GPU for the model-based matching and fine-tuning -- Around 100 GB of disk for datasets, model artifacts, search indexes, manifests, - and run outputs -- OMOP vocabulary files, only if you want to build the concept-linking table - yourself - -## Installation - -### From PyPI (recommended) - -```bash -pip install trialmatchai # or: uv pip install trialmatchai -trialmatchai --help -``` - -That installs the package and the `trialmatchai` CLI with its base dependencies. -For matching with the models, add the runtime extras: - -```bash -pip install "trialmatchai[llm,gpu,entity]" # full GPU runtime (Linux CUDA host) -pip install "trialmatchai[finetune]" # fine-tuning stack -``` +**A configurable CLI for patient-to-trial retrieval and eligibility review.** -| Extra | Adds | -| --- | --- | -| `entity` | biomedical entity extraction | -| `llm` | local embedding and language models | -| `gpu` | GPU inference stack (CUDA / Linux) | -| `finetune` | training dependencies for `trialmatchai finetune` | +[![CI](https://github.com/cbib/TrialMatchAI/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cbib/TrialMatchAI/actions/workflows/ci.yml) +[![Documentation](https://img.shields.io/badge/docs-guides_%26_API-0D9488)](https://cbib.github.io/TrialMatchAI/) +[![PyPI](https://img.shields.io/pypi/v/trialmatchai?color=2563EB)](https://pypi.org/project/trialmatchai/) +[![Python 3.11](https://img.shields.io/badge/python-3.11-2563EB)](https://github.com/cbib/TrialMatchAI/blob/main/pyproject.toml) +[![License: MIT](https://img.shields.io/badge/license-MIT-64748B)](https://github.com/cbib/TrialMatchAI/blob/main/LICENSE) -### From source (for development) +[Try the demo](#try-the-demo) · [Run your own data](#run-your-own-data) · [Architecture](#how-matching-works) · [CLI](#cli-reference) · [Development](#development-and-delivery) -```bash -git clone https://github.com/cbib/TrialMatchAI.git -cd TrialMatchAI -uv sync # base; add --extra llm --extra gpu --extra entity -uv run trialmatchai --help -``` + -> **Calling the CLI:** if you installed from PyPI, run it directly as -> `trialmatchai ...`. From a `uv` source checkout, prefix it with `uv run`. The -> examples below use the `uv run` form, so drop the prefix if you installed from -> PyPI. +TrialMatchAI imports patient information, searches a local clinical-trial index, +and produces ranked results and portable HTML reports. Optional local models add +biomedical entity extraction, query expansion, reranking, and criterion-level +eligibility assessments. -Installing the package only gives you the CLI. Matching also needs the trial -corpus, the model artifacts, and the search index, all of which the **build** step -below produces. +**Release status: beta.** The automated release gates exercise the installed CLI +with synthetic data and real CPU search. They do not qualify GPU inference, +clinical accuracy, or a production clinical deployment. Trial recommendations and +generated explanations require qualified human review; a high retrieval score +does not establish eligibility. -## Quickstart +## Try the demo -Try the full software path on synthetic data with Python 3.11, without downloading -models or a trial corpus (available from this development branch): +Use **Python 3.11**. The base installation includes the CLI and embedded search; +it does not install the optional model stack. ```bash -uv sync --frozen -uv run --frozen trialmatchai demo --workdir /tmp/trialmatchai-demo -uv run --frozen trialmatchai demo --workdir /tmp/trialmatchai-demo --resume +python3.11 -m venv .venv +source .venv/bin/activate +python -m pip install trialmatchai==0.9.0 + +trialmatchai --version +trialmatchai demo --workdir ./demo-workspace +trialmatchai demo --workdir ./demo-workspace --resume ``` -The command creates a real CPU search index, imports a synthetic FHIR patient, -ranks trials, and prints the HTML report path. This demonstrates retrieval and -reporting; it does not validate model accuracy or clinical eligibility. Choose an -empty directory for a new run. See the [delivery runbook](docs/release.md) for -checksums, installed-wheel checks, and release gates, and the -[production roadmap](docs/production-roadmap.md) for remaining qualification work. +Open the printed `results/demo-patient/report.html` path in a browser. Choose an +empty directory for the first run. The demo creates three synthetic trials and a +synthetic FHIR patient, prepares a real LanceDB index, filters and ranks candidates, +and renders a report. It uses BM25 and deterministic hashing embeddings; model +inference and model downloads are disabled. + +Resume preserves completed ranking results and repairs a missing or truncated +patient report. Demo inputs and configuration are checksummed: use a new workspace +for a different configuration. This is a software walkthrough, not a clinical +benchmark. + +## What is available + +| Area | Implemented today | Qualification boundary | +| :--- | :--- | :--- | +| Patient input | Text, FHIR, Phenopacket, and OMOP importers; canonical profiles and summaries | Format coverage is partial; validate mappings against your source data | +| Retrieval | Local LanceDB tables, BM25/vector/hybrid search, multi-channel fusion, structured filters | Retrieval quality depends on corpus, embeddings, filters, and candidate budgets | +| Model stages | Entity extraction, optional query expansion, LLM reranking, criterion assessments | Requires extra dependencies, model access, and runtime qualification | +| Review | Ranked JSON, per-trial outputs, individual and multi-patient HTML reports | No hosted clinical workspace, authentication, review queue, or sign-off system | +| Operations | CLI stages, registry updater, checksum tools, synthetic e2e, verified package publishing | Complete cache invalidation, GPU qualification, and application deployment remain open | +| Agent behavior | A fixed sequence of configurable stages, with optional LLM expansion | Bounded retrieval/review agents are planned; autonomous task planning is not implemented | + +The [codebase review](https://github.com/cbib/TrialMatchAI/blob/main/docs/codebase-review-2026-09-12.md) records the audit findings. +The [production roadmap](https://github.com/cbib/TrialMatchAI/blob/main/docs/production-roadmap.md) turns them into sequenced work +across ranking quality, clinical workflow, agents, and delivery. + +## How matching works + +```mermaid +flowchart LR + P[Patient input] --> I[Canonical profile\nand summary] + I --> L1[First level\nTrial retrieval + fusion] + L1 --> L2[Second level\nCriterion retrieval] + L2 --> F[Final ranking] + F --> H[JSON + HTML\nHuman review] + I -. optional .-> Q[Query expansion] + Q -.-> L1 + L2 -. optional .-> R[LLM reranking] + R -.-> F + L2 -. optional .-> E[Eligibility assessment\nCoT / RAG stage] + E -.-> F + D[(Local trial +\ncriterion index)] --> L1 + D --> L2 + classDef primary fill:#0f766e,color:#fff,stroke:#14b8a6 + classDef optional fill:#eff6ff,color:#1e3a8a,stroke:#60a5fa,stroke-dasharray:5 4 + class L1,L2,F primary + class Q,R,E optional +``` +The build path prepares trial records and criterion rows, then indexes them. +During matching, first-level retrieval gathers trial candidates across query +channels; second-level retrieval selects criterion evidence. Enabled reranking, +constraint scoring, and eligibility assessment contribute to the final output. +The code calls its generated eligibility explanations **CoT reasoning**; these +are model outputs to inspect alongside source evidence, not verified clinical +reasoning or a guarantee that every criterion has been covered. -### 1. Configure the runtime +The default configuration selects `BAAI/bge-m3` embeddings, +`fastino/gliner2-base-v1` entity extraction, `google/gemma-2-2b-it` reranking, and +`microsoft/phi-4` eligibility reasoning with TrialMatchAI adapters. Query expansion +is disabled by default. See the [actual defaults](https://github.com/cbib/TrialMatchAI/blob/main/src/trialmatchai/config/config.json) +and [pipeline guide](https://github.com/cbib/TrialMatchAI/blob/main/docs/pipeline.md) for stage controls and configuration. -```bash -cp .env.example .env # optional local overrides -hf auth login # authenticate once for gated base models (or set HF_TOKEN= instead) -``` +## Run your own data -### 2. Build the system, once +### 1. Select an environment and configuration -```bash -trialmatchai bootstrap-data # download the prepared trial corpus + criteria (~25 GB, one-time) -trialmatchai build # prepare the corpus + build the search index -trialmatchai build --status # see exactly what is built (and what isn't) -``` +The declared runtime is Python 3.11. CI verifies Linux CPU execution. Model memory, +CUDA compatibility, and throughput must be established for the configuration and +hardware you select; there is no universal single-GPU capacity promise. -`build` fails fast if a GPU, an extra, or model access is missing, and it resumes -from where it left off if interrupted. To use your own trials instead of the -bootstrapped corpus, put normalized JSON in `data/trials_jsons/` and `build` will -prepare them. +| Installation | Intended use | +| :--- | :--- | +| `trialmatchai` | CLI, synthetic CPU demo, import/report/artifact utilities | +| `trialmatchai[entity]` | GLiNER2 entity extraction and its model dependencies | +| `trialmatchai[llm]` | Embedding and language-model dependencies | +| `trialmatchai[llm,gpu,entity]` | Optional full model stack on a compatible Linux CUDA host | +| `trialmatchai[finetune]` | Training dependencies; see the [fine-tuning guide](https://github.com/cbib/TrialMatchAI/blob/main/docs/finetuning.md) | -To enable entity-to-concept linking, add `--concepts` (open vocabularies, -downloaded for you). You can also add an OMOP `CONCEPT.csv` for SNOMED, LOINC, and -RxNorm on top: +For the default model pipeline, install its extras in a dedicated environment: ```bash -trialmatchai build --concepts -trialmatchai build --concepts --concepts-csv data/omop/CONCEPT.csv --synonym-csv data/omop/CONCEPT_SYNONYM.csv +python -m pip install 'trialmatchai[llm,gpu,entity]==0.9.0' ``` -#### What gets fetched, and how - -| Resource | How you get it | Automatic? | -| --- | --- | --- | -| Trial corpus (prepared trials + criteria) | `trialmatchai bootstrap-data` | ✅ automatic | -| Fine-tuned adapters ([phi-4 reasoning](https://huggingface.co/majdabd33/trialmatchai-phi4-reasoning-lora), [gemma-2 reranker](https://huggingface.co/majdabd33/trialmatchai-gemma2-reranker-lora)) | downloaded on first use (Hugging Face) | ✅ automatic | -| Fine-tuning datasets (only if you re-train) | `trialmatchai bootstrap-data --finetune-data` | ✅ automatic (opt-in) | -| Embedding model | downloaded on first use | ✅ automatic | -| Concept-linking vocabularies | `trialmatchai build --concepts` | ✅ automatic | -| Base language models | downloaded on first use | ⚠️ the default reranker `google/gemma-2-2b-it` (and `google/medgemma-27b-text-it` in the clinical config) are gated: accept the licence on Hugging Face, then run `hf auth login` once | -| OMOP clinical vocabulary (SNOMED/LOINC/RxNorm) | download `CONCEPT.csv` from [OHDSI Athena](https://athena.ohdsi.org/) | ❌ manual (licensed); linking works without it | +The optional inference stack has unresolved dependency advisories and has not +passed full GPU load/inference qualification in this release. Review the +[validation record](https://github.com/cbib/TrialMatchAI/blob/main/docs/production-validation.md) before deploying it. -From scratch, that is two commands (`bootstrap-data`, then `build --concepts`) -after a one-time `hf auth login`. Everything else is pulled on demand. - -### 3. Match patients, repeatably - -`e2e` ingests the patient (the format is auto-detected) and matches end to end: +Export the packaged configuration into your workspace: ```bash -trialmatchai e2e --input data/patients/raw/patient-1.txt -trialmatchai e2e --input data/patients/raw/patient-1.fhir.json -trialmatchai e2e --input data/patients/omop_extract +python - <<'PY' +from importlib.resources import files +from pathlib import Path +Path("config.json").write_text( + files("trialmatchai").joinpath("config/config.json").read_text(encoding="utf-8"), + encoding="utf-8", +) +PY ``` -Results land in `results//` (ranked trials plus eligibility -explanations). Re-running skips patients that are already matched. +Edit the model, data, output, and device settings for your installation. +For adapters, supply downloaded local directories: the current path normalizer +treats Hub adapter IDs as local paths, an open P09 compatibility issue. The +[configuration schema](https://github.com/cbib/TrialMatchAI/blob/main/src/trialmatchai/config/settings.py) and [environment example](https://github.com/cbib/TrialMatchAI/blob/main/.env.example) +describe overrides. Obtain access to any gated models before running the pipeline. +Model and vocabulary downloads, registry refreshes, and access checks can use the +network; deploying locally does not by itself establish an offline or compliant +environment. -Each run also writes a self-contained HTML report you can open in a browser. -`results/index.html` is a front page over every matched patient, and -`results//report.html` is one patient on its own. (Turn it off with -`reporting.emit_html: false`.) +### 2. Prepare and index trials -### Keeping trials current - -Validate the installation, then fold new or changed ClinicalTrials.gov studies -into the live index. This is incremental and idempotent, so studies that haven't -changed are skipped: +Use normalized trial JSON under `data/trials_jsons/`, then prepare and index it: ```bash -trialmatchai healthcheck # validate config, paths, deps, indexes -trialmatchai update-registry --since 2026-06-01 # one-shot -trialmatchai update-registry --watch --interval 86400 # server: update daily +trialmatchai build --config config.json +trialmatchai build --config config.json --status ``` -The updater also runs from cron, a systemd timer, or GitHub Actions. See -[docs/registry-updater.md](docs/registry-updater.md). - -
-Manual / advanced control (the steps build and e2e wrap) +Alternatively, `bootstrap-data` downloads the configured prepared corpus archives. +For verified downloads, obtain their expected digests through a trusted channel: ```bash -trialmatchai index --prepare # prepare + index from trials_jsons (what `build` runs) -trialmatchai import-patient --input patient.txt # stage a profile only -trialmatchai run # match already-staged profiles -trialmatchai trec --tracks "21 22" # benchmark: official TREC Clinical Trials eval +trialmatchai bootstrap-data --root . --checksum-manifest /path/to/trusted/SHA256SUMS +trialmatchai index --config config.json ``` -
+The manifest path above is a placeholder for your trusted **corpus** manifest; +package-release checksums do not authenticate separately hosted corpus archives. +Bootstrap without digests remains supported for legacy sources and warns that +verification is unavailable. Use compatible embedding settings when indexing +precomputed vectors. -## How it works +Concept linking is optional. `trialmatchai build --config config.json --concepts` +builds the open-vocabulary concept store; licensed OMOP vocabulary files can be +supplied separately. See [pipeline guide](https://github.com/cbib/TrialMatchAI/blob/main/docs/pipeline.md). -The flow below is the **match** path. The one-time **build** step produces the -search index it queries: the trial and criterion embeddings, entity annotations, -and parsed eligibility constraints. +### 3. Import, match, and review -```text -Patient data (text / FHIR / Phenopacket / OMOP) - │ - ▼ -Interoperable importers → canonical patient profile - │ - ▼ -Biomedical entity extraction + variant recognition → concept linking - │ - ▼ -First-level trial retrieval (lexical + semantic) over the local index - │ - ▼ -Multi-channel query fusion for broad candidate recall - │ - ▼ -Criterion retrieval + reranking - │ - ▼ -Constraint-aware criterion scoring - │ - ▼ -Per-criterion eligibility reasoning - │ - ▼ -Final ranking + explanations in results/ +```bash +trialmatchai e2e --config config.json --input patient.fhir.json --format fhir +trialmatchai report --config config.json --patient YOUR_PATIENT_ID +trialmatchai report --config config.json --all ``` -The reranking and eligibility-reasoning stages use language models served locally -on the GPU. Every model component (entity extraction, the reranker, and -eligibility reasoning) is configurable, and you can fine-tune each one (see -[Configuration](#configuration)). - -## Patient inputs +Replace `YOUR_PATIENT_ID` with the imported profile ID. Text notes, Phenopacket JSON, +and OMOP extract directories are also accepted by the importer. FHIR resources +with explicit unresolved patient references are rejected in strict mode or kept +as unsupported input in lenient mode; they are not silently assigned to the sole +patient. Consult the [interoperability guide](https://github.com/cbib/TrialMatchAI/blob/main/docs/interoperability.md) for supported +fields and validation behavior. -The importer supports: +With default paths, the workspace contains: -- free-text notes: `.txt` and `.md` -- GA4GH Phenopacket JSON -- HL7 FHIR R4 bundles, individual FHIR resources, NDJSON, and JSONL -- OMOP CDM extract folders with CSV or Parquet tables - -Importers keep provenance and any unsupported source elements where they can. The -matching summary is built deterministically from the canonical patient profile, -and `trialmatchai run` never reads the raw patient files directly. See -[docs/interoperability.md](docs/interoperability.md) for the format details. - -## Data and storage - -All storage is local and file-based, with no external services. The search index -(`data/search`, with trial and criterion tables) and the concept-linking database -(`data/concepts`) are embedded databases. ClinicalTrials.gov records are normalized -to `data/trials_jsons/.json`, then prepared into one trial row and one row -per eligibility criterion (text, embeddings, entity annotations, and parsed -constraints). The tables carry both full-text and vector columns, so retrieval can -run in `bm25`, `vector`, or `hybrid` mode. Imported patients live under -`data/patients/`. - -## Configuration - -Defaults live in `src/trialmatchai/config/config.json`; runtime overrides go in -`.env` or environment variables. Common overrides: - -```bash -TRIALMATCHAI_SEARCH_MODE=hybrid # bm25 | vector | hybrid -TRIALMATCHAI_OUTPUT_DIR=results -TRIALMATCHAI_TRIALS_JSON_FOLDER=data/trials_jsons -TRIALMATCHAI_FIRST_LEVEL_MAX_TRIALS=1000 -TRIALMATCHAI_LOG_JSON=1 +```text +data/ +├── trials_jsons/ # normalized registry records +├── processed_trials/ # prepared trial records +├── processed_criteria/ # prepared eligibility criteria +├── search/ # embedded LanceDB search tables +└── patients/ # raw copies, canonical profiles, summaries +results/ +├── / +│ ├── ranked_trials.json +│ └── report.html +└── index.html # multi-patient report ``` -The full override list is in [`.env.example`](.env.example). - -Every model swaps through config alone, with no code changes. Pick an embedder -from the built-in catalog with `embedder: {model: "medcpt"}` and the retrieval -metric follows automatically, or point `entity_extraction.model_name`, -`model.reranker_adapter_path`, and `model.cot_adapter_path` at your own -checkpoints. You can also train your own with `trialmatchai finetune -{cot,reranker,ner}`. See the -[fine-tuning guide](https://cbib.github.io/TrialMatchAI/finetuning/). +Reports can contain patient information. Configure access, retention, and sharing +for your environment. The repository does not provide these organizational +controls. ## CLI reference -There is a single entry point, `trialmatchai`, and every capability is a -subcommand. Under the hood they are all slices of **one idempotent pipeline**. - -**The unified pipeline (run any subset)** - -| Command | Purpose | -| --- | --- | -| `trialmatchai pipeline` | Run the whole pipeline, or any slice: `--only` / `--from` / `--to` / `--skip` / `--force` over the stages `prepare → concepts → link → index → ingest → expand → match → eval`. Finished work is skipped. See [docs](https://cbib.github.io/TrialMatchAI/pipeline/). | - -The commands below are convenience presets over that pipeline. - -**Build the system** +Run `trialmatchai --help` for the current options. | Command | Purpose | -| --- | --- | -| `trialmatchai build` | Prepare the corpus (embeddings and entities) and build the search index. Resumable, with `--status`. | -| `trialmatchai bootstrap-data` | Download and extract the prepared trial corpus + criteria (the fine-tuned adapters download from Hugging Face on first use) | -| `trialmatchai build-concepts` | Build the concept table for entity normalization (optional, OMOP) | -| `trialmatchai update-registry` | Fetch changed ClinicalTrials.gov studies and update the index | +| :--- | :--- | +| `demo` | Exercise synthetic import, CPU search, reporting, and resume | +| `pipeline` | Select stages with `--only`, `--from`, `--to`, `--skip`, and `--force` | +| `build` / `index` | Prepare the corpus and build search tables / index prepared data | +| `build-concepts` | Build the optional concept-linking store | +| `bootstrap-data` | Download and extract prepared data and optional model/training archives | +| `import-patient` | Create canonical patient profiles and summaries | +| `e2e` / `run` | Import and match / match staged profiles | +| `report` | Render a patient report or the multi-patient front page | +| `update-registry` | Fetch and upsert ClinicalTrials.gov studies; optional watch mode | +| `trec` | Run TREC Clinical Trials evaluation presets | +| `finetune` | Train supported reasoning, reranker, or NER components | +| `healthcheck` | Inspect configured dependencies, paths, and services | +| `artifacts` | Create or verify portable SHA-256 manifests | + +Resume is stage-specific. Completion files do not yet fingerprint every input, +configuration, and model revision. After changing these, explicitly rebuild or +force the affected stages; successful resume alone does not prove data freshness. +Registry synchronization and deletion/status propagation also need qualification +before use as a live trial directory. + +### Checksums and bootstrap recovery -**Match patients** +```bash +trialmatchai artifacts manifest ./artifacts --json +trialmatchai artifacts verify ./artifacts/SHA256SUMS --require-exact --json +``` -| Command | Purpose | -| --- | --- | -| `trialmatchai e2e` | Ingest a patient and match end to end (idempotent, per-patient resume) | -| `trialmatchai import-patient` | Import text, FHIR, Phenopacket, or OMOP patient data into a profile | -| `trialmatchai run` | Match already-staged patient profiles | -| `trialmatchai report` | Render a self-contained HTML match report: one patient (`--patient `) or a unified front page over all of them (`--all`) | -| `trialmatchai trec` | Benchmark: end-to-end evaluation on the official TREC Clinical Trials tracks | +Verification rejects missing/corrupt files, unsafe paths, and symlinks; exact mode +also rejects unexpected files. Trusted bootstrap digests bind the downloaded +archives to completion markers. Corrupt caches are quarantined with a bounded +retry. Changed archives extract into fresh trees, with previous trees retained as +backups and unrelated model directories preserved. -**Utility** +Bootstrap serializes writers and recovers interrupted publication on the next +run. Replacement uses two directory renames, so stop consumers during updates. +Resume checks marker provenance and managed roots; it does not rehash all extracted +files. Backups and quarantine files consume disk until reviewed and removed. See +the [delivery runbook](https://github.com/cbib/TrialMatchAI/blob/main/docs/release.md) for the exact guarantees and legacy migration. -| Command | Purpose | -| --- | --- | -| `trialmatchai healthcheck` | Validate config, paths, optional model deps, and search tables | -| `trialmatchai index` | Lower-level prepare/index of trial and criteria tables | -| `trialmatchai finetune` | Fine-tune the entity, reranker, or reasoning models (`cot` / `reranker` / `ner`), or `merge` a trained LoRA adapter into its base | +## Evaluation and quality -## Deployment +`trialmatchai trec --tracks "21 22"` and `--tracks "23"` expose the benchmark +presets. Reproducible comparisons require fixed corpus snapshots, topics, qrels, +model revisions, configuration, and a stated treatment of unjudged trials. +The current evaluator and retrieval pipeline have open audit findings; this +release does not claim a newly measured ranking improvement or a validated +comparison against another system. -The supported deployment is a single Python 3.11 GPU server or VM. The search -index and concept-linking database are local files under `data/search` and -`data/concepts`. The registry updater is built for cron, systemd timers, or -GitHub Actions. See [docs/registry-updater.md](docs/registry-updater.md). +The [research paper](https://doi.org/10.1038/s41467-026-70509-w) describes the +published study. Its results should be distinguished from this release's software +checks. The roadmap prioritizes retrieval recall, eligibility semantics, evidence +coverage, benchmark integrity, and a usable review workflow before bounded agent +experiments. -## Development +## Development and delivery ```bash -uv sync -uv run ruff check . -uv run pytest -uv run pre-commit run --all-files # ruff + gitleaks secret scan + hygiene -uv run pip-audit --progress-spinner off --ignore-vuln CVE-2025-3000 -``` - -Install the git hooks once so linting and secret scanning run on every commit: - -```bash -uv run pre-commit install +git clone https://github.com/cbib/TrialMatchAI.git +cd TrialMatchAI +uv sync --frozen # CI uses uv 0.11.24 and Python 3.11 +make lint +make test +make demo +make release-check # full local package/CLI/security rehearsal ``` -## Security +CI and release publishing use the same verification workflow: frozen dependency +resolution, lint, workflow validation, tests, secret/dependency scanning, optional +CPU model imports, package construction, and an isolated installed-wheel CLI e2e. +The smoke checks real indexing, filtering, ranking, HTML, report repair, resume, +and checksum commands with model downloads disabled. -Never commit real credentials, private keys, datasets, models, local search data, -run manifests, or results. Keep runtime values local: +Release tags must match package and module versions. Publishing verifies the tested +wheel and sdist against SHA-256 digests, records build inputs, attaches provenance, +and uses PyPI trusted publishing without rebuilding. These are package delivery +gates; production application deployment, GPU integration tests, complete optional +stack auditing, and clinical acceptance gates remain roadmap work. -```bash -cp .env.example .env -``` +Use the [release runbook](https://github.com/cbib/TrialMatchAI/blob/main/docs/release.md), [validation evidence](https://github.com/cbib/TrialMatchAI/blob/main/docs/production-validation.md), +[security policy](https://github.com/cbib/TrialMatchAI/blob/main/SECURITY.md) when preparing +changes. Report reproducible bugs through [GitHub issues](https://github.com/cbib/TrialMatchAI/issues); +do not include identifiable patient data or credentials. -Artifact bootstrap supports optional SHA-256 verification through -`TRIALMATCHAI_PROCESSED_TRIALS_SHA256`, `TRIALMATCHAI_MODELS_SHA256`, and -`TRIALMATCHAI_CRITERIA_PART__SHA256`. - -Dependency auditing currently ignores `CVE-2025-3000` because the pinned inference -stack lists no fixed version; revisit when upgrading. See [`SECURITY.md`](SECURITY.md) -for the full policy. - -## Documentation - -Deeper guides live on the **[documentation site](https://cbib.github.io/TrialMatchAI/)**: - -- **[Pipeline & CLI](https://cbib.github.io/TrialMatchAI/pipeline/)**: the stage registry, `--only/--skip/--from/--to/--force`, ablation, and presets. -- **[Architecture](https://cbib.github.io/TrialMatchAI/architecture/)**: first-level retrieval, constraint-aware ranking, and storage layout. -- **[Patient interoperability](https://cbib.github.io/TrialMatchAI/interoperability/)**: the text, FHIR, Phenopacket, and OMOP importers. -- **[Fine-tuning & custom models](https://cbib.github.io/TrialMatchAI/finetuning/)**: swap the entity, reranker, and reasoning models, plus the training-data formats. -- **[Registry updater](https://cbib.github.io/TrialMatchAI/registry-updater/)**: keep trials current from ClinicalTrials.gov. -- **[API reference](https://cbib.github.io/TrialMatchAI/api/)**: the Python API. - -## Citation - -If you use TrialMatchAI in your research, please cite the Nature Communications -paper: - -> Abdallah, M. _et al._ TrialMatchAI: an end-to-end AI-powered clinical trial -> recommendation system to streamline patient-to-trial matching. _Nature -> Communications_ **17**, 4472 (2026). - -```bibtex -@article{abdallah2026trialmatchai, - title = {TrialMatchAI: an end-to-end AI-powered clinical trial recommendation system to streamline patient-to-trial matching}, - author = {Abdallah, Majd and Nakken, Sigve and Georges, Mikael and Bierkens, Mariska and Galvis, Johanna and Groppi, Alexis and Karkar, Slim and Meiqari, Lana and Rujano, Maria Alexandra and Canham, Steve and Dienstmann, Rodrigo and Fijneman, Remond and Hovig, Eivind and Meijer, Gerrit and Nikolski, Macha}, - journal = {Nature Communications}, - volume = {17}, - pages = {4472}, - year = {2026}, - doi = {10.1038/s41467-026-70509-w}, - url = {https://doi.org/10.1038/s41467-026-70509-w} -} -``` +## Research and license -## Support +Abdallah, M. *et al.* **TrialMatchAI: an end-to-end AI-powered clinical trial +recommendation system to streamline patient-to-trial matching.** *Nature +Communications* **17**, 4472 (2026). +[Read the paper and citation](https://doi.org/10.1038/s41467-026-70509-w). -- Email: abdallahmajd7@gmail.com +TrialMatchAI code is distributed under the [MIT license](https://github.com/cbib/TrialMatchAI/blob/main/LICENSE). Datasets, +model weights, and clinical vocabularies have their own terms. diff --git a/docs/assets/readme-overview.svg b/docs/assets/readme-overview.svg new file mode 100644 index 00000000..ce6abbce --- /dev/null +++ b/docs/assets/readme-overview.svg @@ -0,0 +1,47 @@ + + TrialMatchAI + Patient profiles flow through trial retrieval and criterion review into ranked reports. A configurable CLI with local search and optional model stages. + + + + + + + + + + + CLINICAL TRIAL RETRIEVAL · OPEN SOURCE + TrialMatchAI + From patient information to a reviewable trial shortlist. + + + + + + + + + + + 01 / IMPORT02 / RETRIEVE + 03 / ASSESS04 / REVIEW + + + Patient profilesTrial candidates + Criterion evidenceRanked reports + + + Text · FHIR · Phenopacket · OMOPLocal lexical + vector search + Optional model assessmentsJSON + HTML + + + + Python 3.11 + + Synthetic CPU demo + + Human review required + BETA / v0.9 + + diff --git a/docs/index.md b/docs/index.md index 82f547f4..96031bd1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,64 +1,53 @@ # TrialMatchAI -[![CI](https://github.com/cbib/TrialMatchAI/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cbib/TrialMatchAI/actions/workflows/ci.yml) -[![Docs](https://github.com/cbib/TrialMatchAI/actions/workflows/docs.yml/badge.svg?branch=main)](https://github.com/cbib/TrialMatchAI/actions/workflows/docs.yml) -[![PyPI](https://img.shields.io/pypi/v/trialmatchai?logo=pypi&logoColor=white&label=PyPI&color=blue)](https://pypi.org/project/trialmatchai/) -[![Python versions](https://img.shields.io/pypi/pyversions/trialmatchai?logo=python&logoColor=white&label=Python)](https://pypi.org/project/trialmatchai/) -[![License: MIT](https://img.shields.io/badge/license-MIT-green)](https://github.com/cbib/TrialMatchAI/blob/main/LICENSE) -[![Linted with Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +![Patient profiles, retrieval, criterion evidence, and ranked reports](assets/readme-overview.svg) -**TrialMatchAI matches patients to the clinical trials they're eligible for.** Give -it a patient — clinical notes, FHIR, Phenopacket, or OMOP — and it returns a ranked -shortlist of trials, each with a transparent, criterion-by-criterion explanation of -why the patient does or doesn't qualify. Everything runs on your own infrastructure: -hybrid retrieval over a local LanceDB index paired with chain-of-thought LLM -reasoning served by vLLM on a single GPU, so sensitive patient data never leaves -your environment. +**A configurable CLI for patient-to-trial retrieval and eligibility review.** +TrialMatchAI imports patient information, searches local LanceDB tables, and writes +ranked results and HTML reports. Optional models add entity extraction, expansion, +reranking, and criterion-level assessments. -!!! warning "Research and informational use only" - TrialMatchAI is not medical advice, not a medical device, and must not replace - review by qualified healthcare professionals. +!!! info "Beta release — declared validation scope" + The release gates test the installed CLI with synthetic data and real CPU search. + GPU inference, clinical accuracy, and production clinical deployment remain + separate qualification work. Review generated assessments against source evidence; + a retrieval score is not an eligibility determination. -## Install +## Start with the synthetic demo -```bash -uv pip install "trialmatchai[llm,gpu,entity]" # full model-backed runtime (GPU host) -# or, lightweight (CLI + base deps only): -uv pip install trialmatchai -``` - -## The two halves - -TrialMatchAI runs in two halves — **build the system once**, then **match patients -many times** — and both are idempotent: finished work is never redone. +Install with Python 3.11 in an activated virtual environment: ```bash -trialmatchai bootstrap-data # fetch the prepared corpus + adapters (Zenodo) -trialmatchai build --concepts # prepare + index + concept store (resumable) -trialmatchai e2e --input patient.txt # ingest + match one patient -# -> results//ranked_trials.json + a self-contained results/index.html +python -m pip install trialmatchai==0.9.0 +trialmatchai demo --workdir ./demo-workspace +trialmatchai demo --workdir ./demo-workspace --resume ``` -## One pipeline, maximally modular - -Under the hood everything is a slice of a single, ordered pipeline of idempotent -stages. Run the whole thing, or any subset: - -```bash -trialmatchai pipeline # run every stage (skipping what's done) -trialmatchai pipeline --only match # just (re)match -trialmatchai pipeline --to index # the build half -trialmatchai pipeline --skip expand # ablation: no query expansion -trialmatchai pipeline --force match # redo a stage even if done -``` - -See **[Pipeline & CLI](pipeline.md)** for the stage list and flag scheme, -**[Architecture](architecture.md)** for how it fits together, and the -**[API reference](api.md)** for the Python API. - -## Cite - -> Abdallah, M. *et al.* TrialMatchAI: an end-to-end AI-powered clinical trial -> recommendation system to streamline patient-to-trial matching. *Nature -> Communications* **17**, 4472 (2026). -> +Open the report path printed by the command. This walkthrough uses BM25 and +hashing embeddings with model stages disabled. It creates synthetic records only; +resume preserves completed ranking and repairs a missing or truncated patient report. + +For your own data, use the [README setup instructions](https://github.com/cbib/TrialMatchAI#run-your-own-data). +Model stages require additional dependencies and configuration. The default adapter +IDs currently need replacement with downloaded local paths. Format mappings, cache +invalidation, and registry freshness have documented limits. + +## Guides + +| Guide | What it covers | +| --- | --- | +| [Architecture](architecture.md) | Runtime components and local storage | +| [Pipeline and CLI](pipeline.md) | Stages, presets, and report commands | +| [Patient interoperability](interoperability.md) | Supported formats and mappings | +| [Registry updater](registry-updater.md) | Fetching and indexing registry studies | +| [Fine-tuning](finetuning.md) | Training entry points and data formats | +| [API reference](api.md) | Python interfaces | +| [Release runbook](release.md) | Checksums, bootstrap recovery, CI, and publishing | +| [Validation record](production-validation.md) | Checks performed and their limits | +| [Codebase review](codebase-review-2026-09-12.md) | Historical audit evidence | +| [Production roadmap](production-roadmap.md) | Sequenced clinical, ranking, agent, and deployment work | + +The current pipeline follows a fixed stage sequence. Bounded agents and a clinical +review workspace are planned. This release does not claim newly improved benchmark +scores; the [paper](https://doi.org/10.1038/s41467-026-70509-w) describes the published +research, while the validation record describes this software release. diff --git a/docs/production-roadmap.md b/docs/production-roadmap.md index ae471211..ddcd842f 100644 --- a/docs/production-roadmap.md +++ b/docs/production-roadmap.md @@ -79,16 +79,21 @@ Maintain separate states for clinical eligibility, evidence completeness, and ex Before claiming quality improvements, establish corrected baselines for all-topic recall/nDCG/eligible precision, per-criterion accuracy, false eligibility/exclusion, evidence fidelity, calibration, successful-run fraction, review time, and cost/latency. Thresholds are to be agreed from measured baselines and clinical error tolerance; no arbitrary production score is invented here. -**Initial implementation status** - -P01 is implemented on the foundation branch with recorded local validation in -[the implementation checks](production-validation.md). Hosted checks and repository -protection configuration must be verified on the draft PR. No release or deployment -has been performed. P02–P14 remain planned work, including production GPU security -and actual model qualification. The local full model environment's dependency audit -reported 23 advisory rows across six packages on 12 September 2026, after the legacy -exceptions; some rows are repeated aliases. Do not describe that stack as security -qualified. A clean base/dev audit is a separate, narrower check. +**Foundation release scope** + +P01 is implemented for the 0.9.0 foundation release with recorded checks in +[the validation record](production-validation.md). This includes recoverable +bootstrap tree replacement, writer locking, corrupt-cache recovery, strict marker +checks, expanded FHIR attribution coverage, query-expansion model preflight, and +demo report repair. The release does not implement full cross-stage snapshot +publication or per-file corpus verification (P05). + +P02–P14 remain planned work, including GPU security and actual model qualification. +The existing local model environment's dependency audit reported 23 advisory rows +across six packages on 12 September 2026, after legacy exceptions; some rows are +repeated aliases. A clean base/dev audit is a separate, narrower check. No production +application deployment is included. Repository protections remain administrator +settings rather than guarantees supplied by the workflow files. **Operational definition of done** diff --git a/docs/production-validation.md b/docs/production-validation.md index a8cd43f8..39a3838c 100644 --- a/docs/production-validation.md +++ b/docs/production-validation.md @@ -4,21 +4,25 @@ Branch: `production/reliability-foundation`, based on `508db33`. Date: 12 Septem 2026. This records checks for the implementation, separately from the historical [baseline review evidence](audit_2026_09_12_validation.md). -- Full local pytest suite after review fixes: **452 passed**, 47.34 seconds. - Includes real CLI e2e and 10 regression cases for manifest redirection, - interrupted initialization, safe publication, and retry instructions. +- Full local pytest suite after the final Copilot fixes: **474 passed**. + Includes real CLI e2e with missing/truncated report recovery, plus regression + coverage for fresh bootstrap replacement across all four stages, retained backups + and unrelated models, failed extraction/publication, interrupted recovery, writer + locking, bounded corrupt-cache retries, non-regular markers, explicit unresolved + FHIR subjects, and explicit/fallback query-expansion model access. Hugging Face and Transformers downloads were disabled. - Initial frozen base/dev check, before review fixes: **440 passed, 1 skipped, 1 e2e deselected**; its dependency audit found **no known vulnerabilities without exceptions**. The skipped test compares the package version with an adjacent `pyproject.toml`, which is absent in this non-editable install. The package job checks version consistency from the source checkout separately. +- Final base/dev advisory recheck: **no known vulnerabilities without exceptions**. - Frozen lockfile validation: passed with uv 0.11.24. - Wheel and sdist build: passed. Packaged MedCPT and bge-m3 catalogs resolve. - Isolated installed-wheel CLI smoke from `/tmp`, with checkout imports forbidden: passed. Covers synthetic FHIR import, real LanceDB indexing, adult/pediatric filtering, ranking, report content, resume preserving ranking output, and - public checksum verification. No model downloads or clinical data. + public checksum verification and missing/truncated report repair. No model downloads or clinical data. - Ruff, actionlint 1.7.12, and strict MkDocs build: passed. - CPU imports of torch, Transformers, GLiNER2, embedding, reranking, reasoning, and fine-tuning modules: passed in the existing local environment. This is not @@ -37,5 +41,7 @@ product defect. The synthetic e2e demonstrates software integration, not retrieval benchmark improvement or clinical validity. Published ranking metrics have not been rerun. -Hosted CI and release execution are separate from these local checks; no production -release, model migration, or application deployment was performed. +Hosted CI and release execution are recorded on [PR #31](https://github.com/cbib/TrialMatchAI/pull/31) +and the [0.9.0 release](https://github.com/cbib/TrialMatchAI/releases/tag/v0.9.0). +A package release is separate from model migration and application deployment; +neither of those is included. diff --git a/docs/release.md b/docs/release.md index 7769876f..f0b06fbf 100644 --- a/docs/release.md +++ b/docs/release.md @@ -25,7 +25,9 @@ resume command. Use `trialmatchai demo --workdir /path/to/empty-directory` to ch the location. Initialization publishes its fixtures atomically; if it is interrupted before completion, retry the same command without `--resume`. Once initialized, `--resume` requires the original, unmodified fixture configuration; -create a new workspace to try a different configuration. Runtime state is retained +create a new workspace to try a different configuration. Resume also regenerates a +missing or truncated patient report from the saved ranking/evidence without rerunning +matching. Runtime state is retained for inspection. No real patient data is used by this example or by CI. `trialmatchai --version` prints the package version. The new artifact commands @@ -65,12 +67,32 @@ The manifest implies `--require-checksums`. Alternatively, strict mode accepts `TRIALMATCHAI_PROCESSED_TRIALS_SHA256`, `TRIALMATCHAI_CRITERIA_PART__SHA256`, `TRIALMATCHAI_MODELS_SHA256`, and `TRIALMATCHAI_FINETUNE_DATA_SHA256`. Every requested digest must exist and contain exactly 64 hexadecimal characters before work starts. -Cached archives are hashed before extraction; mismatches fail. Successful extraction -records archive hashes in its completion marker. Strict mode re-extracts legacy -markers without this provenance. Resume trusts a matching extraction marker; it -does not rehash every extracted file. Atomic publication of complete corpus trees -and per-file corpus verification remain roadmap P05 work. Default bootstrap remains -compatible with legacy archives whose digests have not been supplied. +Cached archives are hashed before extraction. A corrupt cache is retained under a +hidden `.corrupt-` filename and replaced with one fresh download attempt; +a repeatedly corrupt source fails without accepting a different digest. Downloads +use `.part` files; completion markers must be regular files, never symlinks or FIFOs. + +Each changed stage extracts into a new sibling directory before replacing its +managed tree. Dedicated criteria/trial/training trees contain only the new archive +contents. The shared `models/` tree replaces incoming and previously recorded owned +roots, preserving unrelated top-level models. Schema-2 completion markers record +archive hashes and owned roots. Strict mode re-extracts legacy markers to migrate +them; legacy model entries without ownership metadata are conservatively preserved +unless an incoming archive replaces the same top-level name. + +One bootstrap writer holds a workspace lock. Publication renames the old tree to +`.NAME.bootstrap-previous` and then publishes the staged tree. Failure restores the +old tree; if killed between renames, the next bootstrap recovers it. Successful +replacement retains the old tree in `.NAME-backup-*`. Stop readers during updates: +the two renames leave a short gap, and stages are published individually, not as +one atomic trial/criteria/index snapshot. Keep sufficient disk space for staging, +backups, and quarantined downloads. Review these retained files before removal; +unrelated preserved model files may share hardlinks with their backup copies. + +Strict resume checks matching archive provenance and the presence of managed roots. +It does not rehash every extracted file. End-to-end snapshot identity, index +publication, and per-file corpus verification remain roadmap P05 work. Default +bootstrap remains compatible with legacy archives whose digests have not been supplied. ## Local release rehearsal @@ -84,7 +106,8 @@ This validates workflows with actionlint and the lockfile, lints, runs tests, bu checks their manifest, installs the wheel in an isolated environment, exercises the public CLI, scans for secrets, and audits installed dependencies. The installed smoke verifies packaged embedder catalogs, synthetic patient import, a real index, -adult/pediatric filtering, ranked output, HTML content, resume, and checksum CLI. +adult/pediatric filtering, ranked output, HTML content, resume, missing/truncated +report repair without ranking changes, and checksum CLI. Model downloads are disabled. The smoke is also callable via `scripts/installed_smoke.py --workspace /empty/path --expect-installed` using an installed environment's Python. @@ -111,7 +134,9 @@ The build records the source commit, Python/uv versions, and lockfile hash. The publisher downloads the previously tested artifacts, verifies the manifest, creates [GitHub provenance](https://docs.github.com/en/actions/concepts/security/artifact-attestations), and publishes the wheel/sdist using [PyPI OIDC](https://docs.pypi.org/trusted-publishers/). -It does not rebuild. Only the publish job receives write-capable identity permissions. +The release package build omits uv's housekeeping `.gitignore`, so the manifest +covers the wheel, sdist, and build metadata. These files remain available in the +workflow artifact for its declared retention period. It does not rebuild. Only the publish job receives write-capable identity permissions. A failed verification blocks publication. Diagnose a partial publish before retrying; do not overwrite an existing PyPI version. diff --git a/pyproject.toml b/pyproject.toml index 5f589524..e6d6281f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "trialmatchai" -version = "0.8.2" +version = "0.9.0" description = "AI-driven patient-to-clinical-trial matching: hybrid retrieval + LLM eligibility reasoning." readme = "README.md" requires-python = ">=3.11,<3.12" diff --git a/scripts/installed_smoke.py b/scripts/installed_smoke.py index 3c3e0dc0..b6ffeb3d 100644 --- a/scripts/installed_smoke.py +++ b/scripts/installed_smoke.py @@ -68,8 +68,19 @@ def run(*arguments): run("demo", "--workdir", str(demo), "--resume") assert (ranked.read_bytes(), ranked.stat().st_mtime_ns) == original assert json.loads((demo / "demo-result.json").read_text())["resumed"] is True + report_path = Path(result["report"]) + for state in ("missing", "truncated"): + if state == "missing": + report_path.unlink() + else: + report_path.write_text("") + run("demo", "--workdir", str(demo), "--resume") + repaired = report_path.read_text() + assert repaired.rstrip().endswith("") + assert "demo-patient" in repaired and "NCT00000001" in repaired + assert (ranked.read_bytes(), ranked.stat().st_mtime_ns) == original run("artifacts", "verify", str(demo / "SHA256SUMS"), "--json") - print("Installed CLI smoke passed: catalog, import, real index, retrieval, report, resume, checksums.") + print("Installed CLI smoke passed: catalog, import, real index, retrieval, report repair, resume, checksums.") if __name__ == "__main__": diff --git a/src/trialmatchai/__init__.py b/src/trialmatchai/__init__.py index 785a615f..3f0fbb08 100644 --- a/src/trialmatchai/__init__.py +++ b/src/trialmatchai/__init__.py @@ -1,3 +1,3 @@ from __future__ import annotations -__version__ = "0.8.2" +__version__ = "0.9.0" diff --git a/src/trialmatchai/cli/bootstrap_data.py b/src/trialmatchai/cli/bootstrap_data.py index de6afa07..969b75f7 100644 --- a/src/trialmatchai/cli/bootstrap_data.py +++ b/src/trialmatchai/cli/bootstrap_data.py @@ -3,11 +3,15 @@ import argparse import json import os +import shutil import stat import sys import tarfile +import tempfile +import uuid import zipfile from collections.abc import Sequence +from contextlib import contextmanager from pathlib import Path import requests @@ -143,73 +147,163 @@ def bootstrap_data( if require_checksums and missing: raise ValueError(f"Missing required SHA-256 checksums: {', '.join(missing)}") - def stage_checksums(names): - return {name: checksums[name] for name in names if checksums[name] is not None} - - def complete(path, expected): - return _extract_complete(path, expected_checksums=expected if require_checksums else None) - - data_dir = root / "data" - models_dir = root / "models" - data_dir.mkdir(parents=True, exist_ok=True) - - criteria_dir = data_dir / "processed_criteria" - criteria_sums = stage_checksums(f"{CHUNK_PREFIX}_{i}.zip" for i in range(criteria_chunks)) - if force or not complete(criteria_dir, criteria_sums): - criteria_dir.mkdir(parents=True, exist_ok=True) - (criteria_dir / _EXTRACT_MARKER).unlink(missing_ok=True) - for index in range(criteria_chunks): - chunk_name = f"{CHUNK_PREFIX}_{index}.zip" - chunk_path = data_dir / chunk_name - _download_if_missing( - f"{criteria_base_url.rstrip('/')}/{chunk_name}?download=1", - chunk_path, - ) - _verify_sha256( - chunk_path, - checksums[chunk_name], - ) - _safe_extract_zip(chunk_path, criteria_dir) - _mark_extract_complete(criteria_dir, checksums=criteria_sums) - - processed_trials_dir = data_dir / "processed_trials" - trial_sums = stage_checksums([PROCESSED_TRIALS_ARCHIVE]) - if force or not complete(processed_trials_dir, trial_sums): - (processed_trials_dir / _EXTRACT_MARKER).unlink(missing_ok=True) - processed_archive = data_dir / PROCESSED_TRIALS_ARCHIVE - _download_if_missing(data_url, processed_archive) - _verify_sha256( - processed_archive, checksums[PROCESSED_TRIALS_ARCHIVE] - ) - _safe_extract_tar_gz(processed_archive, data_dir) - _mark_extract_complete(processed_trials_dir, checksums=trial_sums) + root = Path(root).resolve() + root.mkdir(parents=True, exist_ok=True) + with _bootstrap_lock(root): + data_dir = root / "data" + data_dir.mkdir(parents=True, exist_ok=True) + stages = [ + (data_dir / "processed_criteria", [ + (f"{CHUNK_PREFIX}_{i}.zip", f"{criteria_base_url.rstrip('/')}/{CHUNK_PREFIX}_{i}.zip?download=1", "zip") + for i in range(criteria_chunks) + ], False), + (data_dir / "processed_trials", [(PROCESSED_TRIALS_ARCHIVE, data_url, "trials")], False), + ] + if with_models: + stages.append((root / "models", [(MODELS_ARCHIVE, models_url, "tar")], True)) + if finetune_data: + stages.append((data_dir / "finetune", [(FINETUNE_ARCHIVE, finetune_data_url, "zip")], False)) + for target, archives, shared in stages: + _recover_publication(target) + expected = {name: checksums[name] for name, _, _ in archives if checksums[name] is not None} + if not force and _extract_complete(target, expected_checksums=expected if require_checksums else None): + continue + _replace_stage(target, archives, data_dir, checksums, expected, shared=shared) + _cleanup_archives(data_dir, criteria_chunks) + + +@contextmanager +def _bootstrap_lock(root: Path): + import fcntl + + fd = os.open(root / ".bootstrap.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise ValueError("Bootstrap lock must be a regular file") + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise ValueError(f"Another bootstrap is already running in {root}") from exc + yield + finally: + os.close(fd) + + +def _download_verified(url: str, destination: Path, expected: str | None) -> None: + # One replacement attempt; never weaken a trusted checksum to accept a file. + for attempt in range(2): + if destination.is_symlink() or (destination.exists() and not destination.is_file()): + raise ValueError(f"Cached archive must be a regular file: {destination}") + _download_if_missing(url, destination) + try: + _verify_sha256(destination, expected) + return + except ValueError: + quarantine = destination.with_name(f".{destination.name}.corrupt-{uuid.uuid4().hex}") + destination.replace(quarantine) + _warn(f"Invalid archive retained at {quarantine}") + if attempt: + raise + _info(f"Retrying {destination.name} with a fresh download.") + + +def _replace_stage(target, archives, data_dir, checksums, expected, *, shared): + if target.is_symlink() or (target.exists() and not target.is_dir()): + raise ValueError(f"Extraction destination must be a directory, not a link: {target}") + with tempfile.TemporaryDirectory(prefix=f".{target.name}-bootstrap-", dir=target.parent) as temp: + staging = Path(temp) / "payload" + staging.mkdir() + for name, url, kind in archives: + archive = data_dir / name + _download_verified(url, archive, checksums[name]) + if kind == "zip": + _safe_extract_zip(archive, staging) + elif kind == "trials": + wrapper = Path(temp) / "trials-archive" + _safe_extract_tar_gz(archive, wrapper) + children = list(wrapper.iterdir()) + if len(children) != 1 or children[0].name != "processed_trials" or not children[0].is_dir(): + raise ValueError("Trial archive must contain only the processed_trials/ directory") + children[0].replace(staging) + else: + _safe_extract_tar_gz(archive, staging) + roots = sorted(path.name for path in staging.iterdir() if path.name != _EXTRACT_MARKER) + if not roots: + raise ValueError(f"Archive produced no artifacts for {target.name}") + if shared and target.exists(): + _preserve_unmanaged_models(target, staging, roots) + _mark_extract_complete(staging, checksums=expected, managed_roots=roots) + _publish_stage(staging, target) + + +def _managed_roots(marker): + if not isinstance(marker, dict) or marker.get("schema") != 2: + return [] + roots = marker.get("managed_roots") + if not isinstance(roots, list) or not roots: + return [] + if any(not isinstance(name, str) or name in {"", ".", "..", _EXTRACT_MARKER} + or "/" in name or "\\" in name for name in roots): + return [] + return roots + + +def _preserve_unmanaged_models(target, staging, incoming): + previous = _read_marker(target / _EXTRACT_MARKER) + managed = set(_managed_roots(previous)) | set(incoming) | {_EXTRACT_MARKER} + # Preserve independent top-level models. Replace whole owned adapter folders, + # including removed files; old trees remain available in the recovery backup. + for path in target.iterdir(): + if path.name in managed: + continue + destination = staging / path.name + if path.is_symlink(): + destination.symlink_to(os.readlink(path)) + elif path.is_dir(): + shutil.copytree(path, destination, symlinks=True, copy_function=_link_or_copy) + elif path.is_file(): + _link_or_copy(path, destination) + else: + raise ValueError(f"Cannot preserve non-regular model entry: {path}") + + +def _link_or_copy(source, destination): + try: + os.link(source, destination) + except OSError: + shutil.copy2(source, destination) + return str(destination) - if with_models: - models_dir.mkdir(parents=True, exist_ok=True) - model_sums = stage_checksums([MODELS_ARCHIVE]) - if force or not complete(models_dir, model_sums): - (models_dir / _EXTRACT_MARKER).unlink(missing_ok=True) - models_archive = data_dir / MODELS_ARCHIVE - _download_if_missing(models_url, models_archive) - _verify_sha256(models_archive, checksums[MODELS_ARCHIVE]) - _safe_extract_tar_gz(models_archive, models_dir) - _mark_extract_complete(models_dir, checksums=model_sums) - if finetune_data: - finetune_dir = data_dir / "finetune" - finetune_sums = stage_checksums([FINETUNE_ARCHIVE]) - if force or not complete(finetune_dir, finetune_sums): - finetune_dir.mkdir(parents=True, exist_ok=True) - (finetune_dir / _EXTRACT_MARKER).unlink(missing_ok=True) - finetune_archive = data_dir / FINETUNE_ARCHIVE - _download_if_missing(finetune_data_url, finetune_archive) - _verify_sha256( - finetune_archive, checksums[FINETUNE_ARCHIVE] - ) - _safe_extract_zip(finetune_archive, finetune_dir) - _mark_extract_complete(finetune_dir, checksums=finetune_sums) - - _cleanup_archives(data_dir, criteria_chunks) +def _previous_path(target): + return target.with_name(f".{target.name}.bootstrap-previous") + + +def _recover_publication(target): + previous = _previous_path(target) + if not previous.exists() and not previous.is_symlink(): + return + if previous.is_symlink() or not previous.is_dir(): + raise ValueError(f"Invalid bootstrap recovery directory: {previous}") + if not target.exists() and not target.is_symlink(): + previous.replace(target) + _info(f"Recovered the previous extraction at {target}") + else: + saved = Path(tempfile.mkdtemp(prefix=f".{target.name}-backup-", dir=target.parent)) + previous.replace(saved) + _info(f"Previous extraction retained at {saved}") + + +def _publish_stage(staging, target): + previous = _previous_path(target) + if target.exists(): + target.replace(previous) + try: + staging.replace(target) + except BaseException: + _recover_publication(target) + raise + _recover_publication(target) def _download_if_missing(url: str, destination: Path) -> None: @@ -222,12 +316,17 @@ def _download_if_missing(url: str, destination: Path) -> None: # Stream to .part and rename on success so a killed download never leaves a truncated # archive at the final path (checksum verification is optional, so can't catch it). partial = destination.with_name(destination.name + ".part") + if partial.is_symlink() or (partial.exists() and not partial.is_file()): + raise ValueError(f"Partial download must be a regular file: {partial}") response = requests.get(url, stream=True, timeout=120) - response.raise_for_status() - with partial.open("wb") as file: - for chunk in response.iter_content(chunk_size=1024 * 1024): - if chunk: - file.write(chunk) + try: + response.raise_for_status() + with partial.open("wb") as file: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if chunk: + file.write(chunk) + finally: + response.close() partial.replace(destination) @@ -290,24 +389,38 @@ def _cleanup_archives(data_dir: Path, criteria_chunks: int) -> None: _EXTRACT_MARKER = ".bootstrap_complete" -def _extract_complete(path: Path, *, expected_checksums: dict[str, str] | None = None) -> bool: - """True only when a prior extract wrote its completion sentinel. +def _read_marker(marker: Path) -> dict | None: + try: + fd = os.open(marker, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "r", encoding="utf-8") as handle: + metadata = os.fstat(handle.fileno()) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > 1024 * 1024: + return None + value = json.load(handle) + return value if isinstance(value, dict) else None + except (OSError, ValueError): + return None - Presence of *some* entries is not proof: a killed extract leaves a partial tree - that would otherwise bake a truncated corpus into every later run. - """ + +def _extract_complete(path: Path, *, expected_checksums: dict[str, str] | None = None) -> bool: marker = path / _EXTRACT_MARKER - if expected_checksums is None: - return marker.is_file() try: - data = json.loads(marker.read_text(encoding="utf-8")) - return isinstance(data, dict) and data.get("schema") == 1 and data.get("sha256") == expected_checksums - except (OSError, ValueError): + if not stat.S_ISREG(marker.lstat().st_mode): + return False + except OSError: return False + if expected_checksums is None: + return True # Legacy non-strict completion markers remain compatible. + value = _read_marker(marker) + roots = _managed_roots(value) + return bool(roots) and value.get("sha256") == expected_checksums and all( + (path / name).exists() and not (path / name).is_symlink() for name in roots + ) -def _mark_extract_complete(path: Path, *, checksums: dict[str, str] | None = None) -> None: - write_json_file({"schema": 1, "sha256": checksums or {}}, str(path / _EXTRACT_MARKER)) +def _mark_extract_complete(path: Path, *, checksums: dict[str, str] | None = None, managed_roots: list[str] | None = None) -> None: + roots = managed_roots if managed_roots is not None else sorted(p.name for p in path.iterdir() if p.name != _EXTRACT_MARKER) + write_json_file({"schema": 2, "sha256": checksums or {}, "managed_roots": roots}, str(path / _EXTRACT_MARKER)) def _runtime_root() -> Path: diff --git a/src/trialmatchai/cli/demo.py b/src/trialmatchai/cli/demo.py index f7a234cc..1600aafb 100644 --- a/src/trialmatchai/cli/demo.py +++ b/src/trialmatchai/cli/demo.py @@ -115,17 +115,35 @@ def run_demo(workdir: str | Path | None = None, *, resume: bool = False) -> dict raise RuntimeError(f"Demo pipeline failed with exit code {rc}") patient_dir = root / "results/demo-patient" ranked = json.loads((patient_dir / "ranked_trials.json").read_text())["RankedTrials"] + report = patient_dir / "report.html" + if not _report_complete(report): + # Ranking may have completed before reporting was interrupted. Repair + # the derived report from saved evidence without repeating matching. + from trialmatchai.main import _maybe_write_report + + _maybe_write_report(patient_dir, config) result = { "schema_version": 1, "mode": "synthetic_retrieval_only", "workspace": str(root), "config": str(root / "config.json"), "report": str(patient_dir / "report.html"), "ranked_trial_ids": [row["TrialID"] for row in ranked], "resumed": resume, } - if not ranked or not Path(result["report"]).is_file(): + if not ranked or not _report_complete(report): raise RuntimeError("Demo did not produce a non-empty ranking and HTML report") write_json_file(result, str(root / "demo-result.json")) return result +def _report_complete(path: Path) -> bool: + if path.is_symlink(): + raise ValueError("Demo report must not be a symlink") + if not path.is_file(): + return False + try: + return path.read_text(encoding="utf-8").rstrip().endswith("") + except UnicodeError: + return False + + def main(argv=None) -> int: parser = argparse.ArgumentParser(prog="trialmatchai demo", description=__doc__) parser.add_argument("--workdir", type=Path, help="Empty workspace directory (default: a new temporary directory)") diff --git a/src/trialmatchai/interop/importers/fhir.py b/src/trialmatchai/interop/importers/fhir.py index 58ae28e8..c04054d1 100644 --- a/src/trialmatchai/interop/importers/fhir.py +++ b/src/trialmatchai/interop/importers/fhir.py @@ -213,6 +213,10 @@ def _profile_for_resource( # An explicit foreign/unresolved subject must never be attributed to the # sole patient. Only a genuinely absent subject permits that fallback. return None + if any(key in resource for key in ("subject", "patient", "beneficiary")): + # Identifier/display-only or malformed references are explicit subjects, + # even when this importer cannot resolve them to a patient. + return None if len(profiles) == 1: return profiles[0] return None diff --git a/src/trialmatchai/main.py b/src/trialmatchai/main.py index 83c8df8c..fd01075a 100644 --- a/src/trialmatchai/main.py +++ b/src/trialmatchai/main.py @@ -57,7 +57,7 @@ def _maybe_write_report(output_folder, config) -> None: summary_dir=config.get("patient_inputs", {}).get("summary_dir"), trial_meta_folders=[str(Path(trials_json).parent / "processed_trials"), trials_json], ) - Path(output_folder, "report.html").write_text(html, encoding="utf-8") + write_text_file([html], str(Path(output_folder, "report.html"))) except Exception: logger.warning("HTML report generation failed for %s", output_folder, exc_info=True) diff --git a/src/trialmatchai/orchestration.py b/src/trialmatchai/orchestration.py index 189f140d..57fcddd9 100644 --- a/src/trialmatchai/orchestration.py +++ b/src/trialmatchai/orchestration.py @@ -107,7 +107,11 @@ def expand_queries(config: Dict[str, Any], *, force: bool = False) -> int: Idempotent: a summary already marked ``query_expanded`` is skipped. """ from trialmatchai.matching.query_expansion import build_query_expander, enrich_summary + from trialmatchai.services.preflight import check_query_expansion_models + issues = check_query_expansion_models(config) + if issues: + raise ValueError("Query expansion preflight failed: " + "; ".join(issues)) expander = build_query_expander(config) if expander is None: logger.info("Query expansion disabled; using deterministic summaries.") diff --git a/src/trialmatchai/services/preflight.py b/src/trialmatchai/services/preflight.py index f17e2bec..025a0fc8 100644 --- a/src/trialmatchai/services/preflight.py +++ b/src/trialmatchai/services/preflight.py @@ -129,6 +129,7 @@ def run_preflight_checks( requested_models.append(model_cfg.get("reranker_model_path")) if requested_models: issues.extend(check_hf_access(requested_models)) + issues.extend(check_query_expansion_models(config)) search_cfg = config.get("search_backend", {}) if search_cfg: @@ -158,6 +159,18 @@ def run_preflight_checks( return issues +def check_query_expansion_models(config: Dict[str, Any]) -> List[str]: + """Check the actual expansion model before the expansion stage loads it.""" + if not config.get("query_expansion", {}).get("enabled", False): + return [] + from trialmatchai.matching.query_expansion import _resolve_settings + + model = _resolve_settings(config)["model"] + if not model: + return ["Query expansion requires query_expansion.model or model.base_model."] + return check_hf_access([model]) + + def check_cuda(config: Dict[str, Any], *, required: bool) -> List[str]: """Verify a CUDA GPU is present when the build/match needs one.""" issues: List[str] = [] diff --git a/tests/test_audit_fixes.py b/tests/test_audit_fixes.py index afb240c3..db391867 100644 --- a/tests/test_audit_fixes.py +++ b/tests/test_audit_fixes.py @@ -150,7 +150,7 @@ def expand(self, narrative): monkeypatch.setattr(qe, "build_query_expander", lambda cfg: _Expander()) orchestration.expand_queries( { - "query_expansion": {"enabled": True}, + "query_expansion": {"enabled": True, "model": "synthetic-local-model"}, "patient_inputs": {"profile_dir": str(profiles), "summary_dir": str(summaries)}, } ) diff --git a/tests/test_bootstrap_data.py b/tests/test_bootstrap_data.py index 84018b06..859f232b 100644 --- a/tests/test_bootstrap_data.py +++ b/tests/test_bootstrap_data.py @@ -2,6 +2,8 @@ import hashlib import io +import json +import os import tarfile import zipfile from pathlib import Path @@ -111,6 +113,7 @@ def test_strict_bootstrap_corruption_prevents_extraction(tmp_path, monkeypatch): (data / "criteria_part_0.zip").write_bytes(b"corrupt") monkeypatch.setenv("TRIALMATCHAI_PROCESSED_TRIALS_SHA256", "0" * 64) monkeypatch.setenv("TRIALMATCHAI_CRITERIA_PART_0_SHA256", "0" * 64) + monkeypatch.setattr("requests.get", lambda *a, **kw: _Response(b"still corrupt")) monkeypatch.setattr("trialmatchai.cli.bootstrap_data._safe_extract_zip", lambda *a: pytest.fail("Must not extract corrupt data")) with pytest.raises(ValueError, match="Checksum mismatch"): bootstrap_data(root=tmp_path, criteria_chunks=1, require_checksums=True) @@ -168,3 +171,156 @@ def _write_zip(path: Path, files: dict[str, str]) -> None: def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() + + +class _Response: + def __init__(self, content): + self.content = content + + def raise_for_status(self): + pass + + def iter_content(self, **kwargs): + yield self.content + + def close(self): + pass + + +def _snapshot(root, version): + data = root / "data" + data.mkdir(exist_ok=True) + _write_tar_gz(data / PROCESSED_TRIALS_ARCHIVE, {f"processed_trials/{version}.json": b"{}"}) + _write_zip(data / "criteria_part_0.zip", {f"{version}.json": "{}"}) + models = {f"adapter/{version}.bin": version.encode()} + if version == "old": + models["retired/config.json"] = b"{}" + _write_tar_gz(data / MODELS_ARCHIVE, models) + _write_zip(data / "finetuning_datasets.zip", {f"{version}.jsonl": "{}"}) + names = [PROCESSED_TRIALS_ARCHIVE, "criteria_part_0.zip", MODELS_ARCHIVE, "finetuning_datasets.zip"] + manifest = root / "SHA256SUMS" + manifest.write_text("\n".join(f"{_sha256(data / name)} {name}" for name in names)) + return manifest + + +def _bootstrap_snapshot(root, manifest, **kwargs): + bootstrap_data(root=root, criteria_chunks=1, with_models=True, finetune_data=True, checksum_manifest=manifest, **kwargs) + + +def test_changed_snapshot_removes_stale_managed_files_and_preserves_user_models(tmp_path): + user_model = tmp_path / "models/user-model/weights.bin" + user_model.parent.mkdir(parents=True) + user_model.write_bytes(b"independent model") + manifest = _snapshot(tmp_path, "old") + unrelated = tmp_path / "data/user-notes.txt" + unrelated.write_text("preserve") + _bootstrap_snapshot(tmp_path, manifest) + _bootstrap_snapshot(tmp_path, _snapshot(tmp_path, "new")) + + for folder, suffix in (("data/processed_trials", "json"), ("data/processed_criteria", "json"), + ("models/adapter", "bin"), ("data/finetune", "jsonl")): + assert not (tmp_path / folder / f"old.{suffix}").exists() + assert (tmp_path / folder / f"new.{suffix}").exists() + assert not (tmp_path / "models/retired").exists() + assert user_model.read_bytes() == b"independent model" + assert unrelated.read_text() == "preserve" + assert list((tmp_path / "data").glob(".processed_trials-backup-*/old.json")) + marker = json.loads((tmp_path / "models/.bootstrap_complete").read_text()) + assert marker["managed_roots"] == ["adapter"] + + +def test_failed_extraction_keeps_previous_active_snapshot(tmp_path, monkeypatch): + import trialmatchai.cli.bootstrap_data as module + + _bootstrap_snapshot(tmp_path, _snapshot(tmp_path, "old")) + marker = tmp_path / "data/processed_criteria/.bootstrap_complete" + previous_marker = marker.read_bytes() + + def interrupted(archive, target): + (target / "partial.json").write_text("{}") + raise OSError("simulated extraction failure") + + monkeypatch.setattr(module, "_safe_extract_zip", interrupted) + with pytest.raises(OSError, match="simulated extraction failure"): + _bootstrap_snapshot(tmp_path, _snapshot(tmp_path, "new")) + assert (marker.parent / "old.json").exists() + assert not (marker.parent / "partial.json").exists() + assert marker.read_bytes() == previous_marker + + +def test_failed_publication_restores_previous_snapshot(tmp_path, monkeypatch): + _bootstrap_snapshot(tmp_path, _snapshot(tmp_path, "old")) + target = tmp_path / "data/processed_criteria" + original = Path.replace + + def interrupted(source, destination): + if source.name == "payload" and Path(destination) == target: + raise OSError("simulated publication failure") + return original(source, destination) + + monkeypatch.setattr(Path, "replace", interrupted) + with pytest.raises(OSError, match="simulated publication failure"): + _bootstrap_snapshot(tmp_path, _snapshot(tmp_path, "new")) + assert (target / "old.json").exists() + assert not (target / "new.json").exists() + + +def test_next_bootstrap_recovers_an_interrupted_directory_swap(tmp_path, monkeypatch): + _bootstrap_snapshot(tmp_path, _snapshot(tmp_path, "old")) + target = tmp_path / "data/processed_criteria" + target.rename(target.with_name(".processed_criteria.bootstrap-previous")) + monkeypatch.setattr("requests.get", lambda *a, **kw: pytest.fail("valid recovered snapshot must resume")) + _bootstrap_snapshot(tmp_path, tmp_path / "SHA256SUMS") + assert (target / "old.json").exists() + + +@pytest.mark.parametrize("kind", ["symlink", "fifo", "directory"]) +def test_completion_rejects_nonregular_markers(tmp_path, kind): + from trialmatchai.cli.bootstrap_data import _extract_complete + + marker = tmp_path / ".bootstrap_complete" + if kind == "symlink": + external = tmp_path / "external.json" + external.write_text(json.dumps({"schema": 2, "sha256": {}, "managed_roots": ["external.json"]})) + marker.symlink_to(external) + elif kind == "fifo": + os.mkfifo(marker) + else: + marker.mkdir() + assert not _extract_complete(tmp_path) + assert not _extract_complete(tmp_path, expected_checksums={}) + + +def test_corrupt_cached_archive_is_quarantined_and_downloaded_once(tmp_path, monkeypatch): + from trialmatchai.cli.bootstrap_data import _download_verified + + path = tmp_path / "archive.zip" + path.write_bytes(b"good archive") + expected = _sha256(path) + path.write_bytes(b"bad cache") + calls = [] + monkeypatch.setattr("requests.get", lambda *a, **kw: calls.append(a) or _Response(b"good archive")) + _download_verified("https://example.invalid/archive.zip", path, expected) + assert path.read_bytes() == b"good archive" + assert len(calls) == 1 + assert next(tmp_path.glob(".archive.zip.corrupt-*")).read_bytes() == b"bad cache" + + +def test_corrupt_source_has_a_bounded_retry_and_never_accepts_bad_data(tmp_path, monkeypatch): + from trialmatchai.cli.bootstrap_data import _download_verified + + calls = [] + monkeypatch.setattr("requests.get", lambda *a, **kw: calls.append(a) or _Response(b"bad source")) + path = tmp_path / "archive.zip" + with pytest.raises(ValueError, match="Checksum mismatch"): + _download_verified("https://example.invalid/archive.zip", path, "0" * 64) + assert len(calls) == 2 + assert not path.exists() + + +def test_concurrent_bootstrap_is_rejected(tmp_path): + from trialmatchai.cli.bootstrap_data import _bootstrap_lock + + with _bootstrap_lock(tmp_path): + with pytest.raises(ValueError, match="already running"): + bootstrap_data(root=tmp_path, criteria_chunks=1) diff --git a/tests/test_deployment_readiness.py b/tests/test_deployment_readiness.py index aa253281..af399780 100644 --- a/tests/test_deployment_readiness.py +++ b/tests/test_deployment_readiness.py @@ -121,3 +121,32 @@ def test_disabled_llms_do_not_check_model_access(tmp_path, monkeypatch): "LLM_reranker": {"enabled": False}, "rag": {"enabled": False}, "model": {"base_model": "org/private-model", "reranker_model_path": "org/private-reranker"}, }, require_models=True) == [] + + +def test_query_expansion_only_checks_explicit_and_fallback_models(tmp_path, monkeypatch): + from trialmatchai.services import preflight + + calls = [] + monkeypatch.setattr(preflight, "check_hf_access", lambda ids: calls.append(ids) or ["synthetic access failure"]) + config = { + "paths": {"output_dir": str(tmp_path / "results")}, + "LLM_reranker": {"enabled": False}, "rag": {"enabled": False}, + "model": {"base_model": "org/base"}, "query_expansion": {"enabled": True}, + } + assert preflight.run_preflight_checks(config, require_models=True) == ["synthetic access failure"] + assert calls == [["org/base"]] + config["query_expansion"]["model"] = "org/expander" + calls.clear() + assert preflight.run_preflight_checks(config, require_models=True) == ["synthetic access failure"] + assert calls == [["org/expander"]] + + +def test_expansion_access_failure_stops_before_model_loading(monkeypatch): + import pytest + from trialmatchai import orchestration + from trialmatchai.services import preflight + + monkeypatch.setattr(preflight, "check_hf_access", lambda ids: ["synthetic gated-model failure"]) + monkeypatch.setattr("trialmatchai.matching.query_expansion.build_query_expander", lambda cfg: pytest.fail("must not load a blocked model")) + with pytest.raises(ValueError, match="Query expansion preflight failed"): + orchestration.expand_queries({"query_expansion": {"enabled": True, "model": "org/expander"}}) diff --git a/tests/test_fhir_robustness.py b/tests/test_fhir_robustness.py index 84fb7263..91a46812 100644 --- a/tests/test_fhir_robustness.py +++ b/tests/test_fhir_robustness.py @@ -232,12 +232,17 @@ def test_fhir_ndjson_strict_raises_on_malformed_line(tmp_path): @pytest.mark.parametrize("strict", [False, True]) -def test_foreign_subject_never_falls_back_to_only_patient(tmp_path, strict): +@pytest.mark.parametrize("subject", [ + {"reference": "Patient/patient-B"}, + {"identifier": {"system": "urn:synthetic", "value": "patient-B"}}, + {"display": "Synthetic patient B"}, {"reference": ""}, {}, None, +]) +def test_foreign_subject_never_falls_back_to_only_patient(tmp_path, strict, subject): path = tmp_path / "mixed.json" path.write_text(json.dumps({"resourceType": "Bundle", "entry": [ {"resource": {"resourceType": "Patient", "id": "patient-A"}}, {"resource": {"resourceType": "Condition", "id": "foreign-condition", - "subject": {"reference": "Patient/patient-B"}, "code": {"text": "diabetes"}}}, + "subject": subject, "code": {"text": "diabetes"}}}, ]})) if strict: with pytest.raises(ValueError, match="no resolvable patient reference"): diff --git a/uv.lock b/uv.lock index bea571a5..2b0770a2 100644 --- a/uv.lock +++ b/uv.lock @@ -3080,7 +3080,7 @@ wheels = [ [[package]] name = "trialmatchai" -version = "0.8.2" +version = "0.9.0" source = { editable = "." } dependencies = [ { name = "lancedb", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'darwin'" },