diff --git a/.config/mise/tasks/cli/qualify-mise b/.config/mise/tasks/cli/qualify-mise new file mode 100755 index 00000000..e593bc24 --- /dev/null +++ b/.config/mise/tasks/cli/qualify-mise @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +#MISE description="Qualify the candidate CLI with isolated mise uv and pipx installs" +set -euo pipefail + +native="${NATIVE_DIST:-$PWD/dist/native}" +receipts="${QUALIFICATION_DIR:-$PWD/.dev/qualification}" +if [[ ${RELEASE_TAG:-} == python-v* ]]; then + native="$receipts/published-native" +fi +python -m scripts.qualify_mise \ + packages/morphir/dist/morphir-*.whl \ + "$native"/morphir_python_native-*.whl \ + packages/morphir-tools/dist/morphir_python-*.whl \ + --report "$receipts/mise-install.json" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28dfb8ee..2fcadc99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,6 +264,12 @@ jobs: # A python-v release qualifies the published native wheel instead. RELEASE_TAG: ${{ inputs.release-tag }} + - name: Qualify mise uv and pipx candidate installs + if: ${{ !startsWith(inputs.release-tag, 'morphir-v') }} + run: mise run cli:qualify-mise + env: + RELEASE_TAG: ${{ inputs.release-tag }} + - name: Upload morphir distributions uses: actions/upload-artifact@v6 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25989a3b..a4f72300 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,8 @@ jobs: needs: qualify if: >- github.event_name == 'workflow_dispatch' || - startsWith(github.ref, 'refs/tags/native-v') + startsWith(github.ref, 'refs/tags/native-v') || + startsWith(github.ref, 'refs/tags/python-v') strategy: fail-fast: false matrix: @@ -66,19 +67,19 @@ jobs: uses: actions/checkout@v6 - name: Reuse the qualification wheel - if: matrix.reuse + if: matrix.reuse && !startsWith(github.ref, 'refs/tags/python-v') uses: actions/download-artifact@v7 with: name: ${{ matrix.reuse }} path: dist/reused/ - name: Keep only the reused wheel - if: matrix.reuse + if: matrix.reuse && !startsWith(github.ref, 'refs/tags/python-v') shell: bash run: mkdir -p dist/native && cp dist/reused/*.whl dist/native/ - name: Build the wheel - if: ${{ !matrix.reuse }} + if: ${{ !matrix.reuse && !startsWith(github.ref, 'refs/tags/python-v') }} uses: PyO3/maturin-action@v1 env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.target == 'aarch64-apple-darwin' && '11.0' || '10.12' }} @@ -90,6 +91,22 @@ jobs: manylinux: ${{ matrix.manylinux || 'off' }} args: --release --locked --out ../../dist/native + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Download the pinned published native wheel for a CLI release + if: startsWith(github.ref, 'refs/tags/python-v') + shell: bash + run: | + pin="$(python -c 'from scripts.qualify_native import tools_pin; print(tools_pin())')" + uvx --from pip==26.2.2 pip download --no-deps --only-binary=:all: \ + --dest dist/native "morphir-python-native==$pin" + - name: Audit the Linux baseline and bundled libraries if: matrix.manylinux != '' shell: bash @@ -101,14 +118,6 @@ jobs: tr '\n' ' ' < audit.txt | tr -s ' ' \ | grep -q 'consistent with the following platform tag: "${{ matrix.platform }}"' - - name: Set up Python 3.14 - uses: actions/setup-python@v6 - with: - python-version: '3.14' - - - name: Set up uv - uses: astral-sh/setup-uv@v7 - - name: Download the qualified core and tools wheels uses: actions/download-artifact@v7 with: @@ -125,6 +134,31 @@ jobs: python scripts/qualify_native.py install \ dist/pure/morphir-*.whl "$wheel" dist/pure/morphir_python-*.whl + # The Linux reuse already passed this in qualify. Other platforms and + # CLI releases exercise the package-manager path with their exact wheels. + - name: Set up mise without installing development toolchains + if: ${{ !matrix.reuse || startsWith(github.ref, 'refs/tags/python-v') }} + uses: jdx/mise-action@v3 + with: + version: '2026.10.20' + install: false + + - name: Qualify mise uv and actual pipx installs + if: ${{ !matrix.reuse || startsWith(github.ref, 'refs/tags/python-v') }} + shell: bash + run: | + python -m scripts.qualify_mise \ + dist/pure/morphir-*.whl dist/native/morphir_python_native-*.whl \ + dist/pure/morphir_python-*.whl --report dist/mise-install.json + + - name: Upload mise qualification evidence + if: ${{ !matrix.reuse || startsWith(github.ref, 'refs/tags/python-v') }} + uses: actions/upload-artifact@v6 + with: + name: mise-install-${{ matrix.platform }} + path: dist/mise-install.json + if-no-files-found: error + - name: Upload the qualified wheel uses: actions/upload-artifact@v6 with: @@ -135,7 +169,9 @@ jobs: native-candidates: name: Verify native release candidates needs: [qualify, native-wheels] - if: needs.native-wheels.result == 'success' + if: >- + needs.native-wheels.result == 'success' && + !startsWith(github.ref, 'refs/tags/python-v') runs-on: ubuntu-latest steps: - name: Checkout repository @@ -228,6 +264,8 @@ jobs: "$bundle/morphir-python-binding-$version.wasm.sha256" dist/bundle/ cp "$bundle/release.json" "dist/bundle/morphir-python-binding-$version.release.json" cp dist/bundle-check/.dev/qualification/bundle*.json dist/bundle/ + # Seal the final asset names, including the renamed descriptor. + (cd dist/bundle && sha256sum -- morphir-python-binding-* bundle*.json > SHA256SUMS) - name: Upload the release set uses: actions/upload-artifact@v6 @@ -256,10 +294,16 @@ jobs: name: extension-release path: dist/ + - name: Verify the final release assets + run: | + cd dist + sha256sum --strict --check SHA256SUMS + - name: Create the GitHub release uses: softprops/action-gh-release@v2 with: files: dist/* + fail_on_unmatched_files: true generate_release_notes: true prerelease: true @@ -293,7 +337,7 @@ jobs: github.event_name == 'push' && github.repository == 'finos/morphir-python' && startsWith(github.ref, 'refs/tags/python-v') - needs: qualify + needs: [qualify, native-wheels] runs-on: ubuntu-latest permissions: id-token: write @@ -336,6 +380,7 @@ jobs: uses: softprops/action-gh-release@v2 with: files: dist/* + fail_on_unmatched_files: true generate_release_notes: true draft: false prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') || contains(github.ref, 'rc') }} diff --git a/README.md b/README.md index a8974bad..f64c27c8 100644 --- a/README.md +++ b/README.md @@ -25,17 +25,21 @@ The library and CLI are separate distributions: Install just the library and SDK: ```bash -pip install morphir==0.0.2 -# or: uv add morphir==0.0.2 +pip install morphir==0.1.0 +# or: uv add morphir==0.1.0 ``` Install the CLI, which also installs the library: ```bash -pip install morphir-python==0.0.3 -# or: uv add morphir-python==0.0.3 +python -m pip install --only-binary=:all: morphir-python==0.1.0 +# or: uv add morphir-python==0.1.0 ``` +For an isolated command-line installation, use the +[mise/uv or mise/pipx configuration](docs/installation.md#install-the-cli-with-mise). +mise manages Python and the CLI environment for you. + Python 3.14 or later is required. The CLI inspects and converts IR, compiles a supported subset of Python to Morphir IR, and generates Python from Morphir IR. The compiler runs in process from prebuilt wheels; you do not need Rust. See the diff --git a/docs/installation.md b/docs/installation.md index 2af4a67c..ca9b5ff4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,6 +1,78 @@ # Install Morphir Python -Use CPython 3.14. Choose a package according to what your application needs. +Use CPython 3.14. For command-line use, mise manages the interpreter and an +isolated CLI environment for you. For Python application development, install +the packages into your application's environment. + +## Install the CLI with mise + +With [mise](https://mise.jdx.dev/getting-started.html) installed, copy this into +your project's `mise.toml`: + + +```toml +[tools] +python = "3.14" +uv = "0.12.10" +"pypi:morphir-python" = { version = "0.1.0", uvx_args = "--no-build" } +``` + + +Then run: + +```bash +mise trust +mise install +mise exec -- morphir-python --help +``` + +This uses mise's `pypi:` backend with uv, installs the published package from +PyPI, and exposes `morphir-python`. `--no-build` refuses source distributions: +the native compiler must have a compatible wheel, so installation never needs +a Rust toolchain. This does not install from a GitHub source checkout or a +standalone executable archive. GitHub Releases also retain the CLI wheel and +source archive; see [the release process](releasing.md). + +To use actual pipx instead of uv, use this alternative `mise.toml`: + + +```toml +[tool_alias] +pipx = "pypi:pipx" + +[tools] +python = "3.14" +uv = "0.12.10" +pipx = { version = "1.18.4", python = "3.14", expose = ["pipx"] } +"pipx:morphir-python" = { version = "0.1.0", python = "3.14", uvx = false, pipx_args = "--pip-args='--only-binary=:all:'" } +``` + + +Run the same three commands above. The alias installs pipx itself through the +portable PyPI backend using uv, including on Windows, and gives mise the correct +dependency ordering. The CLI still installs through actual pipx: +`uvx = false` selects pipx, `python = "3.14"` +on the tool selects its interpreter, and `--only-binary=:all:` enforces the same +wheel-only policy. Choose one configuration. `pipx:` and `pypi:` are accepted +backend names, but mise treats them as separate installed tools. The +[mise backend documentation](https://mise.jdx.dev/dev-tools/backends/pypi.html) +explains these options. + +For a global installation, put the chosen entries in your global mise +configuration, normally `~/.config/mise/config.toml` on macOS/Linux, then run +`mise install`. `mise config ls` shows the configuration files in use. With mise shell +activation enabled, run `morphir-python` directly. Without shell activation, +`mise exec -- morphir-python ...` works from a directory using the configuration. +The CLI's environment is separate from your project's Python dependencies. + +The examples require mise 2026.10.20 or later. The versions are intentionally +pinned; update the CLI version when adopting another release. Pinning the CLI +does not freeze every transitive dependency. The version-only examples use the +installer flags above; they do not claim a fully locked dependency graph. + +## Install Python packages + +Choose a package according to what your application needs. | Package | Install for | What it installs | | --- | --- | --- | @@ -61,3 +133,9 @@ whether the interpreter is free-threaded. If `morphir-python` cannot be found, activate the environment and check its `bin` or `Scripts` directory. A model error such as `PY004` is a compiler diagnostic; reinstalling packages will not make unsupported syntax compile. + +For mise, check `mise which morphir-python` and run with `mise exec --` if the +command is absent from your shell. If its managed interpreter was removed or +replaced, reinstall with `mise install --force pypi:morphir-python` or the +`pipx:` name you configured. A wheel-only refusal is an explicit unsupported +environment, not an instruction to install Rust. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 00000000..6bcdb77f --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,78 @@ +# Release Python packages and the WASM extension + +The package versions are independent. A maintainer approves tags and publication. +The [Release workflow](https://github.com/finos/morphir-python/actions/workflows/release.yml) +qualifies the tagged commit before publishing. Its manual run qualifies a +candidate, including the platform matrix, without publishing a release. + +| Tag | Published output | Installation | +| --- | --- | --- | +| `morphir-v` | Core wheel and source archive on PyPI and GitHub Releases | Python package installer | +| `native-v` | Five native compiler wheels and a source archive on PyPI | Exact dependency of the CLI | +| `python-v` | CLI wheel and source archive on PyPI and GitHub Releases | mise `pypi:`/`pipx:`, uv or pip | +| `extension/python/v` | WASM, SHA-256 checksum, release descriptor, qualification reports and `SHA256SUMS` on GitHub Releases | Morphir extension host | + +The CLI release is a Python distribution. mise installs it from PyPI into an +isolated environment; GitHub release assets are distribution mirrors, not +standalone executables. The [installation guide](installation.md) gives the +tested configurations. The WASM guest is a separate artifact and does not +install through the CLI's Python package backend. + +## Prepare a release + +1. Set the package version and its corresponding constants and dependency pins. + Keep the native crate, native package metadata and the CLI's exact native + requirement consistent. Run `mise run check`. +2. Run the Release workflow manually on the candidate branch. Inspect every + required job, the platform install checks and qualification receipts. Missing, + failed, cancelled or skipped required evidence blocks publication. +3. Release the core library and native package before a CLI release that needs + their new versions. The CLI gate requires published matching dependencies and + refuses native source changes after its pinned `native-v` tag. +4. After maintainer approval, push the corresponding version tag at the reviewed + commit. A manual run does not authorize or perform publication. Check the + tagged run and the resulting PyPI files or GitHub release assets. + +## What qualification proves + +Normal CI checks a fresh candidate installation with both mise/uv and actual +mise/pipx on Linux. It reuses the existing core, CLI and native wheels. The +candidate install is restricted to an offline wheel directory, with dependencies +resolved under the workspace release constraints, so PyPI cannot substitute a +published package with the same version for the candidate. + +Native releases and manual release qualification cover Linux x86_64/aarch64, +macOS arm64/x86_64 and Windows x86_64. A CLI release also runs these platforms, +using its pinned published native wheels instead of rebuilding Rust. Every +selected platform must pass before the CLI publishes. The reused Linux wheel +already passes the same installer checks in the qualification workflow. + +Each installer has separate temporary mise configuration, state, cache and +package environments outside the checkout. Tests check installed versions, +compile and generate both IR v3 and draft v4, recompile generated source and run +the generated model with Rust absent from `PATH`. Receipts retain candidate and +dependency hashes. This complements the existing API, source archive, license, +vulnerability and host checks. + +The WASM gate checks the tag's compiler version, source commit, descriptor digest +and checksum contents. Publication rechecks `SHA256SUMS` for the final release +asset names, including the renamed release descriptor. + +## Reproduce the installer checks + +Check the published configurations without changing your global mise setup: + +```bash +uv run python -m scripts.qualify_mise --report .dev/qualification/mise-released.json +``` + +For candidates, first build the distributions using the existing tasks, then +run `mise run cli:qualify-mise`. CI passes the existing Linux native producer's +wheel; a `python-v` release uses the published-native directory qualified by +`native:qualify`. The script also accepts three explicit wheel paths for another +platform, as shown in the workflow. + +Keep successful run IDs and the artifact receipts with the release record. +For a failed publication, inspect which files were accepted before retrying. +Do not replace a published version's bytes or move a released tag. Correct the +source, qualify a new version and publish a new tag. diff --git a/examples/install/mise-pipx.toml b/examples/install/mise-pipx.toml new file mode 100644 index 00000000..384c3354 --- /dev/null +++ b/examples/install/mise-pipx.toml @@ -0,0 +1,8 @@ +[tool_alias] +pipx = "pypi:pipx" + +[tools] +python = "3.14" +uv = "0.12.10" +pipx = { version = "1.18.4", python = "3.14", expose = ["pipx"] } +"pipx:morphir-python" = { version = "0.1.0", python = "3.14", uvx = false, pipx_args = "--pip-args='--only-binary=:all:'" } diff --git a/examples/install/mise-uv.toml b/examples/install/mise-uv.toml new file mode 100644 index 00000000..cb4318b6 --- /dev/null +++ b/examples/install/mise-uv.toml @@ -0,0 +1,4 @@ +[tools] +python = "3.14" +uv = "0.12.10" +"pypi:morphir-python" = { version = "0.1.0", uvx_args = "--no-build" } diff --git a/openspec/changes/add-process-mep-extension/.openspec.yaml b/openspec/changes/add-process-mep-extension/.openspec.yaml new file mode 100644 index 00000000..a26e6f26 --- /dev/null +++ b/openspec/changes/add-process-mep-extension/.openspec.yaml @@ -0,0 +1,2 @@ +schema: html-design +created: 2026-10-28 diff --git a/openspec/changes/add-process-mep-extension/design.html b/openspec/changes/add-process-mep-extension/design.html new file mode 100644 index 00000000..55297801 --- /dev/null +++ b/openspec/changes/add-process-mep-extension/design.html @@ -0,0 +1,35 @@ + + + + + +add-process-mep-extension + + + + + + + + +
+

add-process-mep-extension

+

OpenSpec change · Bead morphir-5hxy · 2026-10-28 · Proposed; no process or new-draft implementation

+ +

Context

See proposal.md for motivation and the two spec deltas for requirements. PythonExtension already implements native Rust frontend/backend traits. The published SDK 0.3.0 exposes NativeExtension::frontend_backend, open_protocol, model methods and admission helpers. Its wire request type still uses a numeric u64 ID, while the canonical contract admits numeric and string IDs; adopting helpers alone is not full draft conformance. The Python implementation does not opt into model transport and its manifest declares MEP 0.1 only. finos/morphir c552dac801f23f2b2b566d29e08780723498ce36 defines exact MEP 0.2.0-draft.1 and model envelope 0.1.0-draft.1. These versions are independent of SDK 0.3.0, compiler 0.4.2, claims format and IR version.

+

Goals and non-goals

Add a native Rust process guest and current-draft model adapters while retaining WASM and legacy behavior. Keep language semantics shared. Initial model tuples use morphir-ir/morphir-json and supported Library IR 3.0.0 and draft 4.0.0. Rich Ion profiles, unsupported distribution kinds, additional Python syntax, cancellation/progress and performance claims are not introduced. A Python CLI stdio wrapper can follow separately; the selected CLI installation remains mise/pipx.

+

Decisions

RNT-1. Add a thin native Rust guest

Add a process guest crate/entry point, depending on the maintained compiler and published SDK, with no PyO3 or CPython requirement. The proposed command is:

morphir-python-extension --stdio

Reuse the SDK's session endpoint and dispatch. Keep model adaptation in the shared PythonExtension so native, WASM and process paths agree. Put stdio effects in the guest layer. A Python CLI wrapper would reuse the engine but retain an interpreter/launcher dependency; a frozen Python CLI would bundle an unnecessary runtime for this guest.

BoundaryCurrent releaseProposed change
MEPLegacy 0.1Retain 0.1 and add exact 0.2.0-draft.1
Model envelopeNot advertised0.1.0-draft.1, admitted JSON profile
Extension runtimeWASMWASM and native process
SDK0.3.0, numeric-only wire IDsQualified published pin covering the canonical wire contract
Semantic IRSupported v3 and draft v4 Library modelsSame semantic scope, explicit admitted tuples

SDK-1. Resolve concrete SDK wire gaps before claiming conformance

Pin a published SDK that supports canonical numeric and string request IDs and the required draft behavior. SDK 0.3.0 currently has numeric-only ExtensionRequest.id, so link and qualify an upstream SDK update if no suitable release exists. Do not vendor a sibling checkout or silently coerce wire IDs. Both WASM and process must pass the same ID-preservation vectors before the new draft is advertised. Related upstream work is MEP protocol type generation, issue 241; its existence is not evidence that the wire gap is fixed.

MEP-1. Negotiate the new draft explicitly

Retain legacy 0.1 operations and exact 0.2.0-draft.1 offers. Opt in only after supported model methods and executable conformance pass. Reuse SDK model admission and session rules; method constants alone are not support. Never enable model methods under legacy negotiation. Keep optional flags false until implemented.

MOD-1. Start with the explicit JSON model profile

Implement compile_model/generate_model over the existing compile/generate core. Admit complete operation/profile/codec/envelope/IR tuples and their bilateral limits. The initial profile is morphir-ir/morphir-json with envelope 0.1.0-draft.1. Use the canonical base64 payload, exact byte length/hash and critical-field rules. Reject unadvertised Ion profiles, guarantees and unsupported semantic forms. This achieves bounded new-draft support without implying rich Ion preservation or silently converting rich input.

IO-1. Treat framing and lifetime as a bounded server boundary

Use Content-Length JSON-RPC framing, byte-based lengths, an 8 KiB header ceiling and an initial 16 MiB complete-body ceiling. Advertised decoded/encoded model limits must fit those outer bounds, including envelope overhead. Enforce local bounds even if the peer advertises more. Check partial input, duplicate lengths, integer overflow, invalid UTF-8/JSON and EOF before unbounded allocation. Reserve stdout for frames and flush responses. Follow initialization, capability locks, shutdown and exit; do not respond to notifications. Host supervision owns deadlines and descendant cleanup.

RLS-1. Qualify dual-runtime distribution

Retain the portable WASM guest and publish native process executables for the advertised targets, with --stdio locked as an argument. Use a canonical host-readable descriptor with runtime/platform/mode/digest/claims bound per artifact. Validate the staged descriptor with pinned host tooling; do not assume that either the existing WASM-only packager or native-only bundle command can package a mixed release unchanged. Preserve historical assets. Release exact native/CLI package pairs if shared compiler changes affect them.

HST-1. Qualify hosts independently

The Rust host provides ProcessLaunch/ProcessChannel and verified-byte staging. It clears the child environment, favoring a directly executable Rust guest. Qualify a pinned conforming Rust host and test each documented MoonBit route independently. Use a pinned source build when a released host lacks the new draft, and state that prerequisite. If a route needs upstream work, record that dependency and label it unsupported until it passes. Existing MoonBit process wrappers demonstrate framing, not Python extension activation.

+

Risks and trade-offs

RiskMitigation
Draft, SDK and runtime support are confusedExact independent version table, live negotiation and candidate-bound conformance before claims.
Model content or guarantees are lostAdmit only explicit JSON tuples initially. Refuse unsupported profiles, critical fields and guarantees; no silent fallback.
Malformed input exhausts resourcesBound headers, wire bodies and decoded bytes before allocation; test boundary and partial cases.
Process mode is mistaken for a sandboxDocument ordinary OS privileges and the host's explicit supervision, distinct from WASM isolation.
Packaging or host support lagsPin descriptor readers and host builds, qualify installed bytes, record upstream blockers and make only verified support claims.
Native execution is assumed fasterMeasure startup and representative compile/generate workloads before making a performance claim.
+

Migration and rollout

  1. Add negotiation and bounded JSON model adapters with tests and compatibility documentation.
  2. Add the Rust process server with framing and lifecycle tests; preserve the existing WASM path.
  3. Build and qualify runtime artifacts and descriptors with matching claims and source receipts.
  4. Run pinned MEP guest conformance and real host installation/operation cases, including legacy hosts and explicit refusals.
  5. After maintainer approval, publish a new extension version and document runtime selection. Existing users can stay on the prior WASM release; never replace its bytes.

The current proposal does not change protocol claims or release a guest. Every process task remains unstarted.

+

Repository evidence and references

  • rust/crates/morphir-python-binding/src/lib.rs: shared PythonExtension traits and truthful current capabilities.
  • rust/crates/morphir-python-native/src/lib.rs: current private two-function JSON bridge; it is not a full MEP transport.
  • Published morphir-extension-sdk 0.3.0: native.rs, traits.rs and model admission helpers.
  • finos/morphir-rust 26d5e707e5ec94d16421db6380edbdc865d85c04: process launch/channel and stdio frontend fixture. Fixture framing must not be copied as a hardened server.
  • finos/morphir-moonbit bd3adc52e6b8f6368fdaee50f76535799732f8d6: portable MEP embedding wrapper. No claim of Python host compatibility follows.
  • Canonical MEP contract and model transport rules.

Related work: morphir-bj9r owns the selected CLI installation route; morphir-ukvn.5.98 tracks the earlier MoonBit extension qualification context.

+
+ + diff --git a/openspec/changes/add-process-mep-extension/proposal.md b/openspec/changes/add-process-mep-extension/proposal.md new file mode 100644 index 00000000..9bb7cf18 --- /dev/null +++ b/openspec/changes/add-process-mep-extension/proposal.md @@ -0,0 +1,33 @@ +# Proposal + +## Why + +The Python compiler currently exposes a WASM MEP guest advertising legacy `0.1`, while the canonical upstream contract is `0.2.0-draft.1`. Users need a process alternative sharing the compiler and an explicit path to the newer contract, rather than a transport change that leaves its model methods unsupported. + +## What Changes + +- Add a native Rust `morphir-python-extension --stdio` guest around the existing compiler and SDK session endpoint, without a Python runtime dependency. +- Preserve the WASM guest, extension identity, source subset, diagnostics and legacy `0.1` operations. +- Add exact `0.2.0-draft.1` negotiation and bilateral model transport admission, initially for the `morphir-ir`/`morphir-json` profile with the compiler's supported IR v3/v4 Library models. +- Implement bounded, integrity-checked model envelopes and compileModel/generateModel adapters for the admitted tuples. Refuse unsupported profiles and guarantees explicitly; rich Ion transport is not advertised by this initial increment. +- Publish and qualify both runtime choices, with accurate platform/runtime descriptors and host compatibility documentation. Qualify each documented host route independently. +- Keep this change proposed: no process guest or newer-MEP implementation has been added by PR #52. + +## Capabilities + +### New Capabilities + +- `python/model-transport`: New-draft negotiation, complete model tuples, envelope integrity and Python compiler adapters with explicit unsupported-profile refusals. + +### Modified Capabilities + +- `python/extension`: Shared WASM/process compiler behavior, bounded process sessions and qualified dual-runtime distribution. + +## Beads + +- **Tracked by**: `morphir-5hxy` +- **Related**: `morphir-bj9r` (mise/pipx CLI installation and release assurance), `morphir-ukvn.5.98` (MoonBit Extism qualification context). + +## Impact + +A qualified published SDK pin covering canonical numeric/string wire IDs, Rust compiler adapters and a process guest entry point, extension metadata/claims, platform release packaging, MEP conformance fixtures and host documentation. The canonical reference is finos/morphir `c552dac801f23f2b2b566d29e08780723498ce36`, `spec/mep/README.md` and `model-transport.md`. Package, protocol, model envelope and IR versions remain independent. A Python CLI stdio wrapper, additional Python syntax, rich Ion preservation, cancellation/progress and performance claims are outside this increment. Upstream host changes require their own repository work; unqualified routes must remain labeled unsupported. diff --git a/openspec/changes/add-process-mep-extension/specs/python/extension/spec.md b/openspec/changes/add-process-mep-extension/specs/python/extension/spec.md new file mode 100644 index 00000000..838fe53d --- /dev/null +++ b/openspec/changes/add-process-mep-extension/specs/python/extension/spec.md @@ -0,0 +1,53 @@ +# Spec Delta + +## MODIFIED Requirements + +### Requirement: MEP behavior remains compatible +The extension SHALL retain identity `morphir-python`, Python language/target identifiers, initialization and capability negotiation, legacy compile/generate methods and advertised IR v3/v4 support. Native Rust, Python, packaged WASM and process invocations SHALL agree on supported model semantics and language diagnostic codes. New model methods MUST require their negotiated draft and admitted capabilities. (bead: morphir-5hxy) + +#### Scenario: Existing host installs the replacement bundle +- **WHEN** a compatible Morphir host installs the bundle built from morphir-python +- **THEN** initialization, capability discovery, Python compilation and Python generation succeed through the existing protocol + +#### Scenario: Users select different supported runtimes +- **WHEN** qualified WASM and process guests handle equivalent supported compilation or generation requests +- **THEN** they agree on compiler identity, supported semantics, deterministic artifacts and language diagnostic codes + +## ADDED Requirements + +### Requirement: A native process guest serves framed MEP sessions +A released native process guest SHALL serve JSON-RPC MEP over Content-Length-framed stdin/stdout without CPython or Rust installed by the user. It MUST enforce documented header/body bounds, UTF-8 byte lengths, request identity and the SDK lifecycle. Stdout MUST contain only protocol frames; notifications MUST NOT receive responses. (bead: morphir-5hxy) + +#### Scenario: Multiple operations in one process +- **WHEN** a host initializes the guest, performs supported operations, requests shutdown and sends exit +- **THEN** responses retain request IDs, session capabilities remain fixed, and the process exits cleanly + +#### Scenario: Numeric and string request identity +- **WHEN** a supported request uses either a numeric or string JSON-RPC ID +- **THEN** its response preserves that exact ID type and value in both runtime routes + +#### Scenario: Malformed or oversized frame +- **WHEN** a frame has conflicting lengths, truncated data, invalid JSON, invalid UTF-8 or exceeds a declared bound +- **THEN** the guest refuses it without unbounded allocation or logging non-protocol text to stdout + +#### Scenario: Operation outside its lifecycle +- **WHEN** an operation is requested before initialization or after shutdown +- **THEN** the guest applies the protocol lifecycle refusal and does not invoke the compiler + +### Requirement: Both runtime choices have qualified release artifacts +Extension releases SHALL retain portable WASM and provide qualified process artifacts for each advertised process platform. Descriptors MUST bind runtime, platform, arguments, executable mode, digest, protocol versions and capability claims to the selected bytes. Unsupported runtime/platform combinations MUST fail explicitly. (bead: morphir-5hxy) + +#### Scenario: Process artifact is selected offline +- **WHEN** a compatible host selects a qualified installed process artifact +- **THEN** it verifies and launches the locked executable and arguments without obtaining Python, Rust or another artifact from the network + +#### Scenario: Existing user continues with WASM +- **WHEN** a compatible host selects the retained WASM runtime +- **THEN** the extension remains portable and its operations preserve the qualified compiler behavior + +### Requirement: Host support claims follow actual qualification +Each documented successful host route SHALL have retained candidate-bound installation, lifecycle and operation evidence for the stated host revision and runtime. Unqualified or incompatible Rust or MoonBit routes MUST be labeled unsupported with their observed limitations. A transport wrapper alone MUST NOT establish model-envelope support. (bead: morphir-5hxy) + +#### Scenario: A host requires unavailable model behavior +- **WHEN** a host route requires a profile or model operation that the selected guest cannot admit +- **THEN** documentation records that route as unsupported rather than claiming that process transport resolves the incompatibility diff --git a/openspec/changes/add-process-mep-extension/specs/python/model-transport/spec.md b/openspec/changes/add-process-mep-extension/specs/python/model-transport/spec.md new file mode 100644 index 00000000..954de7d3 --- /dev/null +++ b/openspec/changes/add-process-mep-extension/specs/python/model-transport/spec.md @@ -0,0 +1,58 @@ +# Spec Delta + +## Purpose + +Expose the Python compiler through the current canonical MEP draft with explicit model transport negotiation, bounded envelopes and truthful supported-profile claims. + +## ADDED Requirements + +### Requirement: New model operations require exact draft negotiation +The extension SHALL support MEP `0.2.0-draft.1` for model operations while preserving legacy `0.1` behavior. Draft negotiation MUST match the full version exactly. A session negotiated as `0.1` MUST NOT enable model operations. Package, claims, model-envelope and IR versions MUST remain independent. (bead: morphir-5hxy) + +#### Scenario: Host offers the supported new draft +- **WHEN** a host and extension both offer `0.2.0-draft.1` +- **THEN** initialization selects the exact draft before any model operation is admitted + +#### Scenario: Legacy or different draft session +- **WHEN** the session selects `0.1` or the host offers only another draft version +- **THEN** model operations are unavailable and an incompatible-only handshake is refused + +### Requirement: Model operation admission is bilateral +Each model operation SHALL require an initialized, advertised tuple matching operation, profile, codec, envelope version and semantic IR version. Session tuples and limits MUST remain within both peers' offers and permitted static claims. Unknown required guarantees and unadmitted tuples MUST be refused. (bead: morphir-5hxy) + +#### Scenario: Compile support does not grant generation +- **WHEN** only a compile tuple was admitted and the host requests model generation +- **THEN** the operation is refused without invoking the generator + +#### Scenario: Independent lists would produce an unsupported combination +- **WHEN** an operation combines otherwise known tuple members in a combination not admitted as a complete tuple +- **THEN** the extension refuses that combination rather than forming a Cartesian product + +### Requirement: Model envelopes bind exact bounded bytes +Model payloads SHALL use the canonical `0.1.0-draft.1` envelope and validate codec, layout, canonical padded base64, decoded length, SHA-256, semantic IR version and required critical fields before compiler invocation. Advertised and local message/decoded limits MUST be enforced before unbounded allocation or decoding. (bead: morphir-5hxy) + +#### Scenario: Payload length or digest disagrees +- **WHEN** an envelope's declared length or digest differs from its decoded bytes +- **THEN** admission fails without compiling or generating + +#### Scenario: Oversized or unknown critical input +- **WHEN** input exceeds an applicable bound or names a critical field the receiver does not understand +- **THEN** the extension refuses the request and does not silently ignore that condition + +### Requirement: Initial profile support is explicit and lossless +The initial model adapter SHALL support `morphir-ir` with `morphir-json` for supported v3/v4 Library models through `compileModel` and `generateModel`. Unsupported rich Ion profiles, distribution forms or preservation guarantees MUST be refused explicitly. No automatic lossy conversion or fallback SHALL occur. (bead: morphir-5hxy) + +#### Scenario: Supported JSON model round trip +- **WHEN** a qualified JSON-profile model is compiled, generated and recompiled using an admitted tuple +- **THEN** its supported semantics and diagnostics agree with the existing compiler for the selected IR version + +#### Scenario: Rich model requires unsupported preservation +- **WHEN** a host requests an Ion profile or guarantee not advertised by this increment +- **THEN** the extension refuses it rather than converting to JSON and discarding information + +### Requirement: New protocol claims require executable conformance evidence +The extension SHALL advertise the new draft and model tuples only after candidate-bound executable guest qualification covers negotiation, lifecycle, supported operations and negative admission cases. Documentation MUST distinguish implemented profiles from unimplemented optional MEP capabilities. (bead: morphir-5hxy) + +#### Scenario: Protocol constants exist without operation support +- **WHEN** an SDK exposes model types or method names but the Python adapter has not passed qualification +- **THEN** those symbols do not establish or justify an advertised support claim diff --git a/openspec/changes/add-process-mep-extension/tasks.md b/openspec/changes/add-process-mep-extension/tasks.md new file mode 100644 index 00000000..3d78eb74 --- /dev/null +++ b/openspec/changes/add-process-mep-extension/tasks.md @@ -0,0 +1,39 @@ +# Tasks + +This change is proposed. No process guest or newer-MEP implementation has started. + +## 1. Negotiate current and legacy MEP (morphir-5hxy.1) + +- [ ] 1.1 Resolve the SDK 0.3.0 numeric-only wire ID gap with a qualified published SDK pin, linking an upstream prerequisite if needed; add explicit legacy `0.1` and exact `0.2.0-draft.1` offers using the pinned SDK's session helpers; verify numeric/string ID preservation, both handshakes, incompatible drafts and unchanged legacy compile/generate behavior. +- [ ] 1.2 Add bilateral complete-tuple and limit negotiation without widening static claims; verify cross-product, unadvertised method, unsupported guarantee and legacy-session refusals. +- [ ] 1.3 Document independent MEP, envelope, IR, claims and package versions; verify the capability reference matches live discovery and initialized sessions. + +## 2. Adapt supported model envelopes (morphir-5hxy.2) + +- [ ] 2.1 Implement bounded canonical JSON model-envelope admission using SDK helpers; verify base64, byte length, hash, critical fields, semantic IR version and size-limit negative cases. +- [ ] 2.2 Adapt compileModel/generateModel for admitted JSON-profile v3/v4 Library tuples through the shared compiler; verify compile/generate/recompile semantics and diagnostic parity with legacy operations. +- [ ] 2.3 Refuse unadvertised Ion profiles, unsupported distributions and guarantees without lossy fallback; test these refusals and document the initial support matrix. + +## 3. Serve native stdio sessions (morphir-5hxy.3) + +- [ ] 3.1 Add the thin Rust process guest and documented `morphir-python-extension --stdio` entry point; verify it builds and executes without CPython/PyO3 or a user Rust toolchain. +- [ ] 3.2 Implement bounded Content-Length framing around the SDK session; test Unicode byte lengths, partial/truncated messages, duplicate/invalid lengths, overflow, invalid UTF-8/JSON and header/body limits. +- [ ] 3.3 Preserve request IDs, notification silence, fixed session capabilities, shutdown/exit and protocol-only stdout; verify a real multi-request process test and document stderr and supervision behavior. + +## 4. Distribute both runtime choices (morphir-5hxy.4) + +- [ ] 4.1 Build process artifacts for each advertised platform and retain the WASM artifact; verify both use the shared compiler and produce matching supported results and diagnostics. +- [ ] 4.2 Stage canonical runtime/platform/mode/argument/digest/claim descriptors and source receipts with pinned tooling; verify host readers, installed-byte integrity, offline activation and explicit unsupported-platform refusal. +- [ ] 4.3 Add release assurance and runtime-selection documentation alongside each artifact producer, including exact native/CLI pairing where affected; verify missing or mismatched evidence blocks promotion and historical WASM assets remain available. + +## 5. Qualify host interoperability (morphir-5hxy.5) + +- [ ] 5.1 Run the pinned executable MEP guest suite for both runtimes and protocol offers, including supported model operations and negative admission; retain source/artifact-bound conformance receipts. +- [ ] 5.2 Qualify the concrete Rust host process and WASM routes using a pinned compatible release or source build; verify initialization, v3/v4 operations, parity, timeout and child cleanup with a clean child environment. +- [ ] 5.3 Test documented MoonBit routes independently, record prerequisite upstream work for any unsupported path and publish only evidenced compatibility claims; verify documentation recipes against the specified host revisions. +- [ ] 5.4 Measure representative cold-start and compile/generate workloads for process and WASM without making an unmeasured speed claim; retain the reproducible inputs, toolchain and timing report. + +## Workflow follow-up + +- Implementation requires separately applying this proposal after review. +- Merge, tags and publication require maintainer approval; sync/archive only after implementation and qualification are complete. diff --git a/openspec/changes/enable-mise-cli-installation/.openspec.yaml b/openspec/changes/enable-mise-cli-installation/.openspec.yaml new file mode 100644 index 00000000..a26e6f26 --- /dev/null +++ b/openspec/changes/enable-mise-cli-installation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: html-design +created: 2026-10-28 diff --git a/openspec/changes/enable-mise-cli-installation/design.html b/openspec/changes/enable-mise-cli-installation/design.html new file mode 100644 index 00000000..5421fe7a --- /dev/null +++ b/openspec/changes/enable-mise-cli-installation/design.html @@ -0,0 +1,35 @@ + + + + + +enable-mise-cli-installation + + + + + + + + +
+

enable-mise-cli-installation

+

OpenSpec change · Bead morphir-bj9r · 2026-10-28 · Implementation in PR 52; five-platform qualification passed

+ +

Context

See proposal.md for motivation. Release already publishes pure CLI wheel/source artifacts to PyPI and GitHub and a separate WASM bundle. It builds qualified native wheels for five platforms. The Python API pins native 0.1.1; the current CLI/core versions are 0.1.0. The existing Build category owns installed package checks. PR 52 implements installer assurance at 48bf62a. OpenSpec records the delivered behavior and its qualification evidence.

+

Goals and non-goals

Keep mise installation on the published Python package route and verify exact candidates before promotion. Preserve independent versions, existing native qualifications and the dependency-free core library. Frozen Python executables, MEP process serving and new protocol support belong outside this change; see add-process-mep-extension.

+

Decisions

INS-1. Use the Python package backend

Provide canonical configurations in examples/install for uv and actual pipx, with managed CPython 3.14, explicit CLI version and wheel-only policy. Keep PyPI as the installation source and GitHub assets as mirrors. A standalone CLI archive would add embedded-runtime maintenance and platform packaging; the maintainer selected the package route.

INS-2. Bootstrap pipx through a portable explicit backend

The first matrix run proved Windows CLI wheel and mise/uv behavior, then failed when mise's short pipx name selected a macOS/Linux-only backend. Bootstrap pipx through the PyPI backend with uv, give the pipx dependency an explicit alias so installation order is enforced, and keep the CLI tool's uvx=false selection. The corrected canonical configuration passed clean Windows qualification for both uv and actual pipx in run 40651808051.

REL-1. Keep qualification with its existing producers

Use scripts/qualify_mise.py from the Build category and the existing release matrix. Candidate identity validators run before dependency lookup. Download dependencies under the release constraints once, verify supplied wheel bytes, then install from that offline wheelhouse in separate temporary mise environments. Retain package versions and hashes. Exercise both IR formats and generated semantics without Rust on the execution PATH.

Ordinary PR/main CI owns Linux checks. Manual/native qualification extends the existing five-platform matrix. CLI tags consume already published native wheels and require the matrix before PyPI and GitHub publication. Reuse compatible Linux producer evidence. No new schedule or publication privilege is needed.

REL-2. Seal the final WASM asset names

Validate the actual .sha256 contents, not only their syntax. Keep source/version/descriptor checks, then compute SHA256SUMS after staging the versioned descriptor name and reports. The existing publisher rechecks that receipt immediately before uploading. A generic checksum implementation provides the byte check; it is not reimplemented in the application.

+

Risks and trade-offs

RiskMitigation
Same-version PyPI packages mask a candidateValidate all three wheel identities, constrain resolution, use an offline candidate wheelhouse and retain hashes.
Tool installer differs by platformExplicit portable pipx bootstrap and actual five-platform checks. A missing, failed, cancelled or skipped required matrix blocks CLI promotion.
Python runtime changes break an installed launcherDocument managed 3.14 and reinstall commands. Version-only examples do not claim a frozen transitive graph.
Release artifact changes after qualificationSame-run artifact transfer, existing source checks and final asset receipt verification.
+

Build assessment and evidence

Impact is high when invalid artifacts are promoted. Candidate identity/checksum regressions are merge-required; actual installation on advertised targets is release-required. Maintainers own qualification failures and reruns. No nightly deferral or exception is introduced.

CriterionWeightCurrent / proposedEvidence and confidence
Risk/coverage251 / 4High confidence: checksum refusal, candidate identities, selectors and both installers verified on all five advertised targets.
Feedback/placement202 / 3Medium confidence: no extra PR platform matrix. One corrected PR run took 8m25s and full manual qualification 16m05s, including queue and setup; no speedup claim.
Evidence/trust202 / 4High confidence: isolated offline candidate installs, retained wheel digests and final-byte checksum refusal; same-run gates inspected. No public release was exercised.
Efficiency153 / 3Medium confidence: compatible Linux evidence reused in a 20s matrix job; CLI tags consume published native wheels. No new PR platform builds.
Ownership103 / 4Existing Build/matrix/publisher responsibilities retained.
Recovery102 / 3Receipts and qualification-only manual runs; no tag movement.

Design assessment: current 50/100, proposed 89/100, denominator 100. All applicable evidence requirements pass for qualification; promotion remains subject to approval and the exact tag candidate's release gates. Adopt the producer-owned checks. Recovery remains 3/4 because real publication rollback was not exercised.

Corrected source 48bf62abdf47a7950933f16ed79eed2bf29685bb: PR CI 40651590127 and qualification-only Release 40651808051 succeeded. Both actual installers passed Linux x86_64/aarch64, macOS arm64/x86_64 and Windows x86_64. Retained per-platform receipts include wheel/dependency digests and check identity; Linux x86_64 reuses the same-run Build receipt in release-scan-reports. In this single sample, installer execution ranged from 4 to 19 seconds on the four independently run targets, excluding shared wheelhouse preparation. Matrix execution ranged from 20 seconds for reuse to 463 seconds for Intel macOS, including build and setup. These are execution durations, not billed cost.

Local final checks: 2,526 tests passed with 3 pre-existing skips, 34 BDD scenarios and 145 steps, mypy/pyright, lint/format, actionlint and site checks. Initial run 40647620264 exposed Windows' nonportable pipx bootstrap and a macOS Intel network download failure. The explicit portable alias fixed Windows; the replacement run passed both targets. Final OpenSpec-only changes preserve runtime, dependencies, toolchains, workflow and qualification inputs relative to 48bf62a; platform evidence is reused under that checked equivalence. Final-head CI is recorded in PR 52.

+

Migration and rollback

Deliver canonical examples, documentation and gates in PR 52. The Windows bootstrap is corrected and the complete candidate is qualified. Retain final-head CI and source-equivalence evidence for the subsequent OpenSpec-only edits. Maintainer approval remains necessary to merge and publish tags. Existing PyPI versions and GitHub assets stay available; restore a previous supported CLI pin rather than replacing published bytes. Archive the OpenSpec change only after delivery review.

+
+ + diff --git a/openspec/changes/enable-mise-cli-installation/proposal.md b/openspec/changes/enable-mise-cli-installation/proposal.md new file mode 100644 index 00000000..3068d317 --- /dev/null +++ b/openspec/changes/enable-mise-cli-installation/proposal.md @@ -0,0 +1,32 @@ +# Proposal + +## Why + +The CLI already ships Python distributions on PyPI and GitHub Releases, but users lack a documented, qualified mise installation path. Release qualification must prove that both supported Python package installers consume the intended wheels and that the final WASM assets retain their verified checksums. + +## What Changes + +- Document pinned mise/uv and actual mise/pipx configurations with managed CPython 3.14 and wheel-only dependencies. +- Qualify isolated installs of the exact candidate core/native/CLI wheels, including v3/v4 compilation, generation, recompilation and generated-model execution without Rust. +- Gate CLI publication on the existing five-platform qualification matrix, consuming published native wheels for CLI tags and reusing compatible producer artifacts. +- Retain CLI wheel/source mirrors and independent WASM GitHub Releases; verify the WASM checksum contents and final named asset receipt before publication. +- Record the implementation already delivered in PR #52; completed tasks refer to actual evidence. No standalone CLI executable or process MEP guest is introduced here. + +## Capabilities + +### New Capabilities + +- `tools/cli-installation`: Isolated, wheel-only CLI installation through mise and qualification of installer behavior before publication. + +### Modified Capabilities + +- `python/extension`: Final release asset checksum validation before publishing the WASM extension. + +## Beads + +- **Tracked by**: `morphir-bj9r` +- **Related**: `morphir-ukvn.9` (documentation site), `morphir-5hxy` (separate process/current-MEP proposal). + +## Impact + +Installation and release documentation, canonical examples, installed-artifact qualification, the existing CI Build category and release platform matrix, and WASM publication validation. Runtime APIs, package namespaces and release tag families remain compatible. PyPI remains the mise installation source; GitHub Releases retain the CLI distribution mirrors. Merges, tags and public publication require maintainer approval. diff --git a/openspec/changes/enable-mise-cli-installation/specs/python/extension/spec.md b/openspec/changes/enable-mise-cli-installation/specs/python/extension/spec.md new file mode 100644 index 00000000..4ad8e386 --- /dev/null +++ b/openspec/changes/enable-mise-cli-installation/specs/python/extension/spec.md @@ -0,0 +1,18 @@ +# Spec Delta + +## ADDED Requirements + +### Requirement: Final extension release assets retain verified integrity +The extension release SHALL verify that the WASM checksum contents match its bytes and descriptor. A digest receipt covering final published asset names, including a renamed descriptor, MUST be checked immediately before GitHub publication. A checksum or source-provenance mismatch MUST block publication. (bead: morphir-bj9r) + +#### Scenario: Checksum syntax is valid but its digest is wrong +- **WHEN** the checksum file contains a correctly formatted digest that differs from the WASM bytes +- **THEN** candidate qualification refuses the bundle + +#### Scenario: Descriptor receives its public asset name +- **WHEN** the verified descriptor is staged under its versioned GitHub asset name +- **THEN** the final receipt names that asset and publication verifies its bytes along with the WASM, checksum and staged qualification reports + +#### Scenario: Staged asset changes before publication +- **WHEN** a file named by the final release receipt is missing or has different bytes +- **THEN** publication fails before creating or updating the GitHub release diff --git a/openspec/changes/enable-mise-cli-installation/specs/tools/cli-installation/spec.md b/openspec/changes/enable-mise-cli-installation/specs/tools/cli-installation/spec.md new file mode 100644 index 00000000..ad08ced8 --- /dev/null +++ b/openspec/changes/enable-mise-cli-installation/specs/tools/cli-installation/spec.md @@ -0,0 +1,54 @@ +# Spec Delta + +## Purpose + +Provide a documented, isolated CLI installation through mise and require verified installer behavior for the distributions that a release promotes. + +## ADDED Requirements + +### Requirement: Isolated wheel-only CLI installation +The CLI SHALL be installable through documented mise configurations using uv or actual pipx with managed CPython 3.14. Each configuration MUST install the compatible native compiler from a wheel, expose `morphir-python`, and keep its dependencies separate from application environments. (bead: morphir-bj9r) + +#### Scenario: Clean managed installation +- **WHEN** a user applies either documented configuration on a qualified platform +- **THEN** mise installs the interpreter and CLI environment and `mise exec -- morphir-python --help` succeeds without a Rust toolchain + +#### Scenario: No compatible native wheel +- **WHEN** the selected interpreter or platform has no compatible compiler wheel +- **THEN** installation fails rather than compiling Rust from a source distribution + +### Requirement: Installer qualification uses identified candidate artifacts +Release qualification SHALL install the exact identified core, native and CLI candidate wheels in fresh environments outside the checkout. Candidate installation MUST use a constrained offline wheel set, reject missing or mismatched candidate identities, and retain input and dependency digests with installed versions. (bead: morphir-bj9r) + +#### Scenario: A same-version package exists on PyPI +- **WHEN** the candidate and a published package have the same version but different bytes +- **THEN** qualification installs the supplied candidate bytes without substituting the PyPI package + +#### Scenario: Native candidate is missing +- **WHEN** supplied candidate inputs do not contain the required native wheel +- **THEN** qualification refuses the set before resolving a replacement from an index + +### Requirement: Installed commands preserve compiler behavior +Both supported installers SHALL qualify installed CLI compilation, inspection, generation and recompilation for supported IR v3/v4 models, including generated-model execution, with Rust absent from the execution PATH. (bead: morphir-bj9r) + +#### Scenario: Installed compiler round trip +- **WHEN** each installer processes the supported multi-module fixture in v3 and v4 +- **THEN** generated Python recompiles to equivalent IR and the generated model returns the expected values + +### Requirement: CLI publication requires platform installer evidence +CLI publication SHALL require successful installer qualification for every advertised release platform using that release's core/CLI wheels and exact published native dependency. Missing, failed, cancelled or unexpectedly skipped required qualification MUST block publication. Normal CI SHALL qualify the Linux candidate path. (bead: morphir-bj9r) + +#### Scenario: One platform fails qualification +- **WHEN** any advertised platform fails, is cancelled, or lacks required installer evidence +- **THEN** the CLI is not promoted to PyPI or its GitHub release + +#### Scenario: Compatible producer artifacts exist +- **WHEN** the same qualified candidate wheel is available to a compatible consumer +- **THEN** the consumer can reuse it with its identity retained rather than rebuilding the compiler + +### Requirement: Installation channels remain explicit +The CLI SHALL retain Python wheel/source distribution publication on PyPI and GitHub Releases. The documented mise package route SHALL use PyPI; it MUST NOT imply that a standalone executable or the WASM plugin is installed by that route. (bead: morphir-bj9r) + +#### Scenario: User chooses the mise route +- **WHEN** a user follows the installation guide +- **THEN** the guide identifies managed Python, the isolated CLI environment, compatible wheel requirements and the distinct WASM extension channel diff --git a/openspec/changes/enable-mise-cli-installation/tasks.md b/openspec/changes/enable-mise-cli-installation/tasks.md new file mode 100644 index 00000000..b35a992b --- /dev/null +++ b/openspec/changes/enable-mise-cli-installation/tasks.md @@ -0,0 +1,33 @@ +# Tasks + +Implementation began in PR #52 before this OpenSpec formalization. Checked items name completed work; the portable bootstrap and both installers now pass all five platforms in qualification run 40651808051. CI run 40651590127 passed on the qualified implementation. Final OpenSpec-only edits preserve all runtime and qualification inputs. + +## 1. Document managed CLI installation (morphir-bj9r.1) + +- [x] 1.1 Add pinned mise/uv and actual mise/pipx examples, wheel-only policy and installation guidance; verify snippet synchronization and clean released installs locally. +- [x] 1.2 Bootstrap pipx from an explicit portable backend with correct dependency ordering; verify a clean Windows actual-pipx installation as well as the uv route and document the bootstrap. +- [x] 1.3 Document package channels, independent release tags, qualification and rollback; verify site build/link checks and navigation. + +## 2. Qualify candidate installer behavior (morphir-bj9r.2) + +- [x] 2.1 Validate all three candidate identities before dependency lookup, preserve exact wheel bytes, constrain dependency resolution and install offline; verify missing-native refusal and same-version candidate receipts. +- [x] 2.2 Isolate mise/Python installer state outside the checkout and qualify both installer environments; verify installed versions, v3/v4 CLI round trips and generated-model execution without Rust on PATH. +- [x] 2.3 Retain candidate/dependency digests and check identity in qualification receipts, add regression tests and a reproduction task, and verify the installed qualification script against local and Linux CI candidates. + +## 3. Gate platform and WASM publication (morphir-bj9r.3) + +- [x] 3.1 Add installer checks to the existing Build category and path selector; verify focused selector/aggregate regressions and actionlint. +- [x] 3.2 Require the advertised release matrix before CLI publication, reuse compatible Linux evidence and consume published native wheels for CLI tags; verify workflow dependencies and the manual run's refusal to publish. +- [x] 3.3 Reject a well-formed incorrect WASM checksum and seal/check final release asset names; verify the negative checksum regression and a changed-asset receipt refusal. +- [x] 3.4 Complete current-candidate qualification on Linux x86_64/aarch64, macOS arm64/x86_64 and Windows x86_64 with both installers; retain successful platform receipts and confirm unavailable required evidence prevents promotion. + +## 4. Verify integrated delivery (morphir-bj9r.4) + +- [x] 4.1 Run code quality, BDD, installed examples and site checks; record actual local results and the initial current-code CI run. +- [x] 4.2 Validate both OpenSpec changes strictly, browser-check their designs and keep bead states consistent; verify the proposal separates completed installer work from unstarted process work. +- [x] 4.3 Push the final PR candidate and retain its CI/review/platform qualification evidence; reassess the provisional build assessment using actual results and verify all changes are synchronized remotely. + +## Workflow follow-up + +- Merge and publish only under maintainer approval covering those actions. +- Sync and archive this completed change after delivery review; leave the process/current-MEP proposal open until separately applied. diff --git a/packages/morphir-tools/README.md b/packages/morphir-tools/README.md index 8f3f2b7b..2fa1c872 100644 --- a/packages/morphir-tools/README.md +++ b/packages/morphir-tools/README.md @@ -16,8 +16,15 @@ This package provides: ## Installation +For an isolated CLI installation managed by mise, use one of the +[verified mise/uv or mise/pipx configurations](https://finos.github.io/morphir-python/docs/installation/). +mise manages Python 3.14 and installs the compiled native dependency from a +wheel. No Rust toolchain is required on supported platforms. + +To install the CLI and compiler API into an existing Python environment: + ```bash -pip install morphir-python +python -m pip install --only-binary=:all: morphir-python==0.1.0 ``` Or with uv: diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py index 8172e14d..1614ba74 100644 --- a/scripts/ci_changes.py +++ b/scripts/ci_changes.py @@ -58,6 +58,7 @@ ("site/*", ("docs",)), ("examples/extension.sh", ("docs", "docs-examples", "docs-hosts")), ("examples/model/*", ("docs", "docs-examples", "docs-hosts")), + ("examples/install/*", ("python", "native", "docs", "docs-examples")), ("examples/*", ("docs", "docs-examples")), ("scripts/docs_hosts.py", ("docs", "docs-examples", "docs-hosts")), ("scripts/docs_*.py", ("docs", "docs-examples")), diff --git a/scripts/docs_examples.py b/scripts/docs_examples.py index 2ca63d6a..4a9eeb8a 100644 --- a/scripts/docs_examples.py +++ b/scripts/docs_examples.py @@ -85,7 +85,7 @@ def synchronize(root: Path = ROOT, *, write: bool = False) -> list[str]: def snippet(match: re.Match[str]) -> str: name, region = match.groups() - language = "python" if name.endswith(".py") else "bash" + language = {".py": "python", ".toml": "toml"}.get(Path(name).suffix, "bash") key = name + (f"#{region}" if region else "") return ( f"\n```{language}\n" diff --git a/scripts/qualify_mise.py b/scripts/qualify_mise.py new file mode 100644 index 00000000..c264cc75 --- /dev/null +++ b/scripts/qualify_mise.py @@ -0,0 +1,310 @@ +"""Qualify wheel-only CLI installs through mise's uv and actual pipx installers. + +Run without wheel arguments to check the released examples. With three wheels, +download their dependencies once, then install those exact candidates offline. +Every install uses fresh mise state outside the checkout and exercises v3/v4 +compile, inspect, generate and recompile with no Rust toolchain on PATH. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from scripts.check_distributions import load_packages, validate_archive +from scripts.qualify_native import ( + ROOT, + SAMPLE, + WHEEL_NAME, + rust_free_path, + sha256, + validate_wheel, + venv_python, +) + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + +INSTALLERS = ("uv", "pipx") + + +def validate_candidates(wheels: Sequence[Path]) -> None: + """Require identified core, native and CLI inputs before any dependency lookup.""" + core, tools = load_packages(ROOT) + expected = [ + f"morphir-{core.version}-py3-none-any.whl", + f"morphir_python-{tools.version}-py3-none-any.whl", + ] + if ( + len(wheels) != 3 + or wheels[0].name != expected[0] + or wheels[2].name != expected[1] + or not WHEEL_NAME.fullmatch(wheels[1].name) + ): + raise ValueError("expected the core, native and CLI candidate wheels, in order") + validate_archive(core, wheels[0]) + validate_wheel(wheels[1]) + validate_archive(tools, wheels[2]) + + +def tool_name(installer: str) -> str: + """Return the explicit backend identity being qualified.""" + return f"{'pypi' if installer == 'uv' else 'pipx'}:morphir-python" + + +def configuration(installer: str, version: str, wheelhouse: Path | None = None) -> str: + """Render the documented config, optionally restricted to candidate wheels.""" + template = ROOT / "examples" / "install" / f"mise-{installer}.toml" + document = tomllib.loads(template.read_text()) + tools = document["tools"] + options = cast("dict[str, str | bool]", tools[tool_name(installer)]) + options["version"] = version + if wheelhouse: + location = f"--find-links={wheelhouse.as_uri()}" + if installer == "uv": + options["uvx_args"] = shlex.join(["--no-build", "--no-index", location]) + else: + options["pipx_args"] = shlex.join( + ["--pip-args", f"--only-binary=:all: --no-index {location}"] + ) + lines = [] + for section, entries in document.items(): + lines.append(f"[{section}]") + for name, value in entries.items(): + rendered = ( + "{ " + + ", ".join(f"{k} = {json.dumps(v)}" for k, v in value.items()) + + " }" + if isinstance(value, dict) + else json.dumps(value) + ) + lines.append(f"{json.dumps(name)} = {rendered}") + return "\n".join(lines) + "\n" + + +def isolated_environment( + directory: Path, original: Mapping[str, str] | None = None +) -> dict[str, str]: + """Keep mise state and Python installer state separate from the user's.""" + env = dict(os.environ if original is None else original) + for name in list(env): + if name.startswith(("MISE_", "PYTHON", "UV_", "PIP_", "PIPX_")) or name in { + "VIRTUAL_ENV", + "CONDA_PREFIX", + }: + del env[name] + env.update( + { + "MISE_DATA_DIR": str(directory / "data"), + "MISE_CACHE_DIR": str(directory / "cache"), + "MISE_CONFIG_DIR": str(directory / "config"), + "MISE_STATE_DIR": str(directory / "state"), + "MISE_YES": "1", + "MISE_AUTO_INSTALL": "0", + "PIP_CONFIG_FILE": os.devnull, + "NO_COLOR": "1", + } + ) + return env + + +def run(command: Sequence[str], cwd: Path, env: Mapping[str, str]) -> str: + """Run a required check, retaining stderr and refusing timeouts or failure.""" + print(shlex.join(command), file=sys.stderr, flush=True) + result = subprocess.run( + command, cwd=cwd, env=env, text=True, stdout=subprocess.PIPE, timeout=600 + ) + result.check_returncode() + return result.stdout.strip() + + +def exercise(cli: Path, python: Path, work: Path, env: dict[str, str]) -> None: + """Exercise the installed CLI and execute its generated model.""" + env = {**env, "PATH": rust_free_path(python.parent.parent)} + source = work / "source" + shutil.copytree(SAMPLE, source) + run([str(cli), "--help"], work, env) + for target in ("v3", "v4"): + model, again, output = ( + work / f"{target}.json", + work / f"{target}-again.json", + work / f"{target}-generated", + ) + args = ["--package", "acme/example", "--ir-version", target] + run([str(cli), "compile", str(source), *args, "-o", str(model)], work, env) + run([str(cli), "ir", "inspect", str(model)], work, env) + run([str(cli), "generate", str(model), "--output-dir", str(output)], work, env) + run([str(cli), "compile", str(output), *args, "-o", str(again)], work, env) + if json.loads(model.read_text()) != json.loads(again.read_text()): + raise ValueError( + f"{target} generated source did not compile to the same IR" + ) + result = run( + [ + str(python), + "-I", + "-c", + "import sys; sys.path.insert(0, sys.argv[1]); from acme import rules; " + "print(rules.choose(True, 1, 2), rules.twice_identity(7))", + str(output), + ], + work, + env, + ) + if result != "1 7": + raise ValueError(f"generated model returned {result!r}") + + +def qualify(wheels: Sequence[Path], installers: Sequence[str]) -> dict[str, object]: + """Install and exercise released packages or the three supplied candidates.""" + mise = shutil.which("mise") + if mise is None: + raise ValueError("mise is required; see docs/installation.md") + released = json.loads((ROOT / "examples/manifest.json").read_text())["released"] + version = str(released["morphir-python"]) + if wheels: + validate_candidates(wheels) + project = ROOT / "packages/morphir-tools/pyproject.toml" + version = str(tomllib.loads(project.read_text())["project"]["version"]) + reports = [] + dependencies: dict[str, str] = {} + with tempfile.TemporaryDirectory(prefix="morphir-mise-") as scratch: + base = Path(scratch) + wheelhouse = base / "wheels" if wheels else None + if wheelhouse: + wheelhouse.mkdir() + constraints = base / "constraints.txt" + run( + [ + "uv", + "export", + "--frozen", + "--all-packages", + "--no-default-groups", + "--no-emit-workspace", + "--no-hashes", + "--output-file", + str(constraints), + ], + ROOT, + isolated_environment(base / "download"), + ) + run( + [ + "uvx", + "--from", + "pip==26.2.2", + "pip", + "download", + "--only-binary=:all:", + "--dest", + str(wheelhouse), + "--constraint", + str(constraints), + *[str(w.resolve()) for w in wheels], + # pipx bootstraps its shared pip from the same offline index. + "pip==26.2.2", + ], + base, + isolated_environment(base / "download"), + ) + for wheel in wheels: + if sha256(wheel) != sha256(wheelhouse / wheel.name): + raise ValueError(f"wheel download changed {wheel.name}") + dependencies = {p.name: sha256(p) for p in sorted(wheelhouse.glob("*.whl"))} + for installer in installers: + work = base / installer + work.mkdir() + env = isolated_environment(work) + (work / "mise.toml").write_text( + configuration(installer, version, wheelhouse) + ) + start = time.monotonic() + run([mise, "trust"], work, env) + run([mise, "install"], work, env) + tool = tool_name(installer) + installed = Path(run([mise, "where", f"{tool}@{version}"], work, env)) + cli = Path(run([mise, "which", "morphir-python"], work, env)) + environment = ( + installed / "morphir-python" + if installer == "uv" + else installed / "venvs/morphir-python" + ) + python = venv_python(environment) + if not python.is_file(): + raise ValueError(f"the installed tool environment is missing: {python}") + versions = json.loads( + run( + [ + str(python), + "-I", + "-c", + "import json; from importlib.metadata import version; " + "print(json.dumps({p:version(p) for p in " + "['morphir','morphir-python','morphir-python-native']}))", + ], + work, + env, + ) + ) + expected = dict(released) + if wheels: + from scripts.release_candidates import source_versions + + source = source_versions() + expected = { + "morphir": source["morphir"], + "morphir-python": source["python"], + "morphir-python-native": source["native"], + } + if versions != expected: + raise ValueError( + f"installed versions {versions} differ from {expected}" + ) + exercise(cli, python, work, env) + reports.append( + { + "installer": installer, + "versions": versions, + "seconds": round(time.monotonic() - start, 3), + } + ) + return { + "state": "succeeded", + "installs": reports, + "wheels": {w.name: sha256(w) for w in wheels}, + "wheelhouse": dependencies, + "check_sha256": sha256(Path(__file__)), + } + + +def main() -> None: + """Run installed mise qualification and retain its receipt.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("wheels", nargs="*", type=Path) + parser.add_argument("--installer", choices=(*INSTALLERS, "both"), default="both") + parser.add_argument("--report", type=Path) + args = parser.parse_args() + if len(args.wheels) not in (0, 3): + parser.error("supply no wheels, or exactly the core, native and CLI wheels") + installers = INSTALLERS if args.installer == "both" else (args.installer,) + receipt = qualify(args.wheels, installers) + rendered = json.dumps(receipt, indent=2) + "\n" + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(rendered) + print(rendered, end="") + + +if __name__ == "__main__": + main() diff --git a/scripts/release_candidates.py b/scripts/release_candidates.py index 6386ccae..28c309cb 100644 --- a/scripts/release_candidates.py +++ b/scripts/release_candidates.py @@ -19,7 +19,6 @@ import argparse import hashlib import json -import re import sys import tomllib from dataclasses import dataclass @@ -133,11 +132,8 @@ def bundle_candidates(directory: Path, version: str, commit: str) -> list[str]: raise CandidateError( f"the bundle was built from {descriptor.get('gitCommit')}, not {commit}" ) - if not re.fullmatch( - r"[0-9a-f]{64} " + re.escape(artifact) + "\n", - (directory / f"{artifact}.sha256").read_text(), - ): - raise CandidateError("the .sha256 file is malformed") + if (directory / f"{artifact}.sha256").read_text() != f"{digest} {artifact}\n": + raise CandidateError("the .sha256 checksum disagrees with the WASM") return sorted(expected) diff --git a/site/site.json b/site/site.json index 05919f14..28f86a73 100644 --- a/site/site.json +++ b/site/site.json @@ -46,7 +46,8 @@ "group": "Contribute", "items": [ "site.md", - "site-foundation.md" + "site-foundation.md", + "releasing.md" ] } ], diff --git a/tests/unit/test_ci_changes.py b/tests/unit/test_ci_changes.py index a4294a65..d2c796f7 100644 --- a/tests/unit/test_ci_changes.py +++ b/tests/unit/test_ci_changes.py @@ -29,6 +29,10 @@ {"python", "native", "docs", "docs-examples"}, ), (["uv.lock"], {"python", "native", "docs", "docs-examples"}), + ( + ["examples/install/mise-pipx.toml"], + {"python", "native", "docs", "docs-examples"}, + ), (["rust/extension.toml"], {"bundle", "docs", "docs-examples", "docs-hosts"}), ( ["tests/fixtures/python/loans/acme/rules.py"], diff --git a/tests/unit/test_qualify_mise.py b/tests/unit/test_qualify_mise.py new file mode 100644 index 00000000..d7568190 --- /dev/null +++ b/tests/unit/test_qualify_mise.py @@ -0,0 +1,56 @@ +"""The mise qualification must install the identified candidate, never PyPI substitutes.""" + +from __future__ import annotations + +import tomllib +from typing import TYPE_CHECKING + +import pytest + +from scripts import qualify_mise as q + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.mark.parametrize("installer", ["uv", "pipx"]) +def test_candidate_install_is_offline_and_uses_built_wheels( + tmp_path: Path, installer: str +) -> None: + house = tmp_path / "wheel house" + config = tomllib.loads(q.configuration(installer, "9.8.7", house)) + tool = config["tools"][q.tool_name(installer)] + assert tool["version"] == "9.8.7" + args = tool["uvx_args" if installer == "uv" else "pipx_args"] + assert "--no-index" in args + assert house.as_uri() in args + assert ("--no-build" if installer == "uv" else "--only-binary=:all:") in args + if installer == "pipx": + assert tool["uvx"] is False + + +def test_isolation_does_not_inherit_active_python_environment(tmp_path: Path) -> None: + env = q.isolated_environment( + tmp_path, + {"PATH": "/usr/bin", "VIRTUAL_ENV": "wrong", "PYTHONPATH": "wrong"}, + ) + assert "VIRTUAL_ENV" not in env + assert "PYTHONPATH" not in env + assert env["MISE_DATA_DIR"] == str(tmp_path / "data") + assert env["MISE_CONFIG_DIR"] == str(tmp_path / "config") + + +def test_candidate_refuses_a_missing_native_input_before_resolving( + tmp_path: Path, +) -> None: + with pytest.raises(ValueError, match="core, native and CLI"): + q.validate_candidates([tmp_path / "core.whl"] * 3) + + +@pytest.mark.parametrize("installer", ["uv", "pipx"]) +def test_qualification_preserves_the_published_configuration(installer: str) -> None: + original = tomllib.loads( + (q.ROOT / "examples/install" / f"mise-{installer}.toml").read_text() + ) + version = original["tools"][q.tool_name(installer)]["version"] + assert tomllib.loads(q.configuration(installer, version)) == original diff --git a/tests/unit/test_release_candidates.py b/tests/unit/test_release_candidates.py index 12049c2b..1466e739 100644 --- a/tests/unit/test_release_candidates.py +++ b/tests/unit/test_release_candidates.py @@ -110,6 +110,14 @@ def test_a_bundle_without_provenance_is_refused(tmp_path: Path) -> None: r.bundle_candidates(tmp_path, "0.4.0", "a" * 40) +def test_a_bundle_with_a_wrong_checksum_is_refused(tmp_path: Path) -> None: + bundle(tmp_path) + artifact = "morphir-python-binding-0.4.0.wasm" + (tmp_path / f"{artifact}.sha256").write_text(f"{'0' * 64} {artifact}\n") + with pytest.raises(r.CandidateError, match="checksum"): + r.bundle_candidates(tmp_path, "0.4.0", "a" * 40) + + def test_the_command_writes_a_receipt(tmp_path: Path) -> None: native_set(tmp_path) version = r.source_versions()["native"]