diff --git a/.config/mise/config.toml b/.config/mise/config.toml index a61de704..3cccd6e0 100644 --- a/.config/mise/config.toml +++ b/.config/mise/config.toml @@ -1,6 +1,10 @@ [tools] python = "3.14" uv = "latest" +# The Rust half under rust/ (compiler, CPython bridge and WASM extension guest). +rust = { version = "1.98.1", targets = "wasm32-unknown-unknown", components = "rustfmt,clippy" } +# Validates the release WASM guest before bundling. +wasm-tools = "1.259.0" [env] VIRTUAL_ENV = "{{config_root}}/.venv" diff --git a/.config/mise/tasks/bench b/.config/mise/tasks/bench new file mode 100755 index 00000000..903a27a6 --- /dev/null +++ b/.config/mise/tasks/bench @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +#MISE description="Measure native, bridge, API and CLI compile and generate costs" +# Arguments pass to scripts/bench.py, for example: mise run bench -- --modules 40 +set -euo pipefail + +# Measure the release wheel, not a development build. +NATIVE_SDIST=0 mise run native:build +uv pip install --quiet --no-deps --reinstall dist/native/morphir_python_native-*.whl +uv run --no-sync python scripts/bench.py "$@" diff --git a/.config/mise/tasks/coverage b/.config/mise/tasks/coverage index 9145ec1f..8c7ef5c6 100755 --- a/.config/mise/tasks/coverage +++ b/.config/mise/tasks/coverage @@ -3,3 +3,4 @@ set -euo pipefail uv run pytest tests/ --cov=packages/morphir/src --cov=packages/morphir-tools/src --cov-report=term-missing --cov-report=html --cov-report=xml +uv run python -m mypy.stubtest morphir_python_native._native diff --git a/.config/mise/tasks/extension/bundle b/.config/mise/tasks/extension/bundle new file mode 100755 index 00000000..90216a97 --- /dev/null +++ b/.config/mise/tasks/extension/bundle @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +#MISE description="Build, package and qualify the morphir-python WASM extension bundle offline" +set -euo pipefail + +bundle="${BUNDLE_DIR:-$PWD/.morphir/build/extensions/python}" +receipts="${QUALIFICATION_DIR:-$PWD/.dev/qualification}" +mkdir -p "$receipts" "$(dirname "$bundle")" +rm -rf "$bundle" +(cd rust && cargo build --locked --release -p morphir-python-binding --target wasm32-unknown-unknown) +wasm="rust/target/wasm32-unknown-unknown/release/morphir_python_binding.wasm" +wasm-tools validate "$wasm" +commit=() +if [[ -z "$(git status --porcelain --untracked-files=normal)" ]]; then + commit=(--git-commit "$(git rev-parse HEAD)") +fi +uv run --no-sync python scripts/extension_bundle.py package --wasm "$wasm" --output "$bundle" "${commit[@]}" \ + >"$receipts/bundle.json" +uv run --no-sync python scripts/extension_bundle.py host-check --bundle "$bundle" >"$receipts/bundle-host.json" +cat "$receipts/bundle-host.json" diff --git a/.config/mise/tasks/install b/.config/mise/tasks/install index c1fc27db..f972086a 100755 --- a/.config/mise/tasks/install +++ b/.config/mise/tasks/install @@ -1,8 +1,20 @@ #!/usr/bin/env bash #MISE description="Install all dependencies and workspace packages" +# +# By default uv builds morphir-python-native from source, which needs Rust. +# MORPHIR_NATIVE_WHEEL=PATH installs that prebuilt wheel instead, so a CI job can +# reuse one wheel. MORPHIR_SKIP_NATIVE=1 omits the native compiler for jobs that +# only lint or type-check. set -euo pipefail echo "Syncing dependencies..." -uv sync --frozen --all-packages --all-groups +if [[ -n "${MORPHIR_NATIVE_WHEEL:-}" || "${MORPHIR_SKIP_NATIVE:-0}" == 1 ]]; then + uv sync --frozen --all-packages --all-groups --no-install-package morphir-python-native + if [[ -n "${MORPHIR_NATIVE_WHEEL:-}" ]]; then + uv pip install --no-deps "$MORPHIR_NATIVE_WHEEL" + fi +else + uv sync --frozen --all-packages --all-groups +fi echo "Install complete!" diff --git a/.config/mise/tasks/native/build b/.config/mise/tasks/native/build new file mode 100755 index 00000000..0b36d626 --- /dev/null +++ b/.config/mise/tasks/native/build @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +#MISE description="Build the morphir-python-native wheel and source archive into dist/native" +set -euo pipefail + +out="${NATIVE_DIST:-$PWD/dist/native}" +mkdir -p "$out" +cd packages/morphir-python-native +# Portable wheels: refuse host-specific CPU settings. +if [[ "${RUSTFLAGS:-}" == *target-cpu* || "${CARGO_ENCODED_RUSTFLAGS:-}" == *target-cpu* ]]; then + echo "RUSTFLAGS must not select a target CPU for portable wheels" >&2 + exit 1 +fi +uvx --from maturin==1.15.0 maturin build --release --locked --out "$out" "$@" +if [[ "${NATIVE_SDIST:-1}" == 1 ]]; then + uvx --from maturin==1.15.0 maturin sdist --out "$out" +fi diff --git a/.config/mise/tasks/native/qualify b/.config/mise/tasks/native/qualify new file mode 100755 index 00000000..13c44ee9 --- /dev/null +++ b/.config/mise/tasks/native/qualify @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +#MISE description="Qualify built native, core and tools distributions without Rust on PATH" +set -euo pipefail + +native="${NATIVE_DIST:-$PWD/dist/native}" +receipts="${QUALIFICATION_DIR:-$PWD/.dev/qualification}" +mkdir -p "$receipts" +shopt -s nullglob +if [[ ${RELEASE_TAG:-} == python-v* ]]; then + # A CLI release ships with the published native compiler, so qualify those bytes, + # and refuse if the compiler sources changed after that native release. + pin="$(python3 -c 'import sys; sys.path.insert(0, "scripts"); import qualify_native as q; print(q.tools_pin())')" + git fetch --quiet --depth=1 origin "refs/tags/native-v$pin:refs/tags/native-v$pin" + if ! git diff --quiet "native-v$pin" HEAD -- rust packages/morphir-python-native; then + echo "the native compiler changed after native-v$pin; release a new native version first" >&2 + exit 1 + fi + native="$receipts/published-native" + rm -rf "$native" + uvx pip download --quiet --no-deps --only-binary=:all: --implementation cp \ + --python-version 3.14 --abi abi3 --platform manylinux_2_28_x86_64 \ + --dest "$native" "morphir-python-native==$pin" + export SKIP_SDIST_BUILD=1 +fi +wheels=("$native"/morphir_python_native-*.whl) +if [[ ${#wheels[@]} -ne 1 ]]; then + echo "expected exactly one native wheel in $native" >&2 + exit 1 +fi +python3 scripts/qualify_native.py wheel "${wheels[0]}" >"$receipts/native-wheel.json" +sdists=("$native"/morphir_python_native-*.tar.gz) +if [[ ${#sdists[@]} -eq 1 && "${SKIP_SDIST_BUILD:-0}" != 1 ]]; then + rm -rf "$receipts/from-sdist" + python3 scripts/qualify_native.py sdist "${sdists[0]}" --out "$receipts/from-sdist" \ + >"$receipts/native-sdist.json" + # The wheel built from the archive alone must work when installed, too. + python3 scripts/qualify_native.py install \ + packages/morphir/dist/morphir-*.whl "$receipts"/from-sdist/morphir_python_native-*.whl \ + packages/morphir-tools/dist/morphir_python-*.whl >"$receipts/sdist-install.json" +fi +python3 scripts/qualify_native.py install \ + packages/morphir/dist/morphir-*.whl "${wheels[0]}" packages/morphir-tools/dist/morphir_python-*.whl \ + >"$receipts/clean-install.json" +cat "$receipts"/*.json diff --git a/.config/mise/tasks/release-check b/.config/mise/tasks/release-check index 455eb42a..d34c303a 100755 --- a/.config/mise/tasks/release-check +++ b/.config/mise/tasks/release-check @@ -5,12 +5,25 @@ set -euo pipefail release_dir="$(pwd)/.dev/release-check" mkdir -p "$release_dir" uv lock --check -python scripts/check_distributions.py +python -m scripts.check_distributions uvx --from twine==7.0.0 twine check --strict \ packages/morphir/dist/* packages/morphir-tools/dist/* uv export --frozen --all-packages --no-default-groups --no-emit-workspace \ --no-hashes --output-file "$release_dir/constraints.txt" +# The CLI needs its exact native compiler. Use the wheel CI built, unless this is +# a CLI release, which resolves the already published native release from PyPI. +native_wheels=() +if [[ ${RELEASE_TAG:-} != python-v* ]]; then + shopt -s nullglob + native_wheels=("${NATIVE_DIST:-$PWD/dist/native}"/morphir_python_native-*.whl) + shopt -u nullglob + if [[ ${#native_wheels[@]} -ne 1 ]]; then + echo "expected one native wheel in ${NATIVE_DIST:-dist/native}; run mise run native:build" >&2 + exit 1 + fi +fi + packages=(morphir morphir-python) if [[ ${RELEASE_TAG:-} == morphir-v* ]]; then packages=(morphir) @@ -42,7 +55,7 @@ assert find_spec("typer") is None PY_CORE else uv pip install --python "$release_python" --constraint "$release_dir/constraints.txt" \ - packages/morphir/dist/*.whl packages/morphir-tools/dist/*.whl + packages/morphir/dist/*.whl packages/morphir-tools/dist/*.whl "${native_wheels[@]}" "$release_python" -I - <<'PY_CLI' from importlib.metadata import distribution, version @@ -60,6 +73,7 @@ PY_CLI fi uv pip check --python "$release_python" uv pip freeze --python "$release_python" --exclude morphir --exclude morphir-python \ + --exclude morphir-python-native \ > "$release_dir/$package-requirements.txt" if [[ -s "$release_dir/$package-requirements.txt" ]]; then uvx --from pip-audit==2.10.1 pip-audit --strict --no-deps --disable-pip \ @@ -86,7 +100,7 @@ uv pip install --python "$migration_python" --constraint "$release_dir/constrain 'morphir-python==0.0.1' uv pip uninstall --python "$migration_python" morphir-python morphir uv pip install --python "$migration_python" --constraint "$release_dir/constraints.txt" \ - packages/morphir/dist/*.whl packages/morphir-tools/dist/*.whl + packages/morphir/dist/*.whl packages/morphir-tools/dist/*.whl "${native_wheels[@]}" "$migration_python" -I - <<'PY_MIGRATION' from importlib.metadata import version diff --git a/.config/mise/tasks/rust/check b/.config/mise/tasks/rust/check new file mode 100755 index 00000000..da75167e --- /dev/null +++ b/.config/mise/tasks/rust/check @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +#MISE description="Format, lint and test the Rust workspace; build the WASM guest without CPython" +set -euo pipefail +cd rust + +# mise points VIRTUAL_ENV at .venv, which a Rust-only job does not create. +if [[ -n "${VIRTUAL_ENV:-}" && ! -x "$VIRTUAL_ENV/bin/python" ]]; then + unset VIRTUAL_ENV +fi +export PYO3_PYTHON="${PYO3_PYTHON:-$(command -v python3)}" + +cargo fmt --all --check +cargo clippy --locked --all-targets -- -D warnings +# The CPython module code builds only with the feature Maturin enables. +cargo clippy --locked -p morphir-python-native --features python -- -D warnings +cargo test --locked +# The generated-code execution test runs Python; it is ignored by default. +MORPHIR_TEST_PYTHON="${MORPHIR_TEST_PYTHON:-python3}" \ + cargo test --locked -p morphir-python-binding --test functions -- --ignored +cargo run --locked --quiet -p morphir-python-binding --example python_adt >/dev/null +cargo run --locked --quiet -p morphir-python-binding --example python_modules -- 3 >/dev/null + +# The compiler crate, its native consumers and the WASM guest never link CPython. +if cargo tree --locked -p morphir-python-binding -e normal,build | grep -q pyo3; then + echo "morphir-python-binding must not depend on PyO3" >&2 + exit 1 +fi +cargo build --locked --release -p morphir-python-binding --target wasm32-unknown-unknown +wasm-tools validate target/wasm32-unknown-unknown/release/morphir_python_binding.wasm +if wasm-tools print target/wasm32-unknown-unknown/release/morphir_python_binding.wasm \ + | grep -E '^\s*\(import' | grep -v '"extism:host/env"'; then + echo "the WASM guest imports something other than the Extism host" >&2 + exit 1 +fi diff --git a/.config/mise/tasks/rust/consumer b/.config/mise/tasks/rust/consumer new file mode 100755 index 00000000..cf7ecbd8 --- /dev/null +++ b/.config/mise/tasks/rust/consumer @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +#MISE description="Build and run an independent Cargo consumer of this commit's compiler crate" +set -euo pipefail + +repository="$(git rev-parse --show-toplevel)" +revision="$(git rev-parse HEAD)" +consumer="$(mktemp -d)/consumer" +trap 'rm -rf "$(dirname "$consumer")"' EXIT + +cargo new --quiet "$consumer" +cat >>"$consumer/Cargo.toml" <&2 + exit 1 +fi +echo "independent consumer at $revision compiled and generated Python" diff --git a/.config/mise/tasks/test b/.config/mise/tasks/test index c4cf465d..6c1e1615 100755 --- a/.config/mise/tasks/test +++ b/.config/mise/tasks/test @@ -3,3 +3,5 @@ set -euo pipefail uv run pytest tests/ +# The native stubs must match the installed extension module. +uv run python -m mypy.stubtest morphir_python_native._native diff --git a/.config/mise/tasks/typecheck b/.config/mise/tasks/typecheck index b1903331..81a40264 100755 --- a/.config/mise/tasks/typecheck +++ b/.config/mise/tasks/typecheck @@ -3,7 +3,7 @@ set -euo pipefail echo "Running mypy..." -uv run mypy packages/morphir/src packages/morphir-tools/src scripts +uv run mypy packages/morphir/src packages/morphir-tools/src packages/morphir-python-native/python scripts echo "Running pyright..." -uv run pyright packages/morphir/src packages/morphir-tools/src +uv run pyright packages/morphir/src packages/morphir-tools/src packages/morphir-python-native/python diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc5f63c0..4a540ff8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,45 @@ on: permissions: contents: read +concurrency: + # Superseded pull request runs are cancelled; pushes and release calls are not. + group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: + changes: + name: Select checks + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.select.outputs.jobs }} + reason: ${{ steps.select.outputs.reason }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Select categories + id: select + env: + BASE: ${{ github.event.pull_request.base.sha }} + HEAD: ${{ github.event.pull_request.head.sha }} + run: | + # Pushes and release qualification run everything. + if [[ "${{ github.event_name }}" == pull_request ]]; then + python3 scripts/ci_changes.py select --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + else + python3 scripts/ci_changes.py select --full >> "$GITHUB_OUTPUT" + fi + cat "$GITHUB_OUTPUT" + lint: name: Lint + needs: changes + if: fromJSON(needs.changes.outputs.jobs).lint runs-on: ubuntu-latest + env: + MORPHIR_SKIP_NATIVE: '1' steps: - name: Checkout repository uses: actions/checkout@v6 @@ -37,7 +72,11 @@ jobs: typecheck: name: Type Check + needs: changes + if: fromJSON(needs.changes.outputs.jobs).typecheck runs-on: ubuntu-latest + env: + MORPHIR_SKIP_NATIVE: '1' steps: - name: Checkout repository uses: actions/checkout@v6 @@ -51,8 +90,47 @@ jobs: - name: Run type checkers run: mise run typecheck + native-wheel: + name: Native wheel (Linux x86_64) + needs: changes + if: fromJSON(needs.changes.outputs.jobs)['native-wheel'] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build the manylinux_2_28 wheel and source archive + uses: PyO3/maturin-action@v1 + with: + working-directory: packages/morphir-python-native + maturin-version: '1.15.0' + rust-toolchain: '1.98.1' + manylinux: '2_28' + target: x86_64-unknown-linux-gnu + args: --release --locked --sdist --out ../../dist/native + + - name: Audit the Linux baseline and bundled libraries + shell: bash + run: | + # The build container writes dist/ as root. + sudo chown -R "$(id -u):$(id -g)" dist + pipx run --spec 'auditwheel==6.8.2' auditwheel show dist/native/*.whl | tee dist/native/auditwheel.txt + # The report starts with the file name, so match auditwheel's own verdict. + tr '\n' ' ' < dist/native/auditwheel.txt | tr -s ' ' \ + | grep -q 'consistent with the following platform tag: "manylinux_2_28_x86_64"' + sha256sum dist/native/*.whl dist/native/*.tar.gz | tee dist/native/SHA256SUMS + + - name: Upload the native distributions + uses: actions/upload-artifact@v6 + with: + name: native-linux-x86_64 + path: dist/native/ + if-no-files-found: error + test: name: Test + needs: [changes, native-wheel] + if: fromJSON(needs.changes.outputs.jobs).test runs-on: ubuntu-latest steps: - name: Checkout repository @@ -61,10 +139,17 @@ jobs: - name: Set up mise uses: jdx/mise-action@v3 - - name: Install dependencies - run: mise run install + - name: Download the native distributions + uses: actions/download-artifact@v7 + with: + name: native-linux-x86_64 + path: dist/native/ - - name: Run pytest + - name: Install dependencies with the built native wheel + run: | + MORPHIR_NATIVE_WHEEL="$(ls dist/native/*.whl)" mise run install + + - name: Run pytest and check the native stubs run: mise run coverage - name: Run behave @@ -78,10 +163,76 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + rust: + name: Rust + needs: changes + if: fromJSON(needs.changes.outputs.jobs).rust + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up mise + uses: jdx/mise-action@v3 + + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + + - name: Format, lint, test and build the WASM guest + run: mise run rust:check + + - name: Build an independent Cargo consumer + run: mise run rust:consumer + + bundle: + name: Extension bundle + needs: [changes, native-wheel] + if: fromJSON(needs.changes.outputs.jobs).bundle + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up mise + uses: jdx/mise-action@v3 + + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + rust + .dev/tools/morphir-rust-0aa60eb799d4 + + - name: Download the native distributions + uses: actions/download-artifact@v7 + with: + name: native-linux-x86_64 + path: dist/native/ + + - name: Install dependencies with the built native wheel + run: | + MORPHIR_NATIVE_WHEEL="$(ls dist/native/*.whl)" mise run install + + - name: Package and qualify the bundle offline + run: mise run extension:bundle + + - name: Upload the extension bundle + uses: actions/upload-artifact@v6 + with: + name: morphir-python-extension-bundle + path: | + .morphir/build/extensions/python/ + .dev/qualification/bundle*.json + include-hidden-files: true + if-no-files-found: error + build: name: Build runs-on: ubuntu-latest - needs: [lint, typecheck, test] + needs: [changes, lint, typecheck, test, native-wheel] + if: fromJSON(needs.changes.outputs.jobs).build steps: - name: Checkout repository uses: actions/checkout@v6 @@ -92,11 +243,24 @@ jobs: - name: Build packages run: mise run build + - name: Download the native distributions + uses: actions/download-artifact@v7 + with: + name: native-linux-x86_64 + path: dist/native/ + - name: Check distributions, vulnerabilities and licenses run: mise run release-check env: RELEASE_TAG: ${{ inputs.release-tag }} + - name: Qualify the native archive and a clean install without Rust + run: mise run native:qualify + env: + NATIVE_DIST: ${{ github.workspace }}/dist/native + # A python-v release qualifies the published native wheel instead. + RELEASE_TAG: ${{ inputs.release-tag }} + - name: Upload morphir distributions uses: actions/upload-artifact@v6 with: @@ -116,5 +280,22 @@ jobs: uses: actions/upload-artifact@v6 with: name: release-scan-reports - path: .dev/release-check/*.json + path: | + .dev/release-check/*.json + .dev/qualification/*.json include-hidden-files: true + + all-checks: + name: All Checks + if: always() + needs: [changes, lint, typecheck, native-wheel, test, rust, bundle, build] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Require every selected check + env: + SELECTED: ${{ needs.changes.outputs.jobs || '{}' }} + NEEDS: ${{ toJSON(needs) }} + run: python3 scripts/ci_changes.py aggregate --selected "$SELECTED" --needs "$NEEDS" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5118b2cc..25989a3b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,10 @@ on: tags: - 'morphir-v*' - 'python-v*' - # Qualification only. Publishing requires a tag push. + - 'native-v*' + - 'extension/python/v*' + # Qualification only, including every native wheel platform. Publishing + # requires a tag push. workflow_dispatch: permissions: @@ -22,6 +25,244 @@ jobs: with: release-tag: ${{ github.ref_type == 'tag' && github.ref_name || '' }} + # Every advertised platform builds its own wheel once; the same job installs + # that exact wheel, with the qualified core and tools wheels, into a clean + # environment without Rust before uploading it. + native-wheels: + name: Native wheel (${{ matrix.platform }}) + needs: qualify + if: >- + github.event_name == 'workflow_dispatch' || + startsWith(github.ref, 'refs/tags/native-v') + strategy: + fail-fast: false + matrix: + include: + # Qualification already built and tested this wheel; reuse its bytes. + - platform: manylinux_2_28_x86_64 + runner: ubuntu-latest + target: x86_64-unknown-linux-gnu + manylinux: '2_28' + reuse: native-linux-x86_64 + - platform: manylinux_2_28_aarch64 + runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + manylinux: '2_28' + - platform: macosx_11_0_arm64 + runner: macos-latest + target: aarch64-apple-darwin + manylinux: '' + - platform: macosx_10_12_x86_64 + runner: macos-15-intel + target: x86_64-apple-darwin + manylinux: '' + - platform: win_amd64 + runner: windows-latest + target: x86_64-pc-windows-msvc + manylinux: '' + runs-on: ${{ matrix.runner }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Reuse the qualification wheel + if: matrix.reuse + uses: actions/download-artifact@v7 + with: + name: ${{ matrix.reuse }} + path: dist/reused/ + + - name: Keep only the reused wheel + if: matrix.reuse + shell: bash + run: mkdir -p dist/native && cp dist/reused/*.whl dist/native/ + + - name: Build the wheel + if: ${{ !matrix.reuse }} + uses: PyO3/maturin-action@v1 + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.target == 'aarch64-apple-darwin' && '11.0' || '10.12' }} + with: + working-directory: packages/morphir-python-native + maturin-version: '1.15.0' + rust-toolchain: '1.98.1' + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'off' }} + args: --release --locked --out ../../dist/native + + - name: Audit the Linux baseline and bundled libraries + if: matrix.manylinux != '' + shell: bash + run: | + # The build container writes dist/ as root. + sudo chown -R "$(id -u):$(id -g)" dist + pipx run --spec 'auditwheel==6.8.2' auditwheel show dist/native/*.whl | tee audit.txt + # The report starts with the file name, so match auditwheel's own verdict. + tr '\n' ' ' < audit.txt | tr -s ' ' \ + | grep -q 'consistent with the following platform tag: "${{ matrix.platform }}"' + + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Download the qualified core and tools wheels + uses: actions/download-artifact@v7 + with: + pattern: morphir*-dist + path: dist/pure/ + merge-multiple: true + + - name: Install the exact wheels without Rust and exercise them + shell: bash + run: | + wheel="$(ls dist/native/*.whl)" + [[ "$wheel" == *"-cp314-abi3-${{ matrix.platform }}.whl" ]] + python scripts/qualify_native.py wheel "$wheel" + python scripts/qualify_native.py install \ + dist/pure/morphir-*.whl "$wheel" dist/pure/morphir_python-*.whl + + - name: Upload the qualified wheel + uses: actions/upload-artifact@v6 + with: + name: native-wheel-${{ matrix.platform }} + path: dist/native/*.whl + if-no-files-found: error + + native-candidates: + name: Verify native release candidates + needs: [qualify, native-wheels] + if: needs.native-wheels.result == 'success' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download the qualified platform wheels + uses: actions/download-artifact@v7 + with: + pattern: native-wheel-* + path: dist/candidates/ + merge-multiple: true + + - name: Download the qualified source archive + uses: actions/download-artifact@v7 + with: + name: native-linux-x86_64 + path: dist/ci-native/ + + - name: Add the source archive + run: cp dist/ci-native/*.tar.gz dist/candidates/ + + - name: Refuse missing, extra or mismatched candidates + run: | + tag="${{ github.ref_type == 'tag' && github.ref_name || '' }}" + if [[ -z "$tag" ]]; then + tag="native-v$(python3 -c 'import tomllib; print(tomllib.load(open("rust/crates/morphir-python-native/Cargo.toml","rb"))["package"]["version"])')" + fi + python3 scripts/release_candidates.py native dist/candidates --tag "$tag" + + - name: Upload the release set + uses: actions/upload-artifact@v6 + with: + name: native-release + path: dist/candidates/ + if-no-files-found: error + + publish-native: + name: Publish morphir-python-native to PyPI + if: >- + github.event_name == 'push' && + github.repository == 'finos/morphir-python' && + startsWith(github.ref, 'refs/tags/native-v') + needs: native-candidates + runs-on: ubuntu-latest + permissions: + id-token: write + environment: + name: pypi + url: https://pypi.org/project/morphir-python-native/ + steps: + - name: Download the verified release set + uses: actions/download-artifact@v7 + with: + name: native-release + path: dist/ + + - name: Check the bytes against the receipt + run: | + cd dist + sha256sum --check SHA256SUMS + rm SHA256SUMS + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + extension-candidate: + name: Verify the extension bundle candidate + needs: qualify + if: startsWith(github.ref, 'refs/tags/extension/python/v') + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download the qualified bundle + uses: actions/download-artifact@v7 + with: + name: morphir-python-extension-bundle + path: dist/bundle-check/ + + - name: Refuse a bundle that is not from this tag's commit + run: | + bundle=dist/bundle-check/.morphir/build/extensions/python + python3 scripts/release_candidates.py bundle "$bundle" \ + --tag "${{ github.ref_name }}" --commit "$GITHUB_SHA" + mkdir -p dist/bundle + version="${GITHUB_REF_NAME#extension/python/v}" + # Release assets keep morphir-rust's descriptor name. + cp "$bundle/morphir-python-binding-$version.wasm" \ + "$bundle/morphir-python-binding-$version.wasm.sha256" dist/bundle/ + cp "$bundle/release.json" "dist/bundle/morphir-python-binding-$version.release.json" + cp dist/bundle-check/.dev/qualification/bundle*.json dist/bundle/ + + - name: Upload the release set + uses: actions/upload-artifact@v6 + with: + name: extension-release + path: dist/bundle/ + if-no-files-found: error + + publish-extension: + name: Publish the morphir-python extension bundle + if: >- + github.event_name == 'push' && + github.repository == 'finos/morphir-python' && + startsWith(github.ref, 'refs/tags/extension/python/v') + needs: extension-candidate + runs-on: ubuntu-latest + permissions: + contents: write + environment: + name: extension-release + url: https://github.com/finos/morphir-python/releases + steps: + - name: Download the verified bundle + uses: actions/download-artifact@v7 + with: + name: extension-release + path: dist/ + + - name: Create the GitHub release + uses: softprops/action-gh-release@v2 + with: + files: dist/* + generate_release_notes: true + prerelease: true + publish-morphir: name: Publish morphir library and SDK to PyPI if: >- @@ -45,6 +286,7 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + # Qualification refuses a python-v tag until the pinned native release is on PyPI. publish-morphir-python: name: Publish morphir-python to PyPI if: >- diff --git a/.gitignore b/.gitignore index 52171e00..1d6b575f 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,9 @@ yarn-error.log* # behave writes the pretty formatter to this file when its output is piped pretty.output + +# Rust +rust/target/ + +# Extension bundles built by mise run extension:bundle +.morphir/ diff --git a/README.md b/README.md index 79f93fd4..73666c78 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ The library and CLI are separate distributions: | PyPI distribution | Contents | Python import / command | |-------------------|----------|-------------------------| | [`morphir`](https://pypi.org/project/morphir/) | Core library, IR models, and SDK runtime; no third-party runtime dependencies | `import morphir` | -| [`morphir-python`](https://pypi.org/project/morphir-python/) | CLI and tool integrations; depends on `morphir` | `import morphir_tools`; `morphir-python` | +| [`morphir-python`](https://pypi.org/project/morphir-python/) | CLI, Python compiler API and tool integrations; depends on `morphir` and one exact `morphir-python-native` | `import morphir_tools`; `morphir-python` | +| [`morphir-python-native`](https://pypi.org/project/morphir-python-native/) | Private native compiler wheels, installed automatically by `morphir-python` | none (private) | Install just the library and SDK: @@ -30,10 +31,16 @@ pip install morphir-python==0.0.2 # or: uv add morphir-python==0.0.2 ``` -Python 3.14 or later is required. The CLI currently provides IR inspection and -conversion. It is also the intended home for future MEP extension-protocol CLI -integration and possible WASM build tooling; those features are not implemented -by this packaging change. +Python 3.14 or later is required. The CLI inspects and converts IR, compiles a +supported subset of Python to Morphir IR, and generates Python from Morphir IR. +The compiler runs in process from prebuilt wheels; you do not need Rust. See the +[Python compiler guide](docs/python-compiler.md) for the API, the CLI commands and +the supported subset. + +The same compiler is the Rust crate +[`morphir-python-binding`](rust/crates/morphir-python-binding/README.md) and the +`morphir-python` Morphir extension. This repository builds and releases both; see +the [extension guide](docs/python-extension.md). ### Migrating from the bundled 0.0.1 release @@ -167,6 +174,12 @@ mise run check | `mise run check` | Run all checks | | `mise run build` | Build packages | | `mise run release-check` | Validate built packages and scan runtime dependencies for vulnerabilities and licenses | +| `mise run rust:check` | Format, lint and test the Rust workspace and build the WASM guest | +| `mise run rust:consumer` | Build an independent Cargo consumer of the current commit | +| `mise run native:build` | Build the native wheel and source archive into `dist/native` | +| `mise run native:qualify` | Qualify the native archive and a clean install without Rust | +| `mise run extension:bundle` | Build the MEP extension bundle and check it offline in a Morphir host | +| `mise run bench` | Measure native, bridge, API and CLI compile and generate times | | `mise run clean` | Clean build artifacts | ## Releases @@ -232,6 +245,47 @@ the core has no third-party dependencies. Scan reports are retained as workflow artifacts. Scans use metadata/advisories and do not replace source-code security or license review. +### Native compiler and extension bundle releases + +Two more tag families release the compiler. Their versions come from the Rust +crates and are independent of the Python package versions: + +| Tag | Releases | Version source | Publishes to | +|-----|----------|----------------|--------------| +| `native-v` | `morphir-python-native` wheels and source archive | `rust/crates/morphir-python-native/Cargo.toml` | PyPI, `pypi` environment | +| `extension/python/v` | `morphir-python` MEP extension bundle | `rust/crates/morphir-python-binding/Cargo.toml` | GitHub release, `extension-release` environment | + +Add a trusted publisher for the PyPI project `morphir-python-native` with the +same repository, workflow and `pypi` environment, and allow `native-v*` tags in +that environment. Create an `extension-release` environment that allows +`extension/python/v*` tags and requires a maintainer review. + +A `native-v` tag builds one wheel for each qualified platform (Linux glibc +x86_64 and aarch64, macOS arm64 and x86_64, Windows x86_64) and installs each +exact wheel with the core and CLI wheels into a clean environment without Rust. +The Linux x86_64 wheel and the source archive are the ones CI already tested. +Publication downloads the set that `scripts/release_candidates.py` accepted and +checks it against its `SHA256SUMS` receipt. A missing, failed, skipped, extra or +mismatched candidate stops the release. A manual run of `release.yml` qualifies +every platform without publishing. + +An `extension/python/v` tag publishes the bundle that CI built from that tag's +commit and checked in the pinned Morphir CLI. Its `release.json` must name that +commit and its WASM digest. + +Publish in this order, each step after maintainer approval: + +1. `native-v`, when the native compiler changed. +2. `python-v` for a CLI that pins it. Its qualification refuses to + continue until every platform file of the pinned native version is on PyPI. + It then tests the CLI with the published Linux wheel, and refuses if the Rust + or native package sources changed after the `native-v` tag of that version. +3. `extension/python/v`, when the compiler changed. Then update the + ecosystem pin in finos/morphir (`.config/published-extension-bundles.toml`). + +Never delete or overwrite a published version or an older bundle; select an +earlier version through the pins to roll back. + For targeted qualification, `RELEASE_TAG=morphir-v0.0.1 mise run release-check` validates the library tag. `RELEASE_TAG=python-v0.0.2 mise run release-check` also checks the published core dependency. The latter must wait until the library is diff --git a/docs/python-compiler.md b/docs/python-compiler.md new file mode 100644 index 00000000..2551ac7c --- /dev/null +++ b/docs/python-compiler.md @@ -0,0 +1,244 @@ +# Compile Python to Morphir IR and generate Python + +`morphir-python` compiles a supported subset of Python to Morphir IR, and +generates Python from Morphir IR. It does both in process, through a native +compiler that installs automatically with the package. You do not need Rust or a +separate Morphir executable. + +The compiler is the Rust crate +[`morphir-python-binding`](../rust/crates/morphir-python-binding/README.md). The +same crate is the `morphir-python` Morphir extension, so the Python API, the CLI, +native Rust callers and Morphir extension hosts give the same results and the +same diagnostic codes. + +## Install + +```console +pip install morphir-python +``` + +This installs three distributions: + +| Distribution | Contents | Runtime dependencies | +| --- | --- | --- | +| `morphir` | IR models, codecs and the Morphir SDK | none | +| `morphir-python-native` | private native compiler module | none | +| `morphir-python` | the `morphir-python` CLI and `morphir_tools.python` | `morphir`, one exact `morphir-python-native` version, Typer | + +Install only `morphir` when you need the IR and SDK without the compiler. + +Prebuilt `morphir-python-native` wheels are qualified for CPython 3.14 on: + +| Platform | Wheel tag | +| --- | --- | +| Linux x86_64, glibc 2.28 or later | `manylinux_2_28_x86_64` | +| Linux aarch64, glibc 2.28 or later | `manylinux_2_28_aarch64` | +| macOS 11 or later, Apple silicon | `macosx_11_0_arm64` | +| macOS 10.12 or later, Intel | `macosx_10_12_x86_64` | +| Windows x86_64 | `win_amd64` | + +The wheels use the CPython stable ABI (`cp314-abi3`). The free-threaded +interpreter (`python3.14t`) is not qualified. On other platforms `pip` builds the +compiler from the source archive, which needs Rust 1.98 or later; see +[Build from source](#build-from-source). + +## Use the Python API + +```python +from morphir_tools.python import ( + CompiledV3, + CompileFailure, + Generated, + SourceDocument, + compile_sources, + generate_python, +) + +models = SourceDocument( + "acme/models.py", + """\ +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + +type Decision = Pending | Approved +""", +) +rules = SourceDocument( + "acme/rules.py", + """\ +from __future__ import annotations +from .models import Decision + +def choose(flag: bool, first: Decision, second: Decision) -> Decision: + if flag: + return first + else: + return second +""", +) + +compiled = compile_sources([models, rules], package="acme/example") +if isinstance(compiled, CompileFailure): + for diagnostic in compiled.diagnostics: + print(diagnostic.severity.value, diagnostic.render()) + raise SystemExit(1) +assert isinstance(compiled, CompiledV3) + +generated = generate_python(compiled.distribution) +assert isinstance(generated, Generated) +for artifact in generated.artifacts: + print(artifact.path) +``` + +`compile_sources` takes the source documents of one package, its Morphir +package name, and two options: + +- `exposed_modules`: module paths to make public, such as `("acme/models",)`. + `None`, the default, makes every module public. An empty tuple makes every + module private. +- `ir_version`: `IrVersion.V3` (the default, stable format 3.0.0) or + `IrVersion.V4` (draft format 4.0.0). + +It returns one of three frozen types: + +| Result | Contents | +| --- | --- | +| `CompiledV3` | `distribution`, a `morphir.ir.v3.distribution.Library`, and `warnings` | +| `CompiledV4` | `distribution`, a `morphir.ir.v4.distribution.LibraryDistribution`, and `warnings` | +| `CompileFailure` | `diagnostics`, at least one, and no model | + +`generate_python` takes a v3 `Library` or a v4 `LibraryDistribution`, for example +one read with `morphir.ir.v3.codec.decode_versioned_distribution`. It returns +`Generated`, with one `GeneratedArtifact(path, text)` per module in path order, +or `GenerationFailure`. A Specs or Application distribution fails with +`unsupported_distribution`. Convert legacy formats 1 and 2 with +`morphir-python ir convert` first. + +Generation reads both v4 spellings, the compact one the compiler and other +Morphir tools write and the expanded one the Python codec writes. Two limits +apply: + +- IR v3 integer literals must fit in 64 bits. Compile to v4 for larger integers; + v4 keeps them exactly. +- Generation does not yet accept v4 values that carry attributes. A v4 model + that `morphir.ir.migrate.v3_to_v4` made from typed v3 IR keeps the inferred + types as attributes, so generation rejects it with `PY004`. Generate from the + v3 model instead. Empty documentation, which such a model also carries, is + accepted; documentation text is rejected rather than dropped. + +The order of the source documents does not change the result. Calls hold no +shared state, so several threads can compile at the same time; the compiler runs +without holding Python's global interpreter lock. + +### Diagnostics + +Each `CompilerDiagnostic` has a `code`, a `severity` (`Severity.ERROR`, +`WARNING`, `INFO` or `HINT`), a `message`, and either a `source` location +(path, zero-based line and UTF-16 character) or an `ir_path` into the IR +document. A success can carry warnings; a failure never carries a model. + +| Code | Meaning | +| --- | --- | +| `PY001` | Unsupported request or option | +| `PY002` | Python syntax error, with a source range | +| `PY003` | Invalid name or naming collision | +| `PY004` | Unsupported declaration, annotation or IR feature | +| `PY005` | IR decoding or serialization, or generated syntax failure | +| `PY006` | Parse-stage output was requested but is unavailable | +| `unsupported_distribution` | Generation received a Specs or Application distribution | + +IR reading and writing failures keep their `morphir.ir.diagnostics` codes, such +as `unknown_node`. The constants are in `morphir_tools.python.diagnostics`. + +Two exceptions report installation problems, not problems in your model: +`NativeCompilerUnavailable` when the native compiler is missing or its version +does not match this release, and `NativeCompilerError` for an internal compiler +failure, which is a bug to report. + +## Use the CLI + +```console +morphir-python compile src --package acme/example --output model.json +morphir-python ir inspect model.json +morphir-python generate model.json --output-dir generated +morphir-python compile generated --package acme/example --output rebuilt.json +``` + +`compile SOURCE` accepts one `.py` file or a directory. A directory is read +recursively in sorted order; hidden directories, `__pycache__` and symbolic links +are skipped. A file's path relative to the source root gives its module name: +`src/acme/rules.py` becomes module `acme/rules`. A single file uses its parent +directory as the root. + +| Option | Meaning | +| --- | --- | +| `--package NAME` | Morphir package name. Required. | +| `--output FILE` | Where to write the IR document. Required. | +| `--ir-version v3\|v4` | IR version to write; `v3` by default. `v4` prints a draft notice. | +| `--expose MODULE` | Module to make public. Repeat it for more modules. Without it every module is public. | + +`generate FILE --output-dir DIRECTORY` reads a format 3.0 or draft v4 Library +and writes one Python module per Morphir module, keeping the module directories. +The output directory must not exist or must be empty, and must not be a symbolic +link. All files are written to a staging directory next to it and moved into +place together, so a failure leaves no partial output. + +Both commands write diagnostics to standard error as +`error: CODE at LOCATION: MESSAGE`, and exit with 1 on failure. Warnings are +printed as `warning: ...` and do not change the exit code. A failed compilation +leaves an existing output file unchanged. + +To make the generated modules importable, put the output directory on Python's +import path. The generated code needs no Morphir package at run time. + +## Supported Python + +The compiler accepts this subset of Python 3.12 and later, in both IR versions: + +- frozen dataclasses as records, and named unions of dataclasses + (`type Decision = Pending | Approved`) as custom types; +- fixed tuples and non-generic `type` aliases of tuples; +- scalar fields (`int`, `float`, `str`, `bool`), same-package references and + unary `Callable[[A], B]` types; +- annotated pure functions whose bodies return through `if`/`elif`/`else` + branches or conditional expressions, with literals, comparisons, typed calls + and unary lambdas; +- absolute and relative imports between modules of the same package. + +Anything else is rejected with a diagnostic, never compiled partially. This +includes assignments, loops, methods, defaults, containers, optional fields, +boolean operators, other imports, and constructor calls inside functions. The +[compiler README](../rust/crates/morphir-python-binding/README.md#current-boundary) +lists the exact boundary. + +Compilation reads source text only. It never imports or runs the source, and +generation never runs the generated code. + +A compile, generate and recompile round trip keeps module access, references, +type definitions and function behavior. It does not keep comments, formatting or +equivalent spellings of the same expression. + +## Build from source + +Contributors build everything from a checkout with [mise](https://mise.jdx.dev): + +```console +mise install # Python 3.14, uv, Rust 1.98.1 and wasm-tools +mise run install # builds morphir-python-native into .venv with Maturin +mise run check # Python checks +mise run rust:check # Rust format, lint and tests +``` + +To build only the native wheel, run +`uvx --from maturin==1.15.0 maturin build --release` in +`packages/morphir-python-native`. Its source archive contains the whole Rust +workspace it needs, so `pip install morphir_python_native-*.tar.gz` works without +a checkout when Rust is on `PATH`. diff --git a/docs/python-extension.md b/docs/python-extension.md new file mode 100644 index 00000000..7cfe6644 --- /dev/null +++ b/docs/python-extension.md @@ -0,0 +1,105 @@ +# The morphir-python Morphir extension + +The `morphir-python` extension lets a Morphir host, such as the Rust `morphir` +CLI, compile Python with `morphir compile --language python` and generate Python +with `morphir generate --target python`. It is the +[`morphir-python-binding`](../rust/crates/morphir-python-binding/README.md) +compiler built as a WebAssembly guest that speaks the Morphir extension protocol +(MEP). It is the same compiler that the `morphir-python` Python package runs +natively, so both give the same IR and the same diagnostic codes. + +This repository builds and releases the extension. Bundles up to +`extension/python/v0.4.0` were released from +[finos/morphir-rust](https://github.com/finos/morphir-rust/releases) and remain +available there. + +| Identity | Value | +| --- | --- | +| Extension ID | `morphir-python` | +| Short ID and release tag | `python`, `extension/python/v` | +| Language | `python` (`.py`) | +| Target | `python` | +| IR versions | `3`, `4` | +| MEP version | `0.1` | + +The declaration is in [`rust/extension.toml`](../rust/extension.toml). + +## Install a release bundle + +A release contains the WASM guest, its `.sha256` file and a version-2 release +descriptor. Publishing it needs Morphir CLI `0.4.0-beta.7` or later. Rename the +descriptor to `release.json` before you publish it: + +```console +gh release download extension/python/v --repo finos/morphir-python --dir python-bundle +mv python-bundle/morphir-python-binding-.release.json python-bundle/release.json +morphir extension repository init .morphir/python-index +morphir extension repository add python-local --directory .morphir/python-index +morphir extension repository publish python-local --bundle python-bundle +morphir extension repository verify python-local +morphir extension install --repository python-local morphir-python +``` + +Installation verifies the WASM digest and records its capability claims. The CLI +shows WASM claims as `unchecked`: it records the claims the guest declared and +does not probe them. After installation, compilation and generation work +offline. + +## Compile and generate + +Create a project with `src/acme/models.py` and `src/acme/rules.py` (see the +[Python compiler guide](python-compiler.md#use-the-python-api) for supported +Python) and this `morphir.toml`: + +```toml +[project] +name = "acme/example" +version = "0.1.0" +source_directory = "src" + +[frontend] +language = "python" +emit_parse_stage = false +``` + +Then run: + +```console +morphir compile --language python --input src --ir-version 3.0.0 --output compiled +morphir generate --target python --input compiled/morphir-ir.json --output generated +``` + +Omit `--ir-version`, or use `4`, for draft IR v4. + +## Build a development bundle + +From the repository root: + +```console +mise install +mise run install +mise run extension:bundle +``` + +This task: + +1. builds the release WASM guest and validates it with `wasm-tools`; +2. packages it with finos/morphir-rust's canonical WASM packager + (`scripts/package_extension.py`), run unchanged at the revision pinned in + `rust/extension.toml`, so the bundle has the same descriptor format as + earlier releases. The finos/morphir `morphir extension bundle` command packages + native process extensions only; +3. downloads the Morphir CLI version pinned in `rust/extension.toml`, checks its + SHA-256 digest, publishes and installs the bundle into a new Morphir home, + deletes the repository, blocks network access, and compiles, generates and + recompiles a two-module model in IR v3 and v4; +4. checks that the IR from the extension equals the IR from the Python API for + the same sources. + +The bundle is written to `.morphir/build/extensions/python/`, and JSON receipts +with every digest are written to `.dev/qualification/`. A bundle built from a +clean checkout records its commit as `gitCommit`; a bundle from a modified +checkout does not. + +CI runs this task when the compiler, the declaration or the bundle tooling +changes, and uploads the bundle as `morphir-python-extension-bundle`. diff --git a/features/python_codegen.feature b/features/python_codegen.feature new file mode 100644 index 00000000..a70f0381 --- /dev/null +++ b/features/python_codegen.feature @@ -0,0 +1,43 @@ +Feature: Python compilation and generation + As a Python developer modeling a domain with Morphir + I want to compile supported Python to Morphir IR and generate Python from IR + So that my model can move between Python and other Morphir tools + + Scenario Outline: Compile, generate and recompile a multi-module model + Given the Python sample project "loans" + When I compile it with package "acme/example" to IR "" exposing "acme/models" + Then the command exits with code 0 + And the compiled model declares formatVersion + And module "acme/models" is public and module "acme/rules" is private + When I generate Python from the compiled model + Then the command exits with code 0 + And the generated files are "acme/models.py, acme/rules.py" + And the generated code gives the expected results in a separate process + When I recompile the generated Python with the same options + Then the recompiled model equals the compiled model + + Examples: + | version | format | + | v3 | 3 | + | v4 | 4 | + + Scenario: Unsupported Python is rejected without output + Given a Python source file with the text "total = 1" + When I compile it with package "acme/example" to IR "v3" + Then the command exits with code 1 + And the errors contain "PY004" + And no compiled model is written + + Scenario: Generation refuses a directory that already has files + Given the Python sample project "loans" + When I compile it with package "acme/example" to IR "v3" + And I generate Python into a directory that already contains "notes.txt" + Then the command exits with code 1 + And the errors contain "not empty" + And the directory still contains only "notes.txt" + + Scenario: A Python program compiles and generates through the API + Given the Python sample project "loans" + When a program compiles the project with the Python API + Then the program receives a v3 Library model + And generating Python from that model returns 2 artifacts diff --git a/features/steps/python_codegen_steps.py b/features/steps/python_codegen_steps.py new file mode 100644 index 00000000..e20a1cc4 --- /dev/null +++ b/features/steps/python_codegen_steps.py @@ -0,0 +1,214 @@ +"""Step definitions for Python compilation and generation.""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +from behave import given, then, when +from typer.testing import CliRunner + +from morphir.ir import format_version as fv +from morphir.ir.v3 import access_controlled as v3_ac +from morphir.ir.v3 import codec as v3_codec +from morphir.ir.v3 import distribution as v3_dist +from morphir.ir.v4 import codec as v4_codec +from morphir.ir.v4 import distribution as v4_dist +from morphir_tools.cli import app + +if TYPE_CHECKING: + from behave.runner import Context + +SAMPLES = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "python" + +CHECK = ( + "import sys; sys.path.insert(0, sys.argv[1]);" + "from acme import models, rules;" + "print(rules.choose(True, 1, 2), rules.choose(False, 1, 2)," + " rules.pair(1.0, 2.0), rules.twice_identity(7)," + " rules.pick(False, models.Pending(), models.Approved(2.5)))" +) + + +def _workdir(context: Context) -> Path: + if not hasattr(context, "workdir"): + context.workdir = Path(tempfile.mkdtemp(prefix="morphir-bdd-")) + return context.workdir + + +def _run(context: Context, *args: str) -> None: + context.result = CliRunner().invoke(app, list(args)) + + +def _compile(context: Context, source: Path, output: Path) -> None: + args = ["compile", str(source), "--package", context.package] + args += ["--ir-version", context.version, "--output", str(output)] + for module in context.exposed: + args += ["--expose", module] + _run(context, *args) + + +def _decode(context: Context, path: Path) -> object: + codec = v3_codec if context.version == "v3" else v4_codec + return codec.decode_versioned_distribution(path.read_text()) + + +@given('the Python sample project "{name}"') +def step_sample(context: Context, name: str) -> None: + """Select a checked-in Python source tree.""" + context.source = SAMPLES / name + + +@given('a Python source file with the text "{text}"') +def step_source_file(context: Context, text: str) -> None: + """Write a one-line Python source file.""" + context.source = _workdir(context) / "bad.py" + context.source.write_text(text + "\n") + + +@when('I compile it with package "{package}" to IR "{version}" exposing "{module}"') +def step_compile_exposing( + context: Context, package: str, version: str, module: str +) -> None: + """Compile the source with one exposed module.""" + context.package, context.version, context.exposed = package, version, [module] + context.model = _workdir(context) / "model.json" + _compile(context, context.source, context.model) + + +@when('I compile it with package "{package}" to IR "{version}"') +def step_compile(context: Context, package: str, version: str) -> None: + """Compile the source exposing every module.""" + context.package, context.version, context.exposed = package, version, [] + context.model = _workdir(context) / "model.json" + _compile(context, context.source, context.model) + + +@when("I generate Python from the compiled model") +def step_generate(context: Context) -> None: + """Generate Python into a new directory.""" + context.generated = _workdir(context) / "generated" + _run( + context, "generate", str(context.model), "--output-dir", str(context.generated) + ) + + +@when('I generate Python into a directory that already contains "{name}"') +def step_generate_nonempty(context: Context, name: str) -> None: + """Generate Python into a directory holding a user file.""" + context.generated = _workdir(context) / "generated" + context.generated.mkdir() + (context.generated / name).write_text("keep") + _run( + context, "generate", str(context.model), "--output-dir", str(context.generated) + ) + + +@when("I recompile the generated Python with the same options") +def step_recompile(context: Context) -> None: + """Compile the generated tree again.""" + context.recompiled = _workdir(context) / "recompiled.json" + _compile(context, context.generated, context.recompiled) + assert context.result.exit_code == 0, context.result.output + + +@when("a program compiles the project with the Python API") +def step_api(context: Context) -> None: + """Compile through morphir_tools.python in this process.""" + from morphir_tools.python import SourceDocument, compile_sources + + root = context.source + context.compiled = compile_sources( + [ + SourceDocument(path.relative_to(root).as_posix(), path.read_text()) + for path in sorted(root.rglob("*.py")) + ], + package="acme/example", + ) + + +@then("the compiled model declares formatVersion {major:d}") +def step_format(context: Context, major: int) -> None: + """Check the written format version.""" + version, _ = fv.read_document(context.model.read_text()) + assert version.major == major, version + + +@then('module "{public}" is public and module "{private}" is private') +def step_access(context: Context, public: str, private: str) -> None: + """Check module access in the decoded model.""" + model = _decode(context, context.model) + assert isinstance(model, v3_dist.Library | v4_dist.LibraryDistribution) + access: dict[str, str] = {} + for path, module in model.definition.modules.items(): + name = "/".join("-".join(words) for words in path) + if isinstance(module, v3_ac.AccessControlled): + access[name] = module.access.value + else: + access[name] = type(module).__name__ + assert access == {public: "Public", private: "Private"}, access + + +@then('the generated files are "{names}"') +def step_generated_files(context: Context, names: str) -> None: + """Check the generated relative paths.""" + found = sorted( + path.relative_to(context.generated).as_posix() + for path in context.generated.rglob("*") + if path.is_file() + ) + assert found == [name.strip() for name in names.split(",")], found + + +@then("the generated code gives the expected results in a separate process") +def step_execute(context: Context) -> None: + """Run the generated modules in a new interpreter.""" + completed = subprocess.run( + [sys.executable, "-I", "-c", CHECK, str(context.generated)], + capture_output=True, + text=True, + check=True, + ) + assert completed.stdout.strip() == "1 2 (1.0, 2.0) 7 Approved(amount=2.5)", ( + completed.stdout + completed.stderr + ) + + +@then("the recompiled model equals the compiled model") +def step_equal(context: Context) -> None: + """Compare the decoded models.""" + assert _decode(context, context.model) == _decode(context, context.recompiled) + + +@then("no compiled model is written") +def step_no_model(context: Context) -> None: + """Check that compilation wrote nothing.""" + assert not context.model.exists() + + +@then('the directory still contains only "{name}"') +def step_only(context: Context, name: str) -> None: + """Check that generation left the user's directory alone.""" + assert [p.name for p in context.generated.iterdir()] == [name] + + +@then("the program receives a v3 Library model") +def step_api_model(context: Context) -> None: + """Check the API result type.""" + from morphir_tools.python import CompiledV3 + + assert isinstance(context.compiled, CompiledV3), context.compiled + assert isinstance(context.compiled.distribution, v3_dist.Library) + + +@then("generating Python from that model returns {count:d} artifacts") +def step_api_generate(context: Context, count: int) -> None: + """Generate in process from the API model.""" + from morphir_tools.python import Generated, generate_python + + generated = generate_python(context.compiled.distribution) + assert isinstance(generated, Generated), generated + assert len(generated.artifacts) == count diff --git a/openspec/changes/migrate-python-binding/.openspec.yaml b/openspec/changes/migrate-python-binding/.openspec.yaml new file mode 100644 index 00000000..dda53b8a --- /dev/null +++ b/openspec/changes/migrate-python-binding/.openspec.yaml @@ -0,0 +1,2 @@ +schema: html-design +created: 2026-10-27 diff --git a/openspec/changes/migrate-python-binding/design.html b/openspec/changes/migrate-python-binding/design.html new file mode 100644 index 00000000..50ed7914 --- /dev/null +++ b/openspec/changes/migrate-python-binding/design.html @@ -0,0 +1,172 @@ + + + + + +Migrate the Python binding: design + + + + + + + + +
+

Migrate the Python binding

+

OpenSpec change migrate-python-binding · Bead morphir-ukvn.1.6 · 2026-10-27 · Proposed

+ +

Context and evidence

+

See proposal.md for motivation. Inspection used the clean local morphir-rust checkout at f4298d0d416628ce3c910b662a78e224cf8eba8a, on chore/openspec-html-designs. This is an observed source revision, not a claim that it is the eventual migration base. Before copying, select the current upstream main revision and reconcile binding changes since this snapshot.

+ + + + + + + +
Observed codeConsequence
crates/morphir-python-binding, version 0.4.0; native PythonExtension and SDK WASM exportsThis is a Python language compiler implemented in Rust, currently neither a CPython module nor a PyPI distribution.
Ruff component crates pinned together to 0.0.15; core 0.2.0 and SDK 0.3.0 with sibling pathsMove compiler tests and lockfile resolution together; replace sibling paths with independently resolvable dependencies.
ADT, conditional, tuple, function, IR-version and module tests; Gherkin acceptance; native examplesMigration starts by preserving this behavior. More Python syntax is separate work under morphir-rust-72s.
Rust host tests include Python fixtures from the crate directory; release tasks and CI classify the crateDeleting the directory alone would break host tests, CI selection and bundle publishing.
Python core and tools use separate Hatch distributions; Python minimum 3.14; existing CLI only inspects/converts IRIntroduce compiler access without giving the dependency-free core a native build requirement.
Python supports Library and Specs across its versioned codecs; binding generation accepts only Library and a subset of nodesSuccessful IR decoding does not imply the Python backend can generate it. Report explicit subset diagnostics.
+

The approved ecosystem roadmap records independent language repositories and published Rust dependencies. Its A2 template qualification, morphir-ukvn.1.3.3, remains open. Preserve that dependency for final A4 delivery and reuse the canonical bundler rather than creating a new descriptor format.

+
+

Goals and boundaries

+

Own the compiler here; preserve native Rust and MEP callers; expose in-memory compilation and generation to Python; deliver CLI round trips. Python callers receive the existing Python IR models and ordinary immutable result types.

+

This increment preserves the existing language subset. It does not add general Python execution, arbitrary third-party imports, a second pure-Python compiler, full MCK conformance, new component hosts, or additional evaluation engines. The Python host wrapper in morphir-opua.1.4.4 remains independent. Existing IR inspection/conversion retains its broader version/distribution coverage.

+
+

Community reference implementations

+

Research on 2026-10-27 inspected tagged releases and their immutable commit revisions. These are architectural examples, not dependency versions to copy into Morphir. Tokenizers main already contains an unreleased binding rewrite, so the release snapshot is the reference below. The selected patterns complement the official PyO3 and Maturin documentation.

+ + + + +
Released example and source evidencePattern to adoptLimits of the analogy
Hugging Face Tokenizers v0.23.3, 301277072263fb3830eb66eec7e47adadf76f1dc. Its Python binding crate depends on the separate Rust tokenizers library; its Python package uses Maturin with a nested PyO3 module. The tokenizer methods include owned-input extraction and detached Rust work.Keep the Rust compiler reusable and the CPython binding separate. Extract a complete request before detached execution; return to Python only after Rust finishes. Ship Python-facing types and typing support around the native module.Its source layout uses a local Rust dependency inside its own repository. Morphir must additionally resolve core/SDK crates independently of morphir-rust. Its async runtime and mutable tokenizer objects are unnecessary for a stateless compiler request.
Pydantic v2.14.1, b2212c787fb5c73386dc770681ec49cd09da8e73. The Python distribution uses Hatch and exactly pins pydantic-core; the core distribution uses Maturin and a private extension module. Its version check detects incompatible installed core versions, and the core ships native API stubs.Keep existing pure-Python packaging and add a separately built native dependency. Initially pin the exact qualified native version in each tools release and check bridge compatibility when compiler access initializes. Verify native stubs against the installed extension.The Rust core is a required implementation dependency, not an automatic pure-Python fallback. Morphir keeps its standalone IR/SDK distribution independent. We do not copy Pydantic's editable-install exception or its wider interpreter matrix.
Polars py-2.1.0, 7f902bd3ffd27ff82d14278a09a295ff2f7aed92. The Python distribution exactly depends on a runtime wheel; the runtime package uses Maturin and includes explicit build inputs. Its release workflow builds runtime artifacts, installs source archives and tests installed wheels.Separate the Python package from native wheel production; install the exact produced artifacts during qualification. Include source-build inputs explicitly and prove wheel and sdist behavior outside the checkout.Multiple CPU/runtime variants, dataframe FFI and the broad feature matrix solve Polars-specific needs. Start Morphir with one portable native runtime per supported platform/ABI and add variants only with measured need.
+

Our inference is that the separate Rust core, small PyO3 adapter and separately built native wheel fit Morphir's existing two-package workspace. The JSON/codec boundary in IR-1 remains a Morphir decision: these projects also expose bound native objects and do not establish serialized requests as a universal binding practice. Keep JSON private, submit a whole source set in one call, and benchmark copies and codec work before considering a different adapter. No example proves Morphir's Rust path is faster than an equivalent Python compiler.

+

Apply the official PyO3 parallelism guidance when detaching. Detached work owns its Rust inputs and must not access Python objects or invoke Python callbacks. The Maturin layout guidance covers private modules and typing files; its distribution guidance covers Linux compatibility checking, native-library bundling and builds from source archives. Validate these properties with Morphir's actual dependency graph and target toolchain.

+
+

Decisions

+

REPO-1. Keep a Rust half with independently resolvable dependencies

+
rust/
+  Cargo.toml
+  Cargo.lock
+  crates/morphir-python-binding/  # migrated compiler, tests and examples
+  crates/morphir-python-native/   # CPython bridge only
+packages/
+  morphir/                       # existing dependency-free library
+  morphir-tools/                 # public Python compiler API and CLI
+  morphir-python-native/         # native distribution metadata and typing
+
+

Use a Cargo workspace under rust/; retain the crate name and public Rust API. Keep PyO3 in the bridge crate so Cargo consumers and the WASM guest never link CPython. Use exact compatible published Morphir core/SDK versions after checking the current APIs. If the APIs require unreleased changes, pin all related Morphir crates to one exact git revision and record the publication dependency. No sibling paths, branches, vendored copies of core or hidden local patches. Carry workspace metadata explicitly and pin the four Ruff dependencies as a group. Cargo consumers can use the package from an exact morphir-python git revision before any crates.io publication.

+

Alternative: put PyO3 features in the compiler crate. Rejected because feature unification can pull Python linking into native Rust or WASM builds. A mechanical copy comes before language behavior changes, with source provenance and a file inventory.

+

API-1. Expose a Python API over an in-process bridge

+

Use PyO3 and Maturin to build morphir_python_native._native. The bridge invokes the same SDK frontend/backend traits as Rust callers and exchanges serialized requests/results internally. No host, subprocess, Extism runtime or MEP handshake is needed by Python applications. Extract and own the complete request before using PyO3 detached execution for compilation/generation; convert results after returning to Python. Detached code must not access Python objects or call Python callbacks. Keep request state local and verify concurrent calls do not share mutable state. Convert errors deliberately and do not expose panics or serialized envelopes as the public API.

+

Public entry points live in morphir_tools.python: compile_sources and generate_python. Inputs are frozen source-document and options types. Compilation returns a union of version-specific successes and a diagnostic failure; a success always contains its matching Python IR model. Generation returns a success with a tuple of frozen relative-path/text artifacts, or a diagnostic failure. Keep warnings on successes; use a severity enum and source locations distinct from IR member paths. API objects must not hold Rust AST objects or mutable SDK dictionaries. Compiler diagnostic codes are centralized consistently with morphir.ir.diagnostics.

+

Alternative: run the WASM extension from Python. It could reuse an existing bundle, but would add a runtime, host protocol and loading costs to every Python installation. A pure-Python compiler would duplicate a substantial implementation before evidence justifies it. Ordinary Python integration here means a Python API with a native implementation, not a second parser.

+

IR-1. Use versioned JSON as the internal compatibility boundary

+

Decode bridge output through morphir.ir.format_version and the matching v3/v4 codec before constructing a success. Encode Python models through those codecs before generation. Do not invent parallel Python domain models or encode the wire format in the CLI. Any explicit cross-version conversion goes through morphir.ir.migrate; generation does not silently downgrade or drop unsupported attributes.

+

Default new Python API and CLI compilation to stable v3.0 Library; allow explicit draft v4. Preserve the existing native Rust request choices. Generation initially accepts supported v3.0 and draft v4 Libraries only; Specs, dependency packages and unsupported language nodes retain explicit rejection. Legacy formats remain usable through the existing ir convert workflow before generation. JSON precision and parameter order must survive both directions. Cross-codec fixtures include arbitrary integers, unary nested function types, ADTs, tuples, private modules and independent IR. This is a qualification obligation, not assumed parity.

+

Alternative: expose every Rust model through PyO3. Rejected for the initial migration because it would duplicate this repository's versioned Python types and couple the API to Rust representation changes. JSON adds measurable cost, so benchmark it separately.

+

PKG-1. Add a native dependency without changing the core library

+

Add distribution morphir-python-native, containing the private extension, stubs and py.typed. Each tools release initially declares an exact dependency on its qualified native package version and resolves it automatically; the public API remains in the tools namespace. Native and tools versions remain independent numbers. On first compiler access, verify the installed native version and expected bridge API revision; fail clearly on incompatibility without affecting lazy IR-only commands. Match compiled-module identity, distribution metadata and release inputs. A broader version range requires an explicit tested compatibility contract. Keep the two existing distributions on Hatch, their current tags and independent versions. Give the new native distribution its own version/tag, native-v*, and keep the binding bundle on extension/python/v*. No release version is selected by this proposal. Qualify dependency installation using a local wheelhouse before changing release metadata.

+

Initial wheel qualification targets CPython 3.14 on Linux glibc x86_64/aarch64, macOS x86_64/arm64 and Windows x86_64. Pin a released PyO3/Maturin pair supporting the selected interpreter and use the stable ABI where qualified. Do not infer free-threaded support from an abi3 wheel; CPython 3.14t needs distinct qualification. Document source builds and other platforms separately. Build the native sdist with all referenced Rust workspace/path dependency files and prove a clean archive-only wheel build. Keep compiler imports lazy so existing IR commands do not initialize it. Verify private native stubs against installed exports/signatures with stubtest or an equivalent focused check, and type-check public API examples; do not hide drift with blanket type-checker ignores. Build portable wheels without host-specific CPU assumptions, set and audit the Linux compatibility baseline, inspect bundled shared libraries and test imports on the declared deployment targets. An abi3 tag alone does not prove platform portability.

+

Maturin's mixed-project guidance supports a private native submodule and typing files. PyO3's distribution guidance describes interpreter selection and wheel/ABI choices. These support the packaging approach, not a performance claim.

+

CLI-1. Keep I/O in tools and publish complete output

+

The compile command reads a single file or a sorted directory tree and makes one API request containing relative source paths. A single file uses its parent as root; a directory uses itself. Explicit exposure selections use repeated --expose options; the Python API also retains the distinction between omitted exposure and an explicitly empty tuple. Do not recursively follow symlink source directories or discover installed packages. Require a package identity rather than guessing it from filesystem names.

+

The generate command uses the shared reader/codec before the API. Require an absent or empty destination. Validate every artifact path, including collisions and symlink traversal, and write the complete artifact tree in a sibling temporary directory before publishing it. On failure remove staging, preserve existing files and report I/O errors without a traceback. Compile uses a temporary file and atomic replacement. Keep data/output and diagnostics separate; diagnostics go to stderr.

+

PERF-1. Measure the delivered path before claiming speed

+

Add a reproducible benchmark command with checked-in small and multi-module fixtures plus generated scale parameters. Measure native Rust compilation/generation, private bridge calls, public Python API calls and CLI end-to-end runs independently. Include warmup, repetitions, medians and spread, input bytes/module counts, target version, build profile, hardware, toolchains and serialization costs. Use release builds and separate cold import/startup from steady-state execution.

+

Python ast.parse can be reported only as a parsing baseline; it performs less work than compilation and cannot establish a compiler speed ratio. There is no equivalent pure-Python compiler here. State that limitation in results. Benchmarks report observations; noisy timing thresholds do not gate PRs, and a slower bridge result does not justify silently substituting a compiler.

+
+

Build and release assurance

+

Follow the personal build-master skill; this checkout has no repository copy. Preserve existing checks and add scoped Rust/native/bundle categories. No runtime or cost measurements were collected during planning, so estimated performance gains and CI savings are unknown.

+ + + + + + + +
Category and ownerTrigger, risk and deadlineProducer and consumer
Python, Python maintainersPython/API/IR or shared changes; regression detection before mergeExisting Ruff, mypy, pyright, pytest/coverage and behave; API tests consume one built local native wheel.
Rust/compiler, binding maintainersCompiler/dependency or shared changes; subset and protocol regressions before mergeCargo fmt/clippy/test including migrated acceptance and doc tests; external Cargo consumer proves independent use.
Wheel install, packaging maintainersBridge/packaging changes; ABI and missing-file failures before merge on representative Linux and platform-specific checks when touchedBuild once per selected platform/ABI; clean-environment API/CLI smoke consumes that exact wheel. No rebuild in consumers.
MEP bundle, extension maintainersBinding/bundler changes and cutover; host compatibility before mergeOne release WASM build; canonical bundling; digest/capability validation and isolated offline host round trip consume the exact bundle.
Release qualification, release maintainerTag or explicit qualification request; all advertised wheel platforms and source archive before promotionCandidate-bound wheel matrix, sdist rebuild, license/security scans and bundle receipts. Publication consumes qualified bytes; failed, skipped, absent or mismatched evidence blocks promotion.
Benchmarks, binding maintainersExplicit performance request or release assessment; no claim before resultsFixed-candidate release builds; retain environment and raw measurements with report. No live schedule is created here.
+

Test the change selector and aggregate for relevant/irrelevant paths, shared dependencies, cancellations, missing results and expected skips. Documentation-only changes do not install Rust. Keep release qualification callable without publishing; no wheel matrix, bundle publication or registry promotion runs merely because planning files changed. Cache keys include lockfiles, target, toolchain and features; caches never count as test evidence. Record candidate SHA, dependency lock hash, producer run and artifact digest in qualification receipts.

+
Build-master comparison and evidence limits +

Decision scope: proposed PR/release graph for native wheels and MEP bundles. Failure impact is moderate for compiler regressions and high for a broken or mismatched distribution. Merge checks protect touched behavior; release checks protect every advertised artifact. Maintainers named above respond to failures and block promotion until repaired.

+ + + + + + + +
CriterionWeightAll platforms on every PRScoped PR plus exact-candidate release
Risk and coverage fit25U, selectors unimplementedU, selectors unimplemented
Feedback and placement20U, no run measurementsU, no run measurements
Release evidence and trust20U, receipts unimplementedU, receipts unimplemented
Execution efficiency15U, no measured build graphU, no measured build graph
Ownership and structure102, category owners identified2, category owners identified
Operability and recovery10U, refusal cases untestedU, refusal cases untested
+

Totals withheld; 90% of rubric weight has unknown implementation evidence. Gate preservation, candidate identity, trust and failure routing are all unknown until executable checks verify them. Scoped CI is the provisional choice because the approved roadmap requires it. Reassess with actual run records during implementation. This design provides no merge or release approval.

+
+
+

Risks and trade-offs

+ + + + + + + +
RiskMitigation
Published SDK APIs lag the source snapshotResolve and compile against actual artifacts; use one exact git revision as a documented temporary fallback. Never copy shared core crates here.
Rust and Python v4 codecs differ despite matching namesCross-codec tests are required before marking Python API success; correct the supported mapping and reject unrepresentable attributes rather than dropping them.
Tools now depend on native wheelsKeep the core independent, test wheel-only installations and source archives, publish the compatible native dependency before tools, and state the qualified platform matrix.
JSON bridge overhead erases a possible Rust advantageRetain a replaceable private boundary and measure each stage. Performance remains a hypothesis.
Removing the crate breaks host fixture includes or CI matricesMove compiler tests here, replace host fixtures with independent protocol fixtures or pinned external provider input, and run affected workspace/selector gates.
A2 template qualification and live publication remain openImplement and validate local functionality first; preserve the roadmap dependency and record final release/cutover evidence before closing A4.
+
+

Migration and rollback

+
    +
  1. Select the upstream source revision, record provenance and inventory the binding, fixture includes, bundle registry, release selectors and parent CLI references. Preserve unrelated work in the sibling checkouts.
    2. +
  2. Move the crate and behavior tests here; establish standalone Cargo and WASM proofs using declared dependencies. Until cutover, the old provider remains the published provider.
    3. +
  3. Add the isolated CPython bridge, versioned Python-model boundary, typed API and CLI. Each group lands its tests and documentation. Qualify clean wheels and an archive-only source build.
    4. +
  4. Build and install the replacement bundle through the canonical morphir extension bundle path using pinned compatible tools. Validate checksum, provenance and claims. Run native, Python and offline MEP round trips for both IR versions.
    5. +
  5. Prepare coordinated morphir-rust removal and finos/morphir provider-reference changes on feature branches. Remove old release registrations and documentation ownership; retain historical releases. Validate the target repositories independently. Do not close A4 because only the destination builds.
    6. +
  6. After maintainer approval, publish qualified native wheels and bundle before releasing dependent tools or promoting ecosystem discovery references. Verify live installation resolves to this repository. Link merged revisions, release receipts and the independent Rust consumer proof in the epic, then close it only after all acceptance conditions hold.
    7. +
+

Rollback before publication means reverting the coordinated commits while the old provider remains available. After publication, retain immutable assets and select the last working provider/version through registry metadata and dependency constraints. Never delete or overwrite historical releases. Merges and real releases require the approval recorded by the repository rules; preparing their concrete artifacts and qualification evidence comes first.

+
+

Acceptance examples

+
# Run from a clean installed environment without Rust on PATH.
+morphir-python compile src --package acme/example --output model.json
+morphir-python ir inspect model.json
+morphir-python generate model.json --output-dir generated
+morphir-python compile generated --package acme/example --output rebuilt.json
+
+# Repeat using draft v4 and the same exposure choices.
+morphir-python compile src --package acme/example --ir-version v4 --output model-v4.json
+morphir-python generate model-v4.json --output-dir generated-v4
+
+

Compare decoded models after narrowly documented normalization, then execute generated fixtures in a separate test process to verify conditional branches, tuples, imports and closures. Add negative checks for unsupported constructs, unchanged existing files and malformed versions. Also install the exact MEP bundle into an isolated extension repository and run the ecosystem CLI with networking unavailable. Confirm the provider URL/provenance names finos/morphir-python and the old Rust directory is absent.

+

No unresolved design choice changes the scope or task breakdown. Exact release versions and approval timing are release inputs; source revisions and toolchain pins are implementation evidence to record before their dependent tasks.

+
+ +

Implementation findings

+

Implementation on 2026-10-27 found these facts. They refine the decisions above without changing scope.

+ + + + + + + + + + + +
FindingResolution
The binding at finos/morphir-rust 0aa60eb is unchanged since f4298d0, and published morphir-core 0.2.0 and morphir-extension-sdk 0.3.0 build it. Its examples failed at upstream too, because they skipped the MEP initialize handshake.No git-revision fallback is needed. The examples now initialize, and rust:check runs them.
The finos/morphir morphir extension bundle command packages native process executables only. A WASM guest still needs morphir-rust's scripts/package_extension.py, which reads claims through its workspace.The pinned morphir-rust packager (rust/extension.toml) runs unchanged, with this repository's declaration, so the descriptor format stays the same. Native process bundling remains A2 work, morphir-ukvn.1.3.3.
Rust core writes the compact canonical v4 spelling. The Python v4 reader accepted only the expanded spelling, but the profile at efa36060 requires readers to accept every schema-valid spelling.The Python reader now accepts them (the reader half of morphir-ukvn.8.16). The writer is unchanged.
IR from v3_to_v4 carries empty documentation and inferred-type attributes. The v4 backend rejected any documentation and rejects value attributes.Empty documentation is now treated as absent, as the v3 reader already did. Value attributes keep their explicit PY004, and the limit is documented.
A bundle built here would share version 0.4.0 with different morphir-rust bytes.The compiler crate is now 0.4.1, so the first bundle from this repository gets a new version. The native distribution starts at 0.1.0.
The PyO3 build script needs an interpreter even for Rust-only tests.The CPython module sits behind a python feature that Maturin enables. Bridge logic, tests and benchmarks build without Python.
In the first benchmark, the Python codecs and JSON exchange cost more than native compilation for large inputs. Interpreter start-up sets a CLI floor of about 65 ms on the measured machine.Recorded as observations only. The private JSON boundary stays replaceable.
+
+ + diff --git a/openspec/changes/migrate-python-binding/proposal.md b/openspec/changes/migrate-python-binding/proposal.md new file mode 100644 index 00000000..d421e4e4 --- /dev/null +++ b/openspec/changes/migrate-python-binding/proposal.md @@ -0,0 +1,38 @@ +# Proposal + +## Why + +Python compilation and generation currently live in morphir-rust, separate from the Python IR library and CLI. Moving their implementation here lets Python contributors maintain the binding and gives Python callers and CLI users the same compiler without requiring knowledge of Rust or a separate Morphir executable. + +## What Changes + +- Move `morphir-python-binding`, its fixtures, examples, acceptance tests and supported-subset documentation into this repository's Rust half, preserving its native Rust API and MEP extension identity. +- Add an in-process CPython extension, a typed Python compilation/generation API, and prebuilt wheels. Keep `morphir` dependency-free; put compiler integration in the tools distribution and its native dependency. +- Follow inspected release examples from Hugging Face Tokenizers, Pydantic and Polars for reusable Rust code, a separate PyO3 adapter, native package pairing and wheel qualification. Record immutable source references and Morphir-specific choices in the design. +- Add `morphir-python compile` and `morphir-python generate` for Python source to versioned IR and IR to Python, including multi-module inputs, diagnostics and safe output handling. +- Preserve the existing supported Python subset and IR v3/v4 behavior. Reject unsupported constructs explicitly. Validate the Python/Rust IR boundary against this repository's codecs instead of assuming matching model names imply compatibility. +- Build and qualify the extension bundle here, then cut over ecosystem distribution references. **BREAKING**: remove the old crate, Python bundle ownership and workspace-local fixture references from morphir-rust after replacement qualification. +- Measure compiler and Python integration costs with reproducible benchmarks. Treat a Rust performance advantage as unproven. + +## Capabilities + +### New Capabilities + +- `python/compiler`: Supported Python language mapping, deterministic compilation/generation, typed Python access and diagnostics across IR versions. +- `python/extension`: Independent Rust consumption, MEP compatibility, installable Python wheels and ecosystem extension distribution owned here. +- `tools/python-codegen`: CLI compilation and Python generation through the public Python API. + +### Modified Capabilities + +None. Existing `tools/ir-interchange` requirements continue to govern `ir inspect` and `ir convert`. + +## Beads + +- **Tracked by**: `morphir-ukvn.1.6`, reused as an epic for the existing A4 migration and the Python API/CLI requested here. +- **Related**: `morphir-ukvn.1.3` and `morphir-ukvn.1.3.3`, existing common-layout and release qualification dependencies; `morphir-rust-72s`, existing compiler foundations and broader language backlog; `morphir-nhbd`, prior Python bundle publication; `morphir-ukvn.8`, Python IR compatibility; `morphir-opua.1.4.4`, separate future Python component-host wrapper, not a dependency of the in-process compiler API. + +## Impact + +This repository gains a Rust workspace, a separate native wheel distribution, compiler API and CLI commands, scoped CI and extension release qualification. `morphir` and the current pure-Python tools wheel retain their own namespaces and release tags; each tools release initially pins its exact qualified native dependency. Compiler initialization detects incompatible native versions or bridge API revisions. Native extension and bundle versions are explicit and independent of the Python CLI version. + +Coordinated changes are required in finos/morphir-rust for removal and in finos/morphir for bundle discovery, tests and pinned integration references. Published core/SDK dependencies, or exact committed git revisions where necessary, replace sibling-checkout paths. Real releases, merges and registry promotion remain maintainer-approved; qualification builds and draft changes do not imply publication. diff --git a/openspec/changes/migrate-python-binding/specs/python/compiler/spec.md b/openspec/changes/migrate-python-binding/specs/python/compiler/spec.md new file mode 100644 index 00000000..138377e4 --- /dev/null +++ b/openspec/changes/migrate-python-binding/specs/python/compiler/spec.md @@ -0,0 +1,65 @@ +# Spec Delta + +## Purpose + +Compile supported Python domain models to Morphir IR and generate executable Python through an ordinary typed Python API, sharing behavior with native Rust and MEP callers. + +## ADDED Requirements + +### Requirement: Compile the established Python subset +Compilation SHALL preserve the migrated binding's frozen dataclasses, named unions, fixed tuples, annotated pure functions, scalar literals and comparisons, conditionals, typed calls, unary contextual lambdas, local imports and module exposure behavior. It MUST reject unsupported syntax with structured diagnostics. (bead: morphir-ukvn.1.6) + +#### Scenario: Compile a multi-module model +- **WHEN** a caller submits models and rules modules with imported ADTs, tuple aliases and annotated conditional functions +- **THEN** compilation preserves their module identities, references, declaration order and configured public/private module access + +#### Scenario: Reject an unsupported construct +- **WHEN** a model uses an assignment or an unsupported external import +- **THEN** compilation reports the binding's diagnostic code and available source location without returning a successful partial model + +### Requirement: Source processing is static +Compilation and generation MUST NOT import or execute submitted source or generated Python. (bead: morphir-ukvn.1.6) + +#### Scenario: Input attempts a side effect +- **WHEN** source includes top-level file-writing code +- **THEN** compilation rejects the unsupported construct and no file is written by that source + +### Requirement: Versioned IR interoperates with the Python library +Compilation SHALL produce v3.0 Library IR by default and draft v4 Library IR on request. Generated models MUST decode with this repository's matching codec. Generation SHALL accept supported v3.0 and draft v4 Library models, reject unsupported versions and constructs, and preserve arbitrary-precision integers wherever the selected format permits them. (bead: morphir-ukvn.1.6) + +#### Scenario: Python codec reads compiler output +- **WHEN** a supported model is compiled to each advertised target version +- **THEN** the shared format reader and matching Python codec decode it without losing module, type or value semantics + +#### Scenario: Independently authored IR generates Python +- **WHEN** a Library model using supported constructs comes from the Python codec or an independent ecosystem fixture +- **THEN** generation produces valid Python without requiring that model to originate in this compiler + +#### Scenario: Unsupported distribution or format +- **WHEN** generation receives a Specs distribution or an unsupported format version +- **THEN** it reports an explicit diagnostic rather than inventing value bodies or silently migrating the model + +### Requirement: Typed Python API supports both directions +The Python API SHALL accept in-memory source documents, package identity, exposure and target-version options, and return immutable typed results with a version-specific Python IR model. Generation SHALL accept a supported Python IR model and return immutable relative-path source artifacts. Failures SHALL expose structured diagnostics with code, severity and available source location. (bead: morphir-ukvn.1.6) + +#### Scenario: Python caller compiles and generates +- **WHEN** a caller supplies source strings through the documented Python API and passes the compiled model to generation +- **THEN** both operations work in process without MEP envelopes, Rust types, temporary files, or a separate executable in the caller's code + +#### Scenario: Warning accompanies success +- **WHEN** the compiler returns valid IR with a warning +- **THEN** the result retains the warning and its severity separately from successful model data + +### Requirement: Round trips preserve supported semantics +For supported models, compile/generate/recompile SHALL preserve normalized module access, references, type definitions and value behavior in each target IR version. Source formatting and equivalent expression spelling need not be preserved. (bead: morphir-ukvn.1.6) + +#### Scenario: Conditional and higher-order behavior survives generation +- **WHEN** a model using conditional branches, a typed call and a unary closure is generated and recompiled +- **THEN** normalized IR remains equivalent and executing the generated code in a separate verification process produces the expected results + +### Requirement: Source document order does not affect results +Compilation SHALL produce equivalent models and diagnostics for permutations of the same source set, with deterministic artifact ordering during generation. (bead: morphir-ukvn.1.6) + +#### Scenario: Reordered modules +- **WHEN** callers compile the same dependent source modules in different orders +- **THEN** normalized models and diagnostic ordering agree, and subsequent generation returns the same ordered artifacts diff --git a/openspec/changes/migrate-python-binding/specs/python/extension/spec.md b/openspec/changes/migrate-python-binding/specs/python/extension/spec.md new file mode 100644 index 00000000..530f9d08 --- /dev/null +++ b/openspec/changes/migrate-python-binding/specs/python/extension/spec.md @@ -0,0 +1,53 @@ +# Spec Delta + +## Purpose + +Distribute the Python language compiler from morphir-python for native Rust consumers, ordinary Python installations and Morphir extension hosts, with one maintained implementation. + +## ADDED Requirements + +### Requirement: Rust consumers use an independently buildable crate +The migrated crate SHALL retain the `morphir-python-binding` package name and native `PythonExtension` frontend/backend API. A Rust consumer MUST be able to build and invoke it without a morphir-rust sibling checkout, CPython or Python extension linking. (bead: morphir-ukvn.1.6) + +#### Scenario: External Cargo consumer +- **WHEN** a standalone Cargo project depends on the binding at an exact committed revision in morphir-python +- **THEN** it builds and invokes compile and generate using declared core/SDK dependencies without local source overrides or Python development libraries + +### Requirement: MEP behavior remains compatible +The extension SHALL retain identity `morphir-python`, Python language/target identifiers, initialization and capability negotiation, compile/generate methods and advertised IR v3/v4 support. Native Rust, Python and packaged WASM invocations SHALL agree on supported model semantics and language diagnostic codes. (bead: morphir-ukvn.1.6) + +#### Scenario: Existing host installs the replacement bundle +- **WHEN** a compatible Morphir host installs the bundle built from morphir-python +- **THEN** initialization, capability discovery, Python compilation and Python generation succeed through the existing protocol + +### Requirement: Python installation hides Rust implementation details +Installing `morphir-python` on qualified CPython platforms SHALL install its compatible native compiler dependency automatically from wheels. Compilation and generation MUST require neither a Rust toolchain nor a separately installed Morphir CLI at runtime. The `morphir` distribution SHALL keep zero third-party runtime dependencies. (bead: morphir-ukvn.1.6) + +#### Scenario: Clean wheel-only installation +- **WHEN** a user installs the qualified distributions into a clean CPython 3.14 environment on a supported wheel platform with Rust absent +- **THEN** Python API compilation and generation and both new CLI commands succeed + +#### Scenario: Library-only user +- **WHEN** a user installs only `morphir` +- **THEN** IR and SDK imports work without the compiler dependency or native shared libraries + +### Requirement: Qualified artifacts are self-contained +Native wheels and source distributions SHALL include required runtime files, typing information and build inputs. Published artifact metadata SHALL declare supported interpreter/platform combinations and compatible package versions. Qualification MUST install built artifacts outside the source checkout. (bead: morphir-ukvn.1.6) + +#### Scenario: Build from the source archive +- **WHEN** a contributor builds the native wheel from its source archive with the documented build toolchain +- **THEN** the build succeeds without missing workspace files or sibling repositories and the wheel passes an installed API smoke test + +### Requirement: Python extension releases belong to morphir-python +Replacement Python extension bundles SHALL be built, qualified and released from finos/morphir-python with checksums, capability claims and source provenance. Ecosystem installation references SHALL resolve the replacement provider there; historical release assets MUST remain available. (bead: morphir-ukvn.1.6) + +#### Scenario: Ecosystem installation follows the new owner +- **WHEN** a qualified replacement is published and selected in the ecosystem extension repository +- **THEN** `morphir extension install morphir-python` obtains its bundle from morphir-python, verifies it and supports offline compile/generate/recompile + +### Requirement: Old implementation is removed after replacement qualification +The cutover SHALL remove the Python binding implementation, workspace membership and Python-specific publishing ownership from morphir-rust. Tests and integrations MUST use the new provider or independent host fixtures without depending on the deleted directory. (bead: morphir-ukvn.1.6) + +#### Scenario: Rust workspace after removal +- **WHEN** the replacement provider has passed migration qualification and the coordinated removal lands +- **THEN** the morphir-rust workspace and its CI remain valid with no Python compiler implementation or references requiring the former crate directory diff --git a/openspec/changes/migrate-python-binding/specs/tools/python-codegen/spec.md b/openspec/changes/migrate-python-binding/specs/tools/python-codegen/spec.md new file mode 100644 index 00000000..3d0ee305 --- /dev/null +++ b/openspec/changes/migrate-python-binding/specs/tools/python-codegen/spec.md @@ -0,0 +1,69 @@ +# Spec Delta + +## Purpose + +Let morphir-python CLI users compile supported Python models into versioned Morphir IR and generate Python packages from supported IR using the same Python API as application callers. + +## ADDED Requirements + +### Requirement: Compile Python sources from the CLI +`morphir-python compile SOURCE --package PACKAGE --output FILE` SHALL compile one Python file or a directory source tree through the public Python API. It SHALL default to IR v3 and support `--ir-version v4` and repeated `--expose MODULE` selections. Omitting exposure SHALL expose every module. Output SHALL be a versioned IR document readable by `ir inspect`. (bead: morphir-ukvn.1.6) + +#### Scenario: Compile a source directory +- **WHEN** the user compiles a directory containing supported models and locally importing rules modules +- **THEN** the tool writes one versioned Library document with correct module identities and requested exposure, and exits zero + +#### Scenario: Choose draft v4 +- **WHEN** the user requests `--ir-version v4` +- **THEN** the tool writes draft v4 and reports its draft status + +### Requirement: Generate Python from versioned IR +`morphir-python generate FILE --output-dir DIRECTORY` SHALL read IR with the shared format reader and matching Python codec, then call the public generation API. It SHALL write all generated relative-path Python modules while preserving their directory structure. (bead: morphir-ukvn.1.6) + +#### Scenario: Generate modules with imports +- **WHEN** a supported Library contains models and rules in nested modules +- **THEN** generation produces importable modules with the expected paths and supported behavior + +#### Scenario: Version parsing uses the shared contract +- **WHEN** an input has a malformed, missing or unsupported format version +- **THEN** generation reports the same version diagnostic code as the Python library and exits non-zero + +### Requirement: Report failures and warnings consistently +Both commands SHALL preserve Python API diagnostic codes, severities and available locations. Failures SHALL exit non-zero, with diagnostics on stderr. Success warnings SHALL remain distinguishable from errors and SHALL NOT cause a non-zero exit. (bead: morphir-ukvn.1.6) + +#### Scenario: Invalid source +- **WHEN** compilation encounters a supported parser error or unsupported construct +- **THEN** stderr identifies its code and available source location, and no successful IR output is written + +#### Scenario: Unsupported IR construct +- **WHEN** generation receives an unsupported value form +- **THEN** it reports the API diagnostic and writes no generated modules + +### Requirement: Output publication preserves existing user data +Commands MUST finish validation before publishing output. Compile failures MUST preserve an existing output file. Generation SHALL require a nonexistent or empty output directory, reject escaping or colliding artifact paths, and stage all files before publishing them. Failures MUST leave no partial generated output or overwritten user files. (bead: morphir-ukvn.1.6) + +#### Scenario: Existing generation directory contains files +- **WHEN** the user selects a nonempty output directory +- **THEN** the command fails clearly and preserves every existing file + +#### Scenario: Generation fails on a later module +- **WHEN** generation cannot handle one module after processing another +- **THEN** no final output module is published + +#### Scenario: Artifact path escapes the selected directory +- **WHEN** an artifact path is absolute, traverses a parent directory, or would traverse a symlink +- **THEN** the command rejects the artifact set before writing any output + +### Requirement: Existing IR commands remain independent +`ir inspect` and `ir convert` SHALL retain their existing options, shared library behavior and results without initializing the Python compiler. (bead: morphir-ukvn.1.6) + +#### Scenario: Inspect and convert an existing fixture +- **WHEN** the user runs either existing command after installing the updated tools +- **THEN** its existing compatibility and diagnostic tests continue to pass + +### Requirement: CLI supports a complete round trip +Generated Python in the supported subset SHALL be accepted by `compile` when supplied the original package and exposure options, yielding equivalent normalized IR in the chosen target version. (bead: morphir-ukvn.1.6) + +#### Scenario: Compile generate recompile +- **WHEN** the user compiles a supported multi-module model, generates Python, then recompiles that output with the same package, exposure and IR version +- **THEN** both IR documents describe equivalent supported definitions and generated Python returns the expected results diff --git a/openspec/changes/migrate-python-binding/tasks.md b/openspec/changes/migrate-python-binding/tasks.md new file mode 100644 index 00000000..e23f1d6b --- /dev/null +++ b/openspec/changes/migrate-python-binding/tasks.md @@ -0,0 +1,56 @@ +# Tasks + +Implement against `design.html` and the three delta specs. Follow TDD within each behavior group; claim its bead before work and close it only with passing checks and commit evidence. Keep the A2 final-delivery dependency on the epic. Actual publication and live provider promotion require maintainer approval and are listed under workflow follow-up. + +## 1. Move and qualify the standalone Rust compiler (morphir-ukvn.1.6.1) + +- [x] 1.1 Select and record the current upstream morphir-rust source revision, reconcile changes since inspected `f4298d0`, and inventory the crate, fixture consumers, release selectors and parent CLI provider references; verify the recorded file inventory and existing binding tests against that revision. +- [x] 1.2 Move the compiler, examples, fixtures, acceptance tests and support documentation into `rust/crates/morphir-python-binding`; add the Rust workspace and lockfile using resolvable published core/SDK versions or one documented exact git revision; verify `cargo metadata --locked` and absence of sibling-checkout dependencies. +- [x] 1.3 Preserve the native Rust API, MEP identity, Ruff pins and current v3/v4 subset; verify migrated unit/integration, Gherkin acceptance and doc tests, Cargo formatting/clippy, and a release WASM build without CPython linking. +- [x] 1.4 Add the documented external Rust usage example and build prerequisites; verify an independent temporary Cargo consumer can compile and generate through the declared dependency without morphir-rust source paths or Python development libraries. + +## 2. Expose the compiler through a typed in-process Python API (morphir-ukvn.1.6.2) + +- [x] 2.1 Add failing public API tests for in-memory documents, package/exposure options, v3 default and explicit v4, diagnostics and generated artifacts; implement the isolated `morphir-python-native` PyO3 bridge calling the existing compiler traits, following the pinned Tokenizers example in the design; verify owned request extraction, detached Rust execution without Python callbacks, concurrent request isolation, and that compiler-only native/WASM builds still omit PyO3. +- [x] 2.2 Add frozen result/source/artifact types and `morphir_tools.python.compile_sources` and `generate_python`; decode and encode only with existing versioned Python library APIs; verify strict mypy/pyright checks on the new public API and result states that cannot combine failure with successful model data. +- [x] 2.3 Add cross-codec tests for ADTs, nested tuples, imports, visibility, unary higher-order functions, arbitrary integers and independently authored IR in both versions; verify normalized round trips, deterministic source permutations and explicit unsupported model/attribute diagnostics without silent migration. +- [x] 2.4 Preserve language diagnostic codes, severities and available source locations, and document API examples and subset limits; verify warnings survive success, failure has no partial model, submitted code never executes, and documentation examples run through the public API; verify native stubs match installed extension exports/signatures and public typed examples pass both type checkers. + +## 3. Package the native compiler for Python users (morphir-ukvn.1.6.3) + +- [x] 3.1 Add the Maturin native distribution with a qualified pinned PyO3/Maturin pair, private module, stubs, `py.typed`, licenses and explicit version identity, using the pinned Pydantic packaging example in the design; verify wheel contents and agreement between compiled-module identity, distribution metadata and release inputs while both existing distributions remain on Hatch. +- [x] 3.2 Add an exact qualified native dependency to each tools release and uv workspace wiring, keeping core dependencies empty and compiler loading lazy; verify expected native version/bridge API revision checks, clear rejection of deliberately mismatched installations, library-only installation and existing `ir inspect`/`ir convert` tests without initializing the native compiler. +- [x] 3.3 Produce the native source archive with all referenced Rust workspace files; verify building and installing its wheel in a clean directory with no source checkout or sibling repository available, then run an installed API round trip; compare the included build inputs and archive-only qualification with the pinned Polars example in the design. +- [x] 3.4 Document the qualified CPython/platform matrix, wheel-only installation and contributor source builds; verify clean wheelhouse installation of core, native and tools with Rust absent from PATH and record the exact artifacts used; audit the declared Linux baseline and bundled shared libraries, prohibit build-host-specific CPU assumptions in portable wheels, and run installed import/API checks on declared deployment targets. + +## 4. Add Python compile and generate CLI commands (morphir-ukvn.1.6.4) + +- [x] 4.1 Add compile tests and implement file/directory discovery, required package, repeated exposure selections and version options through the public API; verify deterministic module identities, local import resolution and `ir inspect` on written v3/v4 models. +- [x] 4.2 Add generate tests and implement shared IR decoding plus API generation into an absent/empty output directory; verify nested modules, importable output, unsupported distribution/node diagnostics and parity with API codes. +- [x] 4.3 Stage complete output before publication and render I/O failures consistently; verify injected write failures, late-module rejection, preservation of existing compile files, refusal of nonempty directories and artifact path/collision/symlink checks. +- [x] 4.4 Add behave scenarios and user-facing CLI/API documentation for compile/generate/recompile; verify both versions, visibility settings and executable conditional/tuple/closure behavior in a separate process, plus all existing CLI regression tests. + +## 5. Qualify scoped CI and independently owned extension bundles (morphir-ukvn.1.6.5) + +- [x] 5.1 Add mise tasks and scoped Python, Rust/native and bundle checks following the current common-layout contract; verify selector tests for documentation-only, native-only, compiler, shared dependency and workflow changes, plus aggregate failure/cancellation/missing-result handling. +- [x] 5.2 Build one compatible artifact per target/ABI and reuse it for clean-install tests; verify no consumer rebuilds its input and preserve existing lint/typecheck/coverage/BDD/release checks, with runtime and cache observations recorded rather than estimated savings. +- [x] 5.3 Build and package the WASM extension here with the canonical `morphir extension bundle` command against pinned tools; verify claims, checksums, source provenance and initialization/compile/generate/recompile through an isolated offline MEP host for both IR versions. +- [x] 5.4 Add nonpublishing release qualification for the advertised wheel platforms, source archive and extension bundle, independent version/tag validation and exact-artifact promotion inputs; verify workflow syntax and refusal of failed/skipped/missing/mismatched candidate evidence, then document the approval and native-before-tools publication order. + +## 6. Measure native and Python integration costs (morphir-ukvn.1.6.6) + +- [x] 6.1 Add a reproducible benchmark command with fixed fixtures and scale parameters, comparing native Rust, bridge, public API and CLI at matched target versions; verify warmup/repetition metadata, separate startup and JSON/codec costs, and recorded toolchains/build profile/input sizes. +- [x] 6.2 Run release-build samples and record medians/spread in an HTML report with raw results in the local output directory; verify conclusions distinguish parsing-only `ast.parse` measurements from full compilation and make no Rust-versus-Python compiler claim without an equivalent implementation. + +## 7. Remove the old provider and prepare ecosystem cutover (morphir-ukvn.1.6.7) + +- [x] 7.1 Prepare a coordinated morphir-rust feature branch removing the crate, workspace membership and Python release/CI registrations; move compiler-owned tests here and replace former-directory fixture includes with independent host fixtures or a pinned replacement provider; verify affected workspace tests, CI selector tests and no references that require the deleted directory. +- [x] 7.2 Prepare finos/morphir provider/discovery, provenance, integration fixture and pin updates as required by the current registry implementation; verify the isolated extension install selects the bundle built here and offline ecosystem CLI round trips pass without the old Python crate. +- [x] 7.3 Run final repository quality gates, installed Python API/CLI proofs and standalone Rust/native/WASM proofs for the coordinated revisions; verify recorded model equivalence, artifact digests and target-repository checks, and preserve unrelated sibling-checkout work. +- [x] 7.4 Record reviewable commit ranges, qualification receipts, rollback steps and outstanding A2/release dependencies in beads; verify all implementation task evidence is linked and the epic remains open until approved publication and live provider cutover have actually satisfied its acceptance conditions. + +## Workflow follow-up + +- Present the exact qualified artifacts and coordinated repository changes for maintainer release/merge approval. Publish the native dependency and extension bundle before dependent tools or registry promotion; preserve historical releases. +- Verify live `morphir extension install morphir-python` fetches from finos/morphir-python and the published Python packages install and round trip without Rust. Record release URLs, digests and merged revisions before closing A4 and relink relevant compiler backlog work to this repository. +- Archive this OpenSpec change after implementation and the applicable review process; adjust HTML asset paths for the archive location, update the epic spec id and sync beads. diff --git a/packages/morphir-python-native/LICENSE b/packages/morphir-python-native/LICENSE new file mode 100644 index 00000000..8dada3ed --- /dev/null +++ b/packages/morphir-python-native/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/morphir-python-native/README.md b/packages/morphir-python-native/README.md new file mode 100644 index 00000000..6e18cc09 --- /dev/null +++ b/packages/morphir-python-native/README.md @@ -0,0 +1,13 @@ +# morphir-python-native + +The native compiler behind the `morphir-python` tools. It contains a private +CPython extension module built from the Rust `morphir-python-binding` compiler +with [PyO3](https://pyo3.rs) and [Maturin](https://www.maturin.rs). + +Do not import this package directly. Install `morphir-python` and use the typed +API in `morphir_tools.python` or the `morphir-python compile` and +`morphir-python generate` commands. Each `morphir-python` release requires one +exact version of this distribution and checks it before first use. + +Wheels need no Rust toolchain. Building from the source archive needs Rust 1.98 +or later; see the [repository guide](https://github.com/finos/morphir-python/blob/main/docs/python-compiler.md). diff --git a/packages/morphir-python-native/pyproject.toml b/packages/morphir-python-native/pyproject.toml new file mode 100644 index 00000000..14857622 --- /dev/null +++ b/packages/morphir-python-native/pyproject.toml @@ -0,0 +1,43 @@ +[project] +name = "morphir-python-native" +dynamic = ["version"] +description = "Private native compiler for the morphir-python tools" +readme = "README.md" +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.14" +keywords = ["morphir", "compiler", "native"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Rust", + "Topic :: Software Development :: Compilers", + "Typing :: Typed", +] +authors = [ + { name = "FINOS", email = "morphir@finos.org" } +] +dependencies = [] + +[project.urls] +Homepage = "https://github.com/finos/morphir-python" +Repository = "https://github.com/finos/morphir-python" +Issues = "https://github.com/finos/morphir-python/issues" + +[build-system] +# Pinned with PyO3 =0.30.1 in rust/crates/morphir-python-native/Cargo.toml. +requires = ["maturin==1.15.0"] +build-backend = "maturin" + +[tool.maturin] +# The version comes from this crate, so the compiled module, the wheel metadata +# and the release tag (native-v) share one source. +manifest-path = "../../rust/crates/morphir-python-native/Cargo.toml" +python-source = "python" +module-name = "morphir_python_native._native" +bindings = "pyo3" +features = ["python"] +locked = true diff --git a/packages/morphir-python-native/python/morphir_python_native/__init__.py b/packages/morphir-python-native/python/morphir_python_native/__init__.py new file mode 100644 index 00000000..f0a681c2 --- /dev/null +++ b/packages/morphir-python-native/python/morphir_python_native/__init__.py @@ -0,0 +1,6 @@ +"""Private native compiler used by ``morphir_tools.python``. + +This distribution is an implementation detail of the ``morphir-python`` tools. +Its ``_native`` module and JSON request format are not a public API and can +change in any release; use ``morphir_tools.python`` instead. +""" diff --git a/packages/morphir-python-native/python/morphir_python_native/_native.pyi b/packages/morphir-python-native/python/morphir_python_native/_native.pyi new file mode 100644 index 00000000..c1c0a7e0 --- /dev/null +++ b/packages/morphir-python-native/python/morphir_python_native/_native.pyi @@ -0,0 +1,20 @@ +"""Type information for the private CPython bridge to the Morphir Python compiler.""" + +from typing import Final + +__all__ = ["BRIDGE_API", "COMPILER_VERSION", "__version__", "compile", "generate"] + +BRIDGE_API: Final[int] +"""Revision of the JSON contract with ``morphir_tools.python``.""" + +COMPILER_VERSION: Final[str] +"""Version of the linked ``morphir-python-binding`` compiler.""" + +__version__: Final[str] +"""Version of this native module, equal to the distribution version.""" + +def compile(request: str) -> str: + """Compile a JSON request and return a JSON result.""" + +def generate(ir: str) -> str: + """Generate Python from versioned IR JSON text and return a JSON result.""" diff --git a/packages/morphir-python-native/python/morphir_python_native/py.typed b/packages/morphir-python-native/python/morphir_python_native/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/packages/morphir-tools/pyproject.toml b/packages/morphir-tools/pyproject.toml index 3cbefe69..fabc12bd 100644 --- a/packages/morphir-tools/pyproject.toml +++ b/packages/morphir-tools/pyproject.toml @@ -23,6 +23,8 @@ authors = [ ] dependencies = [ "morphir>=0.0.1", + # Exact qualified native compiler; checked again when the compiler first loads. + "morphir-python-native==0.1.0", "typer>=0.15", # Rich uses Pygments; older versions are affected by CVE-2026-4539. "pygments>=2.20.0", diff --git a/packages/morphir-tools/src/morphir_tools/_output.py b/packages/morphir-tools/src/morphir_tools/_output.py new file mode 100644 index 00000000..96c9ef40 --- /dev/null +++ b/packages/morphir-tools/src/morphir_tools/_output.py @@ -0,0 +1,114 @@ +"""Safe publication of generated files. + +Generation writes a complete tree into a new sibling staging directory and only +then moves it into place, so a failure never leaves partial output or touches +existing user files. +""" + +from __future__ import annotations + +import os +import shutil +import tempfile +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + + +def umask() -> int: + """Return the process umask without changing it.""" + mask = os.umask(0o022) + os.umask(mask) + return mask + + +class OutputError(Exception): + """Output cannot be published safely; nothing was written.""" + + +def validate_artifact_paths(paths: Iterable[str]) -> None: + """Check that relative artifact paths stay inside one new directory tree. + + Each path must be a normalized relative POSIX path. No two paths may name the + same file on a case-insensitive file system, and no path may be a parent + directory of another. + + Raises: + OutputError: For the first invalid path or collision. + """ + seen: dict[str, str] = {} + directories: dict[str, str] = {} + for path in paths: + pure = PurePosixPath(path) + if ( + not path + or "\\" in path + or pure.is_absolute() + or ":" in pure.parts[0] + or any(part in {"", ".", ".."} for part in path.split("/")) + ): + raise OutputError(f"artifact path {path!r} is not a safe relative path") + key = path.casefold() + if key in seen: + raise OutputError(f"artifact paths {seen[key]!r} and {path!r} collide") + seen[key] = path + for parent in pure.parents: + if str(parent) != ".": + directories.setdefault(str(parent).casefold(), path) + for key, path in seen.items(): + if key in directories: + raise OutputError( + f"artifact path {path!r} is also a directory of {directories[key]!r}" + ) + + +def check_destination(directory: Path) -> None: + """Require an absent directory, or an empty real directory. + + Raises: + OutputError: When the destination is a symbolic link, a file or a + nonempty directory. + """ + if directory.is_symlink(): + raise OutputError(f"output directory {directory} is a symbolic link") + if directory.exists(): + if not directory.is_dir(): + raise OutputError(f"output path {directory} is not a directory") + if any(directory.iterdir()): + raise OutputError(f"output directory {directory} is not empty") + + +def publish_tree(directory: Path, files: Sequence[tuple[str, str]]) -> None: + """Write ``files`` to a staging directory, then move it to ``directory``. + + Args: + directory: The absent or empty destination. + files: Relative POSIX paths and their text, already validated. + + Raises: + OutputError: When the destination is unsuitable. + OSError: When writing or moving fails; the staging tree is removed and + the destination is unchanged. + """ + check_destination(directory) + parent = directory.parent + parent.mkdir(parents=True, exist_ok=True) + # mkdtemp creates 0700; publish with the existing directory's mode, or the + # mode an ordinary mkdir would give under the user's umask. + mode = directory.stat().st_mode & 0o7777 if directory.exists() else 0o777 & ~umask() + staging = Path(tempfile.mkdtemp(prefix=f".{directory.name}.", dir=parent)) + try: + staging.chmod(mode) + for relative, text in files: + target = staging.joinpath(*relative.split("/")) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + check_destination(directory) + if directory.exists(): + directory.rmdir() + os.replace(staging, directory) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise diff --git a/packages/morphir-tools/src/morphir_tools/cli.py b/packages/morphir-tools/src/morphir_tools/cli.py index 9a3369d4..c75c4ae4 100644 --- a/packages/morphir-tools/src/morphir_tools/cli.py +++ b/packages/morphir-tools/src/morphir_tools/cli.py @@ -2,7 +2,9 @@ Every IR operation goes through the `morphir` library: the shared format-version reader, the v3 (3.0.x and 3.1.x) and draft v4 codecs, and `morphir.ir.migrate`. -The CLI adds file handling and reporting only. +``compile`` and ``generate`` go through the public `morphir_tools.python` API, +which loads the native compiler only when one of them runs. The CLI adds file +handling and reporting only. Exit codes: 0 for success (including success with warnings), 1 when the library reports a diagnostic, 2 for a usage error. @@ -14,7 +16,7 @@ import tempfile from enum import Enum from pathlib import Path -from typing import Annotated +from typing import TYPE_CHECKING, Annotated import typer @@ -30,6 +32,13 @@ from morphir.ir.v4.name import Name as Name4 from morphir.ir.v4.path import Path as Path4 +from ._output import OutputError, publish_tree, umask, validate_artifact_paths + +if TYPE_CHECKING: + from .python import CompilerDiagnostic + +__all__ = ["OutputError", "app", "main", "validate_artifact_paths"] + app = typer.Typer( name="morphir-python", help="Command-line tools for Morphir IR.", @@ -63,6 +72,9 @@ def _read(path: Path) -> str: except OSError as error: typer.echo(f"error: cannot read {path}: {error.strerror}", err=True) raise typer.Exit(1) from None + except UnicodeDecodeError: + typer.echo(f"error: cannot read {path}: the file is not UTF-8", err=True) + raise typer.Exit(1) from None def _write_atomically(target: Path, text: str) -> None: @@ -71,6 +83,9 @@ def _write_atomically(target: Path, text: str) -> None: prefix=f".{target.name}.", suffix=".tmp", dir=directory ) try: + # mkstemp creates 0600; keep an existing file's mode, or use the umask. + mode = target.stat().st_mode & 0o7777 if target.exists() else 0o666 & ~umask() + os.chmod(fd, mode) with os.fdopen(fd, "w", encoding="utf-8") as handle: handle.write(text) Path(temp_name).replace(target) @@ -218,6 +233,165 @@ def convert( typer.echo(f"wrote {output} (format {to.value})") +def _report_compiler(diagnostics: tuple[CompilerDiagnostic, ...]) -> None: + for diagnostic in diagnostics: + typer.echo(f"{diagnostic.severity.value}: {diagnostic.render()}", err=True) + + +def _fail(message: str) -> typer.Exit: + typer.echo(f"error: {message}", err=True) + return typer.Exit(1) + + +def _discover(source: Path) -> tuple[Path, list[Path]]: + """Return the source root and its Python files, sorted, without following links.""" + if source.is_dir() and not source.is_symlink(): + files: list[Path] = [] + for directory, names, filenames in os.walk(source, followlinks=False): + names[:] = sorted( + name + for name in names + if not name.startswith(".") and name != "__pycache__" + ) + files.extend( + Path(directory) / name + for name in sorted(filenames) + if name.endswith(".py") and not (Path(directory) / name).is_symlink() + ) + if not files: + raise _fail(f"no Python source files under {source}") + return source, files + return source.parent, [source] + + +@app.command("compile") +def compile_command( + source: Annotated[ + Path, typer.Argument(help="A Python file, or a directory of Python files.") + ], + package: Annotated[ + str, typer.Option(help="Morphir package name, such as acme/example.") + ], + output: Annotated[ + Path, typer.Option("--output", "-o", help="Where to write the IR document.") + ], + ir_version: Annotated[ + Target, typer.Option("--ir-version", help="The IR version to write.") + ] = Target.V3, + expose: Annotated[ + list[str] | None, + typer.Option( + "--expose", + help="Module to make public, such as acme/models. Repeat for more. " + "Without it every module is public.", + ), + ] = None, +) -> None: + """Compile supported Python source to a versioned Morphir IR Library. + + A directory is compiled as one package; a single file uses its parent + directory as the source root. Sources are parsed, never imported or run. + The output is written to a temporary file and renamed only on success, so a + failed compilation leaves an existing output file unchanged. + """ + from .python import ( # Loads the compiler on demand. + CompiledV3, + CompileFailure, + IrVersion, + NativeCompilerError, + NativeCompilerUnavailable, + SourceDocument, + compile_sources, + ) + + root, files = _discover(source) + documents = [ + SourceDocument(path.relative_to(root).as_posix(), _read(path)) for path in files + ] + version = IrVersion.V3 if ir_version is Target.V3 else IrVersion.V4 + try: + result = compile_sources( + documents, + package=package, + exposed_modules=expose, + ir_version=version, + ) + except (ValueError, NativeCompilerUnavailable) as error: + raise _fail(str(error)) from None + except NativeCompilerError as error: + raise _fail(f"internal compiler error: {error}") from None + if isinstance(result, CompileFailure): + _report_compiler(result.diagnostics) + raise typer.Exit(1) + if isinstance(result, CompiledV3): + document = v3_codec.encode_versioned_distribution(result.distribution) + else: + document = v4_codec.encode_versioned_distribution(result.distribution) + try: + _write_atomically(output, fv.dump_json(document)) + except OSError as error: + raise _fail(f"cannot write {output}: {error.strerror}") from None + _report_compiler(result.warnings) + typer.echo(f"wrote {output} (format {ir_version.value})") + if ir_version is Target.V4: + typer.echo(f"note: {v4_codec.DRAFT_NOTICE}") + + +@app.command("generate") +def generate_command( + file: Annotated[Path, typer.Argument(help="A versioned morphir-ir.json file.")], + output_dir: Annotated[ + Path, + typer.Option( + "--output-dir", help="A new or empty directory for the Python modules." + ), + ], +) -> None: + """Generate Python modules from a format 3.0 or draft v4 Library. + + Module paths are preserved under the output directory. All files are written + to a staging directory first and published together, so a failure leaves no + partial output. Legacy formats 1 and 2 must be upgraded with ir convert first. + """ + from .python import ( # Loads the compiler on demand. + GenerationFailure, + NativeCompilerError, + NativeCompilerUnavailable, + generate_python, + ) + + text = _read(file) + try: + version, _ = fv.read_document(text) + model: object = ( + v4_codec.decode_versioned_distribution(text) + if _is_v4(version) + else v3_codec.decode_versioned_distribution(text) + ) + except DiagnosticError as error: + _report(error.diagnostics) + raise typer.Exit(1) from None + try: + result = generate_python(model) + except NativeCompilerUnavailable as error: + raise _fail(str(error)) from None + except NativeCompilerError as error: + raise _fail(f"internal compiler error: {error}") from None + if isinstance(result, GenerationFailure): + _report_compiler(result.diagnostics) + raise typer.Exit(1) + files = [(artifact.path, artifact.text) for artifact in result.artifacts] + try: + validate_artifact_paths(path for path, _ in files) + publish_tree(output_dir, files) + except OutputError as error: + raise _fail(str(error)) from None + except OSError as error: + raise _fail(f"cannot write {output_dir}: {error.strerror}") from None + _report_compiler(result.warnings) + typer.echo(f"wrote {len(files)} modules to {output_dir}") + + def main() -> None: """Run the ``morphir-python`` command line.""" app() diff --git a/packages/morphir-tools/src/morphir_tools/python/__init__.py b/packages/morphir-tools/src/morphir_tools/python/__init__.py new file mode 100644 index 00000000..0968a9eb --- /dev/null +++ b/packages/morphir-tools/src/morphir_tools/python/__init__.py @@ -0,0 +1,71 @@ +r"""Compile Python to Morphir IR and generate Python from Morphir IR. + +This is the typed, in-process API behind ``morphir-python compile`` and +``morphir-python generate``. A native compiler does the work; callers need +neither Rust nor a separate Morphir executable, and the native compiler loads +only when one of these functions is first called. + +Example: + >>> from morphir_tools.python import SourceDocument, compile_sources + >>> from morphir_tools.python import CompiledV3, generate_python + >>> source = SourceDocument( + ... "rules.py", + ... "def choose(flag: bool, a: int, b: int) -> int:\n" + ... " return a if flag else b\n", + ... ) + >>> compiled = compile_sources([source], package="acme/example") + >>> isinstance(compiled, CompiledV3) + True + >>> generated = generate_python(compiled.distribution) + >>> [artifact.path for artifact in generated.artifacts] + ['rules.py'] +""" + +from ._api import compile_sources, generate_python, native_compiler +from ._bridge import EXPECTED_BRIDGE_API, EXPECTED_NATIVE_VERSION +from ._types import ( + CompiledV3, + CompiledV4, + CompileFailure, + CompilerDiagnostic, + CompileResult, + Generated, + GeneratedArtifact, + GenerateResult, + GenerationFailure, + IrVersion, + LibraryModel, + NativeCompilerError, + NativeCompilerInfo, + NativeCompilerUnavailable, + Severity, + SourceDocument, + SourceLocation, + SourcePosition, +) + +__all__ = [ + "EXPECTED_BRIDGE_API", + "EXPECTED_NATIVE_VERSION", + "CompileFailure", + "CompileResult", + "CompiledV3", + "CompiledV4", + "CompilerDiagnostic", + "GenerateResult", + "Generated", + "GeneratedArtifact", + "GenerationFailure", + "IrVersion", + "LibraryModel", + "NativeCompilerError", + "NativeCompilerInfo", + "NativeCompilerUnavailable", + "Severity", + "SourceDocument", + "SourceLocation", + "SourcePosition", + "compile_sources", + "generate_python", + "native_compiler", +] diff --git a/packages/morphir-tools/src/morphir_tools/python/_api.py b/packages/morphir-tools/src/morphir_tools/python/_api.py new file mode 100644 index 00000000..5a89e19f --- /dev/null +++ b/packages/morphir-tools/src/morphir_tools/python/_api.py @@ -0,0 +1,215 @@ +"""Compile Python sources to Morphir IR and generate Python from IR.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, cast + +from morphir.ir import format_version as fv +from morphir.ir.diagnostics import DiagnosticError +from morphir.ir.v3 import codec as v3_codec +from morphir.ir.v3 import distribution as v3_dist +from morphir.ir.v4 import codec as v4_codec +from morphir.ir.v4 import distribution as v4_dist + +from . import _bridge +from . import diagnostics as codes + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping, Sequence + +from ._types import ( + CompiledV3, + CompiledV4, + CompileFailure, + CompilerDiagnostic, + CompileResult, + Generated, + GeneratedArtifact, + GenerateResult, + GenerationFailure, + IrVersion, + LibraryModel, + NativeCompilerError, + NativeCompilerInfo, + Severity, + SourceDocument, + SourceLocation, + SourcePosition, +) + + +def native_compiler() -> NativeCompilerInfo: + """Load the native compiler and report its identity. + + Raises: + NativeCompilerUnavailable: When it is missing or incompatible. + """ + return _bridge.load() + + +def compile_sources( + sources: Iterable[SourceDocument], + *, + package: str, + exposed_modules: Sequence[str] | None = None, + ir_version: IrVersion = IrVersion.V3, +) -> CompileResult: + """Compile Python source documents to a Morphir IR Library. + + The sources are parsed statically: nothing is imported or executed. The + order of ``sources`` does not affect the result. + + Args: + sources: The documents of one package, with paths relative to its root. + package: The Morphir package name, such as ``"acme/example"``. + exposed_modules: Module paths to make public, such as + ``("acme/models",)``. ``None`` exposes every module; an empty + sequence exposes none. + ir_version: The IR version to write. Defaults to stable v3. + + Returns: + A `CompiledV3` or `CompiledV4` holding the decoded Library, or a + `CompileFailure` with at least one diagnostic and no model. + + Raises: + ValueError: When ``package`` is empty or two documents share a path. + NativeCompilerUnavailable: When the native compiler is missing or + incompatible. + NativeCompilerError: When the native compiler fails internally. + """ + documents = sorted(sources, key=lambda document: document.path) + if not package: + raise ValueError("package must name a Morphir package") + paths = [document.path for document in documents] + if len(set(paths)) != len(paths): + raise ValueError("source documents contain a duplicate path") + request = { + "package": package, + "sources": [{"path": d.path, "text": d.text} for d in documents], + "exposedModules": None if exposed_modules is None else list(exposed_modules), + "irVersion": ir_version.value, + } + output = _call(_bridge.module().compile, json.dumps(request)) + diagnostics = _diagnostics(output) + ir = output.get("ir") + if not output.get("success") or not isinstance(ir, str): + return CompileFailure(diagnostics or (_missing_diagnostic(),)) + try: + if ir_version is IrVersion.V3: + classic = v3_codec.decode_versioned_distribution(ir) + if isinstance(classic, v3_dist.Library): + return CompiledV3(classic, diagnostics) + else: + draft = v4_codec.decode_versioned_distribution(ir) + if isinstance(draft, v4_dist.LibraryDistribution): + return CompiledV4(draft, diagnostics) + except DiagnosticError as error: + return CompileFailure(_ir_diagnostics(error)) + raise NativeCompilerError("the native compiler returned a non-Library model") + + +def generate_python(distribution: LibraryModel | object) -> GenerateResult: + """Generate Python modules from a v3 or draft v4 Library model. + + Args: + distribution: A `morphir.ir.v3.distribution.Library` or + `morphir.ir.v4.distribution.LibraryDistribution`. Specs and + Application distributions are rejected with a diagnostic. + + Returns: + `Generated` with one artifact per module, in path order, or a + `GenerationFailure` with at least one diagnostic and no artifacts. + + Raises: + NativeCompilerUnavailable: When the native compiler is missing or + incompatible. + NativeCompilerError: When the native compiler fails internally. + """ + try: + match distribution: + case v3_dist.Library(): + document = v3_codec.encode_versioned_distribution(distribution) + case v4_dist.LibraryDistribution(): + document = v4_codec.encode_versioned_distribution(distribution) + case _: + kind = type(distribution).__name__ + return GenerationFailure( + ( + CompilerDiagnostic( + codes.UNSUPPORTED_DISTRIBUTION, + Severity.ERROR, + "Python generation needs a Library distribution, " + f"not {kind}", + ), + ) + ) + except DiagnosticError as error: + return GenerationFailure(_ir_diagnostics(error)) + output = _call(_bridge.module().generate, fv.dump_json(document)) + diagnostics = _diagnostics(output) + raw = output.get("artifacts") + if not output.get("success") or not isinstance(raw, list): + return GenerationFailure(diagnostics or (_missing_diagnostic(),)) + artifacts = tuple( + sorted( + ( + GeneratedArtifact(str(item["path"]), str(item["content"])) + for item in cast("list[Mapping[str, object]]", raw) + ), + key=lambda artifact: artifact.path, + ) + ) + return Generated(artifacts, diagnostics) + + +def _call(function: Callable[[str], str], argument: str) -> Mapping[str, object]: + output = cast("Mapping[str, object]", json.loads(function(argument))) + internal = output.get("internalError") + if internal is not None: + raise NativeCompilerError(str(internal)) + return output + + +def _missing_diagnostic() -> CompilerDiagnostic: + return CompilerDiagnostic( + codes.PY_SERIALIZATION, + Severity.ERROR, + "the native compiler failed without a diagnostic", + ) + + +def _diagnostics(output: Mapping[str, object]) -> tuple[CompilerDiagnostic, ...]: + items = cast("list[Mapping[str, object]]", output.get("diagnostics") or []) + return tuple(_diagnostic(item) for item in items) + + +def _diagnostic(item: Mapping[str, object]) -> CompilerDiagnostic: + location = cast("Mapping[str, object] | None", item.get("location")) + source = None + if location is not None: + start = cast("Mapping[str, int]", location["start"]) + end = cast("Mapping[str, int]", location["end"]) + source = SourceLocation( + str(location["path"]), + SourcePosition(start["line"], start["character"]), + SourcePosition(end["line"], end["character"]), + ) + return CompilerDiagnostic( + code=str(item.get("code") or codes.PY_SERIALIZATION), + severity=Severity(str(item["severity"])), + message=str(item["message"]), + source=source, + ) + + +def _ir_diagnostics(error: DiagnosticError) -> tuple[CompilerDiagnostic, ...]: + return tuple( + CompilerDiagnostic( + code=item.code, + severity=Severity.ERROR, + message=item.message, + ir_path=item.location, + ) + for item in error.diagnostics + ) diff --git a/packages/morphir-tools/src/morphir_tools/python/_bridge.py b/packages/morphir-tools/src/morphir_tools/python/_bridge.py new file mode 100644 index 00000000..8d151a4c --- /dev/null +++ b/packages/morphir-tools/src/morphir_tools/python/_bridge.py @@ -0,0 +1,84 @@ +"""Lazy loading and compatibility checks for the private native compiler.""" + +from __future__ import annotations + +import functools +from importlib import import_module, metadata +from typing import Protocol, cast + +from ._types import NativeCompilerInfo, NativeCompilerUnavailable + +EXPECTED_NATIVE_VERSION = "0.1.0" +"""The exact ``morphir-python-native`` version this release was qualified with. + +It must equal the pin in this distribution's ``pyproject.toml``. +""" + +EXPECTED_BRIDGE_API = 1 +"""The bridge contract revision this release speaks.""" + +_DISTRIBUTION = "morphir-python-native" + + +class NativeModule(Protocol): + """The private ``morphir_python_native._native`` module.""" + + BRIDGE_API: int + COMPILER_VERSION: str + __version__: str + + def compile(self, request: str) -> str: + """Compile a JSON request.""" + ... + + def generate(self, ir: str) -> str: + """Generate Python from IR JSON text.""" + ... + + +def _unavailable(detail: str) -> NativeCompilerUnavailable: + return NativeCompilerUnavailable( + f"the Morphir Python compiler is unavailable: {detail}. Reinstall " + f"morphir-python, which requires {_DISTRIBUTION}=={EXPECTED_NATIVE_VERSION}." + ) + + +@functools.cache +def load() -> NativeCompilerInfo: + """Import the native module once and check that this release can use it. + + Raises: + NativeCompilerUnavailable: When the module is missing, or its + distribution version, module version or bridge revision differ from + what this release expects. + """ + try: + installed = metadata.version(_DISTRIBUTION) + except metadata.PackageNotFoundError: + raise _unavailable(f"{_DISTRIBUTION} is not installed") from None + try: + native = cast("NativeModule", import_module("morphir_python_native._native")) + except ImportError as error: + raise _unavailable(f"its native module cannot be imported ({error})") from None + if installed != EXPECTED_NATIVE_VERSION: + raise _unavailable( + f"{_DISTRIBUTION} {installed} is installed but this release needs " + f"{EXPECTED_NATIVE_VERSION}" + ) + if native.__version__ != installed: + raise _unavailable( + f"the native module reports {native.__version__} but its distribution " + f"is {installed}" + ) + if native.BRIDGE_API != EXPECTED_BRIDGE_API: + raise _unavailable( + f"bridge revision {native.BRIDGE_API} differs from the expected " + f"{EXPECTED_BRIDGE_API}" + ) + return NativeCompilerInfo(installed, native.BRIDGE_API, native.COMPILER_VERSION) + + +def module() -> NativeModule: + """Return the checked native module, loading it on first use.""" + load() + return cast("NativeModule", import_module("morphir_python_native._native")) diff --git a/packages/morphir-tools/src/morphir_tools/python/_types.py b/packages/morphir-tools/src/morphir_tools/python/_types.py new file mode 100644 index 00000000..2d181ee7 --- /dev/null +++ b/packages/morphir-tools/src/morphir_tools/python/_types.py @@ -0,0 +1,201 @@ +"""Immutable inputs and results of the Python compiler API.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from morphir.ir.v3 import distribution as v3_dist +from morphir.ir.v4 import distribution as v4_dist + + +class IrVersion(Enum): + """The Morphir IR version a compilation writes.""" + + V3 = "3" + """Stable format 3.0.0.""" + V4 = "4" + """Draft format 4.0.0.""" + + +class Severity(Enum): + """How serious a compiler diagnostic is.""" + + ERROR = "error" + WARNING = "warning" + INFO = "info" + HINT = "hint" + + +@dataclass(frozen=True) +class SourceDocument: + """One Python source file to compile. + + Attributes: + path: Path relative to the source root, with ``/`` separators, such as + ``"acme/rules.py"``. It decides the Morphir module name. + text: The complete source text. It is parsed, never imported or run. + """ + + path: str + text: str + + +@dataclass(frozen=True) +class SourcePosition: + """A zero-based line and UTF-16 character offset in a source document.""" + + line: int + character: int + + +@dataclass(frozen=True) +class SourceLocation: + """A half-open range in one source document.""" + + path: str + start: SourcePosition + end: SourcePosition + + +@dataclass(frozen=True) +class CompilerDiagnostic: + """A problem reported by compilation or generation. + + Attributes: + code: Stable code from `morphir_tools.python.diagnostics` or + `morphir.ir.diagnostics`. + severity: How serious the problem is. + message: Human-readable explanation. Callers must not parse it. + source: The source range, when the problem is in a source document. + ir_path: Path from the IR document root, when the problem is in IR. + """ + + code: str + severity: Severity + message: str + source: SourceLocation | None = None + ir_path: tuple[str | int, ...] = () + + def render(self) -> str: + """Render as ``code at location: message``, like IR diagnostics. + + Source locations are written ``path:line:column`` with one-based line and + column numbers. + """ + if self.source is not None: + start = self.source.start + where = f"{self.source.path}:{start.line + 1}:{start.character + 1}" + elif self.ir_path: + where = "/".join(str(part) for part in self.ir_path) + else: + where = "" + return f"{self.code} at {where}: {self.message}" + + +def _warnings_only(diagnostics: tuple[CompilerDiagnostic, ...]) -> None: + if any(item.severity is Severity.ERROR for item in diagnostics): + raise ValueError("a successful result cannot carry error diagnostics") + + +def _not_empty(diagnostics: tuple[CompilerDiagnostic, ...]) -> None: + if not diagnostics: + raise ValueError("a failure needs at least one diagnostic") + + +@dataclass(frozen=True) +class CompiledV3: + """A successful compilation to a format 3.0.0 Library.""" + + distribution: v3_dist.Library + warnings: tuple[CompilerDiagnostic, ...] = () + + def __post_init__(self) -> None: + _warnings_only(self.warnings) + + +@dataclass(frozen=True) +class CompiledV4: + """A successful compilation to a draft format 4.0.0 Library.""" + + distribution: v4_dist.LibraryDistribution + warnings: tuple[CompilerDiagnostic, ...] = () + + def __post_init__(self) -> None: + _warnings_only(self.warnings) + + +@dataclass(frozen=True) +class CompileFailure: + """A compilation that produced no model.""" + + diagnostics: tuple[CompilerDiagnostic, ...] + + def __post_init__(self) -> None: + _not_empty(self.diagnostics) + + +type CompileResult = CompiledV3 | CompiledV4 | CompileFailure + + +@dataclass(frozen=True) +class GeneratedArtifact: + """One generated Python module. + + Attributes: + path: Path relative to the output root, with ``/`` separators. + text: The module source. + """ + + path: str + text: str + + +@dataclass(frozen=True) +class Generated: + """Successful generation: every module of the input Library.""" + + artifacts: tuple[GeneratedArtifact, ...] + warnings: tuple[CompilerDiagnostic, ...] = () + + def __post_init__(self) -> None: + _warnings_only(self.warnings) + + +@dataclass(frozen=True) +class GenerationFailure: + """A generation that produced no artifacts.""" + + diagnostics: tuple[CompilerDiagnostic, ...] + + def __post_init__(self) -> None: + _not_empty(self.diagnostics) + + +type GenerateResult = Generated | GenerationFailure + +type LibraryModel = v3_dist.Library | v4_dist.LibraryDistribution +"""A Library model that generation accepts.""" + + +@dataclass(frozen=True) +class NativeCompilerInfo: + """Identity of the loaded native compiler. + + Attributes: + version: Version of the ``morphir-python-native`` distribution and module. + bridge_api: Revision of the private bridge contract. + compiler_version: Version of the linked ``morphir-python-binding`` crate. + """ + + version: str + bridge_api: int + compiler_version: str + + +class NativeCompilerUnavailable(ImportError): # noqa: N818 - reads as a state + """The native compiler is missing or incompatible with this release.""" + + +class NativeCompilerError(RuntimeError): + """The native compiler failed internally; this is a bug, not a user error.""" diff --git a/packages/morphir-tools/src/morphir_tools/python/diagnostics.py b/packages/morphir-tools/src/morphir_tools/python/diagnostics.py new file mode 100644 index 00000000..80322ee2 --- /dev/null +++ b/packages/morphir-tools/src/morphir_tools/python/diagnostics.py @@ -0,0 +1,33 @@ +"""Diagnostic codes reported by the Python compiler API. + +Every code the API can report lives in this module, as `morphir.ir.diagnostics` +does for the IR readers and codecs. + +- ``PY001`` to ``PY006`` come from the native ``morphir-python-binding`` compiler. + They are the same codes its MEP extension reports, so Rust, WASM and Python + callers see one vocabulary. +- ``unsupported_distribution`` is local to this API: generation accepts only a + Library distribution. +- Decoding and encoding failures keep the `morphir.ir.diagnostics` code that the + codec reported, such as ``unknown_node`` or ``no_v4_spelling``. +""" + +from __future__ import annotations + +# Native compiler (morphir-python-binding). +PY_REQUEST = "PY001" +"""Unsupported request or option.""" +PY_SYNTAX = "PY002" +"""Python syntax error, with a source range.""" +PY_NAME = "PY003" +"""Invalid name or naming collision.""" +PY_UNSUPPORTED = "PY004" +"""Unsupported declaration, annotation or IR feature.""" +PY_SERIALIZATION = "PY005" +"""IR decoding or serialization, or generated syntax failure.""" +PY_PARSE_STAGE = "PY006" +"""Parse-stage output was requested but is unavailable.""" + +# Python API (morphir-python). +UNSUPPORTED_DISTRIBUTION = "unsupported_distribution" +"""Generation received a Specs or Application distribution.""" diff --git a/packages/morphir/src/morphir/ir/v4/codec.py b/packages/morphir/src/morphir/ir/v4/codec.py index 317ebfa4..25b0f9de 100644 --- a/packages/morphir/src/morphir/ir/v4/codec.py +++ b/packages/morphir/src/morphir/ir/v4/codec.py @@ -9,15 +9,83 @@ the distribution wrapped by its kind (``Library``, ``Specs``, or ``Application``). The codec never writes a format-3 tagged-array shape. -Every node is written in the schema's expanded form, ``{"Kind": {members}}``, with -``attributes`` omitted when empty. The reader accepts exactly the spellings listed -in `TYPE_NODES`, `VALUE_NODES`, `PATTERN_NODES`, and the other tables below, and -fails with a located diagnostic on anything else; it does not guess. +The writer emits every node in the schema's expanded form, ``{"Kind": {members}}``, +with ``attributes`` omitted when empty. The profile says that readers MUST accept +every spelling the v4 JSON Schema marks as valid, so the reader also accepts the +compact canonical vocabulary that Rust ``morphir-core`` writes, and the legacy +spellings that the schema itself lists. Each spelling of a node gives the same +Python model. The reader fails with a located diagnostic on anything else; it +does not guess. + +Spellings the reader accepts (compact first, then expanded): + +- Name: ``"value-in-USD"``, or the legacy word array ``["value", "in", "u", "s", + "d"]``. Path: ``"morphir/SDK"``, or an array of names. FQName: + ``"morphir/SDK:list#map"``, or the legacy ``[package, module, name]`` array. +- Type: a bare FQName string (``Reference``); a bare lowercase name such as + ``"a"`` (``Variable``); a bare array (``Tuple``); ``{"Variable": name}``; + ``{"Reference": "fqn"}``; ``{"Reference": [fqn, arg, ...]}`` with one or more + arguments; ``{"Tuple": [type, ...]}``; and the expanded payload of every kind. + ``Record``, ``ExtensibleRecord``, ``Function`` and ``Unit`` have only the + expanded payload. A string is an FQName when it has ``:`` or ``#``, as the + schema patterns separate them. +- Value (decision 0009): ``true``/``false`` (``BoolLiteral``); a number, which is + an ``IntegerLiteral`` when its lexeme has no point and no exponent and a + ``FloatLiteral`` otherwise; an FQName string (``Reference``); a name string + (``Variable``); a bare array (``List``). Wrapped compact payloads: + ``{"Literal": scalar | literal}``, ``{"Constructor": fqn}``, ``{"Tuple": [...]}``, + ``{"List": [...]}``, ``{"Variable": name}``, ``{"Reference": fqn}``, + ``{"FieldFunction": name}``. ``{"Literal": {"literal": scalar | literal}}`` is + the expanded literal. A bare string is never a string literal: ``{"Literal": + "a"}`` is. Every other kind has only its expanded payload. +- Pattern: a bare array (``TuplePattern``); ``{"TuplePattern": [...]}``; + ``{"LiteralPattern": scalar | literal}``; and the expanded payloads. A + ``DocumentLiteral`` is not a pattern (decision 0013). +- Literal: ``{"IntegerLiteral": 42}`` or ``{"IntegerLiteral": {"value": 42}}``, and + the same for every scalar kind; members beside ``value`` are ignored (kit + values-0046). ``WholeNumberLiteral`` reads as ``IntegerLiteral``. A + ``DecimalLiteral`` must spell a decimal lexeme. A ``DocumentLiteral`` payload is + the document itself. +- Constructors: ``{"name": [[arg, type], ...]}`` or the legacy array + ``[["name", [[arg, type], ...]], ...]``. +- Type and value specifications and definitions: ``{"Kind": {...}}`` with an + optional flattened ``doc`` beside the kind (decision 0010), the nested + ``{"doc", "value"}`` wrapper, and the legacy tagged arrays + ``["TypeAliasDefinition", params, type]``, ``["CustomTypeDefinition", params, + access-controlled constructors]``, ``["TypeAliasSpecification", params, type]``, + ``["OpaqueTypeSpecification", params]``, ``["CustomTypeSpecification", params, + constructors]`` and ``["DerivedTypeSpecification", params, {...}]``. A value + specification's ``inputs`` may be omitted or be a legacy ``[[name, type], ...]`` + array, and its ``doc`` sits beside ``inputs`` and ``output``. +- AccessControlled (module types and values, package modules): ``{"Public": x}`` + and ``{"Private": x}`` (canonical), ``{"public": x}``, ``{"pub": x}``, + ``{"private": x}``, the legacy ``{"access": level, "value": x}``, and the + flattened ``{"access": level, ...members of x}``, with ``level`` any of those + five spellings. A ``CustomTypeDefinition`` body's own ``access`` stays + ``Public`` or ``Private``, as the schema says. +- Distributions: ``packageName`` and entry-point targets take the legacy arrays. + +Schema-valid spellings the reader still refuses, deliberately: + +- A ``doc`` on a value definition inside ``LetDefinition`` or ``LetRecursion`` + (``unknown_member``): the Python model has no slot for it there, so reading it + would lose data. +- A ``doc`` both beside and inside a nested ``{"doc", "value"}`` wrapper + (``duplicate_member``): two strings for one slot. +- Members the schema leaves open (it omits ``additionalProperties: false`` on some + objects, and on the legacy tagged arrays it sets no length): the reader rejects + unknown members and wrong lengths, following the kit's ``unknown_member`` cases. + The expanded literal ``{"value": ...}`` is the exception, as the kit requires. +- A non-empty ``TypeAttributes.constraints`` object: the schema calls it a + placeholder and the deprecated Python ``TypeConstraints`` has no reading. An + empty object is accepted. +- An ``{"access": level}`` object with no other member, and a payload under an + access level that is not the expected node: the schema leaves it unconstrained. The codec covers the value-definition bodies (``ExpressionBody``, ``NativeBody``, ``ExternalBody``, ``IncompleteBody``), ``IncompleteTypeDefinition`` with typed hole reasons, annotations on specifications, and the draft ``Union`` type (from -``docs/spec/draft/types.md``, not yet in the pinned schema). +``docs/spec/draft/types.md``, not yet in the pinned schema; expanded only). Known gaps between the Python v4 model and upstream v4, each handled explicitly: @@ -25,9 +93,18 @@ (upstream makes them definition bodies); encoding fails with ``no_v4_spelling``. - Structured ``TypeConstraints`` (deprecated) have no spelling; encoding fails with ``no_v4_spelling``. -- Only the expanded spelling is read and written. Compact spellings, the legacy - spellings readers accept for one release, and the 4.1.0 linked annotations - wait for the re-sync when upstream v4 settles. +- The writer emits only the expanded spelling, not the compact canonical one the + profile prefers. +- The reader refuses the decision-0006 legacy-window spellings that the pinned + schema marks invalid (``attrs``, ``thenBranch``, ``argumentType``, a record's + field map directly under ``Record``, the nested ``{"value": ...}`` wrapper with + no ``doc``, and others) with ``unknown_member`` or ``unknown_node``. The 4.1.0 + linked annotations wait for the re-sync when upstream v4 settles. +- Python's ``json`` module keeps the integer-or-float kind of a number lexeme, but + not the lexeme itself: a ``FloatLiteral`` or a ``DocumentLiteral`` number such + as ``0.10`` reads as a Python ``float``, and a ``DecimalLiteral`` keeps its value + as a ``Decimal`` (``.5`` is written back as ``0.5``). Integers keep full + precision. - Records are objects keyed by field name (finos decision 0004); duplicate field names fail encoding with ``duplicate_member``. - ``ExpressionBody.inputTypes`` has no attribute slot: upstream spells an input as @@ -42,15 +119,17 @@ single-letter words is written as an uppercase initialism (``value-in-USD``). Names the word-list model cannot represent, such as a one-letter initialism, an initialism with digits, adjacent initialisms, or adjacent single-letter - words, fail with ``invalid_name``. + words (the published ``complete-example.json`` has ``u-s/f-r-2052-a``), fail + with ``invalid_name``. - An entry point's ``doc`` is written from ``EntryPoint.doc.value`` and decodes as ``Documented(None, doc)``. """ from __future__ import annotations +import math import re -from collections.abc import Callable +from collections.abc import Callable, Collection from decimal import Decimal, InvalidOperation from typing import Any @@ -205,6 +284,27 @@ def _m(required: str = "", optional: str = "attributes") -> Members: ) ACCESS_TAGS = {"Public": Public, "Private": Private} +"""The ``access`` member of a ``CustomTypeDefinition`` body.""" + +ACCESS_SPELLINGS = { + "Public": True, + "public": True, + "pub": True, + "Private": False, + "private": False, +} +"""Access levels of an access-controlled wrapper, mapped to whether it is public.""" + +LEGACY_TYPE_SPECIFICATION_ARITY = { + "TypeAliasSpecification": 3, + "OpaqueTypeSpecification": 2, + "CustomTypeSpecification": 3, + "DerivedTypeSpecification": 3, +} +"""Legacy tagged-array type specifications and their lengths, tag included.""" + +LEGACY_TYPE_DEFINITION_ARITY = {"TypeAliasDefinition": 3, "CustomTypeDefinition": 3} +"""Legacy tagged-array type definitions and their lengths, tag included.""" ENTRY_POINT_KINDS = {kind: kind.value.lower() for kind in EntryPointKind} @@ -259,9 +359,10 @@ def _check_members( ) -def _node( - j: object, loc: Location, family: str, table: dict[str, Members] -) -> tuple[str, dict[str, object], Location]: +def _tagged( + j: object, loc: Location, family: str, tags: Collection[str] +) -> tuple[str, object, Location]: + """Read ``{"Kind": payload}`` and return the kind, its payload and location.""" obj = _expect_object(j, loc, family) if len(obj) != 1: raise fail( @@ -271,17 +372,71 @@ def _node( f"expected a {family} wrapped by its kind, found members {sorted(obj)}", ) ((tag, payload),) = obj.items() - if tag not in table: + if tag not in tags: raise fail( d.UNKNOWN_NODE, family, (*loc, tag), f"unknown {family} kind {tag!r} in the draft v4 codec", ) - here = (*loc, tag) + return tag, payload, (*loc, tag) + + +def _expanded( + payload: object, here: Location, tag: str, members: Members +) -> dict[str, object]: + """Read the expanded payload of a node and check its members.""" body = _expect_object(payload, here, tag) - _check_members(body, table[tag], here, tag) - return tag, body, here + _check_members(body, members, here, tag) + return body + + +def _node( + j: object, loc: Location, family: str, table: dict[str, Members] +) -> tuple[str, dict[str, object], Location]: + tag, payload, here = _tagged(j, loc, family, table) + return tag, _expanded(payload, here, tag, table[tag]), here + + +def _refuse_tagged_array( + items: list[object], loc: Location, family: str, tags: Collection[str] +) -> None: + """Refuse a format-3 tagged array where a v4 bare array belongs. + + No v4 bare-array element can be a string that names a node kind, so the + refusal only gives a clearer diagnostic. + """ + if items and isinstance(items[0], str) and items[0] in tags: + raise fail( + d.UNKNOWN_NODE, + family, + loc, + f"an array headed by {items[0]!r} is a format-3 tagged array, " + f"not a v4 {family}", + ) + + +def _legacy_array( + j: list[object], loc: Location, family: str, arity: dict[str, int] +) -> str: + """Check a legacy tagged array such as ``["OpaqueTypeSpecification", []]``.""" + head = j[0] if j else None + if not isinstance(head, str) or head not in arity: + raise fail( + d.UNKNOWN_NODE, + family, + loc, + f"expected a {family} object, or a legacy array headed by one of " + f"{sorted(arity)}", + ) + if len(j) != arity[head]: + raise fail( + d.INVALID_JSON, + head, + loc, + f"a legacy {head} array has {arity[head]} items, found {len(j)}", + ) + return head def _wrap(tag: str, members: dict[str, Json]) -> Json: @@ -330,6 +485,23 @@ def _unkeyed[K, A]( _INITIALISM = re.compile(r"^[A-Z]{2,}$") _LOWER_SEGMENT = re.compile(r"^[a-z0-9]+$") +# Schema patterns at FINOS_MORPHIR_PIN, used with ``fullmatch``. +_LEGACY_WORD = re.compile(r"[a-z0-9]+") +_NAME_STRING = re.compile(r"(?:[a-z0-9]+|[A-Z0-9]+)(?:-(?:[a-z0-9]+|[A-Z0-9]+))*") +_TYPE_VARIABLE_STRING = re.compile(r"[a-z][a-z0-9]*(?:-[a-z0-9]+)*") +_DECIMAL_LEXEME = re.compile( + r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?" +) + + +def _spells_fqname(text: str) -> bool: + """Tell an FQName string from a name string, as the schema patterns do. + + An FQName has ``:`` and ``#``; a name has neither. A string with only one of + them is a malformed FQName, which `decode_fqname` reports. + """ + return ":" in text or "#" in text + def _single_letter(word: str) -> bool: return len(word) == 1 and word.isalpha() @@ -372,8 +544,21 @@ def flush() -> None: def decode_name(j: object, loc: Location = ()) -> Name: - """Read a canonical name string such as ``value-in-USD``.""" - text = _expect_str(j, loc, "Name") + """Read a canonical name string such as ``value-in-USD``, or a legacy array. + + The legacy word array ``["value", "in", "u", "s", "d"]`` is the Python word + list itself, so it reads as ``value-in-USD`` (kit names-0001). + """ + if isinstance(j, list): + return _decode_legacy_name(j, loc) + if not isinstance(j, str): + raise fail( + d.INVALID_JSON, + "Name", + loc, + "expected a name string or a legacy word array for Name", + ) + text = j words: list[str] = [] previous = "" for segment in text.split("-"): @@ -403,18 +588,46 @@ def decode_name(j: object, loc: Location = ()) -> Name: return Name(tuple(words)) +def _decode_legacy_name(items: list[object], loc: Location) -> Name: + if not items: + raise fail( + d.INVALID_NAME, "Name", loc, "a legacy name array needs at least one word" + ) + words: list[str] = [] + for i, word in enumerate(items): + if not isinstance(word, str) or not _LEGACY_WORD.fullmatch(word): + raise fail( + d.INVALID_NAME, + "Name", + (*loc, i), + f"legacy name word {word!r} is not lowercase letters and digits", + ) + words.append(word) + return Name(tuple(words)) + + def encode_path(path: Path, loc: Location = ()) -> str: return "/".join(encode_name(n, loc) for n in path) def decode_path(j: object, loc: Location = ()) -> Path: + """Read ``morphir/SDK``, or a legacy array of names such as ``[["morphir"], ...]``.""" + if isinstance(j, list): + if not j: + raise fail( + d.INVALID_NAME, + "Path", + loc, + "a legacy path array needs at least one name", + ) + return Path(tuple(decode_name(item, (*loc, i)) for i, item in enumerate(j))) text = _expect_str(j, loc, "Path") if not text: return Path(()) return Path(tuple(decode_name(part, loc) for part in text.split("/"))) -_FQNAME = re.compile(r"^([^:#]*):([^:#]*)#([^:#/]+)$") +_FQNAME = re.compile(r"^([^:#]+):([^:#]+)#([^:#/]+)$") def encode_fqname(fqn: FQName, loc: Location = ()) -> str: @@ -425,6 +638,20 @@ def encode_fqname(fqn: FQName, loc: Location = ()) -> str: def decode_fqname(j: object, loc: Location = ()) -> FQName: + """Read ``package:module#name``, or the legacy ``[package, module, name]`` array.""" + if isinstance(j, list): + if len(j) != 3: + raise fail( + d.INVALID_JSON, + "FQName", + loc, + f"a legacy FQName array is [package, module, name], found {len(j)} items", + ) + return FQName( + decode_path(j[0], (*loc, 0)), + decode_path(j[1], (*loc, 1)), + decode_name(j[2], (*loc, 2)), + ) text = _expect_str(j, loc, "FQName") match = _FQNAME.match(text) if match is None: @@ -505,7 +732,9 @@ def _decode_type_attributes(body: dict[str, object], loc: Location) -> t.TypeAtt return t.TypeAttributes() here = (*loc, "attributes") obj = _expect_object(body["attributes"], here, "TypeAttributes") - _check_members(obj, _m("", "source extensions"), here, "TypeAttributes") + _check_members(obj, _m("", "source constraints extensions"), here, "TypeAttributes") + if "constraints" in obj: + _check_no_constraints(obj["constraints"], (*here, "constraints")) source = ( t.SourceLocation(*_decode_source(obj["source"], (*here, "source"))) if "source" in obj @@ -519,6 +748,19 @@ def _decode_type_attributes(body: dict[str, object], loc: Location) -> t.TypeAtt return t.TypeAttributes(source=source, extensions=ext) +def _check_no_constraints(j: object, loc: Location) -> None: + """Accept an empty ``constraints`` object, the schema's placeholder.""" + constraints = _expect_object(j, loc, "TypeConstraints") + if constraints: + raise fail( + d.UNKNOWN_MEMBER, + "TypeConstraints", + (*loc, next(iter(constraints))), + "structured type constraints have no reading in the draft v4 codec; " + "only an empty constraints object is accepted", + ) + + def _encode_value_attributes(a: v.ValueAttributes, loc: Location) -> dict[str, Json]: out: dict[str, Json] = {} if a.source is not None: @@ -652,8 +894,35 @@ def _encode_type_fields(fields: list[t.Field], loc: Location) -> dict[str, Json] def decode_type(j: object, loc: Location = ()) -> t.Type: - """Decode a v4 type from the expanded draft spelling.""" - tag, body, here = _node(j, loc, "Type", TYPE_NODES) + """Decode a v4 type from any spelling the pinned schema accepts. + + A bare FQName string is a ``Reference``, a bare lowercase name a ``Variable``, + and a bare array a ``Tuple``. ``Variable``, ``Reference`` and ``Tuple`` also + take a compact payload; every kind takes its expanded payload. + + Raises: + DiagnosticError: With ``invalid_name`` for a string that is neither an + FQName nor a type variable, ``unknown_node`` for an unknown kind, + and ``invalid_json``, ``unknown_member`` or ``missing_member`` for a + malformed node. + """ + if isinstance(j, str): + return _decode_type_string(j, loc) + if isinstance(j, list): + _refuse_tagged_array(j, loc, "Type", TYPE_NODES) + return t.Tuple(t.TypeAttributes(), _decode_list(decode_type, j, loc, "Type")) + if not isinstance(j, dict): + raise fail( + d.INVALID_JSON, + "Type", + loc, + "a Type is a string, an array, or an object wrapped by its kind", + ) + tag, payload, here = _tagged(j, loc, "Type", TYPE_NODES) + compact = _decode_compact_type(tag, payload, here) + if compact is not None: + return compact + body = _expanded(payload, here, tag, TYPE_NODES[tag]) attrs = _decode_type_attributes(body, here) def fields() -> list[t.Field]: @@ -703,6 +972,46 @@ def fields() -> list[t.Field]: return t.Unit(attrs) +def _decode_type_string(text: str, loc: Location) -> t.Type: + if _spells_fqname(text): + return t.Reference(t.TypeAttributes(), decode_fqname(text, loc)) + if not _TYPE_VARIABLE_STRING.fullmatch(text): + raise fail( + d.INVALID_NAME, + "Type", + loc, + f"{text!r} is neither an FQName nor a type variable name " + "(lowercase words joined by '-')", + ) + return t.Variable(t.TypeAttributes(), decode_name(text, loc)) + + +def _decode_compact_type(tag: str, payload: object, here: Location) -> t.Type | None: + """Decode a compact wrapper payload, or return None for an expanded one.""" + attrs = t.TypeAttributes() + if tag == "Variable" and isinstance(payload, (str, list)): + return t.Variable(attrs, decode_name(payload, here)) + if tag == "Reference" and isinstance(payload, str): + return t.Reference(attrs, decode_fqname(payload, here)) + if tag == "Reference" and isinstance(payload, list): + if len(payload) < 2: + raise fail( + d.INVALID_JSON, + "Reference", + here, + "a Reference array is an FQName followed by one or more type " + "arguments; a Reference with no arguments is the FQName string", + ) + return t.Reference( + attrs, + decode_fqname(payload[0], (*here, 0)), + [decode_type(arg, (*here, i)) for i, arg in enumerate(payload) if i], + ) + if tag == "Tuple" and isinstance(payload, list): + return t.Tuple(attrs, _decode_list(decode_type, payload, here, "Type")) + return None + + def _decode_list[A]( decode: Callable[[object, Location], A], j: object, loc: Location, construct: str ) -> list[A]: @@ -770,7 +1079,13 @@ def encode_literal(literal: lit.Literal, loc: Location = ()) -> Json: def decode_literal(j: object, loc: Location = ()) -> lit.Literal: - """Decode a literal; ``WholeNumberLiteral`` reads as ``IntegerLiteral``.""" + """Decode a literal wrapper such as ``{"IntegerLiteral": 42}``. + + A scalar kind also takes the expanded ``{"value": 42}`` payload, and members + beside ``value`` are ignored (kit values-0046 and values-0047). A + ``DocumentLiteral`` payload is the document itself. ``WholeNumberLiteral`` + reads as ``IntegerLiteral``. + """ obj = _expect_object(j, loc, "Literal") if len(obj) != 1 or next(iter(obj)) not in LITERAL_KINDS: raise fail( @@ -781,7 +1096,22 @@ def decode_literal(j: object, loc: Location = ()) -> lit.Literal: ) ((tag, payload),) = obj.items() here = (*loc, tag) + if tag == "DocumentLiteral": + return lit.DocumentLiteral(_decode_document(payload)) + if isinstance(payload, dict): + if "value" not in payload: + raise fail( + d.INVALID_LITERAL, + tag, + here, + f"{tag} expects a value, or an object with a 'value' member", + ) + payload = payload["value"] + here = (*here, "value") + return _decode_scalar_kind(tag, payload, here) + +def _decode_scalar_kind(tag: str, payload: object, here: Location) -> lit.Literal: def bad(message: str) -> DiagnosticError: return fail(d.INVALID_LITERAL, tag, here, message) @@ -803,18 +1133,99 @@ def bad(message: str) -> DiagnosticError: raise bad(f"{tag} expects an integer") return lit.IntegerLiteral(payload) case "FloatLiteral": - if isinstance(payload, bool) or not isinstance(payload, (int, float)): - raise bad("FloatLiteral expects a number") - return lit.FloatLiteral(float(payload)) - case "DecimalLiteral": - if not isinstance(payload, str): - raise bad("DecimalLiteral expects a string") + return lit.FloatLiteral(_float_value(payload, here, tag)) + case _: + if not isinstance(payload, str) or not _DECIMAL_LEXEME.fullmatch(payload): + raise bad( + f"{payload!r} is not a decimal lexeme: " + "[+-]?(digits(.digits?)?|.digits)([eE][+-]?digits)?" + ) try: return lit.DecimalLiteral(Decimal(payload)) except InvalidOperation: - raise bad(f"invalid decimal {payload!r}") from None - case _: - return lit.DocumentLiteral(_decode_document(payload)) + raise bad( + f"{payload!r} is outside the representable decimal range" + ) from None + + +def _float_value(payload: object, loc: Location, construct: str) -> float: + if isinstance(payload, bool) or not isinstance(payload, (int, float)): + raise fail(d.INVALID_LITERAL, construct, loc, f"{construct} expects a number") + try: + value = float(payload) + except OverflowError: + value = math.inf + if not math.isfinite(value): + raise fail( + d.INVALID_LITERAL, + construct, + loc, + f"{construct} expects a finite number that a float can hold", + ) + return value + + +def _decode_bare_scalar(j: object, loc: Location) -> lit.Literal | None: + """Read a bare boolean or number as a literal (decision 0009). + + A number's kind follows its lexeme: no point and no exponent is an + ``IntegerLiteral``, anything else a ``FloatLiteral``. Python's ``json`` + module keeps that distinction, because it parses an integer lexeme to an + ``int`` of any size and every other number lexeme to a ``float``; so ``0`` + reads as ``int`` and ``0.0`` and ``1e3`` as ``float``. + """ + if isinstance(j, bool): + return lit.BoolLiteral(j) + if isinstance(j, int): + return lit.IntegerLiteral(j) + if isinstance(j, float): + return lit.FloatLiteral(_float_value(j, loc, "FloatLiteral")) + return None + + +def _decode_literal_payload( + payload: object, here: Location, tag: str, *, documents: bool +) -> tuple[v.ValueAttributes, lit.Literal]: + """Read the payload of ``Literal`` or ``LiteralPattern``. + + The payload is a bare scalar, a literal wrapper, or the expanded + ``{"attributes": ..., "literal": scalar | literal}``. + """ + if isinstance(payload, dict) and ("literal" in payload or "attributes" in payload): + body = _expanded(payload, here, tag, _m("literal")) + literal = _decode_literal_spelling( + body["literal"], (*here, "literal"), tag, documents=documents + ) + return _decode_value_attributes(body, here), literal + return v.ValueAttributes(), _decode_literal_spelling( + payload, here, tag, documents=documents + ) + + +def _decode_literal_spelling( + j: object, loc: Location, construct: str, *, documents: bool +) -> lit.Literal: + if isinstance(j, str): + return lit.StringLiteral(j) + bare = _decode_bare_scalar(j, loc) + if bare is not None: + return bare + if not isinstance(j, dict): + raise fail( + d.INVALID_LITERAL, + construct, + loc, + f"{construct} expects a boolean, a number, a string or a typed literal", + ) + literal = decode_literal(j, loc) + if not documents and isinstance(literal, lit.DocumentLiteral): + raise fail( + d.INVALID_LITERAL, + construct, + loc, + "a DocumentLiteral cannot be pattern matched (decision 0013)", + ) + return literal # --- Patterns --- @@ -869,8 +1280,32 @@ def many(items: list[v.Pattern], key: str) -> list[Json]: def decode_pattern(j: object, loc: Location = ()) -> v.Pattern: - """Decode a v4 pattern from the expanded draft spelling.""" - tag, body, here = _node(j, loc, "Pattern", PATTERN_NODES) + """Decode a v4 pattern from any spelling the pinned schema accepts. + + A bare array is a ``TuplePattern``. ``TuplePattern`` also takes an array + payload and ``LiteralPattern`` a bare scalar or a literal wrapper. + """ + if isinstance(j, list): + _refuse_tagged_array(j, loc, "Pattern", PATTERN_NODES.keys() | LITERAL_KINDS) + return v.TuplePattern( + v.ValueAttributes(), _decode_list(decode_pattern, j, loc, "Pattern") + ) + if not isinstance(j, dict): + raise fail( + d.INVALID_JSON, + "Pattern", + loc, + "a Pattern is an array or an object wrapped by its kind", + ) + tag, payload, here = _tagged(j, loc, "Pattern", PATTERN_NODES) + if tag == "TuplePattern" and isinstance(payload, list): + return v.TuplePattern( + v.ValueAttributes(), _decode_list(decode_pattern, payload, here, "Pattern") + ) + if tag == "LiteralPattern": + attrs, literal = _decode_literal_payload(payload, here, tag, documents=False) + return v.LiteralPattern(attrs, literal) + body = _expanded(payload, here, tag, PATTERN_NODES[tag]) attrs = _decode_value_attributes(body, here) def sub(key: str) -> v.Pattern: @@ -894,10 +1329,6 @@ def many(key: str) -> list[v.Pattern]: return v.EmptyListPattern(attrs) case "HeadTailPattern": return v.HeadTailPattern(attrs, sub("head"), sub("tail")) - case "LiteralPattern": - return v.LiteralPattern( - attrs, decode_literal(body["literal"], (*here, "literal")) - ) case _: return v.UnitPattern(attrs) @@ -1052,8 +1483,47 @@ def fields(items: list[tuple[Name, v.Value]], key: str = "fields") -> Json: def decode_value(j: object, loc: Location = ()) -> v.Value: - """Decode a v4 value from the expanded draft spelling.""" - tag, body, here = _node(j, loc, "Value", VALUE_NODES) + """Decode a v4 value from any spelling the pinned schema accepts. + + At value position (decision 0009) a bare boolean or number is a literal, a + bare FQName string a ``Reference``, a bare name string a ``Variable``, and a + bare array a ``List``. ``Literal``, ``Constructor``, ``Tuple``, ``List``, + ``Variable``, ``Reference`` and ``FieldFunction`` also take a compact + payload; every kind takes its expanded payload. + + Raises: + DiagnosticError: With ``invalid_name`` for a bare string that is neither + a name nor an FQName (a string literal needs its ``Literal`` wrapper), + ``invalid_literal`` for a bad literal, ``unknown_node`` for an unknown + kind, and ``invalid_json``, ``unknown_member`` or ``missing_member`` + for a malformed node. + """ + if isinstance(j, str): + return _decode_value_string(j, loc) + if isinstance(j, list): + _refuse_tagged_array(j, loc, "Value", VALUE_NODES.keys() | LITERAL_KINDS) + return v.List(v.ValueAttributes(), _decode_list(decode_value, j, loc, "Value")) + bare = _decode_bare_scalar(j, loc) + if bare is not None: + return v.LiteralValue(v.ValueAttributes(), bare) + if not isinstance(j, dict): + raise fail( + d.INVALID_JSON, + "Value", + loc, + "a Value is a boolean, a number, a string, an array, or an object " + "wrapped by its kind", + ) + tag, payload, here = _tagged(j, loc, "Value", VALUE_NODES) + if tag == "Literal": + literal_attrs, literal = _decode_literal_payload( + payload, here, tag, documents=True + ) + return v.LiteralValue(literal_attrs, literal) + compact = _decode_compact_value(tag, payload, here) + if compact is not None: + return compact + body = _expanded(payload, here, tag, VALUE_NODES[tag]) attrs = _decode_value_attributes(body, here) def sub(key: str) -> v.Value: @@ -1068,10 +1538,6 @@ def fields() -> list[tuple[Name, v.Value]]: ) match tag: - case "Literal": - return v.LiteralValue( - attrs, decode_literal(body["literal"], (*here, "literal")) - ) case "Constructor": return v.Constructor(attrs, decode_fqname(body["fqname"], here)) case "Tuple": @@ -1153,6 +1619,41 @@ def fields() -> list[tuple[Name, v.Value]]: ) +def _decode_value_string(text: str, loc: Location) -> v.Value: + if _spells_fqname(text): + return v.Reference(v.ValueAttributes(), decode_fqname(text, loc)) + if not _NAME_STRING.fullmatch(text): + raise fail( + d.INVALID_NAME, + "Value", + loc, + f"{text!r} is neither a name nor an FQName; a string literal needs its " + "Literal wrapper (decision 0009)", + ) + return v.Variable(v.ValueAttributes(), decode_name(text, loc)) + + +def _decode_compact_value(tag: str, payload: object, here: Location) -> v.Value | None: + """Decode a compact wrapper payload, or return None for an expanded one.""" + attrs = v.ValueAttributes() + if isinstance(payload, list) and tag in ("Tuple", "List"): + items = _decode_list(decode_value, payload, here, "Value") + return v.Tuple(attrs, items) if tag == "Tuple" else v.List(attrs, items) + if not isinstance(payload, (str, list)): + return None + match tag: + case "Constructor": + return v.Constructor(attrs, decode_fqname(payload, here)) + case "Reference": + return v.Reference(attrs, decode_fqname(payload, here)) + case "Variable": + return v.Variable(attrs, decode_name(payload, here)) + case "FieldFunction": + return v.FieldFunction(attrs, decode_name(payload, here)) + case _: + return None + + # --- Hole reasons and incompleteness --- @@ -1386,7 +1887,20 @@ def encode_value_definition(defn: v.ValueDefinitionBody, loc: Location = ()) -> def decode_value_definition(j: object, loc: Location = ()) -> v.ValueDefinitionBody: - """Decode an expression, native, external or incomplete body.""" + """Decode an expression, native, external or incomplete body. + + A module value's ``doc`` is read by the module decoder. Here, as in a + ``LetDefinition``, the Python model has no slot for it, so a ``doc`` member + fails with ``unknown_member`` instead of being dropped. + """ + if isinstance(j, dict) and "doc" in j and len(j) > 1: + raise fail( + d.UNKNOWN_MEMBER, + "ValueDefinition", + (*loc, "doc"), + "the Python model has no documentation slot on this value definition; " + "only module values carry a doc", + ) tag, body, here = _node(j, loc, "ValueDefinition", VALUE_DEFINITION_NODES) inputs = _unkeyed( body["inputTypes"], @@ -1450,15 +1964,49 @@ def encode_value_specification(spec: v.Specification, loc: Location = ()) -> Jso def decode_value_specification(j: object, loc: Location = ()) -> v.Specification: + """Decode ``{"inputs": {...}, "output": type}``. + + ``inputs`` may be omitted when empty (kit definitions-0029) or be the legacy + ``[[name, type], ...]`` array (kit definitions-0004). + """ obj = _expect_object(j, loc, "ValueSpecification") - _check_members(obj, _m("inputs output", "annotations"), loc, "ValueSpecification") + _check_members(obj, _m("output", "annotations inputs"), loc, "ValueSpecification") return v.Specification( - _unkeyed(obj["inputs"], decode_name, decode_type, (*loc, "inputs"), "inputs"), + _decode_spec_inputs(obj["inputs"], (*loc, "inputs")) if "inputs" in obj else [], decode_type(obj["output"], (*loc, "output")), _decode_annotations(obj, loc), ) +def _decode_spec_inputs(j: object, loc: Location) -> list[tuple[Name, t.Type]]: + if not isinstance(j, list): + return _unkeyed(j, decode_name, decode_type, loc, "inputs") + inputs = _decode_list(_decode_name_type_pair, j, loc, "inputs") + _check_unique([name for name, _ in inputs], loc, "inputs") + return inputs + + +def _decode_name_type_pair(j: object, loc: Location) -> tuple[Name, t.Type]: + items = _expect_list(j, loc, "Argument") + if len(items) != 2: + raise fail(d.INVALID_JSON, "Argument", loc, "an argument is [name, type]") + return decode_name(items[0], (*loc, 0)), decode_type(items[1], (*loc, 1)) + + +def _check_unique(names: list[Name], loc: Location, construct: str) -> None: + """Refuse a repeated name that the canonical object spelling cannot carry.""" + seen: set[Name] = set() + for i, name in enumerate(names): + if name in seen: + raise fail( + d.DUPLICATE_MEMBER, + construct, + (*loc, i), + f"{construct} has {encode_name(name)!r} more than once", + ) + seen.add(name) + + # --- Annotations --- @@ -1569,23 +2117,38 @@ def encode_args(args: list[tuple[Name, t.Type]], here: Location) -> Json: def _decode_constructors(j: object, loc: Location) -> ts.Constructors: - def decode_args(j: object, here: Location) -> list[tuple[Name, t.Type]]: - args: list[tuple[Name, t.Type]] = [] - for i, pair in enumerate(_expect_list(j, here, "Constructor")): - items = _expect_list(pair, (*here, i), "ConstructorArg") - if len(items) != 2: - raise fail( - d.INVALID_JSON, - "ConstructorArg", - (*here, i), - "a constructor argument is [name, type]", - ) - args.append( - (decode_name(items[0], (*here, i)), decode_type(items[1], (*here, i))) + """Decode ``{"name": [[arg, type], ...]}`` or the legacy array form. + + The legacy form is ``[["name", [[arg, type], ...]], ...]``; it keeps the + constructor order. + """ + if not isinstance(j, list): + return dict(_unkeyed(j, decode_name, _decode_ctor_args, loc, "Constructors")) + ctors: ts.Constructors = {} + for i, entry in enumerate(j): + here = (*loc, i) + items = _expect_list(entry, here, "Constructor") + if len(items) != 2: + raise fail( + d.INVALID_JSON, + "Constructor", + here, + "a legacy constructor is [name, [[arg, type], ...]]", + ) + name = decode_name(items[0], (*here, 0)) + if name in ctors: + raise fail( + d.DUPLICATE_MEMBER, + "Constructors", + here, + f"constructor {encode_name(name)!r} is given more than once", ) - return args + ctors[name] = _decode_ctor_args(items[1], (*here, 1)) + return ctors - return dict(_unkeyed(j, decode_name, decode_args, loc, "Constructors")) + +def _decode_ctor_args(j: object, loc: Location) -> list[tuple[Name, t.Type]]: + return _decode_list(_decode_name_type_pair, j, loc, "Constructor") def encode_type_specification(spec: ts.Specification, loc: Location = ()) -> Json: @@ -1628,6 +2191,9 @@ def _encode_type_specification(spec: ts.Specification, loc: Location) -> Json: def decode_type_specification(j: object, loc: Location = ()) -> ts.Specification: + """Decode ``{"Kind": {...}}`` or a legacy tagged array (kit definitions-0002).""" + if isinstance(j, list): + return _decode_legacy_type_specification(j, loc) tag, body, here = _node(j, loc, "TypeSpecification", TYPE_SPECIFICATION_NODES) params = ( _decode_params(body["typeParams"], (*here, "typeParams")) @@ -1660,6 +2226,36 @@ def decode_type_specification(j: object, loc: Location = ()) -> ts.Specification ) +def _decode_legacy_type_specification( + j: list[object], loc: Location +) -> ts.Specification: + tag = _legacy_array(j, loc, "TypeSpecification", LEGACY_TYPE_SPECIFICATION_ARITY) + params = _decode_params(j[1], (*loc, 1)) + match tag: + case "TypeAliasSpecification": + return ts.TypeAliasSpecification(params, decode_type(j[2], (*loc, 2))) + case "OpaqueTypeSpecification": + return ts.OpaqueTypeSpecification(params) + case "CustomTypeSpecification": + return ts.CustomTypeSpecification( + params, _decode_constructors(j[2], (*loc, 2)) + ) + case _: + here = (*loc, 2) + details = _expect_object(j[2], here, tag) + _check_members( + details, _m("baseType fromBaseType toBaseType", ""), here, tag + ) + return ts.DerivedTypeSpecification( + params, + ts.DerivedTypeSpecificationDetails( + decode_type(details["baseType"], (*here, "baseType")), + decode_fqname(details["fromBaseType"], (*here, "fromBaseType")), + decode_fqname(details["toBaseType"], (*here, "toBaseType")), + ), + ) + + def encode_type_definition(defn: td.Definition, loc: Location = ()) -> Json: params = _encode_params(defn.type_params, loc) match defn: @@ -1695,6 +2291,17 @@ def encode_type_definition(defn: td.Definition, loc: Location = ()) -> Json: def decode_type_definition(j: object, loc: Location = ()) -> td.Definition: + """Decode ``{"Kind": {...}}`` or a legacy tagged array. + + The legacy arrays are ``["TypeAliasDefinition", params, type]`` and + ``["CustomTypeDefinition", params, access-controlled constructors]``. + """ + if isinstance(j, list): + tag = _legacy_array(j, loc, "TypeDefinition", LEGACY_TYPE_DEFINITION_ARITY) + params = _decode_params(j[1], (*loc, 1)) + if tag == "TypeAliasDefinition": + return td.TypeAliasDefinition(params, decode_type(j[2], (*loc, 2))) + return td.CustomTypeDefinition(params, _dec_access_ctors(j[2], (*loc, 2))) tag, body, here = _node(j, loc, "TypeDefinition", TYPE_DEFINITION_NODES) params = _decode_params(body["typeParams"], (*here, "typeParams")) if tag == "TypeAliasDefinition": @@ -1738,15 +2345,38 @@ def run(item: Documented[A], loc: Location) -> Json: def _decode_documented[A]( decode: Callable[[object, Location], A], ) -> Callable[[object, Location], Documented[A]]: + """Read a ``doc`` flattened beside the node (decision 0010) or nested.""" + def run(j: object, loc: Location) -> Documented[A]: - if isinstance(j, dict) and set(j) == {"doc", "value"}: - text = _expect_str(j["doc"], (*loc, "doc"), "Documented") - return Documented(text, decode(j["value"], (*loc, "value"))) - return Documented(None, decode(j, loc)) + text, rest, here = _split_doc(j, loc) + return Documented(text, decode(rest, here)) return run +def _split_doc(j: object, loc: Location) -> tuple[str | None, object, Location]: + """Split a documented node into its ``doc`` and the node. + + The canonical form puts ``doc`` beside the node's own members. The nested + ``{"doc": ..., "value": node}`` form is also valid; a node that is + documented in both places fails with ``duplicate_member``. + """ + if not isinstance(j, dict) or "doc" not in j: + return None, j, loc + text = _expect_str(j["doc"], (*loc, "doc"), "doc") + if set(j) != {"doc", "value"}: + return text, {key: x for key, x in j.items() if key != "doc"}, loc + inner, here = j["value"], (*loc, "value") + if isinstance(inner, dict) and "doc" in inner: + raise fail( + d.DUPLICATE_MEMBER, + "doc", + (*here, "doc"), + "the node is documented both beside and inside 'value'", + ) + return text, inner, here + + def _encode_access[A]( encode: Callable[[A, Location], Json], ) -> Callable[[AccessControlled[A], Location], Json]: @@ -1760,14 +2390,49 @@ def run(item: AccessControlled[A], loc: Location) -> Json: def _decode_access[A]( decode: Callable[[object, Location], A], ) -> Callable[[object, Location], AccessControlled[A]]: + """Read every access-controlled spelling in `ACCESS_SPELLINGS`. + + The forms are ``{level: x}``, the legacy ``{"access": level, "value": x}``, + and the flattened ``{"access": level, ...members of x}``. + """ + def run(j: object, loc: Location) -> AccessControlled[A]: - tag, payload = _single(j, loc, "AccessControlled", ACCESS_TAGS) - value = decode(payload, (*loc, tag)) - return Public(value) if tag == "Public" else Private(value) + obj = _expect_object(j, loc, "AccessControlled") + if "access" in obj: + public = _decode_access_level(obj["access"], (*loc, "access")) + rest = {key: x for key, x in obj.items() if key != "access"} + if not rest: + raise fail( + d.MISSING_MEMBER, + "AccessControlled", + loc, + "an access-controlled node needs 'value' or its own members " + "beside 'access'", + ) + if set(rest) == {"value"}: + value = decode(rest["value"], (*loc, "value")) + else: + value = decode(rest, loc) + else: + tag, payload = _single(obj, loc, "AccessControlled", ACCESS_SPELLINGS) + public = ACCESS_SPELLINGS[tag] + value = decode(payload, (*loc, tag)) + return Public(value) if public else Private(value) return run +def _decode_access_level(j: object, loc: Location) -> bool: + if not isinstance(j, str) or j not in ACCESS_SPELLINGS: + raise fail( + d.UNKNOWN_NODE, + "Access", + loc, + f"unknown access {j!r}; expected one of {sorted(ACCESS_SPELLINGS)}", + ) + return ACCESS_SPELLINGS[j] + + def _single( j: object, loc: Location, construct: str, tags: dict[str, Any] ) -> tuple[str, object]: @@ -1791,6 +2456,7 @@ def _single( _dec_value_spec = _decode_documented(decode_value_specification) _enc_type_def = _encode_access(_encode_documented(encode_type_definition)) _dec_type_def = _decode_access(_decode_documented(decode_type_definition)) +_dec_access_ctors = _decode_access(_decode_constructors) _enc_value_def = _encode_access(_encode_documented(encode_value_definition)) _dec_value_def = _decode_access(_decode_documented(decode_value_definition)) diff --git a/pyproject.toml b/pyproject.toml index a2e7a866..ccc86744 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,11 @@ test = [ managed = true [tool.uv.workspace] -members = ["packages/morphir", "packages/morphir-tools"] +members = ["packages/morphir", "packages/morphir-python-native", "packages/morphir-tools"] [tool.uv.sources] morphir = { workspace = true } +morphir-python-native = { workspace = true } # ============================================================================= # Ruff Configuration @@ -35,7 +36,8 @@ morphir = { workspace = true } [tool.ruff] target-version = "py314" line-length = 88 -exclude = [".venv", "build", "dist", ".git", "__pycache__"] +# rust/ holds compiler test inputs whose exact text is part of the tests. +exclude = [".venv", "build", "dist", ".git", "__pycache__", "rust"] [tool.ruff.lint] select = [ @@ -60,13 +62,17 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/**/*" = ["D101", "D102", "D103", "E501"] +# The benchmark report embeds the HTML document template's long head lines. +"scripts/bench.py" = ["E501"] +# Compiler input: the supported subset has no TYPE_CHECKING blocks. +"tests/fixtures/python/**/*" = ["TC001", "TC003"] "packages/morphir/src/morphir/ir/**/*.py" = ["D101", "D102", "D103", "UP006", "UP007", "UP035", "UP037", "UP046", "TC001", "TC003", "E501", "C408", "RUF005", "E402"] [tool.ruff.lint.pydocstyle] convention = "google" [tool.ruff.lint.isort] -known-first-party = ["morphir", "morphir_tools"] +known-first-party = ["morphir", "morphir_tools", "morphir_python_native"] [tool.ruff.format] quote-style = "double" @@ -85,7 +91,7 @@ show_error_codes = true show_column_numbers = true namespace_packages = true explicit_package_bases = true -mypy_path = ["packages/morphir/src", "packages/morphir-tools/src"] +mypy_path = ["packages/morphir/src", "packages/morphir-tools/src", "packages/morphir-python-native/python"] [[tool.mypy.overrides]] module = "tests.*" @@ -104,7 +110,7 @@ pythonVersion = "3.14" typeCheckingMode = "standard" reportMissingImports = true reportMissingTypeStubs = false -include = ["packages/morphir/src", "packages/morphir-tools/src", "scripts", "tests"] +include = ["packages/morphir/src", "packages/morphir-tools/src", "packages/morphir-python-native/python", "scripts", "tests"] exclude = ["**/__pycache__", ".venv", "build", "dist"] # ============================================================================= diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..75938590 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,3037 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "annotate-snippets" +version = "0.12.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a5b0a5ab8561c603f16c3b78c24a386b1f70e7fc1b1b1e8c15738828e4a176" +dependencies = [ + "anstyle", + "memchr", + "unicode-width", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow 0.14.1", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow 0.14.1", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.9", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + +[[package]] +name = "console" +version = "0.16.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "cucumber" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a87e18d925b19ebe0fd47ea45316abd216d81ec0879c2448c3f9a0e9da62be" +dependencies = [ + "anyhow", + "clap", + "console", + "cucumber-codegen", + "cucumber-expressions", + "derive_more", + "either", + "futures", + "gherkin", + "globwalk", + "humantime", + "inventory", + "itertools 0.14.0", + "linked-hash-map", + "pin-project", + "ref-cast", + "regex", + "sealed", + "smart-default", +] + +[[package]] +name = "cucumber-codegen" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2fc8a8bbb73af3230db699e8690c5c786655f75eb89e5f18d76055fa1a9a4d" +dependencies = [ + "cucumber-expressions", + "inflections", + "itertools 0.14.0", + "proc-macro2", + "quote", + "regex", + "syn 2.0.119", + "synthez", +] + +[[package]] +name = "cucumber-expressions" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6401038de3af44fe74e6fccdb8a5b7db7ba418f480c8e9ad584c6f65c05a27a6" +dependencies = [ + "derive_more", + "either", + "nom", + "nom_locate", + "regex", + "regex-syntax", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "delegate" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e018fccbeeb50ff26562ece792ed06659b9c2dae79ece77c4456bb10d9bf79b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "extism-convert" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f765cafef8be05cf4518613ae2c6b1113e2a1fa630b632e7e8e53f6af8e6bae9" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytemuck", + "extism-convert-macros", + "prost", + "rmp-serde", + "serde", + "serde_json", +] + +[[package]] +name = "extism-convert-macros" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd35d75555de7859923a276a03540ce90778c57e832f6d33359a6b465f4475a" +dependencies = [ + "manyhow 0.11.4", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "extism-manifest" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d8c1cfe930f60da0875e828261f63093f2379e1c2901b3302b5b3b6893a46a" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", +] + +[[package]] +name = "extism-pdk" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "352fcb5a66eb74145a1c4a01f2bd15d59c62c85be73aac8471880c65b26b798f" +dependencies = [ + "anyhow", + "base64 0.22.1", + "extism-convert", + "extism-manifest", + "extism-pdk-derive", + "serde", + "serde_json", +] + +[[package]] +name = "extism-pdk-derive" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d086daea5fd844e3c5ac69ddfe36df4a9a43e7218cf7d1f888182b089b09806c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str 0.10.0", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "gherkin" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2c0d8c632f8a251ce9a8198079b1022adc586ff4e3d33e18debd40eb463b31" +dependencies = [ + "heck", + "peg", + "quote", + "serde", + "serde_json", + "syn 2.0.119", + "textwrap", + "thiserror 2.0.21", + "typed-builder", +] + +[[package]] +name = "globset" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98b0d85522dc0a824cfebbc9594ff4a9b289c568bea0370319dfbdf20ccdd2d" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "globwalk" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +dependencies = [ + "bitflags", + "ignore", + "walkdir", +] + +[[package]] +name = "granit-parser" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f99e46474f56bd905c56e817ebddcf377a611f94c53ac4649e4d3fa3c0cd0" +dependencies = [ + "arraydeque", + "smallvec", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ice_code" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6524844f553e8da5999f3000cf11d3f1ff926bb03fc087441c7b86dee4a7d48" + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "serde", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "406e9ae5b95d8aa37db6293405931b766a6172d1848daee22d70eb44aa78a43c" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3632ebbd0c5c3637ba1a40bdf8288f2cb9d2268e73441a69d5fc73d7cfbaa87c" + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "serde", + "stable_deref_trait", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b195dc605f81aa181bec9b3bd14e26cc71b903e4644f6af578ca9772c7a710" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913a7d294ad056474749152107309fdee3ec17d2482d1ab70fdc61a81c64b9a3" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ignore" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "484be5657551d7868d67587df967fb4fcbe9d266a761cc855aaa148fda0b3cc7" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inflections" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" + +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ion-rs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "285b4429808446f30da668d210beaeb7ab82defb6045e261c6cdfaeafe565552" +dependencies = [ + "arrayvec", + "base64 0.12.3", + "bumpalo", + "chrono", + "compact_str 0.8.2", + "delegate", + "generic-array", + "ice_code", + "memchr", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "phf", + "rustc-hash", + "smallvec", + "thiserror 1.0.69", + "visibility", + "winnow 0.6.24", +] + +[[package]] +name = "is-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8267aa6001e25494f3015f9663bbd88a18240c74483afa5f0934a1b3e4c388e9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "dashmap", + "hashbrown 0.14.5", + "serde", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros 0.11.4", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "manyhow" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1704b073316272a6785b70599f91f73b49022c2e290b065591ee6b9109ed8c7" +dependencies = [ + "manyhow-macros 0.14.1", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + +[[package]] +name = "manyhow-macros" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31601db71f1f5fb6f831a18aa3052d13a62528ee4f2cba7e8c9a5e258d953725" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "morphir-config" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516498c700b0b1cdcbcc2e2844e336a5a900e46b8d705e4ebc8214b0f0a69b5f" +dependencies = [ + "secrecy", + "serde", + "serde-saphyr", + "serde_json", + "thiserror 2.0.21", + "toml", +] + +[[package]] +name = "morphir-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "215bbd4f9af73a839008dfd1757597a59481dad42b2f5ac30ab05efd903f7146" +dependencies = [ + "anyhow", + "bigdecimal", + "granit-parser", + "indexmap", + "lasso", + "num-bigint 0.5.1", + "schemars", + "serde", + "serde_json", + "sha2", + "stacker", + "thiserror 2.0.21", +] + +[[package]] +name = "morphir-extension-sdk" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fcb8913b6cb5d45fc20aadd638ddb0c1d83efac9868b8409937e4cfca6babc0" +dependencies = [ + "base64 0.22.1", + "extism-pdk", + "ion-rs", + "morphir-json", + "morphir-workspace", + "percent-encoding", + "semver", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", + "url", +] + +[[package]] +name = "morphir-json" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b14a954bd9426e128e1ee7e2eb599d12e57dddb77102b56df692ac4d2eab52fb" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "morphir-python-binding" +version = "0.4.1" +dependencies = [ + "cucumber", + "morphir-core", + "morphir-extension-sdk", + "num-bigint 0.5.1", + "ruff_python_ast", + "ruff_python_codegen", + "ruff_python_parser", + "ruff_text_size", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "morphir-python-native" +version = "0.1.0" +dependencies = [ + "morphir-extension-sdk", + "morphir-python-binding", + "pyo3", + "serde", + "serde_json", +] + +[[package]] +name = "morphir-workspace" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8325724d50379da740f4102718cc222cf376d3b22bfd4f0db7a9c16d016e5e20" +dependencies = [ + "globset", + "morphir-config", + "semver", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "peg" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f76678828272f177ac33b7e2ac2e3e73cc6c1cd1e3e387928aa69562fa51367" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "636d60acf97633e48d266d7415a9355d4389cea327a193f87df395d88cd2b14d" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] + +[[package]] +name = "peg-runtime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555b1514d2d99d78150d3c799d4c357a3e2c2a8062cd108e93a06d9057629c5" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "serde_core", + "writeable", + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "pyo3" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "ruff_python_ast" +version = "0.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7739179c404793f47296e14397478ea03dc07cf3cf6632b3f7afeee80bca10f4" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "char_str", + "compact_str 0.10.0", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror 2.0.21", +] + +[[package]] +name = "ruff_python_codegen" +version = "0.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17c851523888b26872a2c07d736f50e58d408438afe9a8a64ee8e5677ae5489" +dependencies = [ + "ruff_python_ast", + "ruff_python_literal", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", +] + +[[package]] +name = "ruff_python_literal" +version = "0.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6146cbd42f7dd229765b2d97410da14dfb708a7b73a514f6b47db7b4e845a4fc" +dependencies = [ + "bitflags", + "icu_properties", + "itertools 0.15.0", + "ruff_python_ast", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e8801707fbf7f737fb8c924cf7c47c32d6bea0f5cb6aabdef70db9cd4ee05cb" +dependencies = [ + "bitflags", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "rustc-hash", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", + "unicode_names2", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6fa97e6048b7e69ca7784e538af46935a9e3dc64d7f30ea412298a385898b26" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff6d1182e0b5d7c276a3fc835c03c3618b8585b1d6ab409d8a7915f44bda465" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8825dec40aec41929c539f419c427885a1005de8bfcf85284220b6fde41766" +dependencies = [ + "get-size2", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "indexmap", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.9", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sealed" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-saphyr" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8050abb251097357e24aff63ba2c52a6309ecb7d23a5474023df960a02694d8" +dependencies = [ + "annotate-snippets", + "base64 0.23.1", + "encoding_rs_io", + "granit-parser", + "nohash-hasher", + "num-traits", + "serde_core", + "smallvec", + "zmij", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "foldhash", + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "smart-default" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "smawk" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "synthez" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d8a928f38f1bc873f28e0d2ba8298ad65374a6ac2241dabd297271531a736cd" +dependencies = [ + "syn 2.0.119", + "synthez-codegen", + "synthez-core", +] + +[[package]] +name = "synthez-codegen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb83b8df4238e11746984dfb3819b155cd270de0e25847f45abad56b3671047" +dependencies = [ + "syn 2.0.119", + "synthez-core", +] + +[[package]] +name = "synthez-core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906fba967105d822e7c7ed60477b5e76116724d33de68a585681fb253fc30d5c" +dependencies = [ + "proc-macro2", + "quote", + "sealed", + "syn 2.0.119", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "textwrap" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" +dependencies = [ + "icu_segmenter", + "smawk", + "unicode-width", +] + +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "serde_core", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "1.1.8+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" + +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unicode_names2" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +dependencies = [ + "phf", + "unicode_names2_generator", +] + +[[package]] +name = "unicode_names2_generator" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" +dependencies = [ + "getopts", + "log", + "phf_codegen", + "rand", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "serde", + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..40bfa856 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,29 @@ +[workspace] +members = ["crates/morphir-python-binding", "crates/morphir-python-native"] +resolver = "2" + +[workspace.package] +edition = "2024" +rust-version = "1.98" +authors = ["FINOS "] +license = "Apache-2.0" +repository = "https://github.com/finos/morphir-python" +homepage = "https://github.com/finos/morphir-python" + +[workspace.dependencies] +# Published Morphir crates. Never use sibling-checkout paths here. +morphir-core = "=0.2.0" +morphir-extension-sdk = "=0.3.0" +num-bigint = "0.5" +# Ruff component crates evolve together; update this group as one unit. +ruff_python_parser = "=0.0.15" +ruff_python_ast = "=0.0.15" +ruff_python_codegen = "=0.0.15" +ruff_text_size = "=0.0.15" +serde_json = "1.0" +tempfile = "3" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 diff --git a/rust/crates/morphir-python-binding/Cargo.toml b/rust/crates/morphir-python-binding/Cargo.toml new file mode 100644 index 00000000..c9bc3b9a --- /dev/null +++ b/rust/crates/morphir-python-binding/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "morphir-python-binding" +version = "0.4.1" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Python ADT, tuple, function and lambda frontend and backend for Morphir" + +[lib] +crate-type = ["rlib", "cdylib"] + +[dependencies] +num-bigint.workspace = true +morphir-core.workspace = true +morphir-extension-sdk.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true +ruff_python_codegen.workspace = true +ruff_text_size.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true +cucumber = "0.23" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } + +[[test]] +name = "acceptance" +harness = false diff --git a/rust/crates/morphir-python-binding/README.md b/rust/crates/morphir-python-binding/README.md new file mode 100644 index 00000000..1b67cebc --- /dev/null +++ b/rust/crates/morphir-python-binding/README.md @@ -0,0 +1,626 @@ +# Python models, functions and lambdas + +`morphir-python-binding` implements the `morphir-python` extension in Rust. +It provides MEP `frontend/compile` and `backend/generate` through the Morphir +extension SDK, both natively and as an Extism WebAssembly guest. Python source +is parsed with Ruff and never imported or executed. The backend builds +declarations, parses them into Ruff's AST, and uses Ruff's generator to write +Python. All four direct Ruff component dependencies are pinned together. + +This is an initial implementation of a defined subset of Python 3.12 and later +and Morphir IR v3 and v4. It supports ADT declarations, fixed tuples and annotated pure conditional +functions, typed calls and unary lambdas. It does not implement all Python syntax or all Morphir IR nodes, and +does not claim full Morphir Compatibility Kit conformance for this extension. +The crate depends on the published `morphir-core` and `morphir-extension-sdk` crates only. +It does not link CPython; the separate `morphir-python-native` crate in this workspace +adapts it for the `morphir-python` Python package. + +This crate moved from `finos/morphir-rust` (source revision +`0aa60eb799d46f8a317ec47abef4a7a4e61dc8ac`) to `finos/morphir-python`, which now maintains +and releases it. Bundles up to `extension/python/v0.4.0` remain available from the +[finos/morphir-rust releases](https://github.com/finos/morphir-rust/releases). The earlier +`extension/python/v0.1.0` bundle does not include multi-module, private-module, IR v3 or higher-order +function support, and Morphir CLI `0.4.0-beta.1` and later cannot compile with it. Use the +[extension guide](../../../docs/python-extension.md) to build and install the bundle. + +## A complete model + +Save this as `models.py`. It is valid Python and supported frontend input: + +```python +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + +type Decision = Pending | Approved + +@dataclass(frozen=True) +class Application: + decision: Decision + verified: bool + +def choose(flag: bool, first: Decision, second: Decision) -> Decision: + return first if flag else second +``` + +`Decision` becomes a custom type with `Pending` and `Approved` constructors. +`Application` becomes a record type alias. `choose` accepts and returns a +`Decision`; it selects an existing value. Constructing an `Approved` inside a +compiled function is not supported yet. + +After importing the generated module in a Python application, callers can +instantiate its dataclasses and call its functions: + +```python +from models import Approved, Pending, choose + +approved = Approved(amount=125.0) +pending = Pending() +assert choose(True, approved, pending) is approved +assert choose(False, approved, pending) is pending +``` + +This caller is ordinary application code, not input to the Morphir frontend. +The frontend rejects its constructor calls, assignments and assertions. + +## Mapping + +| Python declaration | Morphir definition | +| --- | --- | +| Frozen dataclass outside a sum | Type alias whose body is a record | +| `type Decision = Pending \| Approved` | Custom type with named constructors | +| Dataclass named in that alias | Constructor with its fields as ordered parameters | +| Variant with `pass` | Constructor with no parameters | +| `int`, `float`, `bool`, `str` | SDK `basics#int`, `basics#float`, `basics#bool`, `string#string` | +| `tuple[A, B]` | Tuple type | +| `type Point = tuple[float, float]` | Type alias with a tuple body | +| `(x, y)` or `return x, y` | Tuple value with ordered elements | +| Local or imported type annotation | Fully qualified reference to its defining module in the current package | +| Annotated function | Value definition with typed parameters and an expression body | +| `Callable[[A], B]` | Unary `Function` type from `A` to `B` | +| Named function / function call | Same-package `Reference` / ordered `Apply` nodes | +| `lambda item: body` | `Lambda` with a named wildcard `AsPattern` and a lexically scoped body | +| Returning `if`/`elif`/`else`, or `a if condition else b` | `IfThenElse` with `condition`, `then`, and `else` members | + +The scalar references are exactly `morphir/SDK:basics#int`, +`morphir/SDK:basics#float`, `morphir/SDK:basics#bool`, and +`morphir/SDK:string#string`. With package `acme/example` and file `models.py`, +`Decision` is referenced as `acme/example:models#decision`. + +A variant belongs to exactly one sum. It is not also emitted as a record alias, +and its class name cannot be used as a field type. Refer to its enclosing sum +instead. A one-variant sum is spelled `type Wrapped = Variant`. +Recursive fields work with postponed annotations. These mappings preserve +declarations; Python itself does not enforce annotation types at runtime. + +Each request contains one or more `.py` source documents. A source-relative path +determines its module name: `models.py` becomes `models`, and +`domain/order_items.py` becomes `domain/order-items`. The package name must be a +canonical Morphir package path such as `acme/example`. Omitted `exposedModules` +exposes every module. An explicit empty list makes every module private; a +nonempty list selects the public modules, using canonical paths or dotted names +such as `Domain.OrderItems`. Other modules remain private and usable by siblings. +Unknown exposure names are errors. All emitted types, constructors and functions +within those modules are public. Declaration names use Morphir's +existing name codec, including uppercase initialism segments. Collisions after +normalization are errors. Names must be ASCII identifiers without a leading +underscore; Python keywords and the builtins used by this subset are reserved. +Module filenames also exclude Windows device names such as `con` and `aux`. +Names are checked in their generated spelling too: a field named `Class` would +become the reserved word `class`, so it is rejected. `zip_code` becomes the +Morphir name `zip-code` and generates back as `zip_code`. Type and constructor +names generate in PascalCase; field, function and parameter names use snake_case. + +The backend returns one relative `.py` artifact per module. It rejects names that cannot +roundtrip through this spelling, including collisions across types, constructors +and functions after Morphir name normalization. The common Morphir pattern where +a type and its single constructor share a name therefore needs distinct names +for this first Python subset. + +## Multiple modules and imports + +Module privacy is package metadata, not Python source syntax. Both the frontend +and backend support public and private modules. For a project with +`src/domain/models.py` and `src/domain/rules.py`, this configuration exposes only +the rules module: + +```toml +[project] +name = "acme/example" +version = "0.1.0" +source_directory = "src" +exposed_modules = ["domain.rules"] + +[frontend] +language = "python" +``` + +`rules.py` may import types from the private `models.py`. The backend generates +both files. Keep the exposure configuration when recompiling generated files to +preserve their Morphir visibility; ordinary Python imports do not enforce that +visibility at runtime. Private types, constructors, and functions are still +outside the supported subset. + +Compile all source files together in one request. For example, save the complete +model above as `models.py` and this function as `rules.py`: + +```python +from __future__ import annotations +from models import Decision as Result + +def select_decision(flag: bool, first: Result, second: Result) -> Result: + if flag: + return first + else: + return second +``` + +The function's parameter and result types reference +`acme/example:models#decision`. Import aliases do not create new IR types. +Imported record, sum and tuple types work in fields, constructor parameters and +function signatures. Tuple aliases resolve across modules when checking returns. +The order of source documents does not affect the resulting IR. + +Supported imports include `from models import Decision`, optional `as` aliases, +`import models`, `import domain.models as model`, and relative imports such as +`from .models import Decision` or `from ..models import Decision` inside nested +modules. `from domain import models` also binds a source module. Imports must +resolve to files supplied in the request, and named type or function imports must refer to +declarations in that module. Importing constructors or re-exported +names, wildcard imports, lazy imports and external package imports are rejected. +The standard imports `dataclasses.dataclass`, `__future__.annotations`, and +`Callable` from `typing` or `collections.abc` are supported. A sum's variant +classes must stay in the same module as its alias. + +Relative document paths are relative to the source root. For multiple absolute +paths or file URIs, set `sources.root`; the CLI does this when +compiling a directory. URI queries and fragments do not affect module identity; +percent-encoded path segments are decoded before validating module names. +Dot segments and encoded path separators are rejected. Absolute documents outside that root, module paths that +collide after normalization or case folding, and a file that is also a package directory such as +`models.py` alongside `models/item.py` are errors. A single absolute document +without a root retains the original filename-based behavior. + +Nested modules use Python namespace packages. `__init__.py` modules and package +initialization code are not supported. Generated paths preserve the directory +structure, and generated imports use absolute module paths with collision-free +aliases. Put the generated directory on Python's import path. The backend uses +module imports and postponed annotations, so mutually referring record types +can be generated without eager cross-module type imports. Recursive tuple aliases +remain unsupported. No imports execute during compilation or generation. + +From `rust/`, run `cargo run -p morphir-python-binding --example python_modules -- 3` for a v3 +example, or omit the final argument for v4. Both provide a complete +native MEP example with imported ADTs and tuple aliases. + +## Functions and lambdas + +Both IR v3 and v4 support these declarations: + +```python +from collections.abc import Callable + +def identity(value: int) -> int: + return value + +def apply(transform: Callable[[int], int], value: int) -> int: + return transform(value) + +def retain(value: int) -> Callable[[int], int]: + return lambda ignored: value + +def run(value: int) -> int: + return apply(lambda item: identity(item), value) +``` + +Functions may call other annotated functions in the supplied package, including +forward references, recursion, and functions imported from private modules. +Calls are checked against the declared parameter and return types. Lambdas can +capture enclosing parameters and shadow a parameter with the same source name. +Different spellings that would collapse two lexical bindings to the same Morphir +name are rejected. Generated global function references use module aliases that +avoid parameter and lambda names, including when referencing the current module. +Keep generated files together and import them as modules when executing them. + +Import `Callable` from `collections.abc` or `typing`, optionally with an alias; +qualified forms such as `typing.Callable` also work after importing the module. +`Callable` and `callable` are reserved declaration names. Lambda input types come +from an annotated return, a function argument, or a containing tuple/conditional. +An immediately called lambda can infer its input from its argument. + +Morphir function types are unary. First-class function values therefore use +exactly one parameter: `Callable[[A], B]`, a unary named function, or a unary +lambda. For currying, nest the types and lambdas explicitly: + +```python +def select(flag: bool) -> Callable[[int], Callable[[int], int]]: + return lambda first: lambda second: first if flag else second + +def selected(value: int) -> int: + return select(True)(value)(0) +``` + +Ordinary `def` functions may still have multiple parameters, but their direct +calls must supply all arguments positionally. A zero-parameter `def` call maps +to a value `Reference` and generates as a zero-argument call. Zero-parameter or +multi-parameter definitions cannot themselves be passed as callable values. +Partial applications of multi-parameter definitions, multi-argument `Callable` +annotations or lambdas, keyword arguments, starred arguments, defaults, +variadics, untyped lambda inputs without context, and lambda destructuring +patterns are outside this binding's subset. Nested `def`, assignments and +general Python inference remain unsupported. + +The [function fixture](tests/fixtures/functions.py) is complete frontend input. +The [function tests](tests/functions.rs) cover structural round trips, independently +authored IR and executable Python results. CI also sends these functions through +the packaged WASM extension in both IR versions. + +## Conditional function bodies + +This function can be added to `models.py`: + +```python +def classify(value: int) -> str: + if value < 0: + return "negative" + elif value == 0: + return "zero" + else: + return "positive" +``` + +Functions need parameter and return type annotations. They may return parameters, +`bool`, `int`, `float` or `str` literals, fixed tuples, comparisons, conditional +expressions, typed calls, function references and contextual lambdas. +Parameters can also carry local ADTs or fixed tuples. Conditions must have type +`bool`; both branches must have the same type and match the declared return type. +Scalar comparisons `==`, `!=`, `<`, `<=`, `>` and `>=` map respectively to SDK +Basics `equal`, `not-equal`, `less-than`, `less-than-or-equal`, `greater-than`, and +`greater-than-or-equal`. Each is a `Reference` with two nested `Apply` nodes. +Operands must have matching scalar types; booleans support equality only. + +Nested branches and `elif` chains work. An early-return branch can use a following +return as its fallback, for example `if flag: return first` followed by +`return second`. Every path must return a value. The backend generates explicit +returning `if`/`else` blocks, preserving which branch is evaluated. It may use a +conditional expression inside a condition or comparison operand. + +Function and parameter names generate as snake_case. For IR v4, integer literals preserve +arbitrary precision, including decimal, hexadecimal, octal and binary source +literals. The backend emits decimal integers. Float literals must be finite. Python's implicit +truthiness and numeric coercions are not part of this subset. These restrictions +are checked during compilation and generation. + +The `choose` function in the complete model has this value definition at +`values["choose"]["Public"]`, when compiled with package `acme/example`: + +```json +{ + "ExpressionBody": { + "inputTypes": { + "flag": "morphir/SDK:basics#bool", + "first": "acme/example:models#decision", + "second": "acme/example:models#decision" + }, + "outputType": "acme/example:models#decision", + "body": { + "IfThenElse": { + "condition": {"Variable": "flag"}, + "then": {"Variable": "first"}, + "else": {"Variable": "second"} + } + } + } +} +``` + +This shows the extension's direct output. A host such as the CLI may write the +equivalent expanded type encoding, for example +`{"Reference": {"fqname": "morphir/SDK:basics#bool"}}` in place of the compact +string. Such spelling changes do not change the type. + +For v4, the frontend emits explicit parameter and return types, but does not attach +inferred-type or source-location attributes to each IR value node. Its expression +checks cover the supported subset; they are not a general Morphir type checker. + +## Tuples + +This code can also be added to `models.py`: + +```python +type Point = tuple[float, float] +type LabeledPoint = tuple[Point, str] + +def make_point(x: float, y: float) -> Point: + return (x, y) + +def located(flag: bool, location: Point) -> LabeledPoint: + if flag: + return (location, "known") + else: + return ((0.0, 0.0), "unknown") +``` + +Fixed tuples of two or more elements work in dataclass fields, constructor +parameters, function parameters, return types and function expressions. Elements +can be scalars, local types or nested tuples. Named tuple aliases are expanded +when checking expression types, while their names are retained in IR signatures. +Each tuple element and its position must match the annotation. + +The `Point` alias has `TypeAliasDefinition.typeExp`: + +```json +{"Tuple": ["morphir/SDK:basics#float", "morphir/SDK:basics#float"]} +``` + +The body of `make_point` is a value expression: + +```json +{"Tuple": [{"Variable": "x"}, {"Variable": "y"}]} +``` + +The codec writes the tuple type wrapper shown here; it also accepts the IR v4 +compact bare-array spelling in type positions. A tuple **value** always has a +`Tuple` wrapper. In value positions, a bare array means a list and is outside +this binding's subset. + +Empty tuples, one-element tuples, `tuple[T, ...]`, unparameterized `tuple`, +`typing.Tuple`, `NamedTuple`, starred expansion, tuple indexing, destructuring, +tuple comparison and recursive tuple aliases are not supported. These are limits +of this binding, not claims that Morphir IR cannot represent those concepts. + +## Morphir IR support and conformance + +Both directions support IR v3 and v4 through the shared `morphir-core` models. +The frontend uses one v4 language model and encodes the requested version. The +backend detects the document version, migrates v3 through the shared core API, +and applies the same Python subset checks. IR v1 and v2 are unsupported. +Readers follow the shared compatibility intervals `[3.0.0,3.2.0)` and +`[4.0.0,4.1.0)`, accepting later patches but rejecting later minor versions. +Compilation targets the baseline versions 3.0.0 and 4.0.0. + +| Version | Encoding and attributes | +| --- | --- | +| v3 | `formatVersion: 3`, classic tagged arrays and word-array names; function values carry checked inferred types | +| v4 | `formatVersion: 4`, canonical names and the v4 JSON codec; emitted node attributes remain empty | + +V3 generation accepts typed value annotations, checks them against the supported +expression types, and regenerates them on recompilation. Empty documentation +wrappers normalize to absent documentation. Other documentation and metadata +remain unsupported. Untyped v3 function bodies are not accepted by the current +typed distribution reader. V3 whole-number literals must fit the shared codec's +signed 64-bit range, from -9223372036854775808 to 9223372036854775807. Larger +Python integer literals produce a diagnostic for v3 and remain supported in v4. +These are current binding/codec limits, not limits attributed to Morphir IR. +V3 compilation also checks that every name survives the shared legacy name +conversion. For example, `a_b` would become the initialism `AB`, so it is rejected +with `PY003` for v3; rename it or target v4. Initialisms such as `APIResponse` +remain supported. + +The type and value subset below applies to both versions; the displayed JSON +examples elsewhere in this README use v4. + +| IR area | Frontend emits / backend accepts | +| --- | --- | +| Distribution | `Library`, one or more public or private modules, empty dependency map; definitions within modules are public | +| Type definitions | Non-generic record aliases, fixed tuple aliases, custom types with public constructors | +| Type expressions | The four SDK scalar references, same-package type references across modules, fixed tuples, unary `Function`; a record at a record-alias body | +| Value definitions | V4 `ExpressionBody`, or the classic v3 value-definition object, with annotated inputs and a required output type | +| Value expressions | Parameter `Variable`, scalar `Literal`, fixed `Tuple`, `IfThenElse`, same-package function `Reference`, typed `Apply`, unary `Lambda`, and fully applied two-argument SDK scalar comparisons | +| Lambda patterns | `AsPattern` over `WildcardPattern`, binding one parameter; v3 pattern annotations are checked | +| Literal kinds | `BoolLiteral`, finite `FloatLiteral`, `StringLiteral`; v4 arbitrary-precision `IntegerLiteral` or v3 signed 64-bit `WholeNumberLiteral` | +| Metadata | Empty type attributes; v4 empty value attributes or v3 checked inferred types. Nonempty documentation and other retained metadata are rejected | + +Unsupported IR includes `Specs` and `Application` distributions, dependencies, +private types, constructors and values, generic types, opaque types, empty custom types, extensible +records, unit types, free type variables, and other SDK types such +as List, Maybe and Decimal. Arbitrary aliases such as an alias directly to `int` +are not supported. Type references must resolve within the supplied package. + +Unsupported value forms include record construction or field access, +constructor application, external calls or references, lists, arithmetic, +let bindings, general pattern matching, record updates, holes, and +`NativeBody`, `ExternalBody` or `IncompleteBody` definitions. A valid Morphir IR +document can therefore still be outside this extension's supported subset. + +For supported models, the round-trip contract is preservation of the model +through Python → IR → Python → IR. Tests compare compiled IR structurally and +also exercise independently authored IR inputs. This is not source-text or +byte-for-byte JSON preservation: formatting, comments, declaration spelling, +declaration order and accepted alternate JSON encodings may normalize. Numeric +literals may be respelled. The extension does not preserve arbitrary metadata +that the shared IR reader ignores. + +Current evidence is scoped to the checked-in cases: + +| Evidence | What it establishes | +| --- | --- | +| [ADT integration tests](tests/pipeline.rs) | Expected IR mappings, independent custom-type input, recursive references, naming and rejection boundaries | +| [Function and tuple integration tests](tests/conditionals.rs) | Exact `IfThenElse`/tuple encodings, alias expansion, branch/type validation and round-trips | +| [Functions and lambdas](tests/functions.rs) | V3/v4 calls, closures, independent IR, annotation and scope checks; opt-in Python execution runs in CI | +| [Version integration tests](tests/ir_versions.rs) | V3/v4 roundtrips, private imports, independently authored v3 input, integer bounds and annotation validation | +| [Module integration tests](tests/modules.rs) | Absolute and relative imports, nested modules, cyclic record references, import collisions and cross-module tuple checking | +| [Acceptance scenarios](tests/features/adt.feature) | Supported models pass through the public extension API; unsupported input returns diagnostics | +| [Extension bundle check](../../../docs/python-extension.md#build-a-development-bundle) | Installation, capability negotiation and compile/generate/recompile of the packaged WASM guest through an offline Morphir CLI host | +| Example verification | Generated example IR checked against the v4 JSON Schema, and selected generated Python branch/tuple results checked in Python 3.14.7 | + +There is currently no complete Python-extension MCK report or +full Python/Morphir semantic equivalence claim. Generated Python type annotations +also do not enforce argument types at runtime. + +## Current boundary + +Supported fields are scalars, same-package references, unary `Callable` types and +fixed tuples of at least two elements. The imports described above, frozen dataclasses, non-generic +`type` aliases of dataclass variants or fixed tuples, and annotated pure functions +are the accepted module statements. +Comments and whitespace are not preserved. Methods, field defaults, +inheritance, arbitrary decorators or imports, docstrings, generic parameters, +containers, optional fields, quoted annotations and cross-package dependencies +are rejected. Constructor calls, assignments, loops, bare returns, +decorated or async functions, parameter defaults, variadic parameters, +positional-only or keyword-only parameters, chained comparisons, and boolean +operators are not supported yet. The backend also rejects private types, constructors and values, documentation, +unsupported metadata, dependencies and IR type forms outside this subset. Failures return +diagnostics with no partial IR or artifacts. + +For example, this is valid Python but rejected with `PY004`, because an integer +condition would use Python's implicit truthiness: + +```python +def invalid(value: int) -> int: + return 1 if value else 0 +``` + +Other rejected examples are `return (1, 2)` for `-> tuple[int, str]`, an `if` +whose false path reaches the end of the function without a return, and +`type Point = tuple[float, ...]`. No incomplete model is emitted for these cases. + +Compile functions with `typesOnly=false`. A request containing functions with +`typesOnly=true` returns a diagnostic; bodies are never silently dropped. ADT-only +sources accept either value. `irVersion` accepts `3`, `3.0.0`, `4` or `4.0.0`. The CLI's +`outputDir` string option is accepted as context; `sources.root` determines +module paths for absolute document URIs. Neither grants filesystem access. +`emitParseStage=true` produces warning `PY006`; combining it +with `emitParseStageFatal=true` fails. Other options are rejected. + +| Code | Meaning | +| --- | --- | +| `PY001` | Unsupported request or option | +| `PY002` | Python syntax error, with a source range | +| `PY003` | Invalid name or naming collision | +| `PY004` | Unsupported declaration, annotation or IR feature | +| `PY005` | IR decoding or serialization, or generated syntax failure | +| `PY006` | Parse-stage output was requested but is unavailable | + +Semantic lowering diagnostics identify the source module; request-option +diagnostics can have no source location. Syntax diagnostics +identify the parser's failing range. Source positions use the SDK's UTF-16 +convention. + +## Run the pipeline + +From the `rust/` directory of a `finos/morphir-python` checkout: + +```console +cargo test --locked -p morphir-python-binding +cargo run --locked -p morphir-python-binding --example python_adt +rustup target add wasm32-unknown-unknown +cargo build --locked --release -p morphir-python-binding --target wasm32-unknown-unknown +``` + +The example prints JSON containing compiled IR and generated source for the +[ADT](tests/fixtures/models.py), [conditional](tests/fixtures/conditionals.py), and +[tuple](tests/fixtures/tuples.py) fixtures. Its values and names differ from the +shorter README model. To compile the README model directly, save its complete +model block as `models.py` and run: + +```console +cargo run --locked -p morphir-python-binding --example python_adt -- models.py +``` + +The optional path selects one source file; the example uses package `acme/example` +and returns IR plus generated Python through native MEP. File reading belongs to +this example program, not the extension. The bundle check invokes both +capabilities through the actual WASM guest. The native `NativeExtension::frontend_backend(PythonExtension)` adapter exposes +the same MEP methods without requiring a WASM runtime. + +Build the installable bundle with `mise run extension:bundle` from the repository root. +This builds and validates the WASM guest, packages its checksum and descriptor, and +tests publication, installation and both capabilities with a fresh Morphir Home. +The three bundle files are written to `.morphir/build/extensions/python/`. + +CI uploads these files as `morphir-python-extension-bundle`. Release tags +`extension/python/v*` in `finos/morphir-python` publish the same tested bundle. +The descriptor declares language `python` (`.py`), target `python`, MEP `0.1` +and IR `3` and `4`; these declarations cover only the subset documented above. The +release is a Morphir WASM extension, not a PyPI package or a CPython import module. + +For installation with the parent Morphir CLI, see the +[Python extension guide](../../../docs/python-extension.md). + +## Use the crate from another Rust project + +The crate is not on crates.io yet. Depend on an exact committed revision of +`finos/morphir-python`; Cargo finds the package inside the repository's `rust/` +workspace. Use the same published SDK version as this crate: + +```toml +[dependencies] +morphir-python-binding = { git = "https://github.com/finos/morphir-python", rev = "" } +morphir-extension-sdk = "=0.3.0" +serde_json = "1" +``` + +Build prerequisites are Rust 1.98 or later and, for the WASM guest, the +`wasm32-unknown-unknown` target. Neither CPython nor Python development headers +are needed: this crate does not depend on PyO3. A complete native call looks like this: + +```rust +use morphir_extension_sdk::prelude::*; +use morphir_python_binding::PythonExtension; + +fn main() -> Result<()> { + let source = "def choose(flag: bool, first: int, second: int) -> int:\n return first if flag else second\n"; + let compiled = PythonExtension.compile(CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: vec![SourceDocument { + uri: "rules.py".into(), + language_id: "python".into(), + version: 1, + text: source.into(), + }], + }, + package: CompilePackage { name: "acme/example".into(), exposed_modules: None }, + dependencies: vec![], + options: CompileOptions { types_only: false, ir_version: "3".into(), ..Default::default() }, + baseline: None, + })?; + let ir = compiled.ir.expect("valid source compiles"); + let generated = PythonExtension.generate(GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + })?; + for artifact in generated.artifacts { + println!("{}\n{}", artifact.path, artifact.content); + } + Ok(()) +} +``` + +The [examples](examples) show complete requests, including multi-module sources +and generation from the compiled IR. + +## Specification sources + +The implementation uses `morphir-core::ir::v4` rather than a separate IR model. +The parent repository's [IR knowledgebase](https://github.com/finos/morphir/tree/main/kb/bundles/morphir/morphir-ir) +records the naming, record-field, constructor-parameter and SDK package decisions. +The [IR specification](https://morphir.finos.org/docs/spec/ir/) and the +[extension tutorial](https://github.com/finos/morphir-rust/blob/main/docs/tutorials/extension-tutorial.md) describe the +contracts this crate uses. Ruff's [parser](https://github.com/astral-sh/ruff/tree/main/crates/ruff_python_parser) +and [generator](https://github.com/astral-sh/ruff/tree/main/crates/ruff_python_codegen) +remain internal component crates; upgrades require running both native and WASM tests. + +### Bundle descriptor compatibility + +Newly built bundles include a version-2 `release.json` with capability claims +read from the exact WASM being packaged and checked against +the extension declaration in `rust/extension.toml`. Publishing requires Morphir CLI `0.4.0-beta.7` or +later; older CLIs cannot read this descriptor. The versioned WASM and +`.wasm.sha256` names are unchanged. The downloaded descriptor is still named +`-.release.json`; rename it to `release.json` before +publishing. WASM installs show `Claims: unchecked`. + +See the [packaging contract](https://github.com/finos/morphir-rust/blob/main/docs/extensions/capability-claims.md#packaging-wasm-extensions) +for the descriptor shape and provenance rules. diff --git a/rust/crates/morphir-python-binding/examples/python_adt.rs b/rust/crates/morphir-python-binding/examples/python_adt.rs new file mode 100644 index 00000000..9dde1e9f --- /dev/null +++ b/rust/crates/morphir-python-binding/examples/python_adt.rs @@ -0,0 +1,105 @@ +//! Print a fixture's IR and generated Python using the same MEP endpoint as a host. +use morphir_extension_sdk::{ + prelude::*, + protocol::{InitializeParams, MEP_VERSION, PeerInfo, methods}, +}; +use morphir_python_binding::PythonExtension; + +fn main() -> Result<()> { + let extension = NativeExtension::frontend_backend(PythonExtension)?; + // A host must complete the MEP handshake before compile or generate requests. + let initialized = extension.protocol().handle(ExtensionRequest::new( + methods::INITIALIZE, + InitializeParams { + model_transport: None, + protocol_versions: vec![MEP_VERSION.into()], + host: PeerInfo { + kind: Default::default(), + name: "python-binding-example".into(), + version: env!("CARGO_PKG_VERSION").into(), + }, + }, + 0, + )?); + if let Some(error) = initialized.error { + return Err(ExtensionError::ExecutionFailed(format!("{error:?}"))); + } + let (uri, text) = match std::env::args().nth(1) { + Some(path) => { + let text = std::fs::read_to_string(&path) + .map_err(|error| ExtensionError::ExecutionFailed(format!("{path}: {error}")))?; + (path, text) + } + None => ( + "models.py".into(), + format!( + "{}\n{}\n{}", + include_str!("../tests/fixtures/models.py"), + include_str!("../tests/fixtures/conditionals.py"), + include_str!("../tests/fixtures/tuples.py") + ), + ), + }; + let request = CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: vec![SourceDocument { + uri, + language_id: "python".into(), + version: 1, + text, + }], + }, + package: CompilePackage { + name: "acme/example".into(), + exposed_modules: None, + }, + dependencies: vec![], + options: CompileOptions { + types_only: false, + ir_version: "4".into(), + ..Default::default() + }, + baseline: None, + }; + let compiled = + extension + .protocol() + .handle(ExtensionRequest::new(methods::COMPILE, request, 1)?); + let compiled: CompileResult = serde_json::from_value( + compiled + .result + .ok_or_else(|| ExtensionError::ExecutionFailed(format!("{:?}", compiled.error)))?, + )?; + let ir = compiled + .ir + .ok_or_else(|| ExtensionError::ExecutionFailed(format!("{:?}", compiled.diagnostics)))?; + let generated = extension.protocol().handle(ExtensionRequest::new( + methods::GENERATE, + GenerateRequest { + ir: ir.clone(), + target: "python".into(), + options: Default::default(), + }, + 2, + )?); + let generated: GenerateResult = serde_json::from_value( + generated + .result + .ok_or_else(|| ExtensionError::ExecutionFailed(format!("{:?}", generated.error)))?, + )?; + if !generated.success { + return Err(ExtensionError::ExecutionFailed(format!( + "{:?}", + generated.diagnostics + ))); + } + println!( + "{}", + serde_json::to_string_pretty( + &serde_json::json!({ "ir": ir, "artifacts": generated.artifacts }) + )? + ); + Ok(()) +} diff --git a/rust/crates/morphir-python-binding/examples/python_modules.rs b/rust/crates/morphir-python-binding/examples/python_modules.rs new file mode 100644 index 00000000..c7fed79d --- /dev/null +++ b/rust/crates/morphir-python-binding/examples/python_modules.rs @@ -0,0 +1,101 @@ +//! Compile and generate multiple modules with imported ADTs and tuple aliases through MEP. +use morphir_extension_sdk::{ + prelude::*, + protocol::{InitializeParams, MEP_VERSION, PeerInfo, methods}, +}; +use morphir_python_binding::PythonExtension; + +fn main() -> Result<()> { + let extension = NativeExtension::frontend_backend(PythonExtension)?; + // A host must complete the MEP handshake before compile or generate requests. + let initialized = extension.protocol().handle(ExtensionRequest::new( + methods::INITIALIZE, + InitializeParams { + model_transport: None, + protocol_versions: vec![MEP_VERSION.into()], + host: PeerInfo { + kind: Default::default(), + name: "python-binding-example".into(), + version: env!("CARGO_PKG_VERSION").into(), + }, + }, + 0, + )?); + if let Some(error) = initialized.error { + return Err(ExtensionError::ExecutionFailed(format!("{error:?}"))); + } + let request = CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: vec![ + SourceDocument { + uri: "models.py".into(), + language_id: "python".into(), + version: 1, + text: format!( + "{}\n{}", + include_str!("../tests/fixtures/models.py"), + include_str!("../tests/fixtures/tuples.py") + ), + }, + SourceDocument { + uri: "rules.py".into(), + language_id: "python".into(), + version: 1, + text: include_str!("../tests/fixtures/modules/rules.py").into(), + }, + ], + }, + package: CompilePackage { + name: "acme/example".into(), + exposed_modules: None, + }, + dependencies: vec![], + options: CompileOptions { + types_only: false, + ir_version: std::env::args().nth(1).unwrap_or_else(|| "4".into()), + ..Default::default() + }, + baseline: None, + }; + let compiled = + extension + .protocol() + .handle(ExtensionRequest::new(methods::COMPILE, request, 1)?); + let compiled: CompileResult = serde_json::from_value( + compiled + .result + .ok_or_else(|| ExtensionError::ExecutionFailed(format!("{:?}", compiled.error)))?, + )?; + let ir = compiled + .ir + .ok_or_else(|| ExtensionError::ExecutionFailed(format!("{:?}", compiled.diagnostics)))?; + let generated = extension.protocol().handle(ExtensionRequest::new( + methods::GENERATE, + GenerateRequest { + ir: ir.clone(), + target: "python".into(), + options: Default::default(), + }, + 2, + )?); + let generated: GenerateResult = serde_json::from_value( + generated + .result + .ok_or_else(|| ExtensionError::ExecutionFailed(format!("{:?}", generated.error)))?, + )?; + if !generated.success { + return Err(ExtensionError::ExecutionFailed(format!( + "{:?}", + generated.diagnostics + ))); + } + println!( + "{}", + serde_json::to_string_pretty( + &serde_json::json!({ "ir": ir, "artifacts": generated.artifacts }) + )? + ); + Ok(()) +} diff --git a/rust/crates/morphir-python-binding/src/backend.rs b/rust/crates/morphir-python-binding/src/backend.rs new file mode 100644 index 00000000..00de4a20 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/backend.rs @@ -0,0 +1,306 @@ +use crate::{Outcome, error, names}; +use morphir_core::{ir::v4::*, naming::FQName}; +use morphir_extension_sdk::{Artifact, GenerateRequest}; +use std::collections::{BTreeMap, BTreeSet}; + +mod functions; +mod imports; + +pub(crate) fn generate(request: &GenerateRequest) -> Outcome> { + if request.target != "python" || !request.options.is_empty() { + return Err(error( + "PY001", + "Expected target python without backend options", + )); + } + let ir = crate::ir::decode(request.ir.clone())?; + let Distribution::Library(library) = ir.distribution else { + return Err(error("PY004", "Only Library distributions are supported")); + }; + if !library.dependencies.is_empty() || library.def.modules.is_empty() { + return Err(error( + "PY004", + "Expected at least one module with no dependencies", + )); + } + crate::modules::validate_paths(library.def.modules.keys().map(String::as_str))?; + let identities = library + .def + .modules + .keys() + .map(|name| { + Ok(( + name.clone(), + crate::modules::ModuleIdentity::from_canonical(name)?, + )) + }) + .collect::>>()?; + let aliases = crate::modules::tuple_aliases(&library.package_name, library.def.modules.iter())?; + let signatures = crate::values::signatures(&library)?; + library + .def + .modules + .iter() + .map(|(name, module)| { + render_module(&library, name, module, &identities, &aliases, &signatures) + }) + .collect() +} + +/// Whether documentation has text. Empty text, which IR migrated from v3 carries, +/// documents nothing, as the v3 reader already treats it. +fn documents(doc: &Option) -> bool { + doc.as_ref().is_some_and(|doc| !doc.text().is_empty()) +} + +fn render_module( + library: &LibraryContent, + module_name: &str, + module: &AccessControlled, + identities: &BTreeMap, + tuple_aliases: &crate::values::TupleAliases, + signatures: &crate::values::Signatures, +) -> Outcome { + if documents(&module.value.doc) { + return Err(error( + "PY004", + "Module documentation is not supported by the Python backend", + )); + } + let mut symbols = BTreeSet::new(); + let mut type_names = BTreeMap::new(); + for name in module.value.types.keys() { + let python = + names::type_name(&Name::from_canonical_string(name).map_err(|e| error("PY003", e))?)?; + reserve(&mut symbols, &python)?; + type_names.insert(name.clone(), python); + } + let mut source = String::from( + "from __future__ import annotations\nfrom dataclasses import dataclass\nfrom collections.abc import Callable\n\n", + ); + let mut value_names = BTreeMap::new(); + let imported = imports::prepare( + library, + module_name, + identities, + &mut type_names, + &mut value_names, + )?; + let functions = functions::Context { + names: &value_names, + signatures, + }; + source.push_str(&imported); + let mut aliases = vec![]; + + for (name, definition) in &module.value.types { + require_public(definition.access)?; + if documents(&definition.value.doc) { + return Err(error("PY004", "Type documentation is not supported yet")); + } + let python = &type_names[name]; + match &definition.value.value { + TypeDefinition::TypeAliasDefinition { + type_params, + type_expr: tpe @ Type::Tuple(..), + } if type_params.is_empty() => { + aliases.push(format!( + "type {python} = {}\n", + annotation(tpe, &library.package_name, module_name, &type_names)? + )); + } + TypeDefinition::TypeAliasDefinition { + type_params, + type_expr: Type::Record(attrs, fields), + } if type_params.is_empty() => { + require_empty_attributes(attrs)?; + render_class( + &mut source, + python, + fields.iter().map(|f| (&f.name, &f.tpe)), + &library.package_name, + module_name, + &type_names, + )?; + } + TypeDefinition::CustomTypeDefinition { + type_params, + constructors, + } if type_params.is_empty() => { + require_public(constructors.access)?; + if constructors.value.is_empty() { + return Err(error( + "PY004", + "Empty custom types cannot be represented by a Python union", + )); + } + let mut variants = vec![]; + for constructor in &constructors.value { + let variant = names::type_name(&constructor.name)?; + reserve(&mut symbols, &variant)?; + render_class( + &mut source, + &variant, + constructor.args.iter().map(|a| (&a.name, &a.arg_type)), + &library.package_name, + module_name, + &type_names, + )?; + variants.push(variant); + } + aliases.push(format!("type {python} = {}\n", variants.join(" | "))); + } + _ => { + return Err(error( + "PY004", + "Only non-generic record or tuple aliases and custom types with public constructors are supported", + )); + } + } + } + source.push_str(&aliases.join("\n")); + for (name, definition) in &module.value.values { + require_public(definition.access)?; + if documents(&definition.value.doc) { + return Err(error( + "PY004", + "Function documentation is not supported yet", + )); + } + let name = + names::field_name(&Name::from_canonical_string(name).map_err(|e| error("PY003", e))?)?; + reserve(&mut symbols, &name)?; + source.push_str(&functions::render( + &name, + &definition.value.value, + &library.package_name, + module_name, + &type_names, + tuple_aliases, + &functions, + )?); + } + // Ruff validates the constructed declarations and owns AST-to-source rendering. + // No Python interpreter or user imports run during generation. + let content = ruff_python_codegen::round_trip(&source) + .map_err(|e| error("PY005", format!("Generated Python is invalid: {e}")))?; + Ok(Artifact { + path: identities[module_name].filename(), + content: format!("{}\n", content.trim_end()), + binary: false, + }) +} + +fn render_class<'a>( + source: &mut String, + name: &str, + fields: impl Iterator, + package: &PackageName, + module: &str, + types: &BTreeMap, +) -> Outcome<()> { + source.push_str(&format!("@dataclass(frozen=True)\nclass {name}:\n")); + let mut seen = BTreeSet::new(); + for (name, tpe) in fields { + let field = names::field_name(name)?; + reserve(&mut seen, &field)?; + source.push_str(&format!( + " {field}: {}\n", + annotation(tpe, package, module, types)? + )); + } + if seen.is_empty() { + source.push_str(" pass\n"); + } + source.push('\n'); + Ok(()) +} + +fn annotation( + tpe: &Type, + package: &PackageName, + module: &str, + types: &BTreeMap, +) -> Outcome { + require_empty_attributes(tpe.attributes())?; + match tpe { + Type::Function(_, input, output) => Ok(format!( + "Callable[[{}], {}]", + annotation(input, package, module, types)?, + annotation(output, package, module, types)? + )), + Type::Reference(_, fq, args) if args.is_empty() => reference(fq, package, module, types), + Type::Tuple(_, items) if items.len() >= 2 => Ok(format!( + "tuple[{}]", + items + .iter() + .map(|t| annotation(t, package, module, types)) + .collect::>>()? + .join(", ") + )), + _ => Err(error( + "PY004", + "Unsupported Python field type; expected scalar, local reference, or fixed tuple", + )), + } +} + +fn reference( + fq: &FQName, + package: &PackageName, + module: &str, + types: &BTreeMap, +) -> Outcome { + let canonical = fq.to_canonical_string(); + let scalar = match canonical.as_str() { + "morphir/SDK:basics#int" => Some("int"), + "morphir/SDK:basics#float" => Some("float"), + "morphir/SDK:basics#bool" => Some("bool"), + "morphir/SDK:string#string" => Some("str"), + _ => None, + }; + if let Some(scalar) = scalar { + return Ok(scalar.into()); + } + if let Some(imported) = types.get(&canonical) { + return Ok(imported.clone()); + } + let prefix = format!("{}:{module}#", package.to_canonical_string()); + canonical + .strip_prefix(&prefix) + .and_then(|local| types.get(local)) + .cloned() + .ok_or_else(|| { + error( + "PY004", + format!("Unresolved or external type reference: {canonical}"), + ) + }) +} + +fn reserve(seen: &mut BTreeSet, name: &str) -> Outcome<()> { + if !seen.insert(names::identifier(name)?.to_canonical_string()) { + return Err(error("PY003", format!("Python name collision: {name}"))); + } + Ok(()) +} + +fn require_public(access: Access) -> Outcome<()> { + if access != Access::Public { + return Err(error( + "PY004", + "Private definitions are not supported by this Python subset", + )); + } + Ok(()) +} + +fn require_empty_attributes(attrs: &TypeAttributes) -> Outcome<()> { + if *attrs != TypeAttributes::default() { + return Err(error( + "PY004", + "Type attributes cannot yet be preserved in Python", + )); + } + Ok(()) +} diff --git a/rust/crates/morphir-python-binding/src/backend/functions.rs b/rust/crates/morphir-python-binding/src/backend/functions.rs new file mode 100644 index 00000000..67c95902 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/backend/functions.rs @@ -0,0 +1,160 @@ +use super::{annotation, reserve}; +use crate::{ + Outcome, error, names, + values::{self, unsupported}, +}; +use morphir_core::ir::v4::*; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) fn render( + name: &str, + definition: &ValueDefinition, + package: &PackageName, + module: &str, + types: &BTreeMap, + aliases: &values::TupleAliases, + context: &Context<'_>, +) -> Outcome { + let ValueBody::Expression(body) = &definition.body else { + return Err(unsupported( + "Only expression function bodies can be generated as Python", + )); + }; + let output_type = definition + .output_type + .as_ref() + .ok_or_else(|| unsupported("Function return type is required"))?; + let output = annotation(output_type, package, module, types)?; + let mut seen = BTreeSet::new(); + let inputs = definition + .input_types + .iter() + .map(|(name, entry)| { + let parameter_name = + Name::from_canonical_string(name).map_err(|e| error("PY003", e))?; + let python = names::field_name(¶meter_name)?; + reserve(&mut seen, &python)?; + Ok(format!( + "{python}: {}", + annotation(entry, package, module, types)? + )) + }) + .collect::>>()?; + values::validate_function(definition, aliases, context.signatures)?; + let mut source = format!("\ndef {name}({}) -> {output}:\n", inputs.join(", ")); + context.render_body(&mut source, body, 1)?; + Ok(source) +} + +pub(super) struct Context<'a> { + pub(super) names: &'a BTreeMap, + pub(super) signatures: &'a values::Signatures, +} + +impl Context<'_> { + fn render_body(&self, source: &mut String, body: &Value, depth: usize) -> Outcome<()> { + let indent = " ".repeat(depth); + match body { + Value::IfThenElse(_, condition, then_branch, else_branch) => { + source.push_str(&format!("{indent}if {}:\n", self.expression(condition)?)); + self.render_body(source, then_branch, depth + 1)?; + source.push_str(&format!("{indent}else:\n")); + self.render_body(source, else_branch, depth + 1)?; + } + _ => source.push_str(&format!("{indent}return {}\n", self.expression(body)?)), + } + Ok(()) + } + + fn expression(&self, value: &Value) -> Outcome { + match value { + Value::Lambda(_, Pattern::AsPattern(_, _, name), body) => Ok(format!( + "(lambda {}: {})", + names::field_name(name)?, + self.expression(body)? + )), + Value::Reference(_, name) => { + let signature = &self.signatures[&name.to_canonical_string()]; + let python = self.reference(name)?; + match signature.inputs.len() { + 0 => Ok(format!("{python}()")), + 1 => Ok(python), + _ => Err(unsupported( + "Only unary named functions can be used as callable values", + )), + } + } + Value::Variable(_, name) => names::field_name(name), + Value::Tuple(_, elements) => Ok(format!( + "({})", + elements + .iter() + .map(|value| self.expression(value)) + .collect::>>()? + .join(", ") + )), + Value::Literal(_, Literal::Bool(value)) => { + Ok(if *value { "True" } else { "False" }.into()) + } + Value::Literal(_, Literal::Integer(value)) => Ok(value.to_string()), + Value::Literal(_, Literal::Float(value)) => Ok(format!("{:?}", value.value())), + // JSON escaping is also valid in a Python string literal, including control characters. + Value::Literal(_, Literal::String(value)) => { + serde_json::to_string(value).map_err(|e| error("PY005", e.to_string())) + } + Value::IfThenElse(_, condition, then_branch, else_branch) => Ok(format!( + "({} if {} else {})", + self.expression(then_branch)?, + self.expression(condition)?, + self.expression(else_branch)? + )), + Value::Apply(..) => { + if let Ok((operator, left, right)) = values::comparison(value) { + return Ok(format!( + "({} {} {})", + self.expression(left)?, + operator.python(), + self.expression(right)? + )); + } + let mut head = value; + let mut arguments = vec![]; + while let Value::Apply(_, function, argument) = head { + arguments.push(argument.as_ref()); + head = function; + } + arguments.reverse(); + let (mut source, consumed) = if let Value::Reference(_, name) = head { + let count = self.signatures[&name.to_canonical_string()].inputs.len(); + if arguments.len() < count { + return Err(unsupported( + "Partial applications of multi-parameter functions are not supported", + )); + } + let args = arguments[..count] + .iter() + .map(|arg| self.expression(arg)) + .collect::>>()?; + ( + format!("{}({})", self.reference(name)?, args.join(", ")), + count, + ) + } else { + (self.expression(head)?, 0) + }; + for argument in &arguments[consumed..] { + source = format!("{source}({})", self.expression(argument)?); + } + Ok(source) + } + _ => Err(unsupported("Unsupported Python function expression")), + } + } + + fn reference(&self, name: &morphir_core::naming::FQName) -> Outcome { + self.names + .get(&name.to_canonical_string()) + .cloned() + .ok_or_else(|| unsupported("Unknown function reference")) + } +} diff --git a/rust/crates/morphir-python-binding/src/backend/imports.rs b/rust/crates/morphir-python-binding/src/backend/imports.rs new file mode 100644 index 00000000..a1830556 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/backend/imports.rs @@ -0,0 +1,140 @@ +//! Generate module-qualified type annotations to avoid imported name collisions. + +use crate::{Outcome, error, modules::ModuleIdentity, names}; +use morphir_core::ir::v4::*; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) fn prepare( + library: &LibraryContent, + current: &str, + identities: &BTreeMap, + scope: &mut BTreeMap, + values: &mut BTreeMap, +) -> Outcome { + let definition = &library.def.modules[current].value; + let mut used: BTreeSet<_> = definition + .types + .keys() + .chain(definition.values.keys()) + .cloned() + .collect(); + let mut references = BTreeSet::new(); + let mut value_references = BTreeSet::new(); + for entry in definition.types.values() { + match &entry.value.value { + TypeDefinition::TypeAliasDefinition { type_expr, .. } => { + collect(type_expr, &mut references) + } + TypeDefinition::CustomTypeDefinition { constructors, .. } => { + for constructor in &constructors.value { + used.insert(constructor.name.to_canonical_string()); + for arg in &constructor.args { + collect(&arg.arg_type, &mut references); + } + } + } + _ => {} + } + } + for entry in definition.values.values() { + used.extend(entry.value.value.input_types.keys().cloned()); + if let ValueBody::Expression(body) = &entry.value.value.body { + collect_value(body, &mut value_references, &mut used); + } + for input in entry.value.value.input_types.values() { + collect(input, &mut references); + } + if let Some(output) = &entry.value.value.output_type { + collect(output, &mut references); + } + } + let mut source = String::new(); + let mut next_alias = 1; + for (module, identity) in identities { + let prefix = format!("{}:{module}#", library.package_name.to_canonical_string()); + if !(module != current && references.iter().any(|fq| fq.starts_with(&prefix))) + && !value_references.iter().any(|fq| fq.starts_with(&prefix)) + { + continue; + } + let alias = loop { + let candidate = format!("morphir_module_{next_alias}"); + next_alias += 1; + if used.insert(Name::from(candidate.as_str()).to_canonical_string()) { + break candidate; + } + }; + source.push_str(&format!("import {} as {alias}\n", identity.python)); + for name in library.def.modules[module].value.types.keys() { + let python = names::type_name( + &Name::from_canonical_string(name).map_err(|e| error("PY003", e))?, + )?; + scope.insert(format!("{prefix}{name}"), format!("{alias}.{python}")); + } + for name in library.def.modules[module].value.values.keys() { + let python = names::field_name( + &Name::from_canonical_string(name).map_err(|e| error("PY003", e))?, + )?; + values.insert(format!("{prefix}{name}"), format!("{alias}.{python}")); + } + } + if !source.is_empty() { + source.push('\n'); + } + Ok(source) +} + +fn collect(tpe: &Type, references: &mut BTreeSet) { + match tpe { + Type::Function(_, input, output) => { + collect(input, references); + collect(output, references); + } + Type::Reference(_, fq, args) => { + references.insert(fq.to_canonical_string()); + for arg in args { + collect(arg, references); + } + } + Type::Tuple(_, items) => { + for item in items { + collect(item, references); + } + } + Type::Record(_, fields) => { + for field in fields { + collect(&field.tpe, references); + } + } + _ => {} + } +} + +fn collect_value(value: &Value, references: &mut BTreeSet, used: &mut BTreeSet) { + match value { + Value::Reference(_, name) => { + references.insert(name.to_canonical_string()); + } + Value::Lambda(_, pattern, body) => { + if let Pattern::AsPattern(_, _, name) = pattern { + used.insert(name.to_canonical_string()); + } + collect_value(body, references, used); + } + Value::Apply(_, function, argument) => { + collect_value(function, references, used); + collect_value(argument, references, used); + } + Value::Tuple(_, elements) => { + for value in elements { + collect_value(value, references, used); + } + } + Value::IfThenElse(_, condition, yes, no) => { + for value in [condition, yes, no] { + collect_value(value, references, used); + } + } + _ => {} + } +} diff --git a/rust/crates/morphir-python-binding/src/frontend.rs b/rust/crates/morphir-python-binding/src/frontend.rs new file mode 100644 index 00000000..74a6f631 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/frontend.rs @@ -0,0 +1,390 @@ +use crate::{Outcome, error, names}; +use morphir_core::{ir::v4::*, naming::FQName}; +use morphir_extension_sdk::{ + CompileRequest, Diagnostic, SourceLocation, SourcePosition, SourceRange, +}; +use ruff_python_ast::{Expr, Operator, Stmt, StmtClassDef}; +use ruff_text_size::TextRange; +use std::collections::{BTreeMap, BTreeSet}; + +mod functions; +mod imports; +mod project; + +#[derive(Clone)] +enum Symbol { + Type(FQName), + Function(FQName), + Callable, +} +type TypeScope = BTreeMap; + +pub(crate) fn compile(request: &CompileRequest) -> Outcome<(IRFile, Vec)> { + crate::ir::Version::parse(&request.options.ir_version)?; + if request.language_id != "python" || !request.dependencies.is_empty() { + return Err(error("PY001", "Expected Python and no dependencies")); + } + for (key, value) in &request.options.extra { + let valid = match key.as_str() { + "outputDir" => value.is_string(), + "emitParseStage" | "emitParseStageFatal" => value.is_boolean(), + _ => false, + }; + if !valid { + return Err(error( + "PY001", + format!("Unknown or invalid compile option: {key}"), + )); + } + } + if parse_stage_requested(request) + && request + .options + .extra + .get("emitParseStageFatal") + .and_then(serde_json::Value::as_bool) + == Some(true) + { + return Err(error( + "PY004", + "Python parse-stage output is not implemented", + )); + } + project::compile(request) +} + +pub(crate) fn parse_stage_requested(request: &CompileRequest) -> bool { + request + .options + .extra + .get("emitParseStage") + .and_then(serde_json::Value::as_bool) + == Some(true) +} + +fn lower( + statements: &[Stmt], + types: &TypeScope, + aliases: Option<&crate::values::TupleAliases>, + signatures: &crate::values::Signatures, +) -> Outcome { + let mut classes = BTreeMap::new(); + let mut sums = BTreeMap::new(); + let mut tuple_aliases = BTreeMap::new(); + let mut declared = BTreeSet::new(); + let mut functions = BTreeMap::new(); + let mut dataclass_imported = false; + for statement in statements { + match statement { + Stmt::ImportFrom(import) => { + // The package resolver validates all imports and their binding names. + if import.level == 0 + && import.module.as_ref().map(|m| m.as_str()) == Some("dataclasses") + { + dataclass_imported = true; + } + } + Stmt::Import(_) => {} + Stmt::ClassDef(class) => { + if !dataclass_imported { + return Err(error( + "PY004", + "Import dataclass before declaring a product or variant", + )); + } + declare(&mut declared, class.name.as_str())?; + names::type_name(&names::identifier(class.name.as_str())?)?; + validate_class(class)?; + classes.insert(class.name.to_string(), class); + } + Stmt::TypeAlias(alias) if alias.type_params.is_none() => { + let name = expr_name(&alias.name)?; + declare(&mut declared, name)?; + names::type_name(&names::identifier(name)?)?; + if matches!(alias.value.as_ref(), Expr::Subscript(subscript) if matches!(subscript.value.as_ref(), Expr::Name(name) if name.id.as_str() == "tuple")) + { + tuple_aliases.insert(name.to_owned(), alias.value.as_ref()); + continue; + } + let mut variants = vec![]; + union_members(&alias.value, &mut variants)?; + sums.insert(name.to_owned(), variants); + } + Stmt::FunctionDef(function) => { + declare(&mut declared, function.name.as_str())?; + names::field_name(&names::identifier(function.name.as_str())?)?; + functions.insert(function.name.to_string(), function); + } + _ => { + return Err(error( + "PY004", + "Only frozen dataclasses, non-generic sum or tuple aliases and annotated pure functions are supported", + )); + } + } + } + let mut owned_variants = BTreeSet::new(); + for variants in sums.values() { + for variant in variants { + if !classes.contains_key(*variant) || !owned_variants.insert(*variant) { + return Err(error( + "PY004", + format!( + "Variant {variant} must name a dataclass and belong to exactly one sum" + ), + )); + } + } + } + let mut definitions = BTreeMap::new(); + + for (name, expression) in &tuple_aliases { + let canonical = names::identifier(name)?.to_canonical_string(); + let type_expr = annotation(expression, types)?; + definitions.insert( + canonical, + TypeDefinition::TypeAliasDefinition { + type_params: vec![], + type_expr, + }, + ); + } + for (name, class) in &classes { + if !owned_variants.contains(name.as_str()) { + definitions.insert( + names::identifier(name)?.to_canonical_string(), + TypeDefinition::TypeAliasDefinition { + type_params: vec![], + type_expr: Type::Record(Default::default(), fields(class, types)?), + }, + ); + } + } + for (name, variants) in &sums { + let constructors = variants + .iter() + .map(|variant| { + Ok(ConstructorDefinition { + name: names::identifier(variant)?, + args: fields(classes[*variant], types)? + .into_iter() + .map(|field| ConstructorArg { + name: field.name, + arg_type: field.tpe, + }) + .collect(), + }) + }) + .collect::>>()?; + definitions.insert( + names::identifier(name)?.to_canonical_string(), + TypeDefinition::CustomTypeDefinition { + type_params: vec![], + constructors: public(constructors), + }, + ); + } + Ok(ModuleDefinition { + types: definitions + .into_iter() + .map(|(name, definition)| (name, public(Documented::new(None, definition)))) + .collect(), + values: functions + .into_iter() + .filter_map(|(name, function)| aliases.map(|aliases| (name, function, aliases))) + .map(|(name, function, aliases)| { + Ok(( + names::identifier(&name)?.to_canonical_string(), + public(Documented::new( + None, + functions::lower(function, types, aliases, signatures)?, + )), + )) + }) + .collect::>()?, + doc: None, + }) +} + +fn declare(declared: &mut BTreeSet, name: &str) -> Outcome<()> { + if !declared.insert(names::identifier(name)?.to_canonical_string()) { + return Err(error( + "PY003", + format!("Duplicate declaration after Morphir name normalization: {name}"), + )); + } + Ok(()) +} + +fn validate_class(class: &StmtClassDef) -> Outcome<()> { + if class.type_params.is_some() || class.arguments.is_some() || class.decorator_list.len() != 1 { + return Err(error( + "PY004", + "Dataclasses cannot have type parameters, bases, metaclasses or additional decorators", + )); + } + let Expr::Call(call) = &class.decorator_list[0].expression else { + return Err(error("PY004", "Expected @dataclass(frozen=True)")); + }; + if expr_name(&call.func)? != "dataclass" + || !call.arguments.args.is_empty() + || call.arguments.keywords.len() != 1 + || call.arguments.keywords[0].arg.as_ref().map(|s| s.as_str()) != Some("frozen") + || !matches!(&call.arguments.keywords[0].value, Expr::BooleanLiteral(v) if v.value) + { + return Err(error("PY004", "Only @dataclass(frozen=True) is supported")); + } + Ok(()) +} + +fn fields(class: &StmtClassDef, types: &TypeScope) -> Outcome> { + let mut seen = BTreeSet::new(); + class + .body + .iter() + .filter(|s| !matches!(s, Stmt::Pass(_))) + .map(|statement| { + let Stmt::AnnAssign(field) = statement else { + return Err(error( + "PY004", + "Dataclass bodies may contain only annotated fields without defaults, or pass", + )); + }; + if field.value.is_some() { + return Err(error("PY004", "Field defaults are not supported")); + } + let name = expr_name(&field.target)?; + declare(&mut seen, name)?; + let name = names::identifier(name)?; + names::field_name(&name)?; + Ok(Field::new(name, annotation(&field.annotation, types)?)) + }) + .collect() +} + +fn annotation(expr: &Expr, types: &TypeScope) -> Outcome { + match expr { + Expr::Name(name) => { + if let Some(Symbol::Type(fq)) = types.get(name.id.as_str()) { + return Ok(Type::Reference(Default::default(), fq.clone(), vec![])); + } + let fq = match name.id.as_str() { + "int" => "morphir/SDK:basics#int", + "float" => "morphir/SDK:basics#float", + "bool" => "morphir/SDK:basics#bool", + "str" => "morphir/SDK:string#string", + other => { + return Err(error( + "PY004", + format!("Unknown type or constructor used as a type: {other}"), + )); + } + }; + Ok(Type::Reference( + Default::default(), + FQName::from_canonical_string(fq).map_err(|e| error("PY003", e))?, + vec![], + )) + } + Expr::Attribute(_) => { + let name = imports::qualified_name(expr)?; + let Some(Symbol::Type(fq)) = types.get(&name) else { + return Err(error("PY004", format!("Unknown imported type: {name}"))); + }; + Ok(Type::Reference(Default::default(), fq.clone(), vec![])) + } + Expr::Subscript(subscript) + if matches!( + types.get(&imports::qualified_name(&subscript.value)?), + Some(Symbol::Callable) + ) => + { + let Expr::Tuple(parts) = subscript.slice.as_ref() else { + return Err(error("PY004", "Expected Callable[[input], output]")); + }; + let [Expr::List(inputs), output] = parts.elts.as_slice() else { + return Err(error("PY004", "Expected Callable[[input], output]")); + }; + let [input] = inputs.elts.as_slice() else { + return Err(error( + "PY004", + "Callable requires exactly one input; nest Callable for curried functions", + )); + }; + Ok(Type::Function( + Default::default(), + Box::new(annotation(input, types)?), + Box::new(annotation(output, types)?), + )) + } + Expr::Subscript(subscript) if expr_name(&subscript.value)? == "tuple" => { + let elements: Vec<&Expr> = match subscript.slice.as_ref() { + Expr::Tuple(tuple) => tuple.elts.iter().collect(), + element => vec![element], + }; + if elements.len() < 2 { + return Err(error("PY004", "Tuples require at least two fixed elements")); + } + Ok(Type::Tuple( + Default::default(), + elements + .into_iter() + .map(|e| annotation(e, types)) + .collect::>()?, + )) + } + _ => Err(error( + "PY004", + "Expected int, float, bool, str, a local type name, or a fixed tuple annotation", + )), + } +} + +fn union_members<'a>(expr: &'a Expr, members: &mut Vec<&'a str>) -> Outcome<()> { + match expr { + Expr::BinOp(binary) if binary.op == Operator::BitOr => { + union_members(&binary.left, members)?; + union_members(&binary.right, members) + } + Expr::Name(name) => { + members.push(name.id.as_str()); + Ok(()) + } + _ => Err(error( + "PY004", + "A sum alias must list dataclass variants separated by |", + )), + } +} + +fn expr_name(expr: &Expr) -> Outcome<&str> { + match expr { + Expr::Name(name) => Ok(name.id.as_str()), + _ => Err(error("PY004", "Expected a simple Python name")), + } +} + +fn public(value: T) -> AccessControlled { + AccessControlled { + access: Access::Public, + value, + } +} + +fn at(mut diagnostic: Diagnostic, uri: &str, source: &str, range: TextRange) -> Diagnostic { + let position = |offset: usize| { + let prefix = &source[..offset.min(source.len())]; + SourcePosition::from_line_prefix( + prefix.bytes().filter(|b| *b == b'\n').count() as u32, + prefix.rsplit('\n').next().unwrap_or(""), + ) + }; + diagnostic.location = Some(SourceLocation { + uri: uri.into(), + range: SourceRange { + start: position(range.start().to_usize()), + end: position(range.end().to_usize()), + }, + }); + diagnostic +} diff --git a/rust/crates/morphir-python-binding/src/frontend/functions.rs b/rust/crates/morphir-python-binding/src/frontend/functions.rs new file mode 100644 index 00000000..81d227b4 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/frontend/functions.rs @@ -0,0 +1,352 @@ +use super::{annotation, declare}; +use crate::{ + Outcome, names, + values::{self, Comparison, unsupported}, +}; +use morphir_core::ir::v4::*; +use num_bigint::BigInt; +use ruff_python_ast::{CmpOp, Expr, Number, Stmt, StmtFunctionDef, UnaryOp}; +use std::collections::BTreeSet; + +pub(super) fn signature( + function: &StmtFunctionDef, + types: &super::TypeScope, +) -> Outcome { + let parameters = &function.parameters; + if function.is_async + || function.type_params.is_some() + || !function.decorator_list.is_empty() + || !parameters.posonlyargs.is_empty() + || !parameters.kwonlyargs.is_empty() + || parameters.vararg.is_some() + || parameters.kwarg.is_some() + { + return Err(unsupported( + "Expected a synchronous, non-generic function with ordinary annotated parameters and no decorators", + )); + } + let mut seen = BTreeSet::new(); + let mut source_names = BTreeSet::new(); + let input_types = parameters + .args + .iter() + .map(|parameter| { + if parameter.default.is_some() { + return Err(unsupported("Parameter defaults are not supported")); + } + let source_name = parameter.parameter.name.as_str(); + declare(&mut seen, source_name)?; + let name = names::identifier(source_name)?; + names::field_name(&name)?; + source_names.insert(source_name); + let tpe = annotation( + parameter.annotation().ok_or_else(|| { + unsupported("Every function parameter needs a type annotation") + })?, + types, + )?; + Ok((name.to_canonical_string(), tpe)) + }) + .collect::>()?; + let output_type = annotation( + function + .returns + .as_deref() + .ok_or_else(|| unsupported("Functions need a return type annotation"))?, + types, + )?; + Ok(ValueSpecification { + annotations: Vec::new().into(), + inputs: input_types, + output: output_type, + }) +} + +pub(super) fn lower( + function: &StmtFunctionDef, + types: &super::TypeScope, + aliases: &values::TupleAliases, + signatures: &values::Signatures, +) -> Outcome { + let signature = signature(function, types)?; + let source_names = function + .parameters + .args + .iter() + .map(|p| p.parameter.name.as_str()) + .collect(); + let context = Context { types, signatures }; + let body = context.block(&function.body, None, &source_names)?; + let definition = ValueDefinition { + input_types: signature.inputs, + output_type: Some(signature.output), + body: ValueBody::Expression(body), + }; + values::validate_function(&definition, aliases, signatures)?; + Ok(definition) +} + +struct Context<'a> { + types: &'a super::TypeScope, + signatures: &'a values::Signatures, +} + +impl Context<'_> { + // The continuation is used only by paths that reach the end of a branch. + fn block( + &self, + statements: &[Stmt], + continuation: Option<&Value>, + parameters: &BTreeSet<&str>, + ) -> Outcome { + let Some((first, rest)) = statements.split_first() else { + return continuation + .cloned() + .ok_or_else(|| unsupported("Every function path must return a value")); + }; + match first { + Stmt::Return(statement) if rest.is_empty() => self.expression( + statement + .value + .as_deref() + .ok_or_else(|| unsupported("Bare return is not supported"))?, + parameters, + ), + Stmt::If(statement) => { + let fallback = if rest.is_empty() { + continuation.cloned() + } else { + Some(self.block(rest, continuation, parameters)?) + }; + let mut otherwise = fallback.clone(); + for clause in statement.elif_else_clauses.iter().rev() { + let branch = self.block(&clause.body, fallback.as_ref(), parameters)?; + otherwise = Some(match &clause.test { + Some(test) => conditional( + self.expression(test, parameters)?, + branch, + otherwise.ok_or_else(|| { + unsupported("Every function path must return a value") + })?, + ), + None => branch, + }); + } + Ok(conditional( + self.expression(&statement.test, parameters)?, + self.block(&statement.body, fallback.as_ref(), parameters)?, + otherwise + .ok_or_else(|| unsupported("Every function path must return a value"))?, + )) + } + _ => Err(unsupported( + "Function bodies may contain only returns and if/elif/else statements; unreachable statements are not supported", + )), + } + } + + fn expression(&self, expr: &Expr, parameters: &BTreeSet<&str>) -> Outcome { + let literal = match expr { + Expr::Lambda(lambda) => { + let p = lambda + .parameters + .as_ref() + .ok_or_else(|| unsupported("A lambda requires one parameter"))?; + if p.args.len() != 1 + || !p.posonlyargs.is_empty() + || !p.kwonlyargs.is_empty() + || p.vararg.is_some() + || p.kwarg.is_some() + || p.args[0].default.is_some() + { + return Err(unsupported( + "Lambdas require exactly one ordinary parameter without a default", + )); + } + let source = p.args[0].parameter.name.as_str(); + let name = names::identifier(source)?; + names::field_name(&name)?; + if parameters.iter().any(|outer| { + *outer != source && names::identifier(outer).is_ok_and(|outer| outer == name) + }) { + return Err(unsupported( + "Lambda parameter spelling collides with an enclosing parameter after name normalization", + )); + } + let mut scope = parameters.clone(); + scope.insert(source); + return Ok(Value::Lambda( + Default::default(), + Pattern::AsPattern( + Default::default(), + Box::new(Pattern::WildcardPattern(Default::default())), + name, + ), + Box::new(self.expression(&lambda.body, &scope)?), + )); + } + Expr::Call(call) => { + if !call.arguments.keywords.is_empty() + || call + .arguments + .args + .iter() + .any(|arg| matches!(arg, Expr::Starred(_))) + { + return Err(unsupported( + "Only positional calls without starred arguments are supported", + )); + } + let function = + if let Some((name, signature)) = self.reference(&call.func, parameters) { + if call.arguments.args.len() != signature.inputs.len() { + return Err(unsupported( + "Named function calls must supply every declared parameter", + )); + } + Value::Reference(Default::default(), name.clone()) + } else { + if call.arguments.args.len() != 1 { + return Err(unsupported("Callable values take exactly one argument")); + } + self.expression(&call.func, parameters)? + }; + return call + .arguments + .args + .iter() + .try_fold(function, |function, arg| { + Ok(Value::Apply( + Default::default(), + Box::new(function), + Box::new(self.expression(arg, parameters)?), + )) + }); + } + Expr::Name(_) | Expr::Attribute(_) if self.reference(expr, parameters).is_some() => { + let (name, signature) = self.reference(expr, parameters).unwrap(); + if signature.inputs.len() != 1 { + return Err(unsupported( + "Only unary named functions can be used as callable values", + )); + } + return Ok(Value::Reference(Default::default(), name.clone())); + } + Expr::Tuple(tuple) if tuple.elts.len() >= 2 => { + return Ok(Value::Tuple( + Default::default(), + tuple + .elts + .iter() + .map(|element| self.expression(element, parameters)) + .collect::>()?, + )); + } + Expr::Name(name) if parameters.contains(name.id.as_str()) => { + return Ok(Value::Variable( + Default::default(), + names::identifier(name.id.as_str())?, + )); + } + Expr::If(expression_if) => { + return Ok(conditional( + self.expression(&expression_if.test, parameters)?, + self.expression(&expression_if.body, parameters)?, + self.expression(&expression_if.orelse, parameters)?, + )); + } + Expr::Compare(compare) if compare.ops.len() == 1 && compare.operands.len() == 2 => { + let operator = match compare.ops[0] { + CmpOp::Eq => Comparison::Equal, + CmpOp::NotEq => Comparison::NotEqual, + CmpOp::Lt => Comparison::Less, + CmpOp::LtE => Comparison::LessEqual, + CmpOp::Gt => Comparison::Greater, + CmpOp::GtE => Comparison::GreaterEqual, + _ => { + return Err(unsupported( + "Identity and membership comparisons are not supported", + )); + } + }; + return Ok(operator.apply( + self.expression(&compare.operands[0], parameters)?, + self.expression(&compare.operands[1], parameters)?, + )); + } + Expr::BooleanLiteral(value) => Literal::Bool(value.value), + Expr::StringLiteral(value) => Literal::String(value.value.to_string()), + Expr::NumberLiteral(value) => number(&value.value, false)?, + Expr::UnaryOp(unary) if unary.op == UnaryOp::USub => { + let Expr::NumberLiteral(value) = unary.operand.as_ref() else { + return Err(unsupported( + "Unary minus is supported only for numeric literals", + )); + }; + number(&value.value, true)? + } + _ => { + return Err(unsupported( + "Expected a parameter, scalar literal, fixed tuple, comparison or conditional expression", + )); + } + }; + Ok(Value::Literal(Default::default(), literal)) + } + + fn reference( + &self, + expr: &Expr, + parameters: &BTreeSet<&str>, + ) -> Option<(&morphir_core::naming::FQName, &ValueSpecification)> { + let name = super::imports::qualified_name(expr).ok()?; + if parameters.contains(name.split('.').next()?) { + return None; + } + let super::Symbol::Function(fq) = self.types.get(&name)? else { + return None; + }; + Some((fq, self.signatures.get(&fq.to_canonical_string())?)) + } +} + +fn conditional(condition: Value, then_branch: Value, else_branch: Value) -> Value { + Value::IfThenElse( + Default::default(), + Box::new(condition), + Box::new(then_branch), + Box::new(else_branch), + ) +} + +fn number(value: &Number, negative: bool) -> Outcome { + match value { + Number::Int(value) => { + // Ruff retains the original token for integers larger than u64. + let token = value.to_string().replace('_', "").to_ascii_lowercase(); + let (digits, radix) = match token.get(..2) { + Some("0x") => (&token[2..], 16), + Some("0o") => (&token[2..], 8), + Some("0b") => (&token[2..], 2), + _ => (token.as_str(), 10), + }; + let magnitude = BigInt::parse_bytes(digits.as_bytes(), radix) + .ok_or_else(|| unsupported("Invalid integer literal"))?; + Ok(Literal::Integer(if negative { + -magnitude + } else { + magnitude + })) + } + Number::Float(value) if value.is_finite() => { + Ok(Literal::Float(FloatLiteral::from_f64(if negative { + -*value + } else { + *value + }))) + } + _ => Err(unsupported( + "Only finite real numeric literals are supported", + )), + } +} diff --git a/rust/crates/morphir-python-binding/src/frontend/imports.rs b/rust/crates/morphir-python-binding/src/frontend/imports.rs new file mode 100644 index 00000000..30ba1a71 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/frontend/imports.rs @@ -0,0 +1,215 @@ +//! Static import resolution. Imports never load or execute Python code. + +use super::{Symbol, TypeScope, declare, union_members}; +use crate::{Outcome, error, modules::ModuleIdentity, names}; +use morphir_core::{ir::v4::PackageName, naming::FQName}; +use ruff_python_ast::{Expr, Stmt}; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) type Exports = BTreeMap; + +pub(super) fn exports( + statements: &[Stmt], + package: &PackageName, + module: &str, +) -> Outcome { + let mut types = BTreeSet::new(); + let mut variants = vec![]; + let mut functions = BTreeSet::new(); + for statement in statements { + match statement { + Stmt::FunctionDef(function) => { + types.insert(function.name.to_string()); + functions.insert(function.name.to_string()); + } + Stmt::ClassDef(class) => { + types.insert(class.name.to_string()); + } + Stmt::TypeAlias(alias) => { + types.insert(super::expr_name(&alias.name)?.to_owned()); + if !matches!(alias.value.as_ref(), Expr::Subscript(_)) { + union_members(&alias.value, &mut variants)?; + } + } + _ => {} + } + } + types + .into_iter() + .filter(|name| !variants.contains(&name.as_str())) + .map(|name| { + let fq = format!( + "{}:{module}#{}", + package.to_canonical_string(), + names::identifier(&name)?.to_canonical_string() + ); + let fq = FQName::from_canonical_string(&fq).map_err(|e| error("PY003", e))?; + let symbol = if functions.contains(&name) { + Symbol::Function(fq) + } else { + Symbol::Type(fq) + }; + Ok((name, symbol)) + }) + .collect() +} + +pub(super) fn scope( + statements: &[Stmt], + identity: &ModuleIdentity, + exports: &Exports, +) -> Outcome { + let mut types = exports[&identity.python].clone(); + let mut bindings = BTreeSet::new(); + let mut module_bindings = BTreeMap::new(); + for statement in statements { + match statement { + Stmt::ClassDef(class) => declare(&mut bindings, class.name.as_str())?, + Stmt::TypeAlias(alias) => declare(&mut bindings, super::expr_name(&alias.name)?)?, + Stmt::FunctionDef(function) => declare(&mut bindings, function.name.as_str())?, + _ => {} + } + } + for (index, statement) in statements.iter().enumerate() { + match statement { + Stmt::ImportFrom(import) => { + let module = import.module.as_ref().map(|m| m.as_str()).unwrap_or(""); + if import.is_lazy { + return Err(error("PY004", "Lazy imports are not supported")); + } + if import.level == 0 && ["typing", "collections.abc"].contains(&module) { + for alias in &import.names { + if alias.name.as_str() != "Callable" { + return Err(error( + "PY004", + "Only Callable is supported from typing or collections.abc", + )); + } + let bound = alias + .asname + .as_ref() + .map(|n| n.as_str()) + .unwrap_or("Callable"); + if bound == "Callable" { + if !bindings.insert("callable".into()) { + return Err(error("PY003", "Duplicate Callable binding")); + } + } else { + declare(&mut bindings, bound)?; + } + types.insert(bound.into(), Symbol::Callable); + } + continue; + } + if import.level == 0 && ["dataclasses", "__future__"].contains(&module) { + let expected = if module == "dataclasses" { + "dataclass" + } else { + "annotations" + }; + if import.names.len() != 1 + || import.names[0].name.as_str() != expected + || import.names[0].asname.is_some() + { + return Err(error( + "PY004", + "Only dataclasses.dataclass and __future__.annotations standard imports are supported", + )); + } + if module == "__future__" && !statements[..index].iter().all(|s| { + matches!(s, Stmt::ImportFrom(i) if i.level == 0 && i.module.as_ref().map(|m| m.as_str()) == Some("__future__")) + }) { + return Err(error("PY004", "Future imports must precede other statements")); + } + continue; + } + let target = absolute_module(&identity.python, module, import.level)?; + for alias in &import.names { + let name = alias.name.as_str(); + let bound = alias.asname.as_ref().map(|a| a.as_str()).unwrap_or(name); + declare(&mut bindings, bound)?; + if let Some(fq) = exports.get(&target).and_then(|types| types.get(name)) { + types.insert(bound.into(), fq.clone()); + } else { + let child = if target.is_empty() { + name.into() + } else { + format!("{target}.{name}") + }; + import_module(&mut types, exports, &child, bound)?; + } + } + } + Stmt::Import(import) => { + if import.is_lazy { + return Err(error("PY004", "Lazy imports are not supported")); + } + for alias in &import.names { + let target = alias.name.as_str(); + let bound = alias.asname.as_ref().map(|a| a.as_str()).unwrap_or(target); + let root = bound.split('.').next().unwrap_or(bound); + let binding = if alias.asname.is_some() { target } else { root }; + match module_bindings.get(root) { + Some(existing) if existing == &binding => {} + _ => { + declare(&mut bindings, root)?; + module_bindings.insert(root, binding); + } + } + if ["typing", "collections.abc"].contains(&target) { + types.insert(format!("{bound}.Callable"), Symbol::Callable); + } else { + import_module(&mut types, exports, target, bound)?; + } + } + } + _ => {} + } + } + Ok(types) +} + +fn absolute_module(current: &str, module: &str, level: u32) -> Outcome { + if level == 0 { + return Ok(module.into()); + } + let mut segments: Vec<_> = current.split('.').collect(); + if level as usize >= segments.len() { + return Err(error("PY004", "Relative import escapes the source package")); + } + segments.truncate(segments.len() - level as usize); + if !module.is_empty() { + segments.push(module); + } + Ok(segments.join(".")) +} + +fn import_module( + scope: &mut TypeScope, + exports: &Exports, + target: &str, + bound: &str, +) -> Outcome<()> { + let types = exports.get(target).ok_or_else(|| { + error( + "PY004", + format!("Unresolved local module or type import: {target}"), + ) + })?; + for (name, fq) in types { + scope.insert(format!("{bound}.{name}"), fq.clone()); + } + Ok(()) +} + +pub(super) fn qualified_name(expr: &Expr) -> Outcome { + match expr { + Expr::Name(name) => Ok(name.id.to_string()), + Expr::Attribute(attribute) => Ok(format!( + "{}.{}", + qualified_name(&attribute.value)?, + attribute.attr + )), + _ => Err(error("PY004", "Expected an imported module and type name")), + } +} diff --git a/rust/crates/morphir-python-binding/src/frontend/project.rs b/rust/crates/morphir-python-binding/src/frontend/project.rs new file mode 100644 index 00000000..b8828aed --- /dev/null +++ b/rust/crates/morphir-python-binding/src/frontend/project.rs @@ -0,0 +1,138 @@ +//! Compile a closed set of source modules in declaration and value passes. + +use super::{at, imports, lower}; +use crate::{ + Outcome, error, + modules::{self, ModuleIdentity}, +}; +use morphir_core::ir::v4::*; +use morphir_extension_sdk::CompileRequest; +use ruff_text_size::Ranged; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) fn compile(request: &CompileRequest) -> Outcome<(IRFile, Vec)> { + if request.sources.documents.is_empty() { + return Err(error("PY001", "Expected at least one Python source module")); + } + let package = + PackageName::from_canonical_string(&request.package.name).map_err(|e| error("PY001", e))?; + if package.is_empty() { + return Err(error("PY001", "Package name must not be empty")); + } + let paths = request + .source_paths() + .map_err(|e| error("PY001", e.to_string()))?; + let parsed = request + .sources + .documents + .iter() + .zip(&paths) + .map(|(document, path)| { + if document.language_id != "python" { + return Err(error("PY001", "Source document language must be python")); + } + let identity = ModuleIdentity::from_relative_path(path.as_str())?; + let syntax = ruff_python_parser::parse_module(&document.text).map_err(|e| { + at( + error("PY002", e.to_string()), + &document.uri, + &document.text, + e.range(), + ) + })?; + Ok((document, identity, syntax)) + }) + .collect::>>()?; + modules::validate_paths(parsed.iter().map(|(_, id, _)| id.canonical.as_str()))?; + let names: BTreeSet<_> = parsed + .iter() + .map(|(_, id, _)| id.canonical.clone()) + .collect(); + let exposed = request + .package + .exposed_modules + .iter() + .flatten() + .map(|name| ModuleName::parse(&name.replace('.', "/")).to_canonical_string()) + .collect::>(); + if !exposed.is_subset(&names) { + return Err(error( + "PY001", + "exposedModules names an unknown source module", + )); + } + let exports = parsed + .iter() + .map(|(document, id, syntax)| { + imports::exports(syntax.suite(), &package, &id.canonical) + .map(|types| (id.python.clone(), types)) + .map_err(|e| at(e, &document.uri, &document.text, syntax.syntax().range())) + }) + .collect::>()?; + let scopes = parsed + .iter() + .map(|(document, id, syntax)| { + imports::scope(syntax.suite(), id, &exports) + .map_err(|e| at(e, &document.uri, &document.text, syntax.syntax().range())) + }) + .collect::>>()?; + let mut signatures = crate::values::Signatures::new(); + for ((document, _, syntax), scope) in parsed.iter().zip(&scopes) { + for statement in syntax.suite() { + if let ruff_python_ast::Stmt::FunctionDef(function) = statement { + let super::Symbol::Function(name) = &scope[function.name.as_str()] else { + unreachable!() + }; + let signature = super::functions::signature(function, scope) + .map_err(|e| at(e, &document.uri, &document.text, function.range()))?; + signatures.insert(name.to_canonical_string(), signature); + } + } + } + let lower_modules = |aliases| -> Outcome> { + parsed + .iter() + .zip(&scopes) + .map(|((document, id, syntax), scope)| { + lower(syntax.suite(), scope, aliases, &signatures) + .map(|module| { + ( + id.canonical.clone(), + AccessControlled { + access: if request.package.exposed_modules.is_none() + || exposed.contains(&id.canonical) + { + Access::Public + } else { + Access::Private + }, + value: module, + }, + ) + }) + .map_err(|e| at(e, &document.uri, &document.text, syntax.syntax().range())) + }) + .collect() + }; + let definitions = lower_modules(None)?; + let aliases = modules::tuple_aliases(&package, definitions.iter())?; + let definitions = lower_modules(Some(&aliases))?; + if request.options.types_only && definitions.values().any(|m| !m.value.values.is_empty()) { + return Err(error( + "PY004", + "typesOnly is not supported for Python functions; compile with typesOnly false", + )); + } + let ir = IRFile { + format_version: FormatVersion::Integer(4), + metadata: None, + distribution: Distribution::Library(LibraryContent { + package_name: package, + dependencies: Default::default(), + def: PackageDefinition { + modules: definitions.into_iter().collect(), + }, + }), + }; + Ok((ir, names.into_iter().collect())) +} diff --git a/rust/crates/morphir-python-binding/src/ir.rs b/rust/crates/morphir-python-binding/src/ir.rs new file mode 100644 index 00000000..e5efea2c --- /dev/null +++ b/rust/crates/morphir-python-binding/src/ir.rs @@ -0,0 +1,58 @@ +//! Version boundaries around the shared Python language mapping. + +mod classic; + +use crate::{Outcome, error}; +use morphir_core::ir::v4::{self, IRFile}; + +#[derive(Clone, Copy)] +pub(crate) enum Version { + V3, + V4, +} + +impl Version { + pub(crate) fn parse(text: &str) -> Outcome { + match text { + "3" | "3.0.0" => Ok(Self::V3), + "4" | "4.0.0" => Ok(Self::V4), + _ => Err(error("PY001", "Expected IR version 3, 3.0.0, 4 or 4.0.0")), + } + } + + pub(crate) fn major(self) -> &'static str { + match self { + Self::V3 => "3", + Self::V4 => "4", + } + } + + pub(crate) fn encode(self, model: IRFile) -> Outcome { + match self { + Self::V3 => serde_json::to_value(classic::encode(&model)?), + Self::V4 => { + v4::with_type_encoding(v4::TypeEncoding::Compact, || serde_json::to_value(model)) + } + } + .map_err(|e| error("PY005", e.to_string())) + } +} + +pub(crate) fn decode(value: serde_json::Value) -> Outcome { + use morphir_core::format_version::{NormalizedFormatVersion, ScalarValue, SupportTable}; + let header = value + .get("formatVersion") + .ok_or_else(|| error("PY005", "Missing formatVersion"))?; + let scalar = ScalarValue::from_json(header).map_err(|e| error("PY005", e.to_string()))?; + let normalized = NormalizedFormatVersion::from_scalar(&scalar, &SupportTable::reference()) + .map_err(|e| error("PY005", e.to_string()))?; + if !normalized.is_supported() { + return Err(error("PY005", "Unsupported Morphir IR version")); + } + if normalized.release.major() == 3 { + classic::decode(value) + } else { + // Let the shared v4 codec enforce its complete version/header contract. + serde_json::from_value(value).map_err(|e| error("PY005", e.to_string())) + } +} diff --git a/rust/crates/morphir-python-binding/src/ir/classic.rs b/rust/crates/morphir-python-binding/src/ir/classic.rs new file mode 100644 index 00000000..cb98e4e3 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/ir/classic.rs @@ -0,0 +1,225 @@ +//! Classic v3 encoding for the Python subset, using the shared concrete IR types. + +mod expressions; +mod read; +pub(super) use read::decode; + +use crate::{Outcome, error, values}; +use morphir_core::{ + ir::{classic as c, v4 as v}, + naming, +}; + +type Type = c::Type; +type Definition = c::ValueDefinition; + +fn name(value: &naming::Name) -> Outcome { + let encoded = c::Name::from_str(&value.to_canonical_string()); + let decoded = morphir_core::migration::migrate_name(&encoded, &Default::default()) + .map_err(|e| error("PY003", e.message))?; + if decoded != *value { + return Err(error( + "PY003", + format!( + "Name '{}' cannot roundtrip through classic IR v3 word arrays; rename it or use v4", + value.to_canonical_string() + ), + )); + } + Ok(encoded) +} + +fn canonical_name(value: &str) -> Outcome { + name(&naming::Name::from_canonical_string(value).map_err(|e| error("PY003", e))?) +} + +fn path(value: &naming::Path) -> Outcome { + Ok(c::Path { + segments: value.segments.iter().map(name).collect::>()?, + }) +} + +fn fqname(value: &naming::FQName) -> Outcome { + Ok(c::FQName { + package_path: path(&value.package_path)?, + module_path: path(&value.module_path)?, + local_name: name(&value.local_name)?, + }) +} + +fn access(value: v::Access) -> c::Access { + match value { + v::Access::Public => c::Access::Public, + v::Access::Private => c::Access::Private, + } +} + +fn tpe(value: &v::Type) -> Outcome { + let a = c::Attrs::None; + Ok(match value { + v::Type::Reference(_, reference, args) => c::Type::Reference( + a, + fqname(reference)?, + args.iter().map(tpe).collect::>()?, + ), + v::Type::Tuple(_, elements) => { + c::Type::Tuple(a, elements.iter().map(tpe).collect::>()?) + } + v::Type::Record(_, fields) => c::Type::Record( + a, + fields + .iter() + .map(|f| { + Ok(c::Field { + name: name(&f.name)?, + ty: tpe(&f.tpe)?, + }) + }) + .collect::>()?, + ), + v::Type::Function(_, input, output) => { + c::Type::Function(a, Box::new(tpe(input)?), Box::new(tpe(output)?)) + } + _ => { + return Err(values::unsupported( + "Type cannot be encoded by the Python v3 subset", + )); + } + }) +} + +fn definition( + value: &v::ValueDefinition, + aliases: &values::TupleAliases, + signatures: &values::Signatures, +) -> Outcome { + let v::ValueBody::Expression(body) = &value.body else { + return Err(values::unsupported("Expected an expression body")); + }; + let mut clean = value.clone(); + clean.body = v::ValueBody::Expression(expressions::erase(body)?); + let typed = values::annotate_function(&clean, aliases, signatures)?; + Ok(Definition { + input_types: value + .input_types + .iter() + .map(|(key, value)| { + Ok(c::value::ValueArgument { + name: canonical_name(key)?, + annotation: tpe(value)?, + ty: tpe(value)?, + }) + }) + .collect::>()?, + output_type: tpe(value + .output_type + .as_ref() + .ok_or_else(|| values::unsupported("Expected a return type"))?)?, + body: expressions::encode(body, &typed, aliases)?, + }) +} + +pub(super) fn encode(ir: &v::IRFile) -> Outcome { + let v::Distribution::Library(library) = &ir.distribution else { + return Err(values::unsupported("Expected a Library")); + }; + let signatures = values::signatures(library)?; + let aliases = crate::modules::tuple_aliases(&library.package_name, library.def.modules.iter())?; + let modules = library + .def + .modules + .iter() + .map(|(key, module)| { + let types = module + .value + .types + .iter() + .map(|(key, item)| { + let value = match &item.value.value { + v::TypeDefinition::TypeAliasDefinition { + type_params, + type_expr, + } if type_params.is_empty() => { + c::TypeDefinition::Alias(vec![], tpe(type_expr)?) + } + v::TypeDefinition::CustomTypeDefinition { + type_params, + constructors, + } if type_params.is_empty() => c::TypeDefinition::Custom( + vec![], + c::AccessControlled { + access: access(constructors.access), + value: constructors + .value + .iter() + .map(|ctor| { + Ok(c::Constructor { + name: name(&ctor.name)?, + args: ctor + .args + .iter() + .map(|arg| { + Ok((name(&arg.name)?, tpe(&arg.arg_type)?)) + }) + .collect::>()?, + }) + }) + .collect::>()?, + }, + ), + _ => { + return Err(values::unsupported( + "Type definition cannot be encoded by the Python v3 subset", + )); + } + }; + Ok(( + canonical_name(key)?, + c::AccessControlled { + access: access(item.access), + value: c::Documented::new("", value), + }, + )) + }) + .collect::>()?; + let values = module + .value + .values + .iter() + .map(|(key, item)| { + Ok(( + canonical_name(key)?, + c::AccessControlled { + access: access(item.access), + value: c::Documented::new( + "", + definition(&item.value.value, &aliases, &signatures)?, + ), + }, + )) + }) + .collect::>()?; + Ok(c::module::ModuleEntry { + path: path( + &naming::Path::from_canonical_string(key).map_err(|e| error("PY003", e))?, + )?, + definition: c::AccessControlled { + access: access(module.access), + value: c::ModuleDefinition { + types, + values, + doc: None, + }, + }, + }) + }) + .collect::>()?; + Ok(c::Distribution { + format_version: 3, + distribution: c::DistributionBody::Library( + path(library.package_name.as_path())?, + vec![], + c::PackageDefinition { modules }, + ), + }) +} diff --git a/rust/crates/morphir-python-binding/src/ir/classic/expressions.rs b/rust/crates/morphir-python-binding/src/ir/classic/expressions.rs new file mode 100644 index 00000000..7668ee62 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/ir/classic/expressions.rs @@ -0,0 +1,136 @@ +//! Infer v3 value annotations and verify annotations supplied by a v3 producer. + +use super::{Type, fqname, name, tpe}; +use crate::{Outcome, values}; +use morphir_core::ir::{classic as c, v4 as v}; + +type Value = c::Value; + +fn annotation( + value: &v::ValueAttributes, + expected: &v::Type, + aliases: &values::TupleAliases, +) -> Outcome { + let attrs = value; + if attrs.source.is_some() || !attrs.extensions.is_empty() { + return Err(values::unsupported( + "Value metadata cannot be preserved in Python", + )); + } + if let Some(actual) = &attrs.inferred_type + && values::resolve_aliases(actual, aliases)? != values::resolve_aliases(expected, aliases)? + { + return Err(values::unsupported( + "IR v3 value annotation does not match its expression type", + )); + } + tpe(expected) +} + +pub(super) fn encode( + value: &v::Value, + typed: &v::Value, + aliases: &values::TupleAliases, +) -> Outcome { + let inferred = typed + .attributes() + .inferred_type + .as_ref() + .expect("checked expression"); + let attrs = annotation(value.attributes(), inferred, aliases)?; + Ok(match value { + v::Value::Variable(_, key) => c::Value::Variable(attrs, name(key)?), + v::Value::Literal(_, literal) => c::Value::Literal(attrs, match literal { + v::Literal::Bool(value) => c::Literal::Bool(*value), + v::Literal::Integer(value) => c::Literal::WholeNumber(i64::try_from(value).map_err(|_| values::unsupported("IR v3 whole-number literals must fit the shared codec's signed 64-bit range; use IR v4 for larger integers"))?), + v::Literal::Float(value) => c::Literal::Float(value.value()), + v::Literal::String(value) => c::Literal::String(value.clone()), + _ => return Err(values::unsupported("Unsupported Python literal")), + }), + v::Value::Reference(_, key) => c::Value::Reference(attrs, fqname(key)?), + v::Value::Tuple(_, elements) => { + let v::Value::Tuple(_, checked) = typed else { unreachable!() }; + c::Value::Tuple(attrs, elements.iter().zip(checked).map(|(value, typed)| encode(value, typed, aliases)).collect::>()?) + } + v::Value::IfThenElse(_, condition, yes, no) => { + let v::Value::IfThenElse(_, tc, ty, tn) = typed else { unreachable!() }; + c::Value::IfThenElse(attrs, Box::new(encode(condition, tc, aliases)?), Box::new(encode(yes, ty, aliases)?), Box::new(encode(no, tn, aliases)?)) + } + v::Value::Apply(_, function, argument) => { + let v::Value::Apply(_, tf, ta) = typed else { unreachable!() }; + c::Value::Apply(attrs, Box::new(encode(function, tf, aliases)?), Box::new(encode(argument, ta, aliases)?)) + } + v::Value::Lambda(_, pattern, body) => { + let v::Value::Lambda(_, tp, tb) = typed else { unreachable!() }; + c::Value::Lambda(attrs, encode_pattern(pattern, tp, aliases)?, Box::new(encode(body, tb, aliases)?)) + } _ => return Err(values::unsupported("Unsupported Python expression")), + }) +} + +/// Remove only inferred types after they have been checked against the shared subset inference. +pub(super) fn erase(value: &v::Value) -> Outcome { + let a = Default::default(); + Ok(match value { + v::Value::Variable(_, name) => v::Value::Variable(a, name.clone()), + v::Value::Reference(_, name) => v::Value::Reference(a, name.clone()), + v::Value::Literal(_, literal) => v::Value::Literal(a, literal.clone()), + v::Value::Tuple(_, elements) => { + v::Value::Tuple(a, elements.iter().map(erase).collect::>()?) + } + v::Value::Lambda(_, pattern, body) => { + v::Value::Lambda(a, erase_pattern(pattern)?, Box::new(erase(body)?)) + } + v::Value::Apply(_, function, argument) => { + v::Value::Apply(a, Box::new(erase(function)?), Box::new(erase(argument)?)) + } + v::Value::IfThenElse(_, condition, yes, no) => v::Value::IfThenElse( + a, + Box::new(erase(condition)?), + Box::new(erase(yes)?), + Box::new(erase(no)?), + ), + _ => { + return Err(values::unsupported( + "IR v3 expression is outside the Python subset", + )); + } + }) +} + +fn encode_pattern( + pattern: &v::Pattern, + typed: &v::Pattern, + aliases: &values::TupleAliases, +) -> Outcome> { + let attrs = annotation( + pattern.attributes(), + typed + .attributes() + .inferred_type + .as_ref() + .expect("checked pattern"), + aliases, + )?; + Ok(match (pattern, typed) { + (v::Pattern::WildcardPattern(_), _) => c::Pattern::Wildcard(attrs), + (v::Pattern::AsPattern(_, inner, key), v::Pattern::AsPattern(_, checked, _)) => { + c::Pattern::As( + attrs, + Box::new(encode_pattern(inner, checked, aliases)?), + name(key)?, + ) + } + _ => return Err(values::unsupported("Unsupported lambda pattern")), + }) +} +fn erase_pattern(pattern: &v::Pattern) -> Outcome { + Ok(match pattern { + v::Pattern::WildcardPattern(_) => v::Pattern::WildcardPattern(Default::default()), + v::Pattern::AsPattern(_, inner, name) => v::Pattern::AsPattern( + Default::default(), + Box::new(erase_pattern(inner)?), + name.clone(), + ), + _ => return Err(values::unsupported("Unsupported lambda pattern")), + }) +} diff --git a/rust/crates/morphir-python-binding/src/ir/classic/read.rs b/rust/crates/morphir-python-binding/src/ir/classic/read.rs new file mode 100644 index 00000000..7b2e62c8 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/ir/classic/read.rs @@ -0,0 +1,102 @@ +//! Read classic IR through the core migration, refusing collisions and annotation loss. + +use crate::{Outcome, error, values}; +use morphir_core::{ + ir::{classic as c, v4 as v}, + migration, +}; +use std::collections::BTreeSet; + +fn distinct<'a>(names: impl Iterator) -> Outcome<()> { + let mut seen = BTreeSet::new(); + for name in names { + let key = migration::migrate_name(name, &Default::default()) + .map_err(|e| error("PY003", e.message))? + .to_canonical_string(); + if !seen.insert(key) { + return Err(error("PY003", "Duplicate name in classic IR")); + } + } + Ok(()) +} + +pub(in crate::ir) fn decode(value: serde_json::Value) -> Outcome { + let classic: c::Distribution = + serde_json::from_value(value).map_err(|e| error("PY005", e.to_string()))?; + let c::DistributionBody::Library(_, dependencies, package) = &classic.distribution else { + return Err(values::unsupported( + "a v3 Specs distribution has no definitions to decode", + )); + }; + if !dependencies.is_empty() { + return Err(values::unsupported( + "External dependencies are not supported", + )); + } + let mut module_names = BTreeSet::new(); + for module in &package.modules { + let key = migration::migrate_path(&module.path, &Default::default()) + .map_err(|e| error("PY003", e.message))? + .to_canonical_string(); + if !module_names.insert(key) { + return Err(error("PY003", "Duplicate module in classic IR")); + } + distinct(module.definition.value.types.iter().map(|(key, _)| key))?; + distinct(module.definition.value.values.iter().map(|(key, _)| key))?; + for (_, function) in &module.definition.value.values { + let function = &function.value.value; + distinct(function.input_types.iter().map(|arg| &arg.name))?; + } + } + let migrated = migration::migrate_distribution(&classic, Default::default()) + .map_err(|e| error("PY004", format!("{}: {}", e.path, e.message)))?; + if !migrated.report.can_publish() || !migrated.report.diagnostics().is_empty() { + return Err(values::unsupported( + "IR v3 migration would lose information", + )); + } + let mut ir = migrated.value; + let v::Distribution::Library(library) = &mut ir.distribution else { + unreachable!() + }; + let signatures = values::signatures(library)?; + let aliases = crate::modules::tuple_aliases(&library.package_name, library.def.modules.iter())?; + for module in &package.modules { + for (_, function) in &module.definition.value.values { + for arg in &function.value.value.input_types { + let convert = |ty| { + migration::migrate_type(ty, &mut Default::default()) + .map_err(|e| error("PY004", e.message)) + }; + if values::resolve_aliases(&convert(&arg.annotation)?, &aliases)? + != values::resolve_aliases(&convert(&arg.ty)?, &aliases)? + { + return Err(values::unsupported( + "IR v3 parameter annotation does not match its declared type", + )); + } + } + } + } + for module in library.def.modules.values_mut() { + clear_empty_doc(&mut module.value.doc); + for definition in module.value.types.values_mut() { + clear_empty_doc(&mut definition.value.doc); + } + for definition in module.value.values.values_mut() { + clear_empty_doc(&mut definition.value.doc); + // Encoding checks every supplied inferred type, including intermediate comparison applications. + super::definition(&definition.value.value, &aliases, &signatures)?; + if let v::ValueBody::Expression(body) = &mut definition.value.value.body { + *body = super::expressions::erase(body)?; + } + } + } + Ok(ir) +} + +fn clear_empty_doc(doc: &mut Option) { + if doc.as_ref().is_some_and(|doc| doc.text().is_empty()) { + *doc = None; + } +} diff --git a/rust/crates/morphir-python-binding/src/lib.rs b/rust/crates/morphir-python-binding/src/lib.rs new file mode 100644 index 00000000..3c8da561 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/lib.rs @@ -0,0 +1,151 @@ +//! Python ADT, tuple, function and lambda extension for Morphir. +//! +//! Source is parsed statically, never imported or executed. +//! See the crate README for the supported Python subset. +//! +//! ``` +//! use morphir_extension_sdk::{Extension, NativeExtension}; +//! use morphir_python_binding::PythonExtension; +//! let extension = NativeExtension::frontend_backend(PythonExtension).unwrap(); +//! assert_eq!(PythonExtension::info().id, "morphir-python"); +//! ``` +//! +//! Annotated function bodies can return through conditional branches: +//! +//! ```python +//! def choose(flag: bool, first: int, second: int) -> int: +//! if flag: +//! return first +//! else: +//! return second +//! ``` +//! +//! Both branches become a Morphir `IfThenElse` expression. Compile with +//! `types_only: false`; every path must return the declared result type. + +mod backend; +mod frontend; +mod ir; +mod modules; +mod names; +mod values; + +use morphir_extension_sdk::prelude::*; + +/// Stateless Python frontend and backend, usable natively or as a WASM guest. +#[derive(Default)] +pub struct PythonExtension; + +impl Extension for PythonExtension { + fn info() -> ExtensionInfo { + ExtensionInfo { + id: "morphir-python".into(), + name: "Morphir Python".into(), + version: env!("CARGO_PKG_VERSION").into(), + types: vec![ExtensionType::Frontend, ExtensionType::Backend], + description: Some("Python algebraic data types, tuples, functions and lambdas".into()), + license: Some("Apache-2.0".into()), + ..Default::default() + } + } + + fn capabilities() -> ExtensionCapabilities { + ExtensionCapabilities { + frontend: Some(FrontendCapability { + languages: vec![LanguageCapability { + id: "python".into(), + file_extensions: vec![".py".into()], + }], + ir_versions: vec!["3".into(), "4".into()], + compile: true, + ..Default::default() + }), + backend: Some(BackendCapability { + targets: vec!["python".into()], + ir_versions: vec!["3".into(), "4".into()], + generate: true, + }), + ..Default::default() + } + } +} + +impl Frontend for PythonExtension { + fn compile(&self, request: CompileRequest) -> Result { + let diagnostics = if frontend::parse_stage_requested(&request) { + vec![Diagnostic { + severity: DiagnosticSeverity::Warning, + ..error( + "PY006", + "Python parse-stage output is not implemented; returning compiled IR only", + ) + }] + } else { + vec![] + }; + let compiled = frontend::compile(&request).and_then(|(model, modules)| { + let version = ir::Version::parse(&request.options.ir_version)?; + Ok((version.encode(model)?, modules, version)) + }); + Ok(match compiled { + Ok((ir, modules, version)) => CompileResult { + success: true, + ir_version: Some(version.major().into()), + ir: Some(ir), + modules, + diagnostics, + module_results: vec![], + context_digest: None, + }, + Err(diagnostic) => CompileResult { + success: false, + ir_version: None, + ir: None, + modules: vec![], + diagnostics: vec![diagnostic], + module_results: vec![], + context_digest: None, + }, + }) + } + fn supported_languages() -> Vec { + vec!["python".into()] + } + fn file_extensions() -> Vec { + vec![".py".into()] + } +} + +impl Backend for PythonExtension { + fn generate(&self, request: GenerateRequest) -> Result { + Ok(match backend::generate(&request) { + Ok(artifacts) => GenerateResult { + success: true, + artifacts, + diagnostics: vec![], + }, + Err(diagnostic) => GenerateResult { + success: false, + artifacts: vec![], + diagnostics: vec![diagnostic], + }, + }) + } + fn target_languages() -> Vec { + vec!["python".into()] + } +} + +morphir_extension_sdk::export_extension!(PythonExtension, frontend, backend); + +type Outcome = std::result::Result; + +fn error(code: &str, message: impl Into) -> Diagnostic { + Diagnostic { + severity: DiagnosticSeverity::Error, + code: Some(code.into()), + message: message.into(), + location: None, + related: vec![], + } +} diff --git a/rust/crates/morphir-python-binding/src/modules.rs b/rust/crates/morphir-python-binding/src/modules.rs new file mode 100644 index 00000000..23391f5b --- /dev/null +++ b/rust/crates/morphir-python-binding/src/modules.rs @@ -0,0 +1,111 @@ +//! Module identity and package-wide type information shared by both directions. + +use crate::{Outcome, error, names, values::TupleAliases}; +use morphir_core::ir::v4::*; + +/// A Python dotted import path and its validated Morphir module name. +pub(crate) struct ModuleIdentity { + pub python: String, + pub canonical: String, +} + +impl ModuleIdentity { + pub fn from_relative_path(relative: &str) -> Outcome { + let stem = relative + .strip_suffix(".py") + .ok_or_else(|| error("PY001", "Source URI must end in .py"))?; + let segments = stem + .split('/') + .map(|part| { + let name = names::identifier(part)?; + names::module_file_stem(&name)?; + Ok(name.to_canonical_string()) + }) + .collect::>>()?; + if segments.first().is_some_and(|s| s == "dataclasses") { + return Err(error("PY003", "A source module cannot shadow dataclasses")); + } + Ok(Self { + python: stem.replace('/', "."), + canonical: segments.join("/"), + }) + } + + pub fn from_canonical(canonical: &str) -> Outcome { + let path = ModuleName::from_canonical_string(canonical).map_err(|e| error("PY003", e))?; + if path.is_empty() { + return Err(error("PY003", "Module name must not be empty")); + } + let parts = path + .as_path() + .segments + .iter() + .map(names::module_file_stem) + .collect::>>()?; + let identity = Self::from_relative_path(&format!("{}.py", parts.join("/")))?; + if identity.canonical != canonical { + return Err(error( + "PY003", + "Module name cannot be represented losslessly", + )); + } + Ok(identity) + } + + pub fn filename(&self) -> String { + format!("{}.py", self.python.replace('.', "/")) + } +} + +/// Reject output collisions and module/package conflicts on every supported OS. +pub(crate) fn validate_paths<'a>(modules: impl Iterator) -> Outcome<()> { + let mut seen = std::collections::BTreeSet::new(); + for module in modules { + if !seen.insert(module.to_ascii_lowercase()) { + return Err(error( + "PY003", + format!("Module paths collide after normalization or case folding: {module}"), + )); + } + } + for module in &seen { + let mut parent = module.as_str(); + while let Some((prefix, _)) = parent.rsplit_once('/') { + if seen.contains(prefix) { + return Err(error( + "PY003", + format!("Module is also a package directory: {prefix}"), + )); + } + parent = prefix; + } + } + Ok(()) +} + +pub(crate) fn tuple_aliases<'a>( + package: &PackageName, + modules: impl Iterator)>, +) -> Outcome { + let aliases: TupleAliases = + modules + .flat_map(|(module, definition)| { + definition.value.types.iter().filter_map( + move |(name, definition)| match &definition.value.value { + TypeDefinition::TypeAliasDefinition { + type_params, + type_expr: tpe @ Type::Tuple(..), + } if type_params.is_empty() => Some(( + format!("{}:{module}#{name}", package.to_canonical_string()), + tpe.clone(), + )), + _ => None, + }, + ) + }) + .collect(); + for alias in aliases.values() { + crate::values::resolve_aliases(alias, &aliases)?; + } + Ok(aliases) +} diff --git a/rust/crates/morphir-python-binding/src/names.rs b/rust/crates/morphir-python-binding/src/names.rs new file mode 100644 index 00000000..98bbbdcb --- /dev/null +++ b/rust/crates/morphir-python-binding/src/names.rs @@ -0,0 +1,99 @@ +use crate::{Outcome, error}; +use morphir_core::naming::Name; + +const RESERVED: &[&str] = &[ + "False", + "None", + "True", + "and", + "as", + "assert", + "async", + "await", + "break", + "class", + "continue", + "def", + "del", + "elif", + "else", + "except", + "finally", + "for", + "from", + "global", + "if", + "import", + "in", + "is", + "lambda", + "nonlocal", + "not", + "or", + "pass", + "raise", + "return", + "try", + "while", + "with", + "yield", + "dataclass", + "int", + "str", + "float", + "bool", + "tuple", + "Callable", + "callable", +]; + +pub(crate) fn identifier(text: &str) -> Outcome { + if text.is_empty() + || !text.as_bytes()[0].is_ascii_alphabetic() + || !text.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_') + || RESERVED.contains(&text) + { + return Err(error( + "PY003", + format!("Unsupported or reserved Python identifier: {text}"), + )); + } + Ok(Name::from(text)) +} + +pub(crate) fn type_name(name: &Name) -> Outcome { + let text = name.to_pascal_case(); + faithful(name, text) +} + +pub(crate) fn field_name(name: &Name) -> Outcome { + faithful(name, name.to_canonical_string().replace('-', "_")) +} + +pub(crate) fn module_file_stem(name: &Name) -> Outcome { + let stem = field_name(name)?; + let lower = stem.to_ascii_lowercase(); + if ["con", "prn", "aux", "nul"].contains(&lower.as_str()) + || ["com", "lpt"].iter().any(|prefix| { + lower.strip_prefix(prefix).is_some_and(|suffix| { + suffix.len() == 1 && matches!(suffix.as_bytes()[0], b'1'..=b'9') + }) + }) + { + return Err(error( + "PY003", + format!("Module name is reserved on Windows: {stem}"), + )); + } + Ok(stem) +} + +fn faithful(name: &Name, text: String) -> Outcome { + if identifier(&text)? != *name { + return Err(error( + "PY003", + format!("Name cannot be represented losslessly in Python: {name}"), + )); + } + Ok(text) +} diff --git a/rust/crates/morphir-python-binding/src/values.rs b/rust/crates/morphir-python-binding/src/values.rs new file mode 100644 index 00000000..4710c29c --- /dev/null +++ b/rust/crates/morphir-python-binding/src/values.rs @@ -0,0 +1,224 @@ +//! Validation shared by Python function compilation and generation. + +use crate::{Outcome, error}; +use morphir_core::{ir::v4::*, naming::FQName}; +use std::collections::{BTreeMap, BTreeSet}; + +mod check; +pub(crate) use check::annotate_function; +pub(crate) type Signatures = BTreeMap; + +pub(crate) fn signatures(library: &LibraryContent) -> Outcome { + library + .def + .modules + .iter() + .flat_map(|(module, definition)| { + definition.value.values.iter().map(move |(name, entry)| { + let definition = &entry.value.value; + Ok(( + format!( + "{}:{module}#{name}", + library.package_name.to_canonical_string() + ), + specification(definition)?, + )) + }) + }) + .collect() +} + +pub(crate) fn specification(definition: &ValueDefinition) -> Outcome { + Ok(ValueSpecification { + annotations: Vec::new().into(), + inputs: definition.input_types.clone(), + output: definition + .output_type + .clone() + .ok_or_else(|| unsupported("Function return type is required"))?, + }) +} + +pub(crate) fn function_type(input: Type, output: Type) -> Type { + Type::Function(Default::default(), Box::new(input), Box::new(output)) +} + +pub(crate) fn signature_type(signature: &ValueSpecification) -> Type { + signature + .inputs + .values() + .rev() + .fold(signature.output.clone(), |output, input| { + function_type(input.clone(), output) + }) +} + +/// Tuple aliases indexed by their canonical fully qualified names. +pub(crate) type TupleAliases = BTreeMap; + +pub(crate) fn resolve_aliases(tpe: &Type, aliases: &TupleAliases) -> Outcome { + fn resolve( + tpe: &Type, + aliases: &TupleAliases, + visiting: &mut BTreeSet, + ) -> Outcome { + match tpe { + Type::Reference(_, name, args) if args.is_empty() => { + let key = name.to_canonical_string(); + if let Some(alias) = aliases.get(&key) { + if !visiting.insert(key.clone()) { + return Err(unsupported("Recursive tuple aliases are not supported")); + } + let result = resolve(alias, aliases, visiting); + visiting.remove(&key); + result + } else { + Ok(tpe.clone()) + } + } + Type::Tuple(attrs, elements) => Ok(Type::Tuple( + attrs.clone(), + elements + .iter() + .map(|element| resolve(element, aliases, visiting)) + .collect::>()?, + )), + Type::Function(attrs, input, output) => Ok(Type::Function( + attrs.clone(), + Box::new(resolve(input, aliases, visiting)?), + Box::new(resolve(output, aliases, visiting)?), + )), + _ => Ok(tpe.clone()), + } + } + resolve(tpe, aliases, &mut BTreeSet::new()) +} + +pub(crate) fn validate_function( + definition: &ValueDefinition, + aliases: &TupleAliases, + signatures: &Signatures, +) -> Outcome<()> { + annotate_function(definition, aliases, signatures).map(|_| ()) +} + +#[derive(Clone, Copy)] +pub(crate) enum Comparison { + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, +} + +impl Comparison { + pub(crate) fn python(self) -> &'static str { + match self { + Self::Equal => "==", + Self::NotEqual => "!=", + Self::Less => "<", + Self::LessEqual => "<=", + Self::Greater => ">", + Self::GreaterEqual => ">=", + } + } + + pub(crate) fn reference(self) -> FQName { + let name = match self { + Self::Equal => "equal", + Self::NotEqual => "not-equal", + Self::Less => "less-than", + Self::LessEqual => "less-than-or-equal", + Self::Greater => "greater-than", + Self::GreaterEqual => "greater-than-or-equal", + }; + FQName::from_canonical_string(&format!("morphir/SDK:basics#{name}")).expect("SDK name") + } + + fn from_reference(reference: &FQName) -> Outcome { + [ + Self::Equal, + Self::NotEqual, + Self::Less, + Self::LessEqual, + Self::Greater, + Self::GreaterEqual, + ] + .into_iter() + .find(|op| op.reference() == *reference) + .ok_or_else(|| unsupported("Only scalar comparison applications are supported")) + } + + pub(crate) fn apply(self, left: Value, right: Value) -> Value { + Value::Apply( + Default::default(), + Box::new(Value::Apply( + Default::default(), + Box::new(Value::Reference(Default::default(), self.reference())), + Box::new(left), + )), + Box::new(right), + ) + } +} + +pub(crate) fn scalar(name: &str) -> Type { + let path = if name == "string" { "string" } else { "basics" }; + Type::Reference( + Default::default(), + FQName::from_canonical_string(&format!("morphir/SDK:{path}#{name}")).expect("SDK type"), + vec![], + ) +} + +pub(crate) fn unsupported(message: &str) -> morphir_extension_sdk::Diagnostic { + error("PY004", message) +} + +pub(crate) fn require_type(actual: &Type, expected: &Type) -> Outcome<()> { + if actual != expected { + return Err(unsupported( + "Function expression does not match the required type", + )); + } + Ok(()) +} + +pub(crate) fn literal_type(literal: &Literal) -> Outcome { + Ok(scalar(match literal { + Literal::Bool(_) => "bool", + Literal::Integer(_) => "int", + Literal::Float(_) => "float", + Literal::String(_) => "string", + _ => { + return Err(unsupported( + "Only bool, int, float and str literals are supported", + )); + } + })) +} + +pub(crate) fn require_empty_attributes(value: &Value) -> Outcome<()> { + if *value.attributes() != ValueAttributes::default() { + return Err(unsupported( + "Value attributes cannot yet be preserved in Python", + )); + } + Ok(()) +} + +pub(crate) fn comparison(value: &Value) -> Outcome<(Comparison, &Value, &Value)> { + let Value::Apply(_, function, right) = value else { + return Err(unsupported("Expected a comparison")); + }; + require_empty_attributes(function)?; + let Value::Apply(_, reference, left) = function.as_ref() else { + return Err(unsupported("Expected two comparison arguments")); + }; + require_empty_attributes(reference)?; + let Value::Reference(_, reference) = reference.as_ref() else { + return Err(unsupported("Expected an SDK comparison reference")); + }; + Ok((Comparison::from_reference(reference)?, left, right)) +} diff --git a/rust/crates/morphir-python-binding/src/values/check.rs b/rust/crates/morphir-python-binding/src/values/check.rs new file mode 100644 index 00000000..7f4b11b9 --- /dev/null +++ b/rust/crates/morphir-python-binding/src/values/check.rs @@ -0,0 +1,208 @@ +//! Bidirectional checking supplies lambda input types from the enclosing annotation. +use super::*; + +pub(crate) fn annotate_function( + definition: &ValueDefinition, + aliases: &TupleAliases, + signatures: &Signatures, +) -> Outcome { + let ValueBody::Expression(body) = &definition.body else { + return Err(unsupported("Only expression function bodies are supported")); + }; + let output = resolve_aliases(&specification(definition)?.output, aliases)?; + let parameters = definition + .input_types + .iter() + .map(|(name, tpe)| Ok((name.clone(), resolve_aliases(tpe, aliases)?))) + .collect::>()?; + Context { + signatures, + aliases, + } + .check(body, Some(&output), ¶meters) +} + +fn attrs(tpe: Type) -> ValueAttributes { + ValueAttributes { + inferred_type: Some(Box::new(tpe)), + ..Default::default() + } +} +fn tpe(value: &Value) -> Type { + *value + .attributes() + .inferred_type + .clone() + .expect("checked expression") +} + +pub(crate) fn binder(pattern: &Pattern) -> Outcome<&Name> { + let Pattern::AsPattern(a, inner, name) = pattern else { + return Err(unsupported("Lambda patterns must bind one named variable")); + }; + if *a != ValueAttributes::default() + || !matches!(inner.as_ref(), Pattern::WildcardPattern(a) if *a == ValueAttributes::default()) + { + return Err(unsupported( + "Lambda patterns must be an unannotated wildcard with a name", + )); + } + crate::names::field_name(name)?; + Ok(name) +} + +struct Context<'a> { + signatures: &'a Signatures, + aliases: &'a TupleAliases, +} + +impl Context<'_> { + fn check( + &self, + value: &Value, + expected: Option<&Type>, + parameters: &BTreeMap, + ) -> Outcome { + require_empty_attributes(value)?; + let result = match value { + Value::Literal(_, literal) => { + Value::Literal(attrs(literal_type(literal)?), literal.clone()) + } + Value::Variable(_, name) => Value::Variable( + attrs( + parameters + .get(&name.to_canonical_string()) + .cloned() + .ok_or_else(|| unsupported("Unknown function parameter"))?, + ), + name.clone(), + ), + Value::Reference(_, name) => { + // Resolve only referenced signatures: rebuilding the whole package for + // every body makes WASM validation quadratic in the number of functions. + let signature = self + .signatures + .get(&name.to_canonical_string()) + .ok_or_else(|| unsupported("Unknown function reference"))?; + Value::Reference( + attrs(resolve_aliases(&signature_type(signature), self.aliases)?), + name.clone(), + ) + } + Value::Lambda(_, pattern, body) => { + let Some(Type::Function(_, input, output)) = expected else { + return Err(unsupported( + "A lambda needs a contextual unary Callable type", + )); + }; + let name = binder(pattern)?; + let mut scope = parameters.clone(); + scope.insert(name.to_canonical_string(), *input.clone()); + let body = self.check(body, Some(output), &scope)?; + let pattern = Pattern::AsPattern( + attrs(*input.clone()), + Box::new(Pattern::WildcardPattern(attrs(*input.clone()))), + name.clone(), + ); + Value::Lambda(attrs(expected.unwrap().clone()), pattern, Box::new(body)) + } + Value::Tuple(_, elements) if elements.len() >= 2 => { + let types = match expected { + Some(Type::Tuple(_, types)) if types.len() == elements.len() => Some(types), + _ => None, + }; + let elements = elements + .iter() + .enumerate() + .map(|(i, element)| self.check(element, types.map(|ts| &ts[i]), parameters)) + .collect::>>()?; + Value::Tuple( + attrs(Type::Tuple( + Default::default(), + elements.iter().map(tpe).collect(), + )), + elements, + ) + } + Value::IfThenElse(_, condition, yes, no) => { + let condition = self.check(condition, Some(&scalar("bool")), parameters)?; + let yes = self.check(yes, expected, parameters)?; + let no = self.check(no, Some(&tpe(&yes)), parameters)?; + Value::IfThenElse( + attrs(tpe(&yes)), + Box::new(condition), + Box::new(yes), + Box::new(no), + ) + } + Value::Apply(_, function, argument) => { + if let Ok((op, left, right)) = comparison(value) { + let left = self.check(left, None, parameters)?; + let operand = tpe(&left); + let right = self.check(right, Some(&operand), parameters)?; + let allowed = ["int", "float", "string"] + .iter() + .any(|name| operand == scalar(name)) + || (matches!(op, Comparison::Equal | Comparison::NotEqual) + && operand == scalar("bool")); + if !allowed { + return Err(unsupported( + "Comparison operands must be matching scalar types; bool supports only equality", + )); + } + let tail = function_type(operand.clone(), scalar("bool")); + let reference = Value::Reference( + attrs(function_type(operand, tail.clone())), + op.reference(), + ); + Value::Apply( + attrs(scalar("bool")), + Box::new(Value::Apply( + attrs(tail), + Box::new(reference), + Box::new(left), + )), + Box::new(right), + ) + } else { + if let Value::Lambda(_, pattern, body) = function.as_ref() { + require_empty_attributes(function)?; + let argument = self.check(argument, None, parameters)?; + let input = tpe(&argument); + let name = binder(pattern)?; + let mut scope = parameters.clone(); + scope.insert(name.to_canonical_string(), input.clone()); + let body = self.check(body, expected, &scope)?; + let output = tpe(&body); + let pattern = Pattern::AsPattern( + attrs(input.clone()), + Box::new(Pattern::WildcardPattern(attrs(input.clone()))), + name.clone(), + ); + let function = Value::Lambda( + attrs(function_type(input, output.clone())), + pattern, + Box::new(body), + ); + return Ok(Value::Apply( + attrs(output), + Box::new(function), + Box::new(argument), + )); + } + let function = self.check(function, None, parameters)?; + let Type::Function(_, input, output) = tpe(&function) else { + return Err(unsupported("Call target must have a function type")); + }; + let argument = self.check(argument, Some(&input), parameters)?; + Value::Apply(attrs(*output), Box::new(function), Box::new(argument)) + } + } + _ => return Err(unsupported("Unsupported Python function expression")), + }; + if let Some(expected) = expected { + require_type(&tpe(&result), expected)?; + } + Ok(result) + } +} diff --git a/rust/crates/morphir-python-binding/tests/acceptance.rs b/rust/crates/morphir-python-binding/tests/acceptance.rs new file mode 100644 index 00000000..9a47e63e --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/acceptance.rs @@ -0,0 +1,171 @@ +use cucumber::{World, given, then, when}; +use morphir_extension_sdk::prelude::*; +use morphir_python_binding::PythonExtension; + +#[derive(Debug, Default)] +struct TestDriver { + source: String, + ir_version: Option, + additional: Vec, + compiled: Option, + generated: Option, +} + +impl TestDriver { + fn compile(&self, text: String) -> CompileResult { + let mut documents = vec![SourceDocument { + uri: "models.py".into(), + language_id: "python".into(), + version: 1, + text, + }]; + documents.extend(self.additional.clone()); + self.compile_documents(documents) + } + + fn compile_documents(&self, documents: Vec) -> CompileResult { + PythonExtension + .compile(CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents, + }, + package: CompilePackage { + name: "acme/example".into(), + exposed_modules: None, + }, + dependencies: vec![], + options: CompileOptions { + ir_version: self.ir_version.clone().unwrap_or_else(|| "4".into()), + types_only: false, + ..Default::default() + }, + baseline: None, + }) + .unwrap() + } + + fn compile_and_generate(&mut self) { + let compiled = self.compile(self.source.clone()); + assert!(compiled.success, "{:?}", compiled.diagnostics); + let generated = PythonExtension + .generate(GenerateRequest { + ir: compiled.ir.clone().unwrap(), + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(generated.success, "{:?}", generated.diagnostics); + self.compiled = Some(compiled); + self.generated = Some(generated); + } + + fn assert_roundtrip(&self) { + let recompiled = self.compile_documents( + self.generated + .as_ref() + .unwrap() + .artifacts + .iter() + .map(|a| SourceDocument { + uri: a.path.clone(), + language_id: "python".into(), + version: 1, + text: a.content.clone(), + }) + .collect(), + ); + assert!(recompiled.success, "{:?}", recompiled.diagnostics); + assert_eq!(recompiled.ir, self.compiled.as_ref().unwrap().ir); + } + + fn assert_rejected(&self) { + let result = self.compiled.as_ref().unwrap(); + assert!(!result.success); + assert!(result.ir.is_none()); + assert!( + result + .diagnostics + .iter() + .any(|d| d.severity == DiagnosticSeverity::Error) + ); + } +} + +#[derive(Debug, Default, World)] +struct PythonWorld { + driver: TestDriver, +} + +#[given(expr = "IR version {word}")] +fn ir_version(world: &mut PythonWorld, version: String) { + world.driver.ir_version = Some(version); +} + +#[given("a Python model with a product and a sum with payloads")] +fn model(world: &mut PythonWorld) { + world.driver.source = include_str!("fixtures/models.py").into(); +} + +#[given("a Python model containing an unannotated function")] +fn function(world: &mut PythonWorld) { + world.driver.source = "def value():\n return 1\n".into(); +} + +#[given("annotated Python functions with conditional bodies")] +fn conditionals(world: &mut PythonWorld) { + world.driver.source = include_str!("fixtures/conditionals.py").into(); +} + +#[given("Python tuple aliases and tuple-valued functions")] +fn tuples(world: &mut PythonWorld) { + world.driver.source = include_str!("fixtures/tuples.py").into(); +} + +#[given("Python higher-order functions and captured lambdas")] +fn higher_order(world: &mut PythonWorld) { + world.driver.source = include_str!("fixtures/functions.py").into(); +} + +#[given("Python modules with imported ADTs and tuple aliases")] +fn modules(world: &mut PythonWorld) { + world.driver.source = format!( + "{}\n{}", + include_str!("fixtures/models.py"), + include_str!("fixtures/tuples.py") + ); + world.driver.additional.push(SourceDocument { + uri: "rules.py".into(), + language_id: "python".into(), + version: 1, + text: include_str!("fixtures/modules/rules.py").into(), + }); +} +#[when("I compile the model and generate Python")] +fn roundtrip(world: &mut PythonWorld) { + world.driver.compile_and_generate(); +} + +#[when("I compile the model")] +fn compile(world: &mut PythonWorld) { + world.driver.compiled = Some(world.driver.compile(world.driver.source.clone())); +} + +#[then("compiling the generated Python preserves the model")] +fn preserved(world: &mut PythonWorld) { + world.driver.assert_roundtrip(); +} + +#[then("compilation fails without partial IR")] +fn rejected(world: &mut PythonWorld) { + world.driver.assert_rejected(); +} + +#[tokio::main] +async fn main() { + PythonWorld::cucumber() + .fail_on_skipped() + .run_and_exit(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/features")) + .await; +} diff --git a/rust/crates/morphir-python-binding/tests/conditionals.rs b/rust/crates/morphir-python-binding/tests/conditionals.rs new file mode 100644 index 00000000..85db7ec3 --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/conditionals.rs @@ -0,0 +1,275 @@ +#[path = "support/native.rs"] +mod native; + +use morphir_extension_sdk::{ + prelude::*, + protocol::{ExtensionRequest, methods}, +}; +use morphir_python_binding::PythonExtension; +use serde_json::{Value, json}; + +fn request(source: &str) -> CompileRequest { + CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: vec![SourceDocument { + uri: "models.py".into(), + language_id: "python".into(), + version: 1, + text: source.into(), + }], + }, + package: CompilePackage { + name: "acme/example".into(), + exposed_modules: None, + }, + dependencies: vec![], + options: CompileOptions { + ir_version: "4".into(), + ..Default::default() + }, + baseline: None, + } +} + +fn compile(source: &str) -> CompileResult { + let extension = native::an_initialized_extension(); + let response = extension + .protocol() + .handle(ExtensionRequest::new(methods::COMPILE, request(source), 1).unwrap()); + serde_json::from_value(response.result.unwrap()).unwrap() +} + +fn generate(ir: Value) -> GenerateResult { + PythonExtension + .generate(GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + }) + .unwrap() +} + +fn roundtrip(source: &str) -> Value { + let compiled = compile(source); + assert!(compiled.success, "{:?}", compiled.diagnostics); + let ir = compiled.ir.unwrap(); + let generated = generate(ir.clone()); + assert!(generated.success, "{:?}", generated.diagnostics); + let again = compile(&generated.artifacts[0].content); + assert!(again.success, "{:?}", again.diagnostics); + assert_eq!(again.ir.as_ref(), Some(&ir)); + ir +} + +#[test] +fn conditionals_lower_to_canonical_if_then_else_and_roundtrip() { + let ir = roundtrip(include_str!("fixtures/conditionals.py")); + let choose = &ir["distribution"]["Library"]["def"]["modules"]["models"]["Public"]["values"]["choose"] + ["Public"]["ExpressionBody"]; + assert_eq!( + choose["inputTypes"], + json!({"flag": "morphir/SDK:basics#bool", "first": "morphir/SDK:basics#int", "second": "morphir/SDK:basics#int"}) + ); + assert_eq!(choose["outputType"], json!("morphir/SDK:basics#int")); + assert_eq!( + choose["body"], + json!({"IfThenElse": {"condition": {"Variable": "flag"}, "then": {"Variable": "first"}, "else": {"Variable": "second"}}}) + ); +} + +#[test] +fn branches_can_select_adt_and_tuple_parameters() { + roundtrip(&format!( + "{}\ndef select(flag: bool, first: Decision, second: Decision) -> Decision:\n return first if flag else second\n\ndef pair(flag: bool, first: tuple[int, str], second: tuple[int, str]) -> tuple[int, str]:\n return first if flag else second\n", + include_str!("fixtures/models.py") + )); +} + +#[test] +fn tuple_types_and_values_have_distinct_ir_encodings() { + let ir = roundtrip( + "def pair(amount: int, label: str) -> tuple[int, str]:\n return amount, label\n", + ); + let definition = &ir["distribution"]["Library"]["def"]["modules"]["models"]["Public"]["values"] + ["pair"]["Public"]["ExpressionBody"]; + assert_eq!( + definition["outputType"], + json!({"Tuple": ["morphir/SDK:basics#int", "morphir/SDK:string#string"]}) + ); + assert_eq!( + definition["body"], + json!({"Tuple": [{"Variable": "amount"}, {"Variable": "label"}]}) + ); +} + +#[test] +fn named_and_nested_tuple_aliases_roundtrip() { + roundtrip(include_str!("fixtures/tuples.py")); + roundtrip( + "type Pair = tuple[int, str]\ndef select(flag: bool, pair: Pair) -> tuple[int, str]:\n return pair if flag else (0, 'none')\n", + ); + roundtrip( + "type Pair = tuple[int, str]\ntype Nested = tuple[Pair, bool]\ndef make_nested(value: int) -> Nested:\n return ((value, 'value'), True)\n", + ); +} + +#[test] +fn invalid_tuple_types_and_values_are_rejected() { + for source in [ + "def f() -> tuple[int, str]:\n return (1, 2)\n", + "def f() -> tuple[int, str]:\n return (1, 'a', True)\n", + "def f(flag: bool) -> tuple[int, str]:\n return (1, 'a') if flag else (2, False)\n", + "def f() -> tuple[int]:\n return (1,)\n", + "def f() -> tuple[()]:\n return ()\n", + "type Pair = tuple[int, ...]\n", + "type Pair = tuple[int, Pair]\n", + "type A = tuple[int, B]\ntype B = tuple[str, A]\n", + "type Pair = tuple[int, str]\ndef f() -> Pair:\n return [1, 'a']\n", + "def f(pair: tuple[int, str]) -> tuple[int, str]:\n return (*pair,)\n", + ] { + let result = compile(source); + assert!(!result.success, "accepted {source}"); + assert!(result.ir.is_none()); + } +} + +#[test] +fn backend_accepts_independent_tuple_value_and_rejects_wrong_elements() { + let mut ir = independent_ir( + json!({"Tuple": [{"Literal": {"IntegerLiteral": 1}}, {"Literal": {"StringLiteral": "one"}}]}), + ); + ir["distribution"]["Library"]["def"]["modules"]["models"]["Public"]["values"]["select"]["Public"] + ["ExpressionBody"]["outputType"] = + json!({"Tuple": ["morphir/SDK:basics#int", "morphir/SDK:string#string"]}); + let generated = generate(ir.clone()); + assert!(generated.success, "{:?}", generated.diagnostics); + assert_eq!( + compile(&generated.artifacts[0].content).ir, + Some(ir.clone()) + ); + ir["distribution"]["Library"]["def"]["modules"]["models"]["Public"]["values"]["select"]["Public"] + ["ExpressionBody"]["body"]["Tuple"][1] = json!({"Literal": {"BoolLiteral": true}}); + let invalid = generate(ir); + assert!(!invalid.success); + assert!(invalid.artifacts.is_empty()); +} + +#[test] +fn comparisons_and_literal_boundaries_roundtrip() { + for operator in ["==", "!=", "<", "<=", ">", ">="] { + roundtrip(&format!( + "def compare(left: int, right: int) -> str:\n return 'yes' if left {operator} right else 'no'\n" + )); + } + roundtrip( + "def limits(flag: bool) -> int:\n return -9223372036854775808 if flag else 9223372036854775807\n", + ); + roundtrip( + "def text(flag: bool) -> str:\n return 'line\\n\\x00\\U0001f600' if flag else '\"quoted\"'\n", + ); +} + +#[test] +fn arbitrary_precision_integers_preserve_exact_values() { + for (literal, decimal) in [ + ("9223372036854775808", "9223372036854775808"), + ("18_446_744_073_709_551_616", "18446744073709551616"), + ("0x1_0000_0000_0000_0000", "18446744073709551616"), + ("0O2000000000000000000000", "18446744073709551616"), + ( + "0B10000000000000000000000000000000000000000000000000000000000000000", + "18446744073709551616", + ), + ( + "1234567890123456789012345678901234567890", + "1234567890123456789012345678901234567890", + ), + ] { + for sign in ["", "-"] { + let ir = roundtrip(&format!( + "def number() -> int:\n return {sign}{literal}\n" + )); + let value = &ir["distribution"]["Library"]["def"]["modules"]["models"]["Public"]["values"] + ["number"]["Public"]["ExpressionBody"]["body"]["Literal"]["IntegerLiteral"]; + assert_eq!(value.to_string(), format!("{sign}{decimal}")); + } + } +} + +#[test] +fn invalid_functions_fail_without_partial_ir() { + for source in [ + "def f(flag: bool) -> int:\n if flag:\n return 1\n", + "def f(flag: bool) -> int:\n return 1 if flag else 'no'\n", + "def f(flag: int) -> int:\n return 1 if flag else 2\n", + "def f(flag: bool) -> int:\n return missing if flag else 2\n", + "def f(flag: bool) -> int:\n if flag:\n print('side effect')\n return 1\n return 2\n", + "def f(flag: bool) -> int:\n result = 1\n return result\n", + "def f(flag: bool = True) -> int:\n return 1\n", + "def f(flag) -> int:\n return 1\n", + "def f(flag: bool):\n return 1\n", + "async def f(flag: bool) -> int:\n return 1\n", + "def f(*, flag: bool) -> int:\n return 1\n", + "def f(flag: bool, /) -> int:\n return 1\n", + "def f(a_b: bool, aB: bool) -> int:\n return 1\n", + "def f(Class: bool) -> int:\n return 1\n", + "def f(flag: bool) -> int:\n return\n", + "def f(flag: bool) -> int:\n return 1\n return 2\n", + "def f(flag: bool) -> float:\n return 1e400\n", + "def f(flag: bool) -> int:\n return 1 if flag < True else 2\n", + "def f(a: int, b: str) -> int:\n return 1 if a == b else 2\n", + "def f(flag: bool) -> int:\n return 1 if 0 < 1 < 2 else 2\n", + ] { + let result = compile(source); + assert!(!result.success, "accepted {source}"); + assert!(result.ir.is_none()); + assert!(!result.diagnostics.is_empty()); + } +} + +#[test] +fn types_only_does_not_silently_emit_function_bodies() { + let mut request = request(include_str!("fixtures/conditionals.py")); + request.options.types_only = true; + let result = PythonExtension.compile(request).unwrap(); + assert!(!result.success); + assert!(result.ir.is_none()); +} + +fn independent_ir(body: Value) -> Value { + let definition = json!({"ExpressionBody": { + "inputTypes": {"flag": "morphir/SDK:basics#bool"}, + "outputType": "morphir/SDK:basics#int", "body": body + }}); + json!({"formatVersion": 4, "distribution": {"Library": { + "packageName": "acme/example", "dependencies": {}, "def": {"modules": { + "models": {"Public": {"types": {}, "values": {"select": {"Public": definition}}}} + }} + }}}) +} + +#[test] +fn backend_accepts_independent_if_then_else_ir() { + let ir = independent_ir( + json!({"IfThenElse": {"condition": {"Variable": "flag"}, "then": {"Literal": {"IntegerLiteral": 1}}, "else": {"Literal": {"IntegerLiteral": 2}}}}), + ); + let result = generate(ir.clone()); + assert!(result.success, "{:?}", result.diagnostics); + assert_eq!(compile(&result.artifacts[0].content).ir, Some(ir)); +} + +#[test] +fn backend_rejects_invalid_bodies_without_artifacts() { + for body in [ + json!({"IfThenElse": {"condition": {"Literal": {"IntegerLiteral": 1}}, "then": {"Literal": {"IntegerLiteral": 1}}, "else": {"Literal": {"IntegerLiteral": 2}}}}), + json!({"IfThenElse": {"condition": {"Variable": "flag"}, "then": {"Literal": {"IntegerLiteral": 1}}, "else": {"Literal": {"StringLiteral": "no"}}}}), + json!({"Variable": "unknown"}), + json!({"Reference": "acme/example:models#select"}), + ] { + let result = generate(independent_ir(body)); + assert!(!result.success); + assert!(result.artifacts.is_empty()); + } +} diff --git a/rust/crates/morphir-python-binding/tests/features/adt.feature b/rust/crates/morphir-python-binding/tests/features/adt.feature new file mode 100644 index 00000000..2c615f1b --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/features/adt.feature @@ -0,0 +1,47 @@ +Feature: Python algebraic data types + Scenario Outline: Higher-order functions and closures roundtrip in both IR versions + Given IR version + And Python higher-order functions and captured lambdas + When I compile the model and generate Python + Then compiling the generated Python preserves the model + + Examples: + | version | + | 3 | + | 4 | + + Scenario: Model declarations survive a Python and Morphir roundtrip + Given a Python model with a product and a sum with payloads + When I compile the model and generate Python + Then compiling the generated Python preserves the model + + Scenario: Unsupported behavior is diagnosed + Given a Python model containing an unannotated function + When I compile the model + Then compilation fails without partial IR + + Scenario: Conditional function bodies survive a Python and Morphir roundtrip + Given annotated Python functions with conditional bodies + When I compile the model and generate Python + Then compiling the generated Python preserves the model + + Scenario: Tuple types and values survive a Python and Morphir roundtrip + Given Python tuple aliases and tuple-valued functions + When I compile the model and generate Python + Then compiling the generated Python preserves the model + + Scenario: Imports preserve types and conditional bodies across modules + Given Python modules with imported ADTs and tuple aliases + When I compile the model and generate Python + Then compiling the generated Python preserves the model + + Scenario Outline: Versioned multi-module models roundtrip + Given IR version + And Python modules with imported ADTs and tuple aliases + When I compile the model and generate Python + Then compiling the generated Python preserves the model + + Examples: + | version | + | 3 | + | 4 | diff --git a/rust/crates/morphir-python-binding/tests/fixtures/conditionals.py b/rust/crates/morphir-python-binding/tests/fixtures/conditionals.py new file mode 100644 index 00000000..49a8b735 --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/fixtures/conditionals.py @@ -0,0 +1,35 @@ +def choose(flag: bool, first: int, second: int) -> int: + if flag: + return first + else: + return second + + +def classify(value: int) -> str: + if value < 0: + return "negative" + elif value == 0: + return "zero" + else: + return "positive" + + +def nested(first: bool, second: bool) -> int: + if first: + if second: + return 1 + else: + return 2 + else: + return 3 + + +def guarded(first: bool, second: bool) -> int: + if first: + if second: + return 1 + return 2 + + +def expression(flag: bool) -> float: + return -1.5 if flag else 2.5 diff --git a/rust/crates/morphir-python-binding/tests/fixtures/functions.py b/rust/crates/morphir-python-binding/tests/fixtures/functions.py new file mode 100644 index 00000000..7e4060fa --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/fixtures/functions.py @@ -0,0 +1,36 @@ +from collections.abc import Callable +def identity(value: int) -> int: + return value +def apply(transform: Callable[[int], int], value: int) -> int: + return transform(value) +def capture(value: int) -> Callable[[int], int]: + return lambda ignored: value +def shadow(value: int) -> Callable[[int], int]: + return lambda value: value +def curry(flag: bool) -> Callable[[int], Callable[[int], int]]: + return lambda first: lambda second: first if flag else second +def choose(flag: bool, first: int, second: int) -> int: + return first if flag else second +def run(value: int) -> int: + return choose(True, apply(lambda item: identity(item), value), curry(False)(1)(2)) +def recurse(flag: bool, value: int) -> int: + return recurse(False, value) if flag else value +def constant() -> int: + return 42 +def read_constant(value: int) -> int: + return constant() + +def immediate(value: int) -> int: + return (lambda item: item)(value) + +def tupled(value: int) -> tuple[Callable[[int], int], int]: + return (lambda ignored: value, value) + +def conditional(flag: bool) -> Callable[[int], int]: + return identity if flag else lambda item: item + +def forward(value: int) -> int: + return later(value) + +def later(value: int) -> int: + return value diff --git a/rust/crates/morphir-python-binding/tests/fixtures/models.py b/rust/crates/morphir-python-binding/tests/fixtures/models.py new file mode 100644 index 00000000..8fb4b26f --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/fixtures/models.py @@ -0,0 +1,24 @@ +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Address: + street: str + zip_code: int + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + address: Address + +type Decision = Pending | Approved + +@dataclass(frozen=True) +class Application: + decision: Decision + verified: bool + coordinates: tuple[float, float] diff --git a/rust/crates/morphir-python-binding/tests/fixtures/modules/rules.py b/rust/crates/morphir-python-binding/tests/fixtures/modules/rules.py new file mode 100644 index 00000000..d00893ec --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/fixtures/modules/rules.py @@ -0,0 +1,13 @@ +from __future__ import annotations +from models import Decision, Point + + +def select_decision(flag: bool, first: Decision, second: Decision) -> Decision: + if flag: + return first + else: + return second + + +def origin() -> Point: + return (0.0, 0.0) diff --git a/rust/crates/morphir-python-binding/tests/fixtures/tuples.py b/rust/crates/morphir-python-binding/tests/fixtures/tuples.py new file mode 100644 index 00000000..f76562b5 --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/fixtures/tuples.py @@ -0,0 +1,13 @@ +type Point = tuple[float, float] +type LabeledPoint = tuple[Point, str] + + +def make_point(x: float, y: float) -> Point: + return (x, y) + + +def located(flag: bool, location: Point) -> LabeledPoint: + if flag: + return (location, "known") + else: + return ((0.0, 0.0), "unknown") diff --git a/rust/crates/morphir-python-binding/tests/functions.rs b/rust/crates/morphir-python-binding/tests/functions.rs new file mode 100644 index 00000000..9d628851 --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/functions.rs @@ -0,0 +1,378 @@ +use morphir_extension_sdk::prelude::*; +use morphir_python_binding::PythonExtension; + +fn request(version: &str, source: &str) -> CompileRequest { + CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: vec![SourceDocument { + uri: "functions.py".into(), + language_id: "python".into(), + text: source.into(), + ..Default::default() + }], + }, + package: CompilePackage { + name: "acme/functions".into(), + ..Default::default() + }, + options: CompileOptions { + ir_version: version.into(), + types_only: false, + ..Default::default() + }, + ..Default::default() + } +} + +const FUNCTIONS: &str = include_str!("fixtures/functions.py"); + +#[test] +fn functions_calls_and_captured_lambdas_roundtrip_in_both_ir_versions() { + for version in ["3", "4"] { + let mut input = request(version, FUNCTIONS); + input.sources.documents.push(SourceDocument { + uri: "consumer.py".into(), language_id: "python".into(), + text: "from typing import Callable as Fn\nfrom functions import identity as ident, apply\ndef use(value: int) -> int:\n return apply(ident, value)\ndef make(value: int) -> Fn[[int], int]:\n return lambda ignored: ident(value)\n".into(), + ..Default::default() + }); + input.package.exposed_modules = Some(vec!["consumer".into()]); + let compiled = PythonExtension.compile(input.clone()).unwrap(); + assert!(compiled.success, "{version}: {:?}", compiled.diagnostics); + let ir = compiled.ir.unwrap(); + let generated = PythonExtension + .generate(GenerateRequest { + target: "python".into(), + ir: ir.clone(), + options: Default::default(), + }) + .unwrap(); + assert!(generated.success, "{version}: {:?}", generated.diagnostics); + assert!( + generated + .artifacts + .iter() + .any(|a| a.content.contains("lambda")) + ); + input.sources.documents = generated + .artifacts + .into_iter() + .map(|artifact| SourceDocument { + uri: artifact.path, + text: artifact.content, + language_id: "python".into(), + ..Default::default() + }) + .collect(); + let recompiled = PythonExtension.compile(input).unwrap(); + assert!( + recompiled.success, + "{version}: {:?}", + recompiled.diagnostics + ); + assert_eq!(recompiled.ir, Some(ir)); + } +} + +#[test] +fn invalid_calls_and_lambda_types_return_diagnostics_without_ir() { + for version in ["3", "4"] { + for source in [ + "def bad(value: int) -> int:\n return missing(value)\n", + "from typing import Callable\ndef many(first: int, second: int) -> int:\n return first\ndef bad(value: int) -> Callable[[int], Callable[[int], int]]:\n return many\n", + "from typing import Callable\ndef constant() -> int:\n return 1\ndef bad(value: int) -> Callable[[int], int]:\n return constant\n", + "from typing import Callable\ndef bad(value: Callable[[int], int]) -> int:\n return value(True)\n", + "from typing import Callable\ndef bad(value: int) -> Callable[[int], int]:\n return lambda item: absent\n", + "def identity(value: int) -> int:\n return value\ndef bad(value: int) -> int:\n return identity(True)\n", + "def choose(a: int, b: int) -> int:\n return a\ndef bad(value: int) -> int:\n return choose(value)\n", + "from collections.abc import Callable\ndef bad(value: int) -> Callable[[int], int]:\n return lambda item: True\n", + "from collections.abc import Callable\ndef bad(value: int) -> Callable[[int], int]:\n return lambda first, second: first\n", + "from collections.abc import Callable\ndef bad(value: int) -> Callable[[int], int]:\n return lambda item=1: item\n", + "from collections.abc import Callable\ndef bad(fooBar: int) -> Callable[[int], int]:\n return lambda foo_bar: fooBar\n", + "from collections.abc import Callable\ndef bad(value: Callable[[int, int], int]) -> int:\n return value(1, 2)\n", + "def identity(value: int) -> int:\n return value\ndef bad(value: int) -> int:\n return identity(value=value)\n", + "def identity(value: int) -> int:\n return value\ndef bad(value: int) -> int:\n return identity(*value)\n", + ] { + let result = PythonExtension.compile(request(version, source)).unwrap(); + assert!(!result.success, "{version}: {source}"); + assert!(result.ir.is_none()); + assert!(!result.diagnostics.is_empty()); + } + } +} + +fn generate(ir: serde_json::Value) -> GenerateResult { + PythonExtension + .generate(GenerateRequest { + target: "python".into(), + ir, + options: Default::default(), + }) + .unwrap() +} + +/// Authored directly against the classic IR contract, independently of the frontend. +fn classic_lambda() -> serde_json::Value { + use serde_json::json; + let int = json!([ + "Reference", + {}, + [[["morphir"], ["s", "d", "k"]], [["basics"]], ["int"]], + [] + ]); + let function = json!(["Function", {}, int, int]); + let identity = json!([ + "Lambda", + function, + ["AsPattern", int, ["WildcardPattern", int], ["item"]], + ["Variable", int, ["item"]] + ]); + json!({"formatVersion":3,"distribution":["Library",[["acme"],["functions"]],[],{"modules":[ + [[["functions"]],{"access":"Public","value":{"doc":null,"types":[],"values":[ + [["run"],{"access":"Public","value":{"doc":"","value":{ + "inputTypes":[[["value"],int,int]],"outputType":int, + "body":["Apply",int,identity,["Variable",int,["value"]]] + }}}] + ]}}] + ]}]}) +} + +fn independent_closure() -> serde_json::Value { + use morphir_core::ir::v4::*; + use morphir_core::naming::FQName; + let int = Type::Reference( + Default::default(), + FQName::from_canonical_string("morphir/SDK:basics#int").unwrap(), + vec![], + ); + let variable = |name: &str| { + Value::Variable( + Default::default(), + Name::from_canonical_string(name).unwrap(), + ) + }; + let identity = ValueDefinition { + input_types: [("value".into(), int.clone())].into_iter().collect(), + output_type: Some(int.clone()), + body: ValueBody::Expression(variable("value")), + }; + let closure = ValueDefinition { + input_types: [ + ("identity".into(), int.clone()), + ("morphir-module-1".into(), int.clone()), + ] + .into_iter() + .collect(), + output_type: Some(Type::Function( + Default::default(), + Box::new(int.clone()), + Box::new(int), + )), + body: ValueBody::Expression(Value::Lambda( + Default::default(), + Pattern::AsPattern( + Default::default(), + Box::new(Pattern::WildcardPattern(Default::default())), + Name::from_canonical_string("morphir-module-2").unwrap(), + ), + Box::new(Value::Apply( + Default::default(), + Box::new(Value::Reference( + Default::default(), + FQName::from_canonical_string("acme/functions:functions#identity").unwrap(), + )), + Box::new(variable("identity")), + )), + )), + }; + let value = |definition| AccessControlled { + access: Access::Public, + value: Documented { + doc: None, + value: definition, + }, + }; + serde_json::to_value(IRFile { + format_version: FormatVersion::Integer(4), + metadata: None, + distribution: Distribution::Library(LibraryContent { + package_name: PackageName::from_canonical_string("acme/functions").unwrap(), + dependencies: Default::default(), + def: PackageDefinition { + modules: [( + "functions".into(), + AccessControlled { + access: Access::Public, + value: ModuleDefinition { + types: Default::default(), + values: [ + ("identity".into(), value(identity)), + ("make".into(), value(closure)), + ] + .into_iter() + .collect(), + doc: None, + }, + }, + )] + .into_iter() + .collect(), + }, + }), + }) + .unwrap() +} + +#[test] +fn independent_ir_keeps_global_references_distinct_from_captured_parameters() { + let ir = independent_closure(); + let generated = generate(ir.clone()); + assert!(generated.success, "{:?}", generated.diagnostics); + assert!( + generated.artifacts[0] + .content + .contains("import functions as morphir_module_3") + ); + let compiled = PythonExtension + .compile(request("4", &generated.artifacts[0].content)) + .unwrap(); + assert!(compiled.success, "{:?}", compiled.diagnostics); + assert_eq!( + serde_json::from_value::(compiled.ir.unwrap()).unwrap(), + serde_json::from_value::(ir).unwrap() + ); +} + +#[test] +fn backend_rejects_unsupported_lambda_patterns_and_unknown_references() { + use serde_json::json; + for (path, replacement) in [ + ("/pattern", json!({"WildcardPattern": {}})), + ( + "/body/Apply/function", + json!({"Reference": "acme/functions:missing#identity"}), + ), + ("/body/Apply/argument", json!({"Variable": "missing"})), + ] { + let mut ir = independent_closure(); + let body = &mut ir["distribution"]["Library"]["def"]["modules"]["functions"]["Public"]["values"] + ["make"]["Public"]["ExpressionBody"]["body"]["Lambda"]; + *body.pointer_mut(path).unwrap() = replacement; + let result = generate(ir); + assert!(!result.success, "accepted {path}"); + assert!(result.artifacts.is_empty()); + } +} + +#[test] +fn function_exports_cannot_be_used_as_type_references() { + let encoded = independent_closure().to_string().replace( + "morphir/SDK:basics#int", + "acme/functions:functions#identity", + ); + let result = generate(serde_json::from_str(&encoded).unwrap()); + assert!( + !result.success, + "a value reference was accepted in a type position" + ); + assert!(result.artifacts.is_empty()); +} + +#[test] +fn callable_qualified_annotations_and_aliases_resolve_statically() { + for version in ["3", "4"] { + for import in ["import typing as types", "import collections.abc as types"] { + let source = format!( + "{import}\ndef apply(transform: types.Callable[[int], int], value: int) -> int:\n return transform(value)\n" + ); + let result = PythonExtension.compile(request(version, &source)).unwrap(); + assert!(result.success, "{:?}", result.diagnostics); + } + } +} + +#[test] +fn independently_authored_classic_lambda_and_application_generate_and_roundtrip() { + let ir = classic_lambda(); + let result = generate(ir.clone()); + assert!(result.success, "{:?}", result.diagnostics); + let compiled = PythonExtension + .compile(request("3", &result.artifacts[0].content)) + .unwrap(); + assert!(compiled.success, "{:?}", compiled.diagnostics); + assert_eq!(compiled.ir, Some(ir)); +} + +#[test] +fn classic_lambda_and_pattern_annotations_are_checked() { + use serde_json::json; + let boolean = json!([ + "Reference", + {}, + [[["morphir"], ["s", "d", "k"]], [["basics"]], ["bool"]], + [] + ]); + for path in ["/1", "/2/1", "/2/2/1", "/3/1"] { + let mut ir = classic_lambda(); + let lambda = &mut ir["distribution"][3]["modules"][0][1]["value"]["values"][0][1]["value"] + ["value"]["body"][2]; + *lambda.pointer_mut(path).unwrap() = boolean.clone(); + let result = generate(ir); + assert!(!result.success, "accepted malformed annotation {path}"); + assert!(result.artifacts.is_empty()); + } +} + +#[test] +#[ignore = "requires MORPHIR_TEST_PYTHON; exercised in the Python extension CI job"] +fn generated_python_executes_functions_and_closures() { + use std::{fs, process::Command}; + let python = std::env::var_os("MORPHIR_TEST_PYTHON") + .expect("set MORPHIR_TEST_PYTHON to a Python 3.12+ executable"); + for version in ["3", "4"] { + let mut input = request(version, FUNCTIONS); + input.sources.documents.push(SourceDocument { uri: "consumer.py".into(), language_id: "python".into(), text: "from functions import apply, identity\ndef use(value: int) -> int:\n return apply(identity, value)\n".into(), ..Default::default() }); + let compiled = PythonExtension.compile(input).unwrap(); + assert!(compiled.success, "{:?}", compiled.diagnostics); + let generated = generate(compiled.ir.unwrap()); + assert!(generated.success, "{:?}", generated.diagnostics); + let directory = tempfile::tempdir().unwrap(); + for artifact in generated.artifacts { + fs::write(directory.path().join(artifact.path), artifact.content).unwrap(); + } + let output = Command::new(&python).current_dir(directory.path()).arg("-c").arg(concat!( + "import functions as f, consumer\n", + "assert f.run(7) == 7\nassert consumer.use(9) == 9\n", + "assert f.capture(7)(2) == 7\nassert f.shadow(7)(2) == 2\n", + "assert f.curry(True)(1)(2) == 1\nassert f.curry(False)(1)(2) == 2\n", + "assert f.recurse(True, 8) == 8\nassert f.read_constant(0) == 42\nassert f.immediate(5) == 5\n", + "assert f.tupled(7)[0](2) == 7\nassert f.tupled(7)[1] == 7\n", + "assert f.conditional(True)(4) == 4\nassert f.conditional(False)(4) == 4\nassert f.forward(6) == 6\n", + )).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + let generated = generate(independent_closure()); + assert!(generated.success, "{:?}", generated.diagnostics); + let directory = tempfile::tempdir().unwrap(); + for artifact in generated.artifacts { + fs::write(directory.path().join(artifact.path), artifact.content).unwrap(); + } + let output = Command::new(&python) + .current_dir(directory.path()) + .args([ + "-c", + "import functions; assert functions.make(7, 99)(3) == 7", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/rust/crates/morphir-python-binding/tests/ir_versions.rs b/rust/crates/morphir-python-binding/tests/ir_versions.rs new file mode 100644 index 00000000..3054a881 --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/ir_versions.rs @@ -0,0 +1,286 @@ +use morphir_core::ir::classic; +use morphir_extension_sdk::prelude::*; +use morphir_python_binding::PythonExtension; +use serde_json::{Value, json}; + +fn a_request(version: &str) -> CompileRequest { + CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: vec![ + SourceDocument { + uri: "domain/models.py".into(), language_id: "python".into(), version: 1, + text: concat!( + "from __future__ import annotations\nfrom dataclasses import dataclass\n", + "@dataclass(frozen=True)\nclass Pending:\n pass\n", + "@dataclass(frozen=True)\nclass Approved:\n amount: float\n", + "type Decision = Pending | Approved\n", + "type Point = tuple[float, float]\n", + "@dataclass(frozen=True)\nclass APIResponse:\n decision: Decision\n point: Point\n", + ).into(), + }, + SourceDocument { + uri: "domain/rules.py".into(), language_id: "python".into(), version: 1, + text: concat!( + "from .models import Decision, Point\n", + "def choose(flag: bool, first: Decision, second: Decision) -> Decision:\n", + " return first if flag else second\n", + "def locate(value: int, point: Point) -> tuple[Point, str]:\n", + " if value < 0:\n return (point, 'negative')\n", + " elif value == 0:\n return ((0.0, 0.0), 'zero')\n", + " else:\n return (point, 'positive')\n", + ).into(), + }, + ], + }, + package: CompilePackage { name: "acme/example".into(), exposed_modules: Some(vec!["domain.rules".into()]) }, + dependencies: vec![], + options: CompileOptions { ir_version: version.into(), types_only: false, ..Default::default() }, + baseline: None, + } +} + +fn compile(request: CompileRequest) -> Value { + let expected = if request.options.ir_version.starts_with('3') { + "3" + } else { + "4" + }; + let result = PythonExtension.compile(request).unwrap(); + assert!(result.success, "{:?}", result.diagnostics); + assert_eq!(result.ir_version.as_deref(), Some(expected)); + result.ir.unwrap() +} + +fn generate(ir: Value) -> GenerateResult { + PythonExtension + .generate(GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + }) + .unwrap() +} + +#[test] +fn both_versions_preserve_imports_privacy_tuples_and_conditionals() { + for version in ["3", "3.0.0", "4", "4.0.0"] { + let mut request = a_request(version); + let ir = compile(request.clone()); + if version.starts_with('3') { + let decoded: classic::Distribution = serde_json::from_value(ir.clone()).unwrap(); + assert_eq!(decoded.format_version, 3); + let classic::DistributionBody::Library(package, _, definition) = decoded.distribution + else { + panic!("a Library") + }; + assert_eq!( + serde_json::to_value(package).unwrap(), + json!([["acme"], ["example"]]) + ); + assert_eq!( + definition.modules[0].definition.access, + classic::Access::Private + ); + assert_eq!( + definition.modules[1].definition.access, + classic::Access::Public + ); + assert_eq!(ir["distribution"][0], "Library"); + } + let generated = generate(ir.clone()); + assert!(generated.success, "{:?}", generated.diagnostics); + assert_eq!(generated.artifacts.len(), 2); + request.sources.documents = generated + .artifacts + .into_iter() + .map(|artifact| SourceDocument { + uri: artifact.path, + text: artifact.content, + language_id: "python".into(), + version: 1, + }) + .collect(); + assert_eq!(compile(request), ir); + } +} + +#[test] +fn independently_authored_v3_record_generates_python() { + let scalar = json!([ + "Reference", + {}, + [[["morphir"], ["s", "d", "k"]], [["basics"]], ["int"]], + [] + ]); + let ir = json!({"formatVersion": 3, "distribution": ["Library", [["acme"], ["example"]], [], {"modules": [ + [[["models"]], {"access":"Public", "value":{"types":[ + [["record"], {"access":"Public", "value":{"doc":"", "value":["TypeAliasDefinition", [], ["Record", {}, [ + {"name":["count"], "tpe":scalar} + ]]]}}] + ], "values":[]}}] + ]}]}); + let result = generate(ir); + assert!(result.success, "{:?}", result.diagnostics); + assert!(result.artifacts[0].content.contains("count: int")); +} + +#[test] +fn independently_authored_typed_v3_conditional_generates_python() { + let scalar = |name| { + json!([ + "Reference", + {}, + [[["morphir"], ["s", "d", "k"]], [["basics"]], [name]], + [] + ]) + }; + let integer = scalar("int"); + let boolean = scalar("bool"); + let ir = json!({"formatVersion":3,"distribution":["Library",[["acme"],["example"]],[],{"modules":[ + [[["models"]],{"access":"Public","value":{"types":[],"values":[ + [["choose"],{"access":"Public","value":{"doc":"","value":{ + "inputTypes":[[["flag"],boolean,boolean]], "outputType":integer, + "body":["IfThenElse",integer,["Variable",boolean,["flag"]], + ["Literal",integer,["WholeNumberLiteral",1]],["Literal",integer,["WholeNumberLiteral",2]]] + }}}] + ]}}] + ]}]}); + let result = generate(ir); + assert!(result.success, "{:?}", result.diagnostics); + assert!( + result.artifacts[0] + .content + .contains("def choose(flag: bool) -> int:") + ); + assert!(result.artifacts[0].content.contains("if flag:")); +} + +#[test] +fn v3_parameter_annotations_accept_an_expanded_tuple_alias() { + let mut ir = compile(a_request("3")); + let float = json!([ + "Reference", + {}, + [[["morphir"], ["s", "d", "k"]], [["basics"]], ["float"]], + [] + ]); + ir["distribution"][3]["modules"][1][1]["value"]["values"][1][1]["value"]["value"]["inputTypes"] + [1][1] = json!(["Tuple", {}, [float.clone(), float]]); + let result = generate(ir); + assert!(result.success, "{:?}", result.diagnostics); +} + +#[test] +fn v3_integer_range_is_checked_without_affecting_v4() { + for number in ["-9223372036854775808", "9223372036854775807"] { + let mut request = a_request("3"); + request.sources.documents[1].text = format!("def number() -> int:\n return {number}\n"); + let result = generate(compile(request)); + assert!(result.success, "{:?}", result.diagnostics); + } + for number in ["9223372036854775808", "-9223372036854775809"] { + let mut request = a_request("3"); + request.sources.documents[1].text = format!("def number() -> int:\n return {number}\n"); + let result = PythonExtension.compile(request.clone()).unwrap(); + assert!(!result.success); + assert!(result.ir.is_none()); + assert!(result.diagnostics[0].message.contains("64-bit")); + request.options.ir_version = "4".into(); + compile(request); + } +} + +#[test] +fn rejects_unsupported_versions_and_inconsistent_v3_value_annotations() { + for version in ["1", "2", "3.1.0", "3.0.1", "4.1.0", "3.0", "03", "latest"] { + let result = PythonExtension.compile(a_request(version)).unwrap(); + assert!(!result.success, "accepted {version}"); + assert!(result.ir.is_none()); + } + let mut ir = compile(a_request("3")); + ir["distribution"][3]["modules"][1][1]["value"]["values"][0][1]["value"]["value"]["body"][1] = + json!(["Unit", {}]); + let result = generate(ir); + assert!(!result.success); + assert!(result.artifacts.is_empty()); +} + +#[test] +fn backend_reads_compatible_patches_but_rejects_other_minor_versions() { + for (baseline, rejected_minor) in [("3", "3.2.0"), ("4", "4.1.0")] { + let mut ir = compile(a_request(baseline)); + ir["formatVersion"] = json!(format!("{baseline}.0.7")); + let result = generate(ir.clone()); + assert!(result.success, "{:?}", result.diagnostics); + ir["formatVersion"] = json!(rejected_minor); + assert!(!generate(ir).success); + } +} + +#[test] +fn v3_rejects_duplicate_names_documentation_and_untyped_function_bodies() { + let source = compile(a_request("3")); + let mut duplicate = source.clone(); + let module = duplicate["distribution"][3]["modules"][0].clone(); + duplicate["distribution"][3]["modules"] + .as_array_mut() + .unwrap() + .push(module); + assert!(!generate(duplicate).success); + let mut docs = source.clone(); + docs["distribution"][3]["modules"][0][1]["value"]["types"][0][1]["value"]["doc"] = + json!("Cannot preserve this"); + assert!(!generate(docs).success); + let mut untyped = source; + untyped["distribution"][3]["modules"][1][1]["value"]["values"][0][1]["value"]["value"]["body"] + [1] = json!({}); + assert!(!generate(untyped).success); +} + +#[test] +fn classic_names_cannot_silently_change_the_python_model() { + let mut request = a_request("4"); + request.sources.documents.truncate(1); + request.package.exposed_modules = None; + request.sources.documents[0].text = + "from dataclasses import dataclass\n@dataclass(frozen=True)\nclass Record:\n a_b: int\n" + .into(); + let expected = generate(compile(request.clone())); + assert!(expected.success); + request.options.ir_version = "3".into(); + let compiled = PythonExtension.compile(request).unwrap(); + if compiled.success { + let actual = generate(compiled.ir.unwrap()); + assert!(actual.success, "{:?}", actual.diagnostics); + assert_eq!(actual.artifacts[0].content, expected.artifacts[0].content); + } else { + assert_eq!(compiled.diagnostics[0].code.as_deref(), Some("PY003")); + } +} + +/// v3 always carries documentation strings, so IR migrated from v3 carries empty v4 +/// documentation. Empty text documents nothing and generates; real text is still +/// rejected rather than dropped. +#[test] +fn v4_empty_documentation_generates_but_text_is_rejected() { + let mut ir = compile(a_request("4")); + let module = &mut ir["distribution"]["Library"]["def"]["modules"]["domain/models"]["Private"]; + module["doc"] = json!(""); + let point = &mut module["types"]["point"]["Public"]; + let body = point.as_object().unwrap().clone(); + *point = json!({"doc": ""}); + point.as_object_mut().unwrap().extend(body); + let choose = &mut ir["distribution"]["Library"]["def"]["modules"]["domain/rules"]["Public"]["values"] + ["choose"]["Public"]; + choose["doc"] = json!(""); + let result = generate(ir.clone()); + assert!(result.success, "{:?}", result.diagnostics); + + ir["distribution"]["Library"]["def"]["modules"]["domain/models"]["Private"]["types"]["point"] + ["Public"]["doc"] = json!("A point."); + let rejected = generate(ir); + assert!(!rejected.success); + assert_eq!(rejected.diagnostics[0].code.as_deref(), Some("PY004")); +} diff --git a/rust/crates/morphir-python-binding/tests/modules.rs b/rust/crates/morphir-python-binding/tests/modules.rs new file mode 100644 index 00000000..b52b756b --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/modules.rs @@ -0,0 +1,376 @@ +use morphir_extension_sdk::prelude::*; +use morphir_python_binding::PythonExtension; +use serde_json::{Value, json}; + +fn request(sources: &[(&str, &str)]) -> CompileRequest { + CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: sources + .iter() + .map(|(uri, text)| SourceDocument { + uri: (*uri).into(), + language_id: "python".into(), + version: 1, + text: (*text).into(), + }) + .collect(), + }, + package: CompilePackage { + name: "acme/example".into(), + exposed_modules: None, + }, + dependencies: vec![], + options: CompileOptions { + ir_version: "4".into(), + types_only: false, + ..Default::default() + }, + baseline: None, + } +} + +fn compile(request: CompileRequest) -> Value { + let result = PythonExtension.compile(request).unwrap(); + assert!(result.success, "{:?}", result.diagnostics); + result.ir.unwrap() +} + +fn roundtrip(request: CompileRequest) -> (Value, Vec) { + let ir = compile(request.clone()); + let mut reversed = request; + reversed.sources.documents.reverse(); + assert_eq!( + ir, + compile(reversed), + "document order must not affect resolution" + ); + let generated = PythonExtension + .generate(GenerateRequest { + ir: ir.clone(), + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(generated.success, "{:?}", generated.diagnostics); + let sources: Vec<_> = generated + .artifacts + .iter() + .map(|a| (a.path.as_str(), a.content.as_str())) + .collect(); + assert_eq!(ir, compile(self::request(&sources))); + (ir, generated.artifacts) +} + +const TYPES: &str = "from dataclasses import dataclass\n@dataclass(frozen=True)\nclass Address:\n zip_code: int\ntype Pair = tuple[int, str]\n"; + +#[test] +fn private_modules_are_available_inside_the_package_and_survive_generation() { + let mut input = request(&[ + ("internal/models.py", TYPES), + ( + "api.py", + "from internal.models import Pair\ndef origin() -> Pair:\n return (1, 'one')\n", + ), + ]); + input.package.exposed_modules = Some(vec!["api".into()]); + let ir = compile(input.clone()); + assert!( + ir["distribution"]["Library"]["def"]["modules"]["internal/models"]["Private"].is_object() + ); + assert!(ir["distribution"]["Library"]["def"]["modules"]["api"]["Public"].is_object()); + let generated = PythonExtension + .generate(GenerateRequest { + ir: ir.clone(), + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(generated.success, "{:?}", generated.diagnostics); + input.sources.documents = generated + .artifacts + .iter() + .map(|artifact| SourceDocument { + uri: artifact.path.clone(), + text: artifact.content.clone(), + language_id: "python".into(), + version: 1, + }) + .collect(); + assert_eq!(ir, compile(input)); +} + +#[test] +fn explicitly_empty_exposure_makes_every_module_private() { + let mut input = request(&[("models.py", TYPES)]); + input.package.exposed_modules = Some(vec![]); + let ir = compile(input); + assert!(ir["distribution"]["Library"]["def"]["modules"]["models"]["Private"].is_object()); +} + +#[test] +fn imported_types_and_tuple_aliases_roundtrip_across_modules() { + let (ir, artifacts) = roundtrip(request(&[ + ("models.py", TYPES), + ( + "rules.py", + "from models import Address as Home, Pair\ndef choose(flag: bool, first: Home, second: Home) -> Home:\n return first if flag else second\ndef make_pair() -> Pair:\n return (1, 'one')\n", + ), + ])); + assert_eq!( + artifacts + .iter() + .map(|a| a.path.as_str()) + .collect::>(), + ["models.py", "rules.py"] + ); + assert_eq!( + ir["distribution"]["Library"]["def"]["modules"]["rules"]["Public"]["values"]["choose"]["Public"] + ["ExpressionBody"]["outputType"], + json!("acme/example:models#address") + ); +} + +#[test] +fn nested_modules_resolve_absolute_relative_and_qualified_imports() { + let mut input = request(&[ + ("file:///project/src/domain/models.py", TYPES), + ( + "file:///project/src/domain/rules.py", + "from .models import Pair\nimport domain.models as model\nfrom dataclasses import dataclass\n@dataclass(frozen=True)\nclass Holder:\n home: model.Address\ndef make_pair() -> Pair:\n return (1, 'one')\n", + ), + ( + "file:///project/src/app.py", + "import domain.models\nfrom domain import rules as rule\nfrom dataclasses import dataclass\n@dataclass(frozen=True)\nclass App:\n home: domain.models.Address\n holder: rule.Holder\n", + ), + ]); + input.sources.root = Some("file:///project/src".into()); + input.package.exposed_modules = Some(vec![ + "App".into(), + "Domain.Models".into(), + "Domain.Rules".into(), + ]); + let (_, artifacts) = roundtrip(input); + assert_eq!( + artifacts + .iter() + .map(|a| a.path.as_str()) + .collect::>(), + ["app.py", "domain/models.py", "domain/rules.py"] + ); +} + +#[test] +fn mutually_referencing_records_and_same_named_types_roundtrip() { + roundtrip(request(&[ + ( + "first.py", + "from dataclasses import dataclass\nimport second\n@dataclass(frozen=True)\nclass Item:\n other: second.Item\n", + ), + ( + "second.py", + "from dataclasses import dataclass\nimport first\n@dataclass(frozen=True)\nclass Item:\n other: first.Item\n", + ), + ])); +} + +#[test] +fn rejects_invalid_module_graphs_without_partial_ir() { + for sources in [ + vec![], + vec![("models.py", TYPES), ("models.py", TYPES)], + vec![("foo_bar.py", ""), ("fooBar.py", "")], + vec![ + ("models.py", TYPES), + ("rules.py", "from missing import Address\n"), + ], + vec![ + ("models.py", TYPES), + ("rules.py", "from models import Missing\n"), + ], + vec![("models.py", TYPES), ("rules.py", "from models import *\n")], + vec![ + ("models.py", TYPES), + ("rules.py", "from models import Address as int\n"), + ], + vec![ + ("models.py", TYPES), + ( + "rules.py", + "from models import Address\nfrom dataclasses import dataclass\n@dataclass(frozen=True)\nclass Address:\n x: int\n", + ), + ], + vec![ + ("models.py", TYPES), + ("rules.py", "from .models import Address\n"), + ], + vec![ + ( + "a.py", + "from b import Pair as Other\ntype Pair = tuple[Other, int]\n", + ), + ( + "b.py", + "from a import Pair as Other\ntype Pair = tuple[Other, int]\n", + ), + ], + vec![("../models.py", TYPES)], + vec![("models/__init__.py", TYPES)], + vec![("models.py", TYPES), ("models/child.py", TYPES)], + ] { + let result = PythonExtension.compile(request(&sources)).unwrap(); + assert!(!result.success, "accepted {sources:?}"); + assert!(result.ir.is_none()); + assert!(result.modules.is_empty()); + } +} + +#[test] +fn rejects_sources_outside_root_and_unknown_exposed_modules() { + let mut input = request(&[("file:///other/models.py", TYPES)]); + input.sources.root = Some("file:///project".into()); + assert!(!PythonExtension.compile(input).unwrap().success); + let mut input = request(&[("models.py", TYPES), ("rules.py", "")]); + input.package.exposed_modules = Some(vec!["Missing".into()]); + assert!(!PythonExtension.compile(input).unwrap().success); +} + +#[test] +fn imports_sharing_a_package_prefix_and_parent_relative_imports_roundtrip() { + roundtrip(request(&[ + ("domain/models.py", TYPES), + ("domain/other.py", TYPES), + ( + "domain/rules/select.py", + "from ..models import Address\nfrom .. import other\nimport domain.models\nimport domain.other\nfrom dataclasses import dataclass\n@dataclass(frozen=True)\nclass Choice:\n first: domain.models.Address\n second: domain.other.Address\n third: other.Address\n fourth: Address\n", + ), + ])); +} + +#[test] +fn generated_module_imports_avoid_declarations_and_constructor_names() { + let (_, artifacts) = roundtrip(request(&[ + ("models.py", TYPES), + ( + "rules.py", + "from models import Address\nfrom dataclasses import dataclass\n@dataclass(frozen=True)\nclass MorphirModule1:\n home: Address\n@dataclass(frozen=True)\nclass MorphirModule2:\n pass\ntype Choice = MorphirModule2\ndef morphir_module_1(value: Address) -> Address:\n return value\ndef morphir_module_2(value: Address) -> Address:\n return value\n", + ), + ])); + assert!( + artifacts[1] + .content + .contains("import models as morphir_module_3") + ); +} + +#[test] +fn reports_every_module_and_locates_bad_imports() { + let result = PythonExtension + .compile(request(&[("b.py", ""), ("a.py", "")])) + .unwrap(); + assert_eq!(result.modules, ["a", "b"]); + let result = PythonExtension + .compile(request(&[ + ("models.py", TYPES), + ("bad.py", "from models import Missing\n"), + ])) + .unwrap(); + assert!(!result.success); + assert_eq!( + result.diagnostics[0].location.as_ref().unwrap().uri, + "bad.py" + ); +} + +#[test] +fn backend_rejects_unresolved_cross_module_references_atomically() { + let mut ir = compile(request(&[ + ("models.py", TYPES), + ( + "rules.py", + "from models import Address\nfrom dataclasses import dataclass\n@dataclass(frozen=True)\nclass Holder:\n home: Address\n", + ), + ])); + ir["distribution"]["Library"]["def"]["modules"]["rules"]["Public"]["types"]["holder"]["Public"] + ["TypeAliasDefinition"]["typeExp"]["Record"]["fields"]["home"] = + json!("acme/example:models#missing"); + let result = PythonExtension + .generate(GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(!result.success); + assert!(result.artifacts.is_empty()); +} + +#[test] +fn types_only_applies_to_every_module() { + let mut input = request(&[ + ("models.py", TYPES), + ("rules.py", "def count() -> int:\n return 1\n"), + ]); + input.options.types_only = true; + assert!(!PythonExtension.compile(input).unwrap().success); +} + +#[test] +fn rejects_module_paths_that_collide_on_case_insensitive_filesystems() { + for sources in [ + vec![("API.py", ""), ("api.py", "")], + vec![("API.py", ""), ("api/models.py", "")], + vec![("domain/API.py", ""), ("domain/api.py", "")], + ] { + assert!( + !PythonExtension.compile(request(&sources)).unwrap().success, + "accepted {sources:?}" + ); + } + let ir = json!({"formatVersion": 4, "distribution": {"Library": { + "packageName": "acme/example", "dependencies": {}, "def": {"modules": { + "API": {"Public": {"types": {}, "values": {}}}, + "api": {"Public": {"types": {}, "values": {}}} + }} + }}}); + let result = PythonExtension + .generate(GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(!result.success); + assert!(result.artifacts.is_empty()); +} + +#[test] +fn document_uri_metadata_and_percent_encoding_do_not_change_module_identity() { + let mut input = request(&[ + ( + "file:///my%20project/src/domain/%6dodels.py?rev=1#selection", + TYPES, + ), + ( + "file:///my%20project/src/domain/rules.py?rev=2", + "from .models import Pair\ndef origin() -> Pair:\n return (0, '')\n", + ), + ]); + input.sources.root = Some("file:///my%20project/src/".into()); + roundtrip(input); + for uri in [ + "file://other/project/src/models.py", + "file:///project/src/../models.py", + "file:///project/src/%2e%2e/models.py", + "file:///project/src/domain%2Fmodels.py", + "file:///project/src/domain%5Cmodels.py", + ] { + let mut input = request(&[(uri, TYPES)]); + input.sources.root = Some("file:///project/src".into()); + assert!( + !PythonExtension.compile(input).unwrap().success, + "accepted {uri}" + ); + } +} diff --git a/rust/crates/morphir-python-binding/tests/pipeline.rs b/rust/crates/morphir-python-binding/tests/pipeline.rs new file mode 100644 index 00000000..c464cb1d --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/pipeline.rs @@ -0,0 +1,341 @@ +#[path = "support/native.rs"] +mod native; + +use morphir_extension_sdk::{ + prelude::*, + protocol::{ExtensionRequest, methods}, +}; +use morphir_python_binding::PythonExtension; +use serde_json::json; + +fn a_request(source: &str) -> CompileRequest { + CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: vec![SourceDocument { + uri: "models.py".into(), + language_id: "python".into(), + version: 1, + text: source.into(), + }], + }, + package: CompilePackage { + name: "acme/example".into(), + exposed_modules: Some(vec!["Models".into()]), + }, + dependencies: vec![], + options: CompileOptions { + ir_version: "4".into(), + types_only: true, + ..Default::default() + }, + baseline: None, + } +} + +fn compile(source: &str) -> CompileResult { + PythonExtension.compile(a_request(source)).unwrap() +} + +#[test] +fn product_and_sum_have_the_expected_ir_and_roundtrip_through_mep() { + let extension = native::an_initialized_extension(); + let response = extension.protocol().handle( + ExtensionRequest::new( + methods::COMPILE, + a_request(include_str!("fixtures/models.py")), + 1, + ) + .unwrap(), + ); + let compiled: CompileResult = serde_json::from_value(response.result.unwrap()).unwrap(); + assert!(compiled.success, "{:?}", compiled.diagnostics); + let ir = compiled.ir.unwrap(); + let types = &ir["distribution"]["Library"]["def"]["modules"]["models"]["Public"]["types"]; + assert_eq!( + types["address"]["Public"]["TypeAliasDefinition"]["typeExp"]["Record"]["fields"]["zip-code"], + json!("morphir/SDK:basics#int") + ); + assert_eq!( + types["decision"]["Public"]["CustomTypeDefinition"]["constructors"]["pending"], + json!([]) + ); + assert!( + types.get("pending").is_none(), + "variants are constructors, not record aliases" + ); + let response = extension.protocol().handle( + ExtensionRequest::new( + methods::GENERATE, + GenerateRequest { + ir: ir.clone(), + target: "python".into(), + options: Default::default(), + }, + 2, + ) + .unwrap(), + ); + let generated: GenerateResult = serde_json::from_value(response.result.unwrap()).unwrap(); + assert!(generated.success, "{:?}", generated.diagnostics); + assert_eq!(generated.artifacts[0].path, "models.py"); + let recompiled = compile(&generated.artifacts[0].content); + assert!(recompiled.success, "{:?}", recompiled.diagnostics); + assert_eq!(recompiled.ir.unwrap(), ir); +} + +#[test] +fn rejects_unsupported_or_ambiguous_source_without_partial_ir() { + for source in [ + "def calculate():\n return 1\n", + "from dataclasses import dataclass\n@dataclass\nclass Mutable:\n x: int\n", + "from dataclasses import dataclass\n@dataclass(frozen=True)\nclass Default:\n x: int = 1\n", + "from dataclasses import dataclass\n@dataclass(frozen=True)\nclass Unknown:\n x: Missing\n", + "import os\n", + "type Generic[T] = T\n", + "from dataclasses import dataclass\nfrom __future__ import annotations\n", + "class Broken(:\n", + ] { + let result = compile(source); + assert!(!result.success, "accepted: {source}"); + assert!(result.ir.is_none()); + assert!(!result.diagnostics.is_empty()); + } +} + +#[test] +fn advertises_and_enforces_the_initial_contract() { + let capabilities = PythonExtension::capabilities(); + assert_eq!(capabilities.frontend.unwrap().ir_versions, ["3", "4"]); + assert_eq!(capabilities.backend.unwrap().targets, ["python"]); + let mut request = a_request(""); + request.options.ir_version = "2".into(); + assert!(!PythonExtension.compile(request).unwrap().success); + assert!( + !PythonExtension + .generate(GenerateRequest { + ir: json!({}), + target: "elm".into(), + options: Default::default() + }) + .unwrap() + .success + ); +} + +#[test] +fn accepts_cli_context_but_rejects_required_parse_stage_output() { + let mut request = a_request(include_str!("fixtures/models.py")); + request.options.extra = [ + ("outputDir".into(), json!("ignored/output")), + ("emitParseStage".into(), json!(true)), + ("emitParseStageFatal".into(), json!(false)), + ] + .into_iter() + .collect(); + let result = PythonExtension.compile(request.clone()).unwrap(); + assert!(result.success, "{:?}", result.diagnostics); + assert_eq!(result.diagnostics[0].severity, DiagnosticSeverity::Warning); + request + .options + .extra + .insert("emitParseStageFatal".into(), json!(true)); + assert!(!PythonExtension.compile(request).unwrap().success); +} + +/// A source root belongs in `sources.root`. The old options key is rejected +/// even when a caller constructs the request directly in Rust. +#[test] +fn legacy_source_root_uri_option_is_rejected_not_ignored() { + let mut request = a_request(include_str!("fixtures/models.py")); + request + .options + .extra + .insert("sourceRootUri".into(), json!("file:///project/src")); + let result = PythonExtension.compile(request).unwrap(); + assert!(!result.success, "{:?}", result.diagnostics); +} + +/// The modern CLI request carries the root with its documents and preserves +/// CLI context options through the protocol boundary. +#[test] +fn a_modern_compile_request_compiles_through_the_protocol() { + let extension = native::an_initialized_extension(); + let response = extension.protocol().handle( + ExtensionRequest::new( + methods::COMPILE, + json!({ + "languageId": "python", + "sources": { + "root": "file:///project/src", + "documents": [{ + "uri": "file:///project/src/domain/models.py", + "languageId": "python", + "version": 1, + "text": include_str!("fixtures/models.py"), + }], + }, + "package": {"name": "acme/example", "exposedModules": ["domain.models"]}, + "dependencies": [], + "options": { + "typesOnly": true, + "irVersion": "4", + "outputDir": "compiled", + "emitParseStage": false, + "emitParseStageFatal": false, + }, + }), + 1, + ) + .unwrap(), + ); + + assert!(response.error.is_none(), "{:?}", response.error); + let compiled: CompileResult = serde_json::from_value(response.result.unwrap()).unwrap(); + assert!(compiled.success, "{:?}", compiled.diagnostics); + // The root was applied, not dropped: dropping it would name the module + // after the bare file and the document would resolve to the same name for + // any directory it sat in. + assert_eq!(compiled.modules, ["domain/models"]); +} + +#[test] +fn rejects_name_collisions_shared_variants_and_constructor_references() { + for source in [ + "@dataclass(frozen=True)\nclass Thing:\n zip_code: int\n zipCode: int\n", + "@dataclass(frozen=True)\nclass A:\n pass\ntype Choice = A | A\n", + "@dataclass(frozen=True)\nclass A:\n pass\ntype Choice = A\ntype Other = A\n", + "@dataclass(frozen=True)\nclass A:\n pass\ntype Choice = A\n@dataclass(frozen=True)\nclass B:\n field: A\n", + "@dataclass(frozen=True)\nclass FooBar:\n pass\n@dataclass(frozen=True)\nclass Foo_Bar:\n pass\n", + "@dataclass(frozen=True)\nclass A(Base):\n pass\n", + "@dataclass(frozen=True)\nclass A:\n field: __import__('os').system('echo unsafe')\n", + ] { + let result = compile(&format!("from dataclasses import dataclass\n{source}")); + assert!(!result.success, "accepted: {source}"); + assert!(result.ir.is_none()); + } +} + +#[test] +fn rejects_names_whose_generated_spelling_is_reserved() { + for body in [ + "@dataclass(frozen=True)\nclass Product:\n Class: str\n", + "@dataclass(frozen=True)\nclass Variant:\n Int: int\ntype Choice = Variant\n", + "@dataclass(frozen=True)\nclass none:\n pass\n", + "@dataclass(frozen=True)\nclass Variant:\n pass\ntype true = Variant\n", + ] { + let result = compile(&format!("from dataclasses import dataclass\n{body}")); + assert!(!result.success, "accepted: {body}"); + assert!(result.ir.is_none()); + assert_eq!(result.diagnostics[0].code.as_deref(), Some("PY003")); + } +} + +fn a_library(definition: serde_json::Value) -> serde_json::Value { + json!({"formatVersion": 4, "distribution": {"Library": { + "packageName": "acme/example", "dependencies": {}, "def": {"modules": { + "models": {"Public": {"types": {"choice": {"Public": definition}}, "values": {}}} + }} + }}}) +} + +#[test] +fn backend_accepts_an_independently_authored_sum() { + let ir = a_library( + json!({"CustomTypeDefinition": {"typeParams": [], "access": "Public", "constructors": { + "absent": [], "present": [["value", "morphir/SDK:string#string"]] + }}}), + ); + let result = PythonExtension + .generate(GenerateRequest { + ir: ir.clone(), + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(result.success, "{:?}", result.diagnostics); + assert_eq!(compile(&result.artifacts[0].content).ir.unwrap(), ir); +} + +#[test] +fn backend_refuses_unsupported_ir_without_partial_artifacts() { + for definition in [ + json!({"TypeAliasDefinition": {"typeParams": ["a"], "typeExp": "a"}}), + json!({"TypeAliasDefinition": {"typeParams": [], "typeExp": {"Function": {"parameterType": "morphir/SDK:basics#int", "returnType": "morphir/SDK:basics#int"}}}}), + json!({"TypeAliasDefinition": {"typeParams": [], "typeExp": {"Record": {"fields": {"value": "elsewhere/pkg:other#type"}}}}}), + json!({"CustomTypeDefinition": {"typeParams": [], "access": "Private", "constructors": {"hidden": []}}}), + json!({"CustomTypeDefinition": {"typeParams": [], "access": "Public", "constructors": {"choice": []}}}), + json!({"CustomTypeDefinition": {"typeParams": [], "access": "Public", "constructors": {}}}), + ] { + let result = PythonExtension + .generate(GenerateRequest { + ir: a_library(definition), + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(!result.success); + assert!(result.artifacts.is_empty()); + assert!(!result.diagnostics.is_empty()); + } +} + +#[test] +fn recursive_types_and_initialisms_roundtrip() { + let result = compile( + "from __future__ import annotations\nfrom dataclasses import dataclass\n@dataclass(frozen=True)\nclass Empty:\n pass\n@dataclass(frozen=True)\nclass Link:\n value_in_USD: float\n rest: Chain\ntype Chain = Empty | Link\n", + ); + assert!(result.success, "{:?}", result.diagnostics); + let ir = result.ir.unwrap(); + let generated = PythonExtension + .generate(GenerateRequest { + ir: ir.clone(), + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(generated.success, "{:?}", generated.diagnostics); + assert_eq!(compile(&generated.artifacts[0].content).ir.unwrap(), ir); +} + +#[test] +fn syntax_errors_identify_the_source_document() { + let result = compile("class Broken(:\n"); + assert_eq!(result.diagnostics[0].code.as_deref(), Some("PY002")); + assert_eq!( + result.diagnostics[0].location.as_ref().unwrap().uri, + "models.py" + ); +} + +#[test] +fn refuses_nonportable_output_module_names() { + for module in [ + "../models", + "nested/con", + "dataclasses", + "con", + "aux", + "COM1", + "lpt9", + ] { + let mut ir = a_library( + json!({"TypeAliasDefinition": {"typeParams": [], "typeExp": {"Record": {"fields": {}}}}}), + ); + let modules = ir["distribution"]["Library"]["def"]["modules"] + .as_object_mut() + .unwrap(); + let definition = modules.remove("models").unwrap(); + modules.insert(module.into(), definition); + let result = PythonExtension + .generate(GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + }) + .unwrap(); + assert!(!result.success, "accepted module {module}"); + assert!(result.artifacts.is_empty()); + } +} diff --git a/rust/crates/morphir-python-binding/tests/support/native.rs b/rust/crates/morphir-python-binding/tests/support/native.rs new file mode 100644 index 00000000..762d5a0c --- /dev/null +++ b/rust/crates/morphir-python-binding/tests/support/native.rs @@ -0,0 +1,29 @@ +//! Native protocol fixture with a completed initialization handshake. + +use morphir_extension_sdk::{ + NativeExtension, + protocol::{ExtensionRequest, InitializeParams, MEP_VERSION, PeerInfo, methods}, +}; +use morphir_python_binding::PythonExtension; + +pub fn an_initialized_extension() -> NativeExtension { + let extension = NativeExtension::frontend_backend(PythonExtension).unwrap(); + let response = extension.protocol().handle( + ExtensionRequest::new( + methods::INITIALIZE, + InitializeParams { + model_transport: None, + protocol_versions: vec![MEP_VERSION.into()], + host: PeerInfo { + kind: Default::default(), + name: "binding-test".into(), + version: "1.0.0".into(), + }, + }, + 0, + ) + .unwrap(), + ); + assert!(response.error.is_none(), "{:?}", response.error); + extension +} diff --git a/rust/crates/morphir-python-native/Cargo.toml b/rust/crates/morphir-python-native/Cargo.toml new file mode 100644 index 00000000..488d1080 --- /dev/null +++ b/rust/crates/morphir-python-native/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "morphir-python-native" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Private CPython bridge to the Morphir Python compiler" +publish = false + +[lib] +name = "_native" +crate-type = ["cdylib", "rlib"] + +[dependencies] +morphir-python-binding = { path = "../morphir-python-binding" } +morphir-extension-sdk.workspace = true +pyo3 = { version = "=0.30.1", features = ["abi3-py314"], optional = true } +serde = { version = "1.0", features = ["derive"] } +serde_json.workspace = true + +[features] +# The CPython module. Maturin enables it; Rust tests and benchmarks of the +# bridge logic build without an interpreter. +python = ["dep:pyo3"] diff --git a/rust/crates/morphir-python-native/examples/bench.rs b/rust/crates/morphir-python-native/examples/bench.rs new file mode 100644 index 00000000..f7b47851 --- /dev/null +++ b/rust/crates/morphir-python-native/examples/bench.rs @@ -0,0 +1,73 @@ +//! Time the compiler in Rust, directly and through the JSON bridge. +//! +//! Usage: `bench REQUEST.json WARMUP REPETITIONS`. REQUEST is a bridge compile +//! request. Prints JSON with one list of nanosecond samples per stage: +//! +//! - `rust_compile`, `rust_generate`: the `Frontend`/`Backend` trait calls on +//! owned SDK values, as a native Rust caller makes them; +//! - `bridge_compile`, `bridge_generate`: the bridge functions the CPython module +//! runs after detaching, including request parsing and result serialization. + +use std::time::Instant; + +use _native::{CompileInput, compile_input, compile_request, generate_input, run_json}; +use morphir_extension_sdk::prelude::*; +use morphir_python_binding::PythonExtension; +use serde_json::{Value, json}; + +fn time(warmup: usize, repetitions: usize, mut operation: F) -> Vec { + for _ in 0..warmup { + operation(); + } + (0..repetitions) + .map(|_| { + let start = Instant::now(); + operation(); + start.elapsed().as_nanos() + }) + .collect() +} + +fn main() { + let mut args = std::env::args().skip(1); + let path = args.next().expect("request path"); + let warmup: usize = args.next().expect("warmup").parse().unwrap(); + let repetitions: usize = args.next().expect("repetitions").parse().unwrap(); + let text = std::fs::read_to_string(path).unwrap(); + let parse = || serde_json::from_str::(&text).unwrap(); + + let request = compile_request(parse()); + let compiled = PythonExtension.compile(request.clone()).unwrap(); + assert!(compiled.success, "{:?}", compiled.diagnostics); + let ir = compiled.ir.unwrap(); + let ir_text = ir.to_string(); + let generate = GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + }; + + // Owned input, as a Rust caller passes it: the timed call includes the clone. + let rust_compile = time(warmup, repetitions, || { + std::hint::black_box(PythonExtension.compile(request.clone()).unwrap()); + }); + let rust_generate = time(warmup, repetitions, || { + std::hint::black_box(PythonExtension.generate(generate.clone()).unwrap()); + }); + let bridge_compile = time(warmup, repetitions, || { + std::hint::black_box(run_json(&text, compile_input)); + }); + let bridge_generate = time(warmup, repetitions, || { + std::hint::black_box(run_json::(&ir_text, generate_input)); + }); + println!( + "{}", + json!({ + "rust_compile": rust_compile, + "rust_generate": rust_generate, + "bridge_compile": bridge_compile, + "bridge_generate": bridge_generate, + "ir_bytes": ir_text.len(), + }) + ); +} diff --git a/rust/crates/morphir-python-native/src/lib.rs b/rust/crates/morphir-python-native/src/lib.rs new file mode 100644 index 00000000..02413695 --- /dev/null +++ b/rust/crates/morphir-python-native/src/lib.rs @@ -0,0 +1,273 @@ +//! Private CPython bridge to the Morphir Python compiler. +//! +//! The `morphir_python_native._native` module exposes two functions that take and +//! return JSON text. They are a private boundary for `morphir_tools.python`; the +//! public, typed Python API lives there. +//! +//! Each call extracts its whole request into owned Rust values while attached to +//! the interpreter, then detaches and runs the compiler. Detached work never +//! touches Python objects or calls back into Python, and holds no state shared +//! between calls, so concurrent calls from several threads are independent. +//! Panics are caught and reported as an internal-error result, never as a Python +//! `PanicException`. + +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use morphir_extension_sdk::prelude::*; +use morphir_python_binding::PythonExtension; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Revision of the JSON contract between this module and `morphir_tools.python`. +/// +/// Increment it whenever a request or result member changes meaning, is added, or +/// is removed. The tools package refuses to use a module with another revision. +pub const BRIDGE_API: u32 = 1; + +/// One source document, identified by its path relative to the source root. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceInput { + pub path: String, + pub text: String, +} + +/// A complete compilation request, owned by Rust before the interpreter is released. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct CompileInput { + pub package: String, + pub sources: Vec, + /// `None` exposes every module; an empty list exposes none. + pub exposed_modules: Option>, + /// `"3"` or `"4"`. + pub ir_version: String, +} + +/// A zero-based line and UTF-16 character offset. +#[derive(Debug, Serialize)] +pub struct PositionOutput { + pub line: u32, + pub character: u32, +} + +/// The source document and range a diagnostic refers to. +#[derive(Debug, Serialize)] +pub struct LocationOutput { + pub path: String, + pub start: PositionOutput, + pub end: PositionOutput, +} + +/// One compiler diagnostic. +#[derive(Debug, Serialize)] +pub struct DiagnosticOutput { + pub severity: &'static str, + pub code: Option, + pub message: String, + pub location: Option, +} + +/// One generated file. +#[derive(Debug, Serialize)] +pub struct ArtifactOutput { + pub path: String, + pub content: String, +} + +/// The result of one bridge call. +/// +/// `internal_error` is set only when the compiler failed in a way that is not a +/// language diagnostic, such as a panic or an invalid bridge request. +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BridgeOutput { + pub success: bool, + /// The compiled IR document as JSON text, so that number lexemes and member + /// order reach the Python codecs unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + pub ir: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub artifacts: Option>, + pub diagnostics: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub internal_error: Option, +} + +impl BridgeOutput { + fn internal(message: impl Into) -> Self { + Self { + internal_error: Some(message.into()), + ..Self::default() + } + } +} + +fn severity(value: &DiagnosticSeverity) -> &'static str { + match value { + DiagnosticSeverity::Error => "error", + DiagnosticSeverity::Warning => "warning", + DiagnosticSeverity::Info => "info", + DiagnosticSeverity::Hint => "hint", + } +} + +fn diagnostics(items: Vec) -> Vec { + items + .into_iter() + .map(|item| DiagnosticOutput { + severity: severity(&item.severity), + code: item.code, + message: item.message, + location: item.location.map(|location| LocationOutput { + path: location.uri, + start: PositionOutput { + line: location.range.start.line, + character: location.range.start.character, + }, + end: PositionOutput { + line: location.range.end.line, + character: location.range.end.character, + }, + }), + }) + .collect() +} + +/// Build the SDK request the bridge sends to the compiler. +pub fn compile_request(input: CompileInput) -> CompileRequest { + CompileRequest { + language_id: "python".into(), + sources: SourceSet { + root: None, + documents: input + .sources + .into_iter() + .map(|source| SourceDocument { + uri: source.path, + language_id: "python".into(), + version: 1, + text: source.text, + }) + .collect(), + }, + package: CompilePackage { + name: input.package, + exposed_modules: input.exposed_modules, + }, + dependencies: vec![], + options: CompileOptions { + types_only: false, + ir_version: input.ir_version, + ..Default::default() + }, + baseline: None, + } +} + +/// Compile an owned request with the native frontend. Pure Rust; never touches Python. +pub fn compile_input(input: CompileInput) -> BridgeOutput { + match PythonExtension.compile(compile_request(input)) { + Ok(result) => BridgeOutput { + success: result.success && result.ir.is_some(), + ir: if result.success { + result.ir.as_ref().map(Value::to_string) + } else { + None + }, + artifacts: None, + diagnostics: diagnostics(result.diagnostics), + internal_error: None, + }, + Err(error) => BridgeOutput::internal(error.to_string()), + } +} + +/// Generate Python from owned IR with the native backend. Pure Rust; never touches Python. +pub fn generate_input(ir: Value) -> BridgeOutput { + let request = GenerateRequest { + ir, + target: "python".into(), + options: Default::default(), + }; + match PythonExtension.generate(request) { + Ok(result) => BridgeOutput { + success: result.success, + ir: None, + artifacts: Some(if result.success { + result + .artifacts + .into_iter() + .map(|artifact| ArtifactOutput { + path: artifact.path, + content: artifact.content, + }) + .collect() + } else { + vec![] + }), + diagnostics: diagnostics(result.diagnostics), + internal_error: None, + }, + Err(error) => BridgeOutput::internal(error.to_string()), + } +} + +/// Parse, run and serialize one request, converting panics into an internal error. +/// +/// The SDK enables serde_json's `arbitrary_precision`, so numbers keep their lexemes. +pub fn run_json(request: &str, operation: F) -> String +where + I: for<'de> Deserialize<'de>, + F: FnOnce(I) -> BridgeOutput, +{ + let output = match serde_json::from_str::(request) { + Ok(input) => catch_unwind(AssertUnwindSafe(|| operation(input))).unwrap_or_else(|panic| { + let detail = panic + .downcast_ref::<&str>() + .map(|text| (*text).to_owned()) + .or_else(|| panic.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".into()); + BridgeOutput::internal(format!("the native compiler panicked: {detail}")) + }), + Err(error) => BridgeOutput::internal(format!("invalid bridge request: {error}")), + }; + serde_json::to_string(&output).unwrap_or_else(|error| { + format!(r#"{{"success":false,"diagnostics":[],"internalError":"cannot serialize result: {error}"}}"#) + }) +} + +#[cfg(feature = "python")] +#[pyo3::pymodule(name = "_native")] +mod native { + use pyo3::prelude::*; + + #[pymodule_export] + const BRIDGE_API: u32 = super::BRIDGE_API; + + #[pymodule_export] + #[allow(non_upper_case_globals)] + const __version__: &str = env!("CARGO_PKG_VERSION"); + + /// Records the version the linked `morphir-python-binding` compiler reports. + #[pymodule_init] + fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { + use morphir_extension_sdk::Extension; + module.add( + "COMPILER_VERSION", + morphir_python_binding::PythonExtension::info().version, + ) + } + + /// Compile a JSON request and return a JSON result. + #[pyfunction] + fn compile(py: Python<'_>, request: String) -> String { + py.detach(move || super::run_json(&request, super::compile_input)) + } + + /// Generate Python from a versioned IR document's JSON text and return a JSON result. + #[pyfunction] + fn generate(py: Python<'_>, ir: String) -> String { + py.detach(move || super::run_json(&ir, super::generate_input)) + } +} diff --git a/rust/crates/morphir-python-native/tests/bridge.rs b/rust/crates/morphir-python-native/tests/bridge.rs new file mode 100644 index 00000000..17288288 --- /dev/null +++ b/rust/crates/morphir-python-native/tests/bridge.rs @@ -0,0 +1,137 @@ +//! The JSON bridge contract, exercised without an interpreter. + +use std::thread; + +use _native::{BridgeOutput, CompileInput, run_json}; +use serde_json::{Value, json}; + +const RULES: &str = "def choose(flag: bool, first: int, second: int) -> int:\n return first if flag else second\n"; + +fn compile(request: Value) -> Value { + serde_json::from_str(&run_json::( + &request.to_string(), + _native::compile_input, + )) + .unwrap() +} + +fn generate(ir: &str) -> Value { + serde_json::from_str(&run_json::(ir, _native::generate_input)).unwrap() +} + +fn request(version: &str) -> Value { + json!({ + "package": "acme/example", + "sources": [{"path": "rules.py", "text": RULES}], + "exposedModules": null, + "irVersion": version, + }) +} + +#[test] +fn compiles_and_generates_in_both_versions() { + for version in ["3", "4"] { + let compiled = compile(request(version)); + assert_eq!(compiled["success"], true, "{compiled}"); + let ir = compiled["ir"].as_str().unwrap(); + let document: Value = serde_json::from_str(ir).unwrap(); + assert_eq!(document["formatVersion"], version.parse::().unwrap()); + assert!(compiled.get("internalError").is_none()); + let generated = generate(ir); + assert_eq!(generated["success"], true, "{generated}"); + let artifacts = generated["artifacts"].as_array().unwrap(); + assert_eq!(artifacts.len(), 1); + assert_eq!(artifacts[0]["path"], "rules.py"); + assert!( + artifacts[0]["content"] + .as_str() + .unwrap() + .contains("def choose(") + ); + } +} + +#[test] +fn language_errors_are_diagnostics_with_locations_and_no_model() { + let mut bad = request("3"); + bad["sources"][0]["text"] = json!("def broken(:\n"); + let compiled = compile(bad); + assert_eq!(compiled["success"], false); + assert!(compiled.get("ir").is_none()); + let diagnostic = &compiled["diagnostics"][0]; + assert_eq!(diagnostic["severity"], "error"); + assert_eq!(diagnostic["code"], "PY002"); + assert_eq!(diagnostic["location"]["path"], "rules.py"); + assert_eq!(diagnostic["location"]["start"]["line"], 0); +} + +#[test] +fn unsupported_statements_are_rejected_without_execution() { + let mut bad = request("3"); + bad["sources"][0]["text"] = json!("open('owned.txt', 'w').write('x')\n"); + let compiled = compile(bad); + assert_eq!(compiled["success"], false); + assert_eq!(compiled["diagnostics"][0]["code"], "PY004"); + assert!(!std::path::Path::new("owned.txt").exists()); +} + +#[test] +fn malformed_requests_are_internal_errors() { + let output = run_json::("{\"package\": 1}", |_| unreachable!()); + let output: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output["success"], false); + assert!( + output["internalError"] + .as_str() + .unwrap() + .starts_with("invalid bridge request") + ); + let mut extra = request("3"); + extra["unexpected"] = json!(true); + assert!(compile(extra)["internalError"].is_string()); +} + +#[test] +fn panics_become_internal_errors() { + let output = run_json::("{}", |_| -> BridgeOutput { panic!("boom") }); + let output: Value = serde_json::from_str(&output).unwrap(); + assert_eq!( + output["internalError"], + "the native compiler panicked: boom" + ); +} + +#[test] +fn integer_lexemes_survive_generation_input() { + let compiled = compile(request("4")); + let ir = compiled["ir"].as_str().unwrap(); + assert!(generate(ir)["success"].as_bool().unwrap()); + let invalid = generate("{\"formatVersion\": 4"); + assert!( + invalid["internalError"] + .as_str() + .unwrap() + .starts_with("invalid bridge request") + ); +} + +#[test] +fn concurrent_requests_do_not_share_state() { + let expected: Vec = (0..8) + .map(|index| compile(request(if index % 2 == 0 { "3" } else { "4" }))) + .collect(); + let handles: Vec<_> = (0..8) + .map(|index| { + thread::spawn(move || { + (0..20) + .map(|_| compile(request(if index % 2 == 0 { "3" } else { "4" }))) + .collect::>() + }) + }) + .collect(); + for (index, handle) in handles.into_iter().enumerate() { + for output in handle.join().unwrap() { + assert_eq!(output, expected[index]); + } + } +} diff --git a/rust/extension.toml b/rust/extension.toml new file mode 100644 index 00000000..6f3dcf62 --- /dev/null +++ b/rust/extension.toml @@ -0,0 +1,34 @@ +# The morphir-python MEP extension bundle built from this repository. +# +# [extension] uses the keys of a finos/morphir-rust `.github/extensions.toml` +# entry, so the canonical WASM packager reads it unchanged. + +[extension] +short_id = "python" +package = "morphir-python-binding" +artifact = "morphir-python-binding" +extension_id = "morphir-python" +name = "Morphir Python" +mep_versions = ["0.1"] +targets = ["python"] +languages = [{ id = "python", file_extensions = [".py"] }] +ir_versions = ["3", "4"] +release_with_workspace = true + +# The finos/morphir `extension bundle` command packages native process +# extensions only, so WASM bundles still come from morphir-rust's packager +# (scripts/package_extension.py). It runs unchanged at this exact revision. +[packager] +repository = "https://github.com/finos/morphir-rust.git" +rev = "0aa60eb799d46f8a317ec47abef4a7a4e61dc8ac" + +# The released Morphir CLI that installs and runs the bundle offline. +[host] +version = "0.4.0-beta.10" +release = "https://github.com/finos/morphir/releases/download/v0.4.0-beta.10" + +[host.sha256] +"x86_64-unknown-linux-gnu" = "3f6dc0559e7570d9cdb9c95761333552593da6bc8c788b9da3bb4a1d93101923" +"aarch64-unknown-linux-gnu" = "76a92a11030cd254d2283e98363cf203f86aaecd583fa480bdf6f508c86710f6" +"aarch64-apple-darwin" = "98635ec74fbdc3c2cef1d4ae184bb1c5ed9e2960575f3d8abb4b97e0e4da363f" +"x86_64-apple-darwin" = "4080dcfb04f6e1c902bce2780756c6af44320ac0dd6ff6bd953ca74a0da8e128" diff --git a/rust/rustfmt.toml b/rust/rustfmt.toml new file mode 100644 index 00000000..cca6df5a --- /dev/null +++ b/rust/rustfmt.toml @@ -0,0 +1,7 @@ +# Rustfmt configuration for Morphir Rust +edition = "2024" +max_width = 100 +tab_spaces = 4 +newline_style = "Unix" +use_small_heuristics = "Default" + diff --git a/scripts/bench.py b/scripts/bench.py new file mode 100644 index 00000000..711fbb3e --- /dev/null +++ b/scripts/bench.py @@ -0,0 +1,527 @@ +"""Measure the Python compiler's cost at each layer, with fixed inputs. + +Stages, all on the same sources and IR version: + +- ``rust_compile`` / ``rust_generate``: the Rust trait calls, release build; +- ``bridge_compile`` / ``bridge_generate``: the same work through the JSON + bridge, in Rust, including request parsing and result serialization; +- ``native_compile`` / ``native_generate``: the CPython module calls; +- ``api_compile`` / ``api_generate``: the public API, including codec work; +- ``codec_decode`` / ``codec_encode``: the Python v3/v4 codecs alone; +- ``cli_compile`` / ``cli_generate``: ``morphir-python`` in a new process; +- ``cold_import``: a new interpreter importing and loading the compiler; +- ``ast_parse``: Python's ``ast.parse`` on the sources, a parsing-only baseline. + +There is no pure-Python compiler, so no stage supports a Rust-versus-Python +compiler speed claim; ``ast_parse`` does far less work than compilation. + +The command writes raw samples and metadata to ``results.json`` and a summary to +``report.html`` in a new directory under ``.dev/bench/``. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import html +import json +import os +import platform +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +import tomllib +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from collections.abc import Callable + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = ROOT / "tests" / "fixtures" / "python" / "loans" + +MODEL_TEMPLATE = """\ +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending{i}: + pass + +@dataclass(frozen=True) +class Approved{i}: + amount: float + count: int + +type Decision{i} = Pending{i} | Approved{i} + +type Point{i} = tuple[float, float] +""" + +FUNCTION_TEMPLATE = """ +def choose{i}{j}(flag: bool, first: Decision{i}, second: Decision{i}) -> Decision{i}: + if flag: + return first + else: + return second + +def pair{i}{j}(left: float, right: float) -> Point{i}: + return (left, right) + +def limit{i}{j}(value: int, bound: int) -> int: + return value if value < bound else bound +""" + + +def tag(index: int) -> str: + """Spell an index with two or more letters; v3 names cannot hold digits.""" + letters = "" + index += 26 + while index: + index, remainder = divmod(index, 26) + letters = chr(ord("a") + remainder) + letters + return letters + + +def generated_sources(modules: int, functions: int) -> dict[str, str]: + """Return ``modules`` model and rules module pairs with ``functions`` each.""" + sources: dict[str, str] = {} + for number in range(modules): + i = tag(number) + sources[f"scale/models{i}.py"] = MODEL_TEMPLATE.format(i=i) + header = ( + "from __future__ import annotations\n" + f"from .models{i} import Decision{i}, Point{i}\n" + ) + body = "".join( + FUNCTION_TEMPLATE.format(i=i, j=tag(j)) for j in range(functions) + ) + sources[f"scale/rules{i}.py"] = header + body + return sources + + +def sample_sources() -> dict[str, str]: + """Return the checked-in two-module sample project.""" + return { + p.relative_to(SAMPLE).as_posix(): p.read_text() + for p in sorted(SAMPLE.rglob("*.py")) + } + + +def summarize(samples: list[int]) -> dict[str, float]: + """Return the median and spread of nanosecond samples, in milliseconds.""" + values = sorted(sample / 1e6 for sample in samples) + quartiles = statistics.quantiles(values, n=4) if len(values) > 1 else values * 3 + return { + "median_ms": statistics.median(values), + "q1_ms": quartiles[0], + "q3_ms": quartiles[-1], + "min_ms": values[0], + "max_ms": values[-1], + "count": len(values), + } + + +def timed(warmup: int, repetitions: int, operation: Callable[[], object]) -> list[int]: + """Return nanosecond samples after warmup calls.""" + for _ in range(warmup): + operation() + samples = [] + for _ in range(repetitions): + start = time.perf_counter_ns() + operation() + samples.append(time.perf_counter_ns() - start) + return samples + + +def _tool(*args: str) -> str: + try: + return subprocess.run( + list(args), capture_output=True, text=True, check=True + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unavailable" + + +def environment() -> dict[str, object]: + """Describe the hardware, toolchains and the loaded native module.""" + import morphir_python_native._native as native + + lock = tomllib.loads((ROOT / "rust" / "Cargo.lock").read_text()) + versions = { + package["name"]: package["version"] + for package in lock["package"] + if package["name"] in {"pyo3", "morphir-core", "morphir-extension-sdk"} + } + module = Path(native.__file__) + cpu = _tool("sysctl", "-n", "machdep.cpu.brand_string") + if cpu == "unavailable" and Path("/proc/cpuinfo").exists(): + cpu = next( + ( + line.split(":", 1)[1].strip() + for line in Path("/proc/cpuinfo").read_text().splitlines() + if line.startswith("model name") + ), + "unavailable", + ) + return { + "timestamp": datetime.now(UTC).isoformat(), + "platform": platform.platform(), + "machine": platform.machine(), + "cpu": cpu, + "cpu_count": os.cpu_count(), + "python": sys.version, + "rustc": _tool("rustc", "--version"), + "cargo_profile": "release (lto, codegen-units=1, opt-level=3)", + "crates": versions, + "native_module": module.name, + "native_module_sha256": hashlib.sha256(module.read_bytes()).hexdigest(), + "native_version": native.__version__, + "compiler_version": native.COMPILER_VERSION, + "git_commit": _tool("git", "-C", str(ROOT), "rev-parse", "HEAD"), + "git_dirty": bool(_tool("git", "-C", str(ROOT), "status", "--porcelain")), + } + + +def rust_samples( + request: dict[str, object], warmup: int, repetitions: int, work: Path +) -> dict[str, object]: + """Run the release-build Rust benchmark example.""" + path = work / "request.json" + path.write_text(json.dumps(request)) + output = subprocess.run( + [ + "cargo", "run", "--quiet", "--locked", "--release", + "-p", "morphir-python-native", "--example", "bench", "--", + str(path), str(warmup), str(repetitions), + ], + cwd=ROOT / "rust", capture_output=True, text=True, check=True, + ).stdout # fmt: skip + return dict(json.loads(output)) + + +def measure( + name: str, + sources: dict[str, str], + version: str, + warmup: int, + repetitions: int, + cli_repetitions: int, +) -> dict[str, object]: + """Measure every stage for one input set and IR version.""" + import morphir_python_native._native as native + from morphir.ir import format_version as fv + from morphir.ir.v3 import codec as v3_codec + from morphir.ir.v4 import codec as v4_codec + from morphir_tools.python import ( + CompileFailure, + Generated, + IrVersion, + SourceDocument, + compile_sources, + generate_python, + ) + + ir_version = IrVersion.V3 if version == "3" else IrVersion.V4 + codec = v3_codec if version == "3" else v4_codec + documents = [SourceDocument(path, text) for path, text in sorted(sources.items())] + request: dict[str, object] = { + "package": "acme/bench", + "sources": [{"path": d.path, "text": d.text} for d in documents], + "exposedModules": None, + "irVersion": version, + } + request_text = json.dumps(request) + compiled = compile_sources(documents, package="acme/bench", ir_version=ir_version) + if isinstance(compiled, CompileFailure): + raise SystemExit(f"{name}: {compiled.diagnostics}") + generated = generate_python(compiled.distribution) + if not isinstance(generated, Generated): + raise SystemExit(f"{name}: {generated}") + ir_text = json.loads(native.compile(request_text))["ir"] + encoded = fv.dump_json(codec.encode_versioned_distribution(compiled.distribution)) + + samples: dict[str, list[int]] = {} + with tempfile.TemporaryDirectory(prefix="morphir-bench-") as scratch: + work = Path(scratch) + rust = rust_samples(request, warmup, repetitions, work) + for key in ( + "rust_compile", + "rust_generate", + "bridge_compile", + "bridge_generate", + ): + samples[key] = [int(value) for value in rust[key]] # type: ignore[attr-defined] + samples["native_compile"] = timed( + warmup, repetitions, lambda: native.compile(request_text) + ) + samples["native_generate"] = timed( + warmup, repetitions, lambda: native.generate(encoded) + ) + samples["api_compile"] = timed( + warmup, + repetitions, + lambda: compile_sources( + documents, package="acme/bench", ir_version=ir_version + ), + ) + samples["api_generate"] = timed( + warmup, repetitions, lambda: generate_python(compiled.distribution) + ) + samples["codec_decode"] = timed( + warmup, repetitions, lambda: codec.decode_versioned_distribution(ir_text) + ) + samples["codec_encode"] = timed( + warmup, + repetitions, + lambda: fv.dump_json( + codec.encode_versioned_distribution(compiled.distribution) + ), + ) + samples["ast_parse"] = timed( + warmup, + repetitions, + lambda: [ast.parse(text) for text in sources.values()], + ) + source_root = work / "src" + for path, text in sources.items(): + target = source_root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + cli = shutil.which("morphir-python", path=str(Path(sys.executable).parent)) + model = work / "model.json" + flag = "v3" if version == "3" else "v4" + compile_cli = [ + str(cli), "compile", str(source_root), "--package", "acme/bench", + "--ir-version", flag, "--output", str(model), + ] # fmt: skip + samples["cli_compile"] = timed( + 1, + cli_repetitions, + lambda: subprocess.run(compile_cli, check=True, capture_output=True), + ) + counter = iter(range(10**6)) + + def generate_cli() -> None: + out = work / f"gen{next(counter)}" + subprocess.run( + [str(cli), "generate", str(model), "--output-dir", str(out)], + check=True, + capture_output=True, + ) + + samples["cli_generate"] = timed(1, cli_repetitions, generate_cli) + cold = [ + sys.executable, "-c", + "from morphir_tools.python import native_compiler; native_compiler()", + ] # fmt: skip + samples["cold_import"] = timed( + 1, cli_repetitions, lambda: subprocess.run(cold, check=True) + ) + return { + "input": name, + "ir_version": version, + "modules": len(sources), + "source_bytes": sum(len(t.encode()) for t in sources.values()), + "ir_bytes_rust": rust["ir_bytes"], + "ir_bytes_python_encoder": len(encoded.encode()), + "generated_bytes": sum(len(a.text.encode()) for a in generated.artifacts), + "warmup": warmup, + "repetitions": repetitions, + "process_repetitions": cli_repetitions, + "samples_ns": samples, + "summary": {stage: summarize(values) for stage, values in samples.items()}, + } + + +STAGE_NOTES = { + "rust_compile": "Rust Frontend::compile on owned SDK values", + "bridge_compile": "Rust bridge: parse request, compile, serialize IR text", + "native_compile": "CPython call to _native.compile, GIL released", + "api_compile": "compile_sources: request JSON, native call, codec decode", + "codec_decode": "Python codec decode of the IR text alone", + "cli_compile": "morphir-python compile in a new process", + "rust_generate": "Rust Backend::generate on owned IR", + "bridge_generate": "Rust bridge: parse IR text, generate, serialize", + "native_generate": "CPython call to _native.generate", + "api_generate": "generate_python: codec encode, native call, artifacts", + "codec_encode": "Python codec encode of the model alone", + "cli_generate": "morphir-python generate in a new process", + "cold_import": "new interpreter: import API and load the native module", + "ast_parse": "ast.parse of the sources (parsing only, no compilation)", +} + + +def observations(runs: list[dict[str, object]]) -> list[str]: + """Derive plain statements from the medians; no thresholds, no speed claims.""" + notes = [] + for run in runs: + summary = cast("dict[str, dict[str, float]]", run["summary"]) + + def median(stage: str, s: dict[str, dict[str, float]] = summary) -> float: + return s[stage]["median_ms"] + + api, native = median("api_compile"), median("native_compile") + notes.append( + f"{run['input']}, IR v{run['ir_version']} ({run['modules']} modules, " + f"{run['ir_bytes_rust']} IR bytes): the API compile median is " + f"{api:.2f} ms, of which the native call is {native:.2f} ms and a " + f"separate codec decode of the same IR takes {median('codec_decode'):.2f} " + f"ms. Generation: API {median('api_generate'):.2f} ms, native " + f"{median('native_generate'):.2f} ms, codec encode " + f"{median('codec_encode'):.2f} ms. The Rust trait call is " + f"{median('rust_compile'):.2f} ms. The CLI compile in a new process is " + f"{median('cli_compile'):.1f} ms, against a cold import of " + f"{median('cold_import'):.1f} ms. ast.parse alone is " + f"{median('ast_parse'):.2f} ms." + ) + return notes + + +def report(results: dict[str, object], path: Path) -> None: + """Write an HTML summary from the agent-document template.""" + env = results["environment"] + runs = results["runs"] + assert isinstance(env, dict) and isinstance(runs, list) + assets = os.path.relpath(ROOT / "docs" / "templates" / "assets", path.parent) + rows = [] + for run in runs: + summary = run["summary"] + for stage, note in STAGE_NOTES.items(): + s = summary[stage] + rows.append( + f"{html.escape(run['input'])}v{run['ir_version']}" + f"{stage}{s['median_ms']:.3f}" + f"{s['q1_ms']:.3f}–{s['q3_ms']:.3f}" + f"{s['min_ms']:.3f}–{s['max_ms']:.3f}" + f"{s['count']}{html.escape(note)}" + ) + inputs = "".join( + f"{html.escape(r['input'])}v{r['ir_version']}" + f"{r['modules']}{r['source_bytes']}" + f"{r['ir_bytes_rust']}{r['ir_bytes_python_encoder']}" + f"{r['generated_bytes']}" + for r in runs + ) + environment_rows = "".join( + f"{html.escape(str(k))}{html.escape(str(v))}" + for k, v in env.items() + ) + document = f""" + + + + +Python compiler integration costs + + + + + + + +
+

Python compiler integration costs

+

morphir-ukvn.1.6.6 · {html.escape(str(env["timestamp"]))} · measurement report

+ +

What these numbers show

+

Each stage ran on the same sources and IR version. Values are milliseconds +per call: the median, the interquartile range and the full range. Raw samples are in +results.json next to this report. They are observations on one machine, +not thresholds.

+

The difference between the Rust, bridge, native and API stages is the cost of +JSON exchange, the CPython call and the Python codecs. The CLI and cold-import +stages include interpreter start-up.

+

ast_parse only parses. It does not resolve names, check types or +build IR, so it is not a compiler. This repository has no pure-Python compiler, so +these results support no claim that Rust compiles faster or slower than an +equivalent Python implementation.

+

Observations

+
    {"".join(f"
  • {html.escape(note)}
    • " for note in observations(runs))}
+

Within a few hundred microseconds, differences between the Rust, bridge and +native stages are within run-order and allocation noise; the Rust stages run +first in a separate process. For larger inputs, the Python codecs and the JSON +exchange cost more than the native compiler itself, and every CLI run adds +interpreter start-up and imports (the cold-import stage). These are the costs to +reduce first if they matter; none of them measures compiler language speed.

+

Inputs

+ + +{inputs}
InputIRModulesSource bytesIR bytes (Rust)IR bytes (Python encoder)Generated bytes
+

Results

+ + +{"".join(rows)}
InputIRStageMedian msIQR msRange msSamplesMeasures
+

Environment

+{environment_rows}
+
+ + +""" + path.write_text(document) + + +def main(argv: list[str] | None = None) -> None: + """Run the benchmark and write its results and report.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--modules", type=int, default=20) + parser.add_argument("--functions", type=int, default=10) + parser.add_argument("--warmup", type=int, default=5) + parser.add_argument("--repetitions", type=int, default=30) + parser.add_argument("--process-repetitions", type=int, default=10) + parser.add_argument("--output", type=Path, default=ROOT / ".dev" / "bench") + parser.add_argument( + "--report-only", type=Path, help="Re-render report.html from a results.json" + ) + args = parser.parse_args(argv) + if args.report_only is not None: + saved = json.loads(args.report_only.read_text()) + report(saved, args.report_only.parent / "report.html") + print(args.report_only.parent / "report.html") + return + inputs = { + "sample": sample_sources(), + f"scale-{args.modules}x{args.functions}": generated_sources( + args.modules, args.functions + ), + } + runs = [ + measure( + name, + sources, + version, + args.warmup, + args.repetitions, + args.process_repetitions, + ) + for name, sources in inputs.items() + for version in ("3", "4") + ] + results: dict[str, object] = { + "environment": environment(), + "parameters": vars(args) | {"output": str(args.output)}, + "runs": runs, + } + stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + directory = args.output / stamp + directory.mkdir(parents=True) + (directory / "results.json").write_text(json.dumps(results, indent=1)) + report(results, directory / "report.html") + print(directory / "report.html") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_distributions.py b/scripts/check_distributions.py index 0b25a5e8..62bbaa88 100644 --- a/scripts/check_distributions.py +++ b/scripts/check_distributions.py @@ -15,6 +15,8 @@ from typing import TYPE_CHECKING from urllib.request import urlopen +from scripts.release_candidates import NATIVE_PLATFORMS, classify + if TYPE_CHECKING: from collections.abc import Mapping from email.message import Message @@ -139,10 +141,38 @@ def match_published_wheel(wheel: Path, published: Mapping[str, str]) -> None: ) +def require_published_native(version: str, published: set[str]) -> None: + """Refuse a CLI release before its exact native dependency is published. + + Args: + version: The exact ``morphir-python-native`` version the CLI pins. + published: File names PyPI lists for that version. + """ + expected = set(native_candidates_for(version)) + if not expected <= published: + raise ValueError( + f"publish morphir-python-native {version} with every qualified platform " + f"before this CLI; missing {sorted(expected - published)}" + ) + + +def native_candidates_for(version: str) -> list[str]: + """Return the file names a complete native release contains.""" + stem = f"morphir_python_native-{version}" + return [f"{stem}-cp314-abi3-{tag}.whl" for tag in NATIVE_PLATFORMS] + [ + f"{stem}.tar.gz" + ] + + def main() -> None: - """Validate both candidates and require the selected CLI's published core.""" - packages = load_packages(Path(__file__).resolve().parents[1]) - selected = select_package(os.environ.get("RELEASE_TAG", ""), packages) + """Validate both candidates and require the selected CLI's published inputs.""" + root = Path(__file__).resolve().parents[1] + packages = load_packages(root) + tag = os.environ.get("RELEASE_TAG", "") + if tag.startswith(("native-v", "extension/python/v")): + classify(tag) + tag = "" + selected = select_package(tag, packages) for package in packages: expected = {package.wheel.name, f"{package.stem}.tar.gz"} actual = { @@ -165,6 +195,18 @@ def main() -> None: } match_published_wheel(core.wheel, digests) print("The CLI's qualified core matches the published library") + pin = next( + requirement.split("==", 1)[1] + for requirement in tomllib.loads( + (packages[1].directory / "pyproject.toml").read_text() + )["project"]["dependencies"] + if requirement.startswith("morphir-python-native==") + ) + url = f"https://pypi.org/pypi/morphir-python-native/{pin}/json" + with urlopen(url, timeout=30) as response: + files = {item["filename"] for item in json.load(response)["urls"]} + require_published_native(pin, files) + print(f"The CLI's native dependency {pin} is published") if __name__ == "__main__": diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py new file mode 100644 index 00000000..1321b138 --- /dev/null +++ b/scripts/ci_changes.py @@ -0,0 +1,178 @@ +"""Select CI categories for a change set and gate on their results. + +``select`` maps changed paths to the categories CI must run and prints +``key=value`` lines for ``$GITHUB_OUTPUT``. Unknown paths, workflow changes, +pushes and release qualification select every category, so selection can only +skip work it knows is unaffected. + +``aggregate`` is the required check. Every selected job must have succeeded and +every unselected job must have been skipped. A failed, cancelled, missing or +unexpectedly skipped job fails the aggregate. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import subprocess +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + +CATEGORIES = ("python", "native", "rust", "bundle") +"""``python``: lint, type check and Python tests; ``native``: the native wheel and +its clean-install qualification; ``rust``: Rust format, lint, tests and the +independent consumer; ``bundle``: the WASM extension bundle and its host check.""" + +JOBS: Mapping[str, tuple[str, ...]] = { + "lint": ("python",), + "typecheck": ("python",), + "native-wheel": ("python", "native", "bundle"), + "test": ("python", "native"), + "rust": ("rust",), + "bundle": ("bundle",), + "build": ("python", "native"), +} +"""Each job runs when any of its categories is selected.""" + +# First match wins. Each rule names the categories a matching path selects. +RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + (".github/*", CATEGORIES), + (".config/mise/*", CATEGORIES), + ("rust/extension.toml", ("bundle",)), + # Lint and type checks cover every script. + ("scripts/extension_bundle.py", ("python", "bundle")), + ("rust/crates/morphir-python-binding/*", CATEGORIES), + ("rust/crates/morphir-python-native/*", ("python", "native", "rust")), + ("rust/*", CATEGORIES), + ("packages/morphir-python-native/*", ("python", "native")), + ("scripts/qualify_native.py", ("python", "native")), + ("tests/fixtures/python/*", ("python", "native", "bundle")), + ("packages/*", ("python", "native")), + ("tests/*", ("python", "native")), + ("features/*", ("python", "native")), + ("scripts/*", ("python", "native")), + ("pyproject.toml", ("python", "native")), + ("uv.lock", ("python", "native")), + ("behave.ini", ("python",)), + (".pre-commit-config.yaml", ("python",)), + ("docs/python-compiler.md", ("python",)), + ("*.md", ()), + ("docs/*", ()), + ("openspec/*", ()), + ("LICENSE", ()), + ("LICENSE.spdx", ()), + ("NOTICE", ()), + ("renovate.json", ()), + ("skills-lock.json", ()), + (".beads/*", ()), + # Hatchling uses .gitignore to choose the files in each source archive. + (".gitignore", ("python", "native")), + (".gitattributes", ()), +) + + +@dataclass(frozen=True) +class Selection: + """The categories a change set needs, and why.""" + + categories: frozenset[str] + reason: str + + def jobs(self) -> dict[str, bool]: + """Return whether each job runs.""" + return { + job: any(category in self.categories for category in categories) + for job, categories in JOBS.items() + } + + +def select(paths: Iterable[str], *, full: bool = False) -> Selection: + """Select categories for changed paths; unknown paths select everything.""" + if full: + return Selection(frozenset(CATEGORIES), "full run") + selected: set[str] = set() + for path in paths: + for pattern, categories in RULES: + if fnmatch.fnmatchcase(path, pattern): + selected.update(categories) + break + else: + return Selection(frozenset(CATEGORIES), f"unclassified path {path}") + return Selection(frozenset(selected), "changed paths") + + +def aggregate( + selected: Mapping[str, bool], results: Mapping[str, Mapping[str, str]] +) -> list[str]: + """Return problems with the job results; empty when the gate passes. + + Args: + selected: Whether each job was selected, from ``select``. + results: The ``needs`` context: job name to ``{"result": ...}``. + """ + problems: list[str] = [] + for job, required in sorted(selected.items()): + result = results.get(job, {}).get("result") + if result is None: + problems.append(f"{job}: no result") + elif required and result != "success": + problems.append(f"{job}: selected but {result}") + elif not required and result != "skipped": + problems.append(f"{job}: not selected but {result}") + for job in sorted(set(results) - set(selected) - {"changes"}): + if results[job].get("result") != "success": + problems.append(f"{job}: {results[job].get('result')}") + changes = results.get("changes", {}).get("result") + if changes != "success": + problems.append(f"changes: {changes}") + return problems + + +def changed_paths(base: str, head: str) -> list[str]: + """Return paths changed between the merge base of ``base`` and ``head``.""" + output = subprocess.run( + ["git", "diff", "--name-only", f"{base}...{head}"], + check=True, + capture_output=True, + text=True, + ).stdout + return [line for line in output.splitlines() if line] + + +def main(argv: list[str] | None = None) -> None: + """Run ``select`` or ``aggregate``.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + chooser = commands.add_parser("select") + chooser.add_argument("--base") + chooser.add_argument("--head", default="HEAD") + chooser.add_argument("--full", action="store_true") + chooser.add_argument("paths", nargs="*") + gate = commands.add_parser("aggregate") + gate.add_argument("--selected", required=True, help="JSON from select") + gate.add_argument("--needs", required=True, help="JSON of the needs context") + args = parser.parse_args(argv) + if args.command == "select": + paths = list(args.paths) + if args.base: + paths += changed_paths(args.base, args.head) + selection = select(paths, full=args.full or not (args.base or args.paths)) + print(f"jobs={json.dumps(selection.jobs(), sort_keys=True)}") + for category in CATEGORIES: + print(f"{category}={str(category in selection.categories).lower()}") + print(f"reason={selection.reason}") + return + problems = aggregate(json.loads(args.selected), json.loads(args.needs)) + for problem in problems: + print(f"::error::{problem}") + if problems: + raise SystemExit(1) + print("all selected checks passed") + + +if __name__ == "__main__": + main() diff --git a/scripts/extension_bundle.py b/scripts/extension_bundle.py new file mode 100644 index 00000000..8d057534 --- /dev/null +++ b/scripts/extension_bundle.py @@ -0,0 +1,391 @@ +"""Package the morphir-python WASM extension and qualify it in a Morphir host. + +Subcommands: + +- ``package --wasm FILE --output DIR [--git-commit SHA]``: run finos/morphir-rust's + canonical WASM packager, unchanged, at the revision pinned in + ``rust/extension.toml``. The packager reads this repository's extension + declaration and crate version, asks the exact guest bytes for their capability + claims, and writes the WASM, its ``.sha256`` file and a version-2 + ``release.json``. +- ``host-check --bundle DIR``: download the pinned Morphir CLI, verify its + digest, publish and install the bundle into a new Morphir home, delete the + repository, block network access, then compile, generate and recompile a + multi-module model in IR v3 and v4. The compiled IR must equal what the Python + API produces for the same sources. + +Each subcommand prints a JSON receipt and exits non-zero on the first failure. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import urllib.request +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DECLARATION = ROOT / "rust" / "extension.toml" +CRATE = ROOT / "rust" / "crates" / "morphir-python-binding" / "Cargo.toml" +SAMPLE = ROOT / "tests" / "fixtures" / "python" / "loans" +CACHE = ROOT / ".dev" / "tools" +# A closed local port: any attempted network access fails fast. +OFFLINE = "http://127.0.0.1:9" + + +class BundleError(Exception): + """A packaging or qualification step failed.""" + + +def declaration() -> dict[str, Any]: + """Return the parsed ``rust/extension.toml``.""" + return tomllib.loads(DECLARATION.read_text()) + + +def crate_version() -> str: + """Return the binding crate version, which is the extension version.""" + return str(tomllib.loads(CRATE.read_text())["package"]["version"]) + + +def sha256(data: bytes) -> str: + """Return a hexadecimal SHA-256 digest.""" + return hashlib.sha256(data).hexdigest() + + +def registry_entry(extension: dict[str, Any]) -> str: + """Render the extension as a morphir-rust ``.github/extensions.toml`` file.""" + short_id = extension["short_id"] + lines = [f"[extensions.{short_id}]"] + for key, value in extension.items(): + if key == "short_id": + continue + lines.append(f"{key} = {_toml(value)}") + return "\n".join(lines) + "\n" + + +def _toml(value: object) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, str): + return json.dumps(value) + if isinstance(value, list): + return "[" + ", ".join(_toml(item) for item in value) + "]" + if isinstance(value, dict): + items = ", ".join(f"{k} = {_toml(v)}" for k, v in value.items()) + return "{ " + items + " }" + raise BundleError(f"unsupported declaration value {value!r}") + + +def _run(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + return subprocess.run(args, check=True, text=True, **kwargs) + + +def packager_checkout() -> Path: + """Fetch the pinned morphir-rust revision and reset it to a clean state.""" + pin = declaration()["packager"] + checkout = CACHE / f"morphir-rust-{pin['rev'][:12]}" + if not (checkout / ".git").is_dir(): + checkout.mkdir(parents=True, exist_ok=True) + _run(["git", "init", "--quiet"], cwd=checkout) + _run(["git", "remote", "add", "origin", pin["repository"]], cwd=checkout) + if ( + subprocess.run( + ["git", "cat-file", "-e", f"{pin['rev']}^{{commit}}"], + cwd=checkout, + capture_output=True, + check=False, + ).returncode + != 0 + ): + _run( + ["git", "fetch", "--quiet", "--depth", "1", "origin", pin["rev"]], + cwd=checkout, + ) + _run(["git", "checkout", "--quiet", "--force", pin["rev"]], cwd=checkout) + _run(["git", "clean", "--quiet", "-fdx", "-e", "target"], cwd=checkout) + head = _run(["git", "rev-parse", "HEAD"], cwd=checkout, capture_output=True) + if head.stdout.strip() != pin["rev"]: + raise BundleError(f"packager checkout is at {head.stdout.strip()}") + return checkout + + +def prepare_packager(checkout: Path) -> None: + """Point the pinned packager at this repository's declaration and version. + + The packager reads ``.github/extensions.toml`` and the registered crate's + ``Cargo.toml``. Replace them with this repository's extension entry and a + dependency-free stub crate carrying the binding's version, then update only + the workspace members in ``Cargo.lock``. + """ + extension = declaration()["extension"] + (checkout / ".github" / "extensions.toml").write_text(registry_entry(extension)) + crate = checkout / "crates" / extension["package"] + shutil.rmtree(crate, ignore_errors=True) + (crate / "src").mkdir(parents=True) + (crate / "src" / "lib.rs").write_text("") + (crate / "Cargo.toml").write_text( + f'[package]\nname = "{extension["package"]}"\n' + f'version = "{crate_version()}"\nedition = "2024"\npublish = false\n' + ) + _run(["cargo", "update", "--quiet", "--workspace"], cwd=checkout) + changed = _run( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=checkout, + capture_output=True, + ).stdout.split("\n") + allowed = ( + ".github/extensions.toml", + "Cargo.lock", + f"crates/{extension['package']}/", + ) + for line in filter(None, changed): + path = line[3:] + if not path.startswith(allowed): + raise BundleError(f"preparing the packager changed {path}") + + +def package(wasm: Path, output: Path, git_commit: str | None) -> dict[str, object]: + """Package the guest with the pinned canonical packager.""" + checkout = packager_checkout() + prepare_packager(checkout) + if output.exists(): + raise BundleError(f"bundle directory {output} already exists") + args = [sys.executable, "-B", "scripts/package_extension.py", "--short-id"] + args += [declaration()["extension"]["short_id"], "--wasm", str(wasm.resolve())] + args += ["--output", str(output.resolve())] + if git_commit: + args += ["--git-commit", git_commit] + _run(args, cwd=checkout) + return receipt(output) + + +def receipt(bundle: Path) -> dict[str, object]: + """Check a bundle's files, digest and claims, and describe them.""" + extension = declaration()["extension"] + version = crate_version() + artifact = f"{extension['artifact']}-{version}.wasm" + names = sorted(p.name for p in bundle.iterdir()) + if names != sorted([artifact, f"{artifact}.sha256", "release.json"]): + raise BundleError(f"unexpected bundle files {names}") + wasm = (bundle / artifact).read_bytes() + digest = sha256(wasm) + if (bundle / f"{artifact}.sha256").read_text() != f"{digest} {artifact}\n": + raise BundleError("the .sha256 file does not match the WASM") + descriptor = json.loads((bundle / "release.json").read_text()) + (entry,) = descriptor["artifacts"] + claims = entry["claims"] + if ( + descriptor["extensionId"] != extension["extension_id"] + or descriptor["version"] != version + or entry["sha256"] != digest + or claims["extension"]["id"] != extension["extension_id"] + or claims["extension"]["version"] != version + ): + raise BundleError("release.json disagrees with the declaration or WASM") + capabilities = claims["capabilities"] + frontend, backend = capabilities.get("frontend"), capabilities.get("backend") + if not ( + isinstance(frontend, dict) + and frontend.get("compile") is True + and frontend.get("irVersions") == extension["ir_versions"] + and isinstance(backend, dict) + and backend.get("generate") is True + and backend.get("targets") == extension["targets"] + ): + raise BundleError("the claims lack the declared compile and generate support") + return { + "bundle": {name: sha256((bundle / name).read_bytes()) for name in names}, + "wasm_sha256": digest, + "extension": descriptor["extensionId"], + "version": version, + "git_commit": descriptor.get("gitCommit"), + "packager_rev": declaration()["packager"]["rev"], + "capabilities": sorted(k for k, v in capabilities.items() if v), + } + + +def _target() -> str: + machine = platform.machine().lower() + arch = {"arm64": "aarch64", "amd64": "x86_64"}.get(machine, machine) + system = {"Darwin": "apple-darwin", "Linux": "unknown-linux-gnu"}.get( + platform.system() + ) + if system is None: + raise BundleError(f"no pinned Morphir CLI for {platform.system()}") + return f"{arch}-{system}" + + +def morphir_cli() -> tuple[Path, str]: + """Download the pinned Morphir CLI for this platform and verify its digest.""" + host = declaration()["host"] + target = _target() + expected = host["sha256"].get(target) + if expected is None: + raise BundleError(f"no pinned Morphir CLI digest for {target}") + directory = CACHE / f"morphir-{host['version']}-{target}" + binary = directory / "morphir" + archive_name = f"morphir-{host['version']}-{target}.tgz" + if not binary.is_file(): + with urllib.request.urlopen( + f"{host['release']}/{archive_name}", timeout=120 + ) as r: + data = r.read() + if sha256(data) != expected: + raise BundleError(f"{archive_name} digest differs from the pin") + directory.mkdir(parents=True, exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(data)) as archive: + member = next( + m for m in archive.getmembers() if Path(m.name).name == "morphir" + ) + extracted = archive.extractfile(member) + if extracted is None: + raise BundleError(f"{archive_name} has no morphir executable") + binary.write_bytes(extracted.read()) + binary.chmod(0o755) + return binary, expected + + +def host_check(bundle: Path) -> dict[str, object]: + """Install the bundle with the pinned CLI and round trip offline.""" + bundle = bundle.resolve() + described = receipt(bundle) + cli, cli_digest = morphir_cli() + with tempfile.TemporaryDirectory(prefix="python-bundle-host-") as scratch: + work = Path(scratch) + project, home, repository = work / "project", work / "home", work / "repo" + shutil.copytree(SAMPLE, project / "src") + (project / "morphir.toml").write_text( + '[project]\nname = "acme/example"\nversion = "0.1.0"\n' + 'source_directory = "src"\n[frontend]\nlanguage = "python"\n' + "emit_parse_stage = false\n" + ) + base = {**os.environ, "MORPHIR_HOME": str(home)} + + def morphir(*args: str, offline: bool = False) -> str: + env = dict(base) + if offline: + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"): + env[name] = env[name.lower()] = OFFLINE + env.pop("NO_PROXY", None) + env.pop("no_proxy", None) + result = subprocess.run( + [str(cli), *args], + cwd=project, + env=env, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise BundleError(f"morphir {' '.join(args)}: {result.stderr}") + return result.stdout + + morphir("extension", "repository", "init", str(repository)) + morphir( + "extension", "repository", "add", "local", "--directory", str(repository) + ) + morphir("extension", "repository", "publish", "local", "--bundle", str(bundle)) + morphir("extension", "repository", "verify", "local") + morphir("extension", "install", "morphir-python", "--repository", "local") + shutil.rmtree(repository) + listed = morphir("extension", "list", offline=True) + if "morphir-python" not in listed: + raise BundleError(f"morphir-python is not installed: {listed}") + results: dict[str, object] = {} + for version in ("3.0.0", "4"): + tag = version.split(".")[0] + compiled, generated = f"compiled-{tag}", f"generated-{tag}" + morphir( + "compile", "--language", "python", "--input", "src", + "--ir-version", version, "--output", compiled, offline=True, + ) # fmt: skip + morphir( + "generate", "--target", "python", "--input", + f"{compiled}/morphir-ir.json", "--output", generated, offline=True, + ) # fmt: skip + morphir( + "compile", "--language", "python", "--input", generated, + "--ir-version", version, "--output", f"re{compiled}", offline=True, + ) # fmt: skip + first = json.loads((project / compiled / "morphir-ir.json").read_text()) + again = json.loads( + (project / f"re{compiled}" / "morphir-ir.json").read_text() + ) + if first != again: + raise BundleError(f"IR v{tag} changed across a host round trip") + files = sorted( + p.relative_to(project / generated).as_posix() + for p in (project / generated).rglob("*.py") + ) + results[f"v{tag}"] = { + "modules": files, + "agrees_with_python_api": agrees_with_api(first, tag), + } + return { + **described, + "host": {"morphir": declaration()["host"]["version"], "sha256": cli_digest}, + "offline_round_trips": results, + } + + +def agrees_with_api(document: dict[str, Any], tag: str) -> bool: + """Compare host output with the Python API's model for the same sources.""" + from morphir.ir.v3 import codec as v3_codec + from morphir.ir.v4 import codec as v4_codec + from morphir_tools.python import ( + CompileFailure, + IrVersion, + SourceDocument, + compile_sources, + ) + + sources = [ + SourceDocument(p.relative_to(SAMPLE).as_posix(), p.read_text()) + for p in sorted(SAMPLE.rglob("*.py")) + ] + version = IrVersion.V3 if tag == "3" else IrVersion.V4 + compiled = compile_sources(sources, package="acme/example", ir_version=version) + if isinstance(compiled, CompileFailure): + raise BundleError(f"the Python API failed: {compiled.diagnostics}") + codec = v3_codec if tag == "3" else v4_codec + host_model = codec.decode_versioned_distribution(json.dumps(document)) + if host_model != compiled.distribution: + raise BundleError(f"host and Python API models differ for IR v{tag}") + return True + + +def main() -> None: + """Run one subcommand and print its receipt.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + pack = commands.add_parser("package") + pack.add_argument("--wasm", type=Path, required=True) + pack.add_argument("--output", type=Path, required=True) + pack.add_argument("--git-commit") + check = commands.add_parser("host-check") + check.add_argument("--bundle", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "package": + result = package(args.wasm, args.output, args.git_commit) + else: + result = host_check(args.bundle) + except (BundleError, subprocess.CalledProcessError, OSError, KeyError) as error: + print(f"bundle check failed: {error}", file=sys.stderr) + raise SystemExit(1) from None + print(json.dumps({"check": args.command, **result}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/qualify_native.py b/scripts/qualify_native.py new file mode 100644 index 00000000..18f22e10 --- /dev/null +++ b/scripts/qualify_native.py @@ -0,0 +1,404 @@ +"""Qualify the native compiler distribution and a clean installation of all three. + +Subcommands: + +- ``wheel WHEEL``: check one ``morphir-python-native`` wheel's contents, tags and + metadata against the Rust crate and the tools pin. +- ``sdist SDIST --out DIR``: check that the source archive holds every Rust + input, then build a wheel from it in a temporary directory outside the + checkout, with no sibling repository reachable. +- ``install WHEEL...``: install the exact core, native and tools wheels into a + new virtual environment with Rust absent from ``PATH``, then exercise the + installed API and CLI, a library-only installation and a deliberately + mismatched native installation. + +Every subcommand prints one JSON receipt naming the artifacts and their SHA-256 +digests, and exits non-zero on the first failed check. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import zipfile +from email.parser import BytesParser +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + +ROOT = Path(__file__).resolve().parents[1] +NATIVE = "morphir-python-native" +NATIVE_CRATE = ROOT / "rust" / "crates" / "morphir-python-native" / "Cargo.toml" +TOOLS_PROJECT = ROOT / "packages" / "morphir-tools" / "pyproject.toml" +SAMPLE = ROOT / "tests" / "fixtures" / "python" / "loans" + +REQUIRED_SDIST_FILES = ( + "pyproject.toml", + "rust/Cargo.toml", + "rust/Cargo.lock", + "rust/crates/morphir-python-native/Cargo.toml", + "rust/crates/morphir-python-native/src/lib.rs", + "rust/crates/morphir-python-binding/Cargo.toml", + "rust/crates/morphir-python-binding/src/lib.rs", + "python/morphir_python_native/__init__.py", + "python/morphir_python_native/_native.pyi", + "python/morphir_python_native/py.typed", +) + +# Wheel tags the release builds; abi3 for CPython 3.14 and later. +WHEEL_NAME = re.compile( + r"^morphir_python_native-(?P[^-]+)-cp314-abi3-(?P[a-z0-9_.]+)\.whl$" +) + + +class QualificationError(Exception): + """A qualification check failed.""" + + +def sha256(path: Path) -> str: + """Return the hexadecimal SHA-256 digest of a file.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def crate_version(manifest: Path = NATIVE_CRATE) -> str: + """Return the native crate version, which is also the distribution version.""" + return str(tomllib.loads(manifest.read_text())["package"]["version"]) + + +def tools_pin(project: Path = TOOLS_PROJECT) -> str: + """Return the exact native version the tools distribution requires.""" + for requirement in tomllib.loads(project.read_text())["project"]["dependencies"]: + match = re.fullmatch(rf"{NATIVE}==([^\s;]+)", requirement.replace(" ", "")) + if match: + return match.group(1) + raise QualificationError(f"{project} has no exact {NATIVE} requirement") + + +def check_versions(version: str, expected_module: str | None = None) -> None: + """Require the crate version, the tools pin and the tools constant to agree.""" + pin = tools_pin() + if version != crate_version() or version != pin: + raise QualificationError( + f"native version {version} differs from crate {crate_version()} " + f"or tools pin {pin}" + ) + bridge = ( + ROOT + / "packages" + / "morphir-tools" + / "src" + / "morphir_tools" + / "python" + / "_bridge.py" + ).read_text() + if f'EXPECTED_NATIVE_VERSION = "{version}"' not in bridge: + raise QualificationError("EXPECTED_NATIVE_VERSION differs from the native pin") + if expected_module is not None and expected_module != version: + raise QualificationError( + f"compiled module reports {expected_module}, distribution is {version}" + ) + + +def validate_wheel(path: Path) -> dict[str, object]: + """Check a native wheel's name, files, tags and metadata. + + Raises: + QualificationError: For the first problem found. + """ + match = WHEEL_NAME.match(path.name) + if match is None: + raise QualificationError(f"{path.name} is not a cp314-abi3 native wheel") + version = match["version"] + check_versions(version) + stem = f"morphir_python_native-{version}" + with zipfile.ZipFile(path) as archive: + names = set(archive.namelist()) + metadata = BytesParser().parsebytes(archive.read(f"{stem}.dist-info/METADATA")) + wheel = archive.read(f"{stem}.dist-info/WHEEL").decode() + modules = [ + name + for name in names + if re.fullmatch(r"morphir_python_native/_native\.(abi3\.so|pyd)", name) + ] + required = { + "morphir_python_native/__init__.py", + "morphir_python_native/_native.pyi", + "morphir_python_native/py.typed", + f"{stem}.dist-info/licenses/LICENSE", + } + if len(modules) != 1 or not required <= names: + raise QualificationError( + f"{path.name} lacks {sorted(required - names)} or one native module" + ) + if metadata["Name"] != NATIVE or metadata["Version"] != version: + raise QualificationError(f"{path.name} has the wrong distribution identity") + if metadata["License-Expression"] != "Apache-2.0": + raise QualificationError(f"{path.name} lacks the Apache-2.0 license") + if metadata.get_all("Requires-Dist"): + raise QualificationError(f"{path.name} must have no runtime dependencies") + if metadata["Requires-Python"] != ">=3.14": + raise QualificationError(f"{path.name} must require Python 3.14 or later") + if "Root-Is-Purelib: false" not in wheel: + raise QualificationError(f"{path.name} must be a platform wheel") + return {"wheel": path.name, "sha256": sha256(path), "version": version} + + +def validate_sdist_names(names: Sequence[str], version: str) -> None: + """Require every Rust and Python build input under the archive root.""" + prefix = f"morphir_python_native-{version}/" + present = {name.removeprefix(prefix) for name in names if name.startswith(prefix)} + missing = [name for name in REQUIRED_SDIST_FILES if name not in present] + if missing: + raise QualificationError(f"the source archive lacks {missing}") + if any(".." in name.split("/") or name.startswith("/") for name in names): + raise QualificationError("the source archive has an unsafe member path") + + +def build_from_sdist(path: Path, out: Path) -> dict[str, object]: + """Build a wheel from the source archive alone, outside the checkout.""" + version = crate_version() + if path.name != f"morphir_python_native-{version}.tar.gz": + raise QualificationError(f"unexpected source archive name {path.name}") + with tarfile.open(path) as archive: + validate_sdist_names(archive.getnames(), version) + with tempfile.TemporaryDirectory(prefix="native-sdist-") as scratch: + isolated = Path(scratch) + if ROOT in isolated.parents: + raise QualificationError("the scratch directory is inside the checkout") + copy = isolated / path.name + shutil.copy2(path, copy) + out.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["uv", "build", "--wheel", str(copy), "--out-dir", str(isolated / "dist")], + cwd=isolated, + check=True, + ) + (wheel,) = (isolated / "dist").glob("*.whl") + validate_wheel(wheel) + target = out / wheel.name + shutil.copy2(wheel, target) + return { + "sdist": path.name, + "sdist_sha256": sha256(path), + "wheel": target.name, + "wheel_sha256": sha256(target), + } + + +def rust_free_path(venv: Path) -> str: + """Return a PATH holding the virtual environment and system tools only.""" + entries = [str(venv / ("Scripts" if os.name == "nt" else "bin"))] + for entry in os.environ.get("PATH", "").split(os.pathsep): + tools = ( + shutil.which(tool, path=entry) for tool in ("cargo", "rustc", "rustup") + ) + managed = any(word in entry for word in ("cargo", "rustup", "mise")) + if entry and not managed and all(found is None for found in tools): + entries.append(entry) + path = os.pathsep.join(entries) + for tool in ("cargo", "rustc", "rustup"): + if shutil.which(tool, path=path) is not None: + raise QualificationError(f"{tool} is still on the qualification PATH") + return path + + +INSTALLED_CHECK = r""" +import json, shutil, subprocess, sys, tempfile +from importlib.metadata import distribution, version +from pathlib import Path + +import morphir_python_native._native as native +from morphir_tools.python import ( + CompiledV3, CompiledV4, Generated, IrVersion, SourceDocument, + compile_sources, generate_python, native_compiler, +) + +sample = Path(sys.argv[1]) +info = native_compiler() +assert info.version == version("morphir-python-native") == native.__version__ +assert not distribution("morphir").requires +docs = [SourceDocument(p.relative_to(sample).as_posix(), p.read_text()) + for p in sorted(sample.rglob("*.py"))] +for ir_version, kind in ((IrVersion.V3, CompiledV3), (IrVersion.V4, CompiledV4)): + compiled = compile_sources(docs, package="acme/example", ir_version=ir_version) + assert isinstance(compiled, kind), compiled + generated = generate_python(compiled.distribution) + assert isinstance(generated, Generated), generated + again = compile_sources( + [SourceDocument(a.path, a.text) for a in generated.artifacts], + package="acme/example", ir_version=ir_version, + ) + assert again == compiled +work = Path(tempfile.mkdtemp()) +cli = [shutil.which("morphir-python", path=str(Path(sys.executable).parent))] +for flag in ("v3", "v4"): + model, out = work / f"{flag}.json", work / f"gen-{flag}" + subprocess.run(cli + ["compile", str(sample), "--package", "acme/example", + "--ir-version", flag, "-o", str(model)], check=True) + subprocess.run(cli + ["ir", "inspect", str(model)], check=True) + subprocess.run(cli + ["generate", str(model), "--output-dir", str(out)], check=True) + result = subprocess.run( + [sys.executable, "-I", "-c", + "import sys; sys.path.insert(0, sys.argv[1]); from acme import rules;" + "print(rules.choose(True, 1, 2), rules.twice_identity(7))", str(out)], + check=True, capture_output=True, text=True) + assert result.stdout.split() == ["1", "7"], result.stdout +print(json.dumps({"native": info.version, "compiler": info.compiler_version, + "bridge_api": info.bridge_api})) +""" + +LIBRARY_CHECK = r""" +from importlib.util import find_spec +from morphir.ir import format_version +from morphir.sdk import basics +assert basics.round(2.5) == 3 and format_version.V3.major == 3 +assert find_spec("morphir_python_native") is None +assert find_spec("morphir_tools") is None +""" + + +def venv_python(directory: Path, system: str = os.name) -> Path: + """Return the interpreter path inside a virtual environment.""" + if system == "nt": + return directory / "Scripts" / "python.exe" + return directory / "bin" / "python" + + +def _venv(directory: Path) -> Path: + subprocess.run( + ["uv", "venv", "--quiet", "--python", "3.14", str(directory)], check=True + ) + return venv_python(directory) + + +def _script(python: Path, name: str) -> str: + found = shutil.which(name, path=str(python.parent)) + if found is None: + raise QualificationError(f"{name} is not installed next to {python}") + return found + + +def _mismatch(python: Path, path: str) -> None: + """Fake an incompatible native install and require a clear refusal.""" + metadata = subprocess.run( + [ + str(python), + "-I", + "-c", + "from importlib.metadata import distribution as d;" + "print(d('morphir-python-native')._path)", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + file = Path(metadata) / "METADATA" + file.write_text(re.sub(r"(?m)^Version: .*$", "Version: 0.0.0", file.read_text())) + result = subprocess.run( + [ + _script(python, "morphir-python"), + "compile", + str(SAMPLE), + "--package", + "acme/example", + "-o", + str(Path(metadata).parent / "x.json"), + ], + capture_output=True, + text=True, + env={**os.environ, "PATH": path}, + check=False, + ) + if result.returncode != 1 or "morphir-python-native 0.0.0" not in result.stderr: + raise QualificationError( + f"a mismatched native install was not refused clearly: {result.stderr}" + ) + + +def install(wheels: Sequence[Path]) -> dict[str, object]: + """Install the exact wheels without Rust and exercise them.""" + by_name = {wheel.name.split("-")[0]: wheel for wheel in wheels} + expected = {"morphir", "morphir_python", "morphir_python_native"} + if set(by_name) != expected: + raise QualificationError(f"expected one wheel each for {sorted(expected)}") + validate_wheel(by_name["morphir_python_native"]) + with tempfile.TemporaryDirectory(prefix="native-install-") as scratch: + work = Path(scratch) + python = _venv(work / "full") + path = rust_free_path(work / "full") + env = {**os.environ, "PATH": path, "CARGO_HOME": str(work / "no-cargo")} + env.pop("VIRTUAL_ENV", None) + subprocess.run( + ["uv", "pip", "install", "--quiet", "--no-build", "--python", str(python)] + + [str(w) for w in wheels], + check=True, + ) + installed = subprocess.run( + [str(python), "-I", "-c", INSTALLED_CHECK, str(SAMPLE)], + env=env, + check=True, + capture_output=True, + text=True, + cwd=work, + ) + library = _venv(work / "library") + subprocess.run( + [ + *("uv", "pip", "install", "--quiet", "--no-build"), + *("--python", str(library), str(by_name["morphir"])), + ], + check=True, + ) + subprocess.run([str(library), "-I", "-c", LIBRARY_CHECK], env=env, check=True) + _mismatch(python, path) + return { + "python": platform.python_version(), + "platform": platform.platform(), + "machine": platform.machine(), + "rust_on_path": False, + "artifacts": {wheel.name: sha256(wheel) for wheel in wheels}, + "installed": json.loads(installed.stdout.strip().splitlines()[-1]), + "library_only": "passed", + "mismatch_refused": True, + } + + +def main(argv: Sequence[str] | None = None) -> None: + """Run one qualification subcommand and print its receipt.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + wheel = commands.add_parser("wheel") + wheel.add_argument("wheel", type=Path) + sdist = commands.add_parser("sdist") + sdist.add_argument("sdist", type=Path) + sdist.add_argument("--out", type=Path, required=True) + installs = commands.add_parser("install") + installs.add_argument("wheels", type=Path, nargs=3) + args = parser.parse_args(argv) + try: + if args.command == "wheel": + receipt = validate_wheel(args.wheel) + elif args.command == "sdist": + receipt = build_from_sdist(args.sdist, args.out) + else: + receipt = install(args.wheels) + except (QualificationError, subprocess.CalledProcessError) as error: + print(f"qualification failed: {error}", file=sys.stderr) + raise SystemExit(1) from None + print(json.dumps({"check": args.command, **receipt}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/release_candidates.py b/scripts/release_candidates.py new file mode 100644 index 00000000..6386ccae --- /dev/null +++ b/scripts/release_candidates.py @@ -0,0 +1,180 @@ +"""Validate release tags and the exact candidate artifacts a release may promote. + +Tags and the versions they must match: + +- ``morphir-v`` and ``python-v``: the pure Python packages, + checked by ``scripts/check_distributions.py``; +- ``native-v``: ``morphir-python-native``, the native crate version; +- ``extension/python/v``: the MEP bundle, the compiler crate version. + +``tag TAG`` prints the release kind and version, or fails for a malformed tag or +a version that differs from the source. ``native DIR --tag TAG`` and +``bundle DIR --tag TAG --commit SHA`` refuse a candidate set with a missing, +extra or mismatched artifact and write a ``SHA256SUMS`` receipt for publication +to verify again. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +NATIVE_PLATFORMS = ( + "manylinux_2_28_x86_64", + "manylinux_2_28_aarch64", + "macosx_11_0_arm64", + "macosx_10_12_x86_64", + "win_amd64", +) +"""The qualified native wheel platforms, documented in docs/python-compiler.md.""" + + +class CandidateError(Exception): + """The candidate artifacts or tag cannot be promoted.""" + + +@dataclass(frozen=True) +class Release: + """A release kind and the version its tag names.""" + + kind: str + version: str + + +def _version(manifest: Path) -> str: + data = tomllib.loads(manifest.read_text()) + return str((data.get("package") or data["project"])["version"]) + + +def source_versions(root: Path = ROOT) -> dict[str, str]: + """Return the version each release kind has in the source tree.""" + return { + "morphir": _version(root / "packages" / "morphir" / "pyproject.toml"), + "python": _version(root / "packages" / "morphir-tools" / "pyproject.toml"), + "native": _version( + root / "rust" / "crates" / "morphir-python-native" / "Cargo.toml" + ), + "extension": _version( + root / "rust" / "crates" / "morphir-python-binding" / "Cargo.toml" + ), + } + + +PREFIXES = ( + ("extension/python/v", "extension"), + ("morphir-v", "morphir"), + ("python-v", "python"), + ("native-v", "native"), +) + + +def classify(tag: str, versions: dict[str, str] | None = None) -> Release: + """Return the release a tag names, checking its version against the source. + + Raises: + CandidateError: For an unknown prefix or a version mismatch. + """ + versions = versions or source_versions() + for prefix, kind in PREFIXES: + if tag.startswith(prefix): + version = tag.removeprefix(prefix) + if version != versions[kind]: + raise CandidateError( + f"tag {tag} names {version} but the source has {versions[kind]}" + ) + return Release(kind, version) + raise CandidateError(f"tag {tag!r} has no known release prefix") + + +def _sums(directory: Path, names: list[str]) -> str: + lines = [ + f"{hashlib.sha256((directory / name).read_bytes()).hexdigest()} {name}" + for name in sorted(names) + ] + return "\n".join(lines) + "\n" + + +def native_candidates(directory: Path, version: str) -> list[str]: + """Require exactly one wheel per qualified platform and one source archive.""" + stem = f"morphir_python_native-{version}" + expected = {f"{stem}-cp314-abi3-{tag}.whl" for tag in NATIVE_PLATFORMS} + expected.add(f"{stem}.tar.gz") + found = {p.name for p in directory.iterdir() if p.suffix in {".whl", ".gz"}} + if found != expected: + raise CandidateError( + f"native candidates differ: missing {sorted(expected - found)}, " + f"unexpected {sorted(found - expected)}" + ) + return sorted(expected) + + +def bundle_candidates(directory: Path, version: str, commit: str) -> list[str]: + """Require the bundle for this version, built from exactly this commit.""" + artifact = f"morphir-python-binding-{version}.wasm" + expected = {artifact, f"{artifact}.sha256", "release.json"} + found = {p.name for p in directory.iterdir() if p.is_file()} + if found != expected: + raise CandidateError(f"bundle files differ: {sorted(found ^ expected)}") + descriptor = json.loads((directory / "release.json").read_text()) + wasm = (directory / artifact).read_bytes() + digest = hashlib.sha256(wasm).hexdigest() + (entry,) = descriptor["artifacts"] + if descriptor.get("version") != version or entry.get("sha256") != digest: + raise CandidateError("release.json disagrees with the tag or the WASM") + if descriptor.get("gitCommit") != commit: + raise CandidateError( + f"the bundle was built from {descriptor.get('gitCommit')}, not {commit}" + ) + if not re.fullmatch( + r"[0-9a-f]{64} " + re.escape(artifact) + "\n", + (directory / f"{artifact}.sha256").read_text(), + ): + raise CandidateError("the .sha256 file is malformed") + return sorted(expected) + + +def main(argv: list[str] | None = None) -> None: + """Run one check and print or write its receipt.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + tag = commands.add_parser("tag") + tag.add_argument("tag") + native = commands.add_parser("native") + native.add_argument("directory", type=Path) + native.add_argument("--tag", required=True) + bundle = commands.add_parser("bundle") + bundle.add_argument("directory", type=Path) + bundle.add_argument("--tag", required=True) + bundle.add_argument("--commit", required=True) + args = parser.parse_args(argv) + try: + release = classify(args.tag) + if args.command == "tag": + print(f"kind={release.kind}\nversion={release.version}") + return + if args.command == "native": + if release.kind != "native": + raise CandidateError(f"{args.tag} is not a native release tag") + names = native_candidates(args.directory, release.version) + else: + if release.kind != "extension": + raise CandidateError(f"{args.tag} is not an extension release tag") + names = bundle_candidates(args.directory, release.version, args.commit) + except (CandidateError, OSError, KeyError, ValueError) as error: + print(f"candidate refused: {error}", file=sys.stderr) + raise SystemExit(1) from None + sums = _sums(args.directory, names) + (args.directory / "SHA256SUMS").write_text(sums) + print(sums, end="") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/ir/python-compiler/PROVENANCE.md b/tests/fixtures/ir/python-compiler/PROVENANCE.md new file mode 100644 index 00000000..b1b36d7f --- /dev/null +++ b/tests/fixtures/ir/python-compiler/PROVENANCE.md @@ -0,0 +1,17 @@ +# Python compiler output fixtures + +These files are the IR documents that the migrated Python compiler wrote for a +small `acme/example` package. The compiler is `morphir-python-binding` 0.4.0 +(`rust/crates/morphir-python-binding`), which writes IR through Rust +`morphir-core` 0.2.0. `morphir-core` writes the compact canonical v4 spelling. + +Each file is the `ir` member of a probe result, written again with two-space +indentation. No value is edited. + +| Fixture | Format | Probe | +| --- | --- | --- | +| `modules-v4.json` | v4 (`"formatVersion": 4`) | modules `models` and `rules` | +| `adt-v4.json` | v4 (`"formatVersion": 4`) | the same modules, with more value bodies | +| `modules-v3.json` | v3 (`"formatVersion": 3`) | the v3 output for `modules-v4.json` | + +`tests/unit/ir/test_v4_compact.py` reads them. diff --git a/tests/fixtures/ir/python-compiler/adt-v4.json b/tests/fixtures/ir/python-compiler/adt-v4.json new file mode 100644 index 00000000..1900bf86 --- /dev/null +++ b/tests/fixtures/ir/python-compiler/adt-v4.json @@ -0,0 +1,381 @@ +{ + "formatVersion": 4, + "distribution": { + "Library": { + "packageName": "acme/example", + "dependencies": {}, + "def": { + "modules": { + "models": { + "Public": { + "types": { + "address": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Record": { + "fields": { + "street": "morphir/SDK:string#string", + "zip-code": "morphir/SDK:basics#int" + } + } + } + } + } + }, + "application": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Record": { + "fields": { + "decision": "acme/example:models#decision", + "verified": "morphir/SDK:basics#bool", + "coordinates": { + "Tuple": [ + "morphir/SDK:basics#float", + "morphir/SDK:basics#float" + ] + } + } + } + } + } + } + }, + "decision": { + "Public": { + "CustomTypeDefinition": { + "typeParams": [], + "access": "Public", + "constructors": { + "pending": [], + "approved": [ + [ + "amount", + "morphir/SDK:basics#float" + ], + [ + "address", + "acme/example:models#address" + ] + ] + } + } + } + }, + "labeled-point": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Tuple": [ + "acme/example:models#point", + "morphir/SDK:string#string" + ] + } + } + } + }, + "point": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Tuple": [ + "morphir/SDK:basics#float", + "morphir/SDK:basics#float" + ] + } + } + } + } + }, + "values": { + "choose": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "flag": "morphir/SDK:basics#bool", + "first": "morphir/SDK:basics#int", + "second": "morphir/SDK:basics#int" + }, + "outputType": "morphir/SDK:basics#int", + "body": { + "IfThenElse": { + "condition": { + "Variable": "flag" + }, + "then": { + "Variable": "first" + }, + "else": { + "Variable": "second" + } + } + } + } + } + }, + "classify": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "value": "morphir/SDK:basics#int" + }, + "outputType": "morphir/SDK:string#string", + "body": { + "IfThenElse": { + "condition": { + "Apply": { + "function": { + "Apply": { + "function": { + "Reference": "morphir/SDK:basics#less-than" + }, + "argument": { + "Variable": "value" + } + } + }, + "argument": { + "Literal": { + "IntegerLiteral": 0 + } + } + } + }, + "then": { + "Literal": { + "StringLiteral": "negative" + } + }, + "else": { + "IfThenElse": { + "condition": { + "Apply": { + "function": { + "Apply": { + "function": { + "Reference": "morphir/SDK:basics#equal" + }, + "argument": { + "Variable": "value" + } + } + }, + "argument": { + "Literal": { + "IntegerLiteral": 0 + } + } + } + }, + "then": { + "Literal": { + "StringLiteral": "zero" + } + }, + "else": { + "Literal": { + "StringLiteral": "positive" + } + } + } + } + } + } + } + } + }, + "expression": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "flag": "morphir/SDK:basics#bool" + }, + "outputType": "morphir/SDK:basics#float", + "body": { + "IfThenElse": { + "condition": { + "Variable": "flag" + }, + "then": { + "Literal": { + "FloatLiteral": -1.5 + } + }, + "else": { + "Literal": { + "FloatLiteral": 2.5 + } + } + } + } + } + } + }, + "guarded": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "first": "morphir/SDK:basics#bool", + "second": "morphir/SDK:basics#bool" + }, + "outputType": "morphir/SDK:basics#int", + "body": { + "IfThenElse": { + "condition": { + "Variable": "first" + }, + "then": { + "IfThenElse": { + "condition": { + "Variable": "second" + }, + "then": { + "Literal": { + "IntegerLiteral": 1 + } + }, + "else": { + "Literal": { + "IntegerLiteral": 2 + } + } + } + }, + "else": { + "Literal": { + "IntegerLiteral": 2 + } + } + } + } + } + } + }, + "located": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "flag": "morphir/SDK:basics#bool", + "location": "acme/example:models#point" + }, + "outputType": "acme/example:models#labeled-point", + "body": { + "IfThenElse": { + "condition": { + "Variable": "flag" + }, + "then": { + "Tuple": [ + { + "Variable": "location" + }, + { + "Literal": { + "StringLiteral": "known" + } + } + ] + }, + "else": { + "Tuple": [ + { + "Tuple": [ + { + "Literal": { + "FloatLiteral": 0.0 + } + }, + { + "Literal": { + "FloatLiteral": 0.0 + } + } + ] + }, + { + "Literal": { + "StringLiteral": "unknown" + } + } + ] + } + } + } + } + } + }, + "make-point": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "x": "morphir/SDK:basics#float", + "y": "morphir/SDK:basics#float" + }, + "outputType": "acme/example:models#point", + "body": { + "Tuple": [ + { + "Variable": "x" + }, + { + "Variable": "y" + } + ] + } + } + } + }, + "nested": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "first": "morphir/SDK:basics#bool", + "second": "morphir/SDK:basics#bool" + }, + "outputType": "morphir/SDK:basics#int", + "body": { + "IfThenElse": { + "condition": { + "Variable": "first" + }, + "then": { + "IfThenElse": { + "condition": { + "Variable": "second" + }, + "then": { + "Literal": { + "IntegerLiteral": 1 + } + }, + "else": { + "Literal": { + "IntegerLiteral": 2 + } + } + } + }, + "else": { + "Literal": { + "IntegerLiteral": 3 + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/tests/fixtures/ir/python-compiler/modules-v3.json b/tests/fixtures/ir/python-compiler/modules-v3.json new file mode 100644 index 00000000..2d29d42a --- /dev/null +++ b/tests/fixtures/ir/python-compiler/modules-v3.json @@ -0,0 +1,1952 @@ +{ + "formatVersion": 3, + "distribution": [ + "Library", + [ + [ + "acme" + ], + [ + "example" + ] + ], + [], + { + "modules": [ + [ + [ + [ + "models" + ] + ], + { + "access": "Public", + "value": { + "types": [ + [ + [ + "address" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": [ + "TypeAliasDefinition", + [], + [ + "Record", + {}, + [ + { + "name": [ + "street" + ], + "tpe": [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "string" + ] + ], + [ + "string" + ] + ], + [] + ] + }, + { + "name": [ + "zip", + "code" + ], + "tpe": [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "int" + ] + ], + [] + ] + } + ] + ] + ] + } + } + ], + [ + [ + "application" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": [ + "TypeAliasDefinition", + [], + [ + "Record", + {}, + [ + { + "name": [ + "decision" + ], + "tpe": [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ] + }, + { + "name": [ + "verified" + ], + "tpe": [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "bool" + ] + ], + [] + ] + }, + { + "name": [ + "coordinates" + ], + "tpe": [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ] + } + ] + ] + ] + } + } + ], + [ + [ + "decision" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": [ + "CustomTypeDefinition", + [], + { + "access": "Public", + "value": [ + [ + [ + "pending" + ], + [] + ], + [ + [ + "approved" + ], + [ + [ + [ + "amount" + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ], + [ + [ + "address" + ], + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "address" + ] + ], + [] + ] + ] + ] + ] + ] + } + ] + } + } + ], + [ + [ + "labeled", + "point" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": [ + "TypeAliasDefinition", + [], + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "point" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "string" + ] + ], + [ + "string" + ] + ], + [] + ] + ] + ] + ] + } + } + ], + [ + [ + "point" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": [ + "TypeAliasDefinition", + [], + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ] + ] + } + } + ] + ], + "values": [ + [ + [ + "located" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": { + "inputTypes": [ + [ + [ + "flag" + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "bool" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "bool" + ] + ], + [] + ] + ], + [ + [ + "location" + ], + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "point" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "point" + ] + ], + [] + ] + ] + ], + "outputType": [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "labeled", + "point" + ] + ], + [] + ], + "body": [ + "IfThenElse", + [ + "Tuple", + {}, + [ + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "string" + ] + ], + [ + "string" + ] + ], + [] + ] + ] + ], + [ + "Variable", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "bool" + ] + ], + [] + ], + [ + "flag" + ] + ], + [ + "Tuple", + [ + "Tuple", + {}, + [ + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "string" + ] + ], + [ + "string" + ] + ], + [] + ] + ] + ], + [ + [ + "Variable", + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + [ + "location" + ] + ], + [ + "Literal", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "string" + ] + ], + [ + "string" + ] + ], + [] + ], + [ + "StringLiteral", + "known" + ] + ] + ] + ], + [ + "Tuple", + [ + "Tuple", + {}, + [ + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "string" + ] + ], + [ + "string" + ] + ], + [] + ] + ] + ], + [ + [ + "Tuple", + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + [ + [ + "Literal", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "FloatLiteral", + 0.0 + ] + ], + [ + "Literal", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "FloatLiteral", + 0.0 + ] + ] + ] + ], + [ + "Literal", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "string" + ] + ], + [ + "string" + ] + ], + [] + ], + [ + "StringLiteral", + "unknown" + ] + ] + ] + ] + ] + } + } + } + ], + [ + [ + "make", + "point" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": { + "inputTypes": [ + [ + [ + "x" + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ], + [ + [ + "y" + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + "outputType": [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "point" + ] + ], + [] + ], + "body": [ + "Tuple", + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + [ + [ + "Variable", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "x" + ] + ], + [ + "Variable", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "y" + ] + ] + ] + ] + } + } + } + ] + ], + "doc": null + } + } + ], + [ + [ + [ + "rules" + ] + ], + { + "access": "Public", + "value": { + "types": [], + "values": [ + [ + [ + "origin" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": { + "inputTypes": [], + "outputType": [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "point" + ] + ], + [] + ], + "body": [ + "Tuple", + [ + "Tuple", + {}, + [ + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ] + ] + ], + [ + [ + "Literal", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "FloatLiteral", + 0.0 + ] + ], + [ + "Literal", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "float" + ] + ], + [] + ], + [ + "FloatLiteral", + 0.0 + ] + ] + ] + ] + } + } + } + ], + [ + [ + "select", + "decision" + ], + { + "access": "Public", + "value": { + "doc": "", + "value": { + "inputTypes": [ + [ + [ + "flag" + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "bool" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "bool" + ] + ], + [] + ] + ], + [ + [ + "first" + ], + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ] + ], + [ + [ + "second" + ], + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ], + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ] + ] + ], + "outputType": [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ], + "body": [ + "IfThenElse", + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ], + [ + "Variable", + [ + "Reference", + {}, + [ + [ + [ + "morphir" + ], + [ + "s", + "d", + "k" + ] + ], + [ + [ + "basics" + ] + ], + [ + "bool" + ] + ], + [] + ], + [ + "flag" + ] + ], + [ + "Variable", + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ], + [ + "first" + ] + ], + [ + "Variable", + [ + "Reference", + {}, + [ + [ + [ + "acme" + ], + [ + "example" + ] + ], + [ + [ + "models" + ] + ], + [ + "decision" + ] + ], + [] + ], + [ + "second" + ] + ] + ] + } + } + } + ] + ], + "doc": null + } + } + ] + ] + } + ] +} diff --git a/tests/fixtures/ir/python-compiler/modules-v4.json b/tests/fixtures/ir/python-compiler/modules-v4.json new file mode 100644 index 00000000..a0ba4a53 --- /dev/null +++ b/tests/fixtures/ir/python-compiler/modules-v4.json @@ -0,0 +1,232 @@ +{ + "formatVersion": 4, + "distribution": { + "Library": { + "packageName": "acme/example", + "dependencies": {}, + "def": { + "modules": { + "models": { + "Public": { + "types": { + "address": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Record": { + "fields": { + "street": "morphir/SDK:string#string", + "zip-code": "morphir/SDK:basics#int" + } + } + } + } + } + }, + "application": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Record": { + "fields": { + "decision": "acme/example:models#decision", + "verified": "morphir/SDK:basics#bool", + "coordinates": { + "Tuple": [ + "morphir/SDK:basics#float", + "morphir/SDK:basics#float" + ] + } + } + } + } + } + } + }, + "decision": { + "Public": { + "CustomTypeDefinition": { + "typeParams": [], + "access": "Public", + "constructors": { + "pending": [], + "approved": [ + [ + "amount", + "morphir/SDK:basics#float" + ], + [ + "address", + "acme/example:models#address" + ] + ] + } + } + } + }, + "labeled-point": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Tuple": [ + "acme/example:models#point", + "morphir/SDK:string#string" + ] + } + } + } + }, + "point": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Tuple": [ + "morphir/SDK:basics#float", + "morphir/SDK:basics#float" + ] + } + } + } + } + }, + "values": { + "located": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "flag": "morphir/SDK:basics#bool", + "location": "acme/example:models#point" + }, + "outputType": "acme/example:models#labeled-point", + "body": { + "IfThenElse": { + "condition": { + "Variable": "flag" + }, + "then": { + "Tuple": [ + { + "Variable": "location" + }, + { + "Literal": { + "StringLiteral": "known" + } + } + ] + }, + "else": { + "Tuple": [ + { + "Tuple": [ + { + "Literal": { + "FloatLiteral": 0.0 + } + }, + { + "Literal": { + "FloatLiteral": 0.0 + } + } + ] + }, + { + "Literal": { + "StringLiteral": "unknown" + } + } + ] + } + } + } + } + } + }, + "make-point": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "x": "morphir/SDK:basics#float", + "y": "morphir/SDK:basics#float" + }, + "outputType": "acme/example:models#point", + "body": { + "Tuple": [ + { + "Variable": "x" + }, + { + "Variable": "y" + } + ] + } + } + } + } + } + } + }, + "rules": { + "Public": { + "types": {}, + "values": { + "origin": { + "Public": { + "ExpressionBody": { + "inputTypes": {}, + "outputType": "acme/example:models#point", + "body": { + "Tuple": [ + { + "Literal": { + "FloatLiteral": 0.0 + } + }, + { + "Literal": { + "FloatLiteral": 0.0 + } + } + ] + } + } + } + }, + "select-decision": { + "Public": { + "ExpressionBody": { + "inputTypes": { + "flag": "morphir/SDK:basics#bool", + "first": "acme/example:models#decision", + "second": "acme/example:models#decision" + }, + "outputType": "acme/example:models#decision", + "body": { + "IfThenElse": { + "condition": { + "Variable": "flag" + }, + "then": { + "Variable": "first" + }, + "else": { + "Variable": "second" + } + } + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/tests/fixtures/ir/v4/PROVENANCE.md b/tests/fixtures/ir/v4/PROVENANCE.md new file mode 100644 index 00000000..290cae88 --- /dev/null +++ b/tests/fixtures/ir/v4/PROVENANCE.md @@ -0,0 +1,17 @@ +# Format-4 example fixtures + +These files are the v4 example distributions that finos/morphir publishes. They +use the compact v4 spelling. They are copied byte-for-byte and are not edited. + +The source repository is licensed under the Apache License, Version 2.0. +Copyright 2020 Morgan Stanley and the FINOS Morphir contributors. + +| Fixture | Repository | Commit | Path | +| --- | --- | --- | --- | +| `finos-books-and-records-example.json` | [finos/morphir](https://github.com/finos/morphir) | `efa360608cf977d807695be33d60571e4f9c3e05` | `website/static/ir/examples/v4/books-and-records-example.json` | +| `finos-complete-example.json` | finos/morphir | `efa36060` | `website/static/ir/examples/v4/complete-example.json` | + +Kit case distributions-0004 says that a reader accepts `complete-example.json`. +Its module path `u-s/f-r-2052-a` has adjacent single-letter words, which the +Python word-list name model cannot represent, so the tests expect `invalid_name` +at that module and decode the rest after a substitution. diff --git a/tests/fixtures/ir/v4/finos-books-and-records-example.json b/tests/fixtures/ir/v4/finos-books-and-records-example.json new file mode 100644 index 00000000..4549775b --- /dev/null +++ b/tests/fixtures/ir/v4/finos-books-and-records-example.json @@ -0,0 +1,211 @@ +{ + "formatVersion": "4.0.0", + "distribution": { + "Library": { + "packageName": "morphir/reference/model", + "dependencies": { + "morphir/SDK": { + "modules": { + "basics": { + "types": { + "int": { + "OpaqueTypeSpecification": {} + }, + "float": { + "OpaqueTypeSpecification": {} + }, + "bool": { + "OpaqueTypeSpecification": {} + } + }, + "values": {} + }, + "string": { + "types": { + "string": { + "OpaqueTypeSpecification": {} + } + }, + "values": {} + }, + "dict": { + "types": { + "dict": { + "TypeAliasSpecification": { + "typeParams": [ + "k", + "v" + ], + "typeExp": { + "Reference": [ + "morphir/SDK:dict#dict", + "k", + "v" + ] + } + } + } + }, + "values": {} + } + } + } + }, + "def": { + "modules": { + "books-and-records": { + "Public": { + "types": { + "product-ID": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": "morphir/SDK:string#string" + } + } + }, + "price": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": "morphir/SDK:basics#float" + } + } + }, + "quantity": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": "morphir/SDK:basics#int" + } + } + }, + "deal": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Record": { + "fields": { + "product": "morphir/reference/model:books-and-records#product-ID", + "price": "morphir/reference/model:books-and-records#price", + "quantity": "morphir/reference/model:books-and-records#quantity" + } + } + } + } + } + }, + "deal-cmd": { + "Public": { + "CustomTypeDefinition": { + "typeParams": [], + "access": "Public", + "constructors": { + "open-deal": [ + ["product", "morphir/reference/model:books-and-records#product-ID"], + ["price", "morphir/reference/model:books-and-records#price"], + ["quantity", "morphir/reference/model:books-and-records#quantity"] + ], + "close-deal": [ + ["product", "morphir/reference/model:books-and-records#product-ID"] + ] + } + } + } + }, + "deal-event": { + "Public": { + "CustomTypeDefinition": { + "typeParams": [], + "access": "Public", + "constructors": { + "deal-opened": [ + ["product", "morphir/reference/model:books-and-records#product-ID"], + ["price", "morphir/reference/model:books-and-records#price"], + ["quantity", "morphir/reference/model:books-and-records#quantity"] + ], + "deal-closed": [ + ["product", "morphir/reference/model:books-and-records#product-ID"] + ], + "deal-not-found": [ + ["product", "morphir/reference/model:books-and-records#product-ID"] + ], + "duplicate-deal": [ + ["product", "morphir/reference/model:books-and-records#product-ID"] + ], + "invalid-price": [ + ["price", "morphir/reference/model:books-and-records#price"] + ], + "invalid-quantity": [ + ["quantity", "morphir/reference/model:books-and-records#quantity"] + ] + } + } + } + }, + "book": { + "Public": { + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Reference": [ + "morphir/SDK:dict#dict", + "morphir/reference/model:books-and-records#product-ID", + "morphir/reference/model:books-and-records#deal" + ] + } + } + } + } + }, + "values": { + "open-deal": { + "Public": { + "IncompleteBody": { + "inputTypes": { + "product": "morphir/reference/model:books-and-records#product-ID", + "price": "morphir/reference/model:books-and-records#price", + "quantity": "morphir/reference/model:books-and-records#quantity", + "book": "morphir/reference/model:books-and-records#book" + }, + "outputType": { + "Tuple": [ + "morphir/reference/model:books-and-records#book", + "morphir/reference/model:books-and-records#deal-event" + ] + }, + "incompleteness": { + "Draft": {} + } + } + } + }, + "close-deal": { + "Public": { + "IncompleteBody": { + "inputTypes": { + "product": "morphir/reference/model:books-and-records#product-ID", + "book": "morphir/reference/model:books-and-records#book" + }, + "outputType": { + "Tuple": [ + "morphir/reference/model:books-and-records#book", + "morphir/reference/model:books-and-records#deal-event" + ] + }, + "incompleteness": { + "Draft": {} + } + } + } + } + }, + "doc": "Books and Records module - manages deal lifecycle including opening, closing, and tracking deals." + } + } + } + } + } + } +} diff --git a/tests/fixtures/ir/v4/finos-complete-example.json b/tests/fixtures/ir/v4/finos-complete-example.json new file mode 100644 index 00000000..e5285a0c --- /dev/null +++ b/tests/fixtures/ir/v4/finos-complete-example.json @@ -0,0 +1,138 @@ +{ + "formatVersion": "4.0.0", + "distribution": { + "Library": { + "packageName": "regulation", + "dependencies": { + "morphir/SDK": { + "modules": { + "basics": { + "types": { + "int": { + "OpaqueTypeSpecification": {} + }, + "float": { + "OpaqueTypeSpecification": {} + }, + "bool": { + "OpaqueTypeSpecification": {} + } + }, + "values": { + "add": { + "inputs": { + "a": "morphir/SDK:basics#int", + "b": "morphir/SDK:basics#int" + }, + "output": "morphir/SDK:basics#int" + } + } + }, + "list": { + "types": { + "list": { + "TypeAliasSpecification": { + "typeParams": [ + "a" + ], + "typeExp": { + "Reference": [ + "morphir/SDK:list#list", + "a" + ] + } + } + } + }, + "values": { + "map": { + "inputs": { + "f": { + "Function": { + "parameterType": "a", + "returnType": "b" + } + }, + "list": { + "Reference": [ + "morphir/SDK:list#list", + "a" + ] + } + }, + "output": { + "Reference": [ + "morphir/SDK:list#list", + "b" + ] + } + } + } + } + } + } + }, + "def": { + "modules": { + "u-s/f-r-2052-a/data-tables": { + "access": "Public", + "value": { + "types": { + "data-tables": { + "access": "Public", + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Record": { + "fields": { + "inflows": "regulation:u-s/f-r-2052-a/data-tables#inflows", + "outflows": "regulation:u-s/f-r-2052-a/data-tables#outflows", + "supplemental": "regulation:u-s/f-r-2052-a/data-tables#supplemental" + } + } + } + } + }, + "inflows": { + "access": "Public", + "TypeAliasDefinition": { + "typeParams": [], + "typeExp": { + "Record": { + "fields": { + "assets": { + "Reference": [ + "morphir/SDK:list#list", + "regulation:u-s/f-r-2052-a/data-tables/inflows#assets" + ] + } + } + } + } + } + } + }, + "values": { + "calculate-total": { + "access": "Public", + "ExpressionBody": { + "inputTypes": { + "tables": "regulation:u-s/f-r-2052-a/data-tables#data-tables" + }, + "outputType": "morphir/SDK:basics#float", + "body": { + "Literal": { + "FloatLiteral": 0.0 + } + } + } + } + }, + "doc": "Data tables module for regulatory reporting" + } + } + } + } + } + } +} diff --git a/tests/fixtures/python/loans/acme/models.py b/tests/fixtures/python/loans/acme/models.py new file mode 100644 index 00000000..aa412d0d --- /dev/null +++ b/tests/fixtures/python/loans/acme/models.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Pending: + pass + + +@dataclass(frozen=True) +class Approved: + amount: float + + +type Decision = Pending | Approved + +type Point = tuple[float, float] diff --git a/tests/fixtures/python/loans/acme/rules.py b/tests/fixtures/python/loans/acme/rules.py new file mode 100644 index 00000000..ef433c52 --- /dev/null +++ b/tests/fixtures/python/loans/acme/rules.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from collections.abc import Callable + +from .models import Decision, Point + + +def choose(flag: bool, first: int, second: int) -> int: + if flag: + return first + else: + return second + + +def pick(flag: bool, first: Decision, second: Decision) -> Decision: + return first if flag else second + + +def pair(left: float, right: float) -> Point: + return (left, right) + + +def apply_twice(step: Callable[[int], int], value: int) -> int: + return step(step(value)) + + +def twice_identity(value: int) -> int: + return apply_twice(lambda item: item, value) diff --git a/tests/tools/test_python_api.py b/tests/tools/test_python_api.py new file mode 100644 index 00000000..9ac59727 --- /dev/null +++ b/tests/tools/test_python_api.py @@ -0,0 +1,287 @@ +"""Tests for the typed Python compiler API (capability python/compiler).""" + +from __future__ import annotations + +import dataclasses +import json +import threading +from pathlib import Path + +import pytest + +from morphir.ir.v3 import codec as v3_codec +from morphir.ir.v3 import distribution as v3_dist +from morphir.ir.v4 import distribution as v4_dist +from morphir_tools import python as api +from morphir_tools.python import diagnostics as codes + +V3_FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "ir" / "v3" + +MODELS = """\ +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + +type Decision = Pending | Approved + +type Point = tuple[float, float] +""" + +RULES = """\ +from __future__ import annotations +from collections.abc import Callable +from .models import Decision, Point + +def choose(flag: bool, first: Decision, second: Decision) -> Decision: + if flag: + return first + else: + return second + +def origin() -> Point: + return (0.0, 0.0) + +def apply_twice(step: Callable[[int], int], value: int) -> int: + return step(step(value)) + +def add_one(value: int) -> int: + return apply_twice(lambda item: item, value) +""" + +SOURCES = ( + api.SourceDocument("acme/models.py", MODELS), + api.SourceDocument("acme/rules.py", RULES), +) + + +def module_access(library: v3_dist.Library) -> dict[str, str]: + return { + "/".join("-".join(name) for name in path): module.access.value + for path, module in library.definition.modules.items() + } + + +def compile_ok( + sources: tuple[api.SourceDocument, ...] = SOURCES, + **options: object, +) -> api.CompiledV3 | api.CompiledV4: + result = api.compile_sources(sources, package="acme/example", **options) # type: ignore[arg-type] + assert not isinstance(result, api.CompileFailure), result + return result + + +class TestCompile: + def test_defaults_to_a_v3_library(self) -> None: + result = compile_ok() + assert isinstance(result, api.CompiledV3) + assert isinstance(result.distribution, v3_dist.Library) + assert result.warnings == () + assert set(module_access(result.distribution)) == {"acme/models", "acme/rules"} + + def test_explicit_v4_returns_the_v4_model(self) -> None: + result = compile_ok(ir_version=api.IrVersion.V4) + assert isinstance(result, api.CompiledV4) + assert isinstance(result.distribution, v4_dist.LibraryDistribution) + + def test_exposure_selects_public_modules(self) -> None: + result = compile_ok(exposed_modules=("acme/models",)) + assert isinstance(result, api.CompiledV3) + assert module_access(result.distribution) == { + "acme/models": "Public", + "acme/rules": "Private", + } + + def test_empty_exposure_differs_from_omitted_exposure(self) -> None: + result = compile_ok(exposed_modules=()) + assert isinstance(result, api.CompiledV3) + assert set(module_access(result.distribution).values()) == {"Private"} + + def test_syntax_error_is_a_located_failure(self) -> None: + result = api.compile_sources( + (api.SourceDocument("broken.py", "def broken(:\n"),), package="acme/x" + ) + assert isinstance(result, api.CompileFailure) + (diagnostic,) = result.diagnostics + assert diagnostic.code == codes.PY_SYNTAX + assert diagnostic.severity is api.Severity.ERROR + assert diagnostic.source is not None + assert diagnostic.source.path == "broken.py" + assert diagnostic.source.start.line == 0 + assert diagnostic.render().startswith("PY002 at broken.py:1:") + + @pytest.mark.parametrize( + "text", + [ + "count = 1\n", + "import os\n", + "from pathlib import Path\nPath('owned.txt').write_text('x')\n", + ], + ) + def test_unsupported_constructs_fail_without_a_model( + self, text: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + result = api.compile_sources( + (api.SourceDocument("bad.py", text),), package="acme/x" + ) + assert isinstance(result, api.CompileFailure) + assert not hasattr(result, "distribution") + assert result.diagnostics[0].severity is api.Severity.ERROR + assert result.diagnostics[0].code in {codes.PY_UNSUPPORTED, codes.PY_NAME} + assert list(tmp_path.iterdir()) == [] + + def test_source_permutations_give_equal_models(self) -> None: + first = compile_ok() + second = compile_ok(tuple(reversed(SOURCES))) + assert first == second + + def test_concurrent_calls_are_independent(self) -> None: + expected = { + version: compile_ok(ir_version=version) for version in api.IrVersion + } + failures: list[str] = [] + + def work(version: api.IrVersion) -> None: + for _ in range(5): + if compile_ok(ir_version=version) != expected[version]: + failures.append(version.value) + + threads = [ + threading.Thread(target=work, args=(version,)) + for version in list(api.IrVersion) * 4 + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert failures == [] + + def test_invalid_options_are_rejected_before_compiling(self) -> None: + with pytest.raises(ValueError, match="package"): + api.compile_sources(SOURCES, package="") + with pytest.raises(ValueError, match="duplicate"): + api.compile_sources(SOURCES + SOURCES[:1], package="acme/x") + + +class TestResultTypes: + def test_results_are_frozen(self) -> None: + result = compile_ok() + with pytest.raises(dataclasses.FrozenInstanceError): + result.warnings = () # type: ignore[misc] + with pytest.raises(dataclasses.FrozenInstanceError): + SOURCES[0].text = "" # type: ignore[misc] + + def test_failure_cannot_be_empty(self) -> None: + with pytest.raises(ValueError): + api.CompileFailure(()) + with pytest.raises(ValueError): + api.GenerationFailure(()) + + def test_success_cannot_hold_errors(self) -> None: + error = api.CompilerDiagnostic( + code="PY004", severity=api.Severity.ERROR, message="no" + ) + with pytest.raises(ValueError): + api.Generated(artifacts=(), warnings=(error,)) + + +class TestGenerate: + @pytest.mark.parametrize("version", list(api.IrVersion)) + def test_generates_one_artifact_per_module(self, version: api.IrVersion) -> None: + compiled = compile_ok(ir_version=version) + generated = api.generate_python(compiled.distribution) + assert isinstance(generated, api.Generated), generated + assert [artifact.path for artifact in generated.artifacts] == [ + "acme/models.py", + "acme/rules.py", + ] + rules = generated.artifacts[1].text + assert "def choose(" in rules + assert "lambda" in rules + + @pytest.mark.parametrize("version", list(api.IrVersion)) + def test_generated_python_recompiles_to_the_same_model( + self, version: api.IrVersion + ) -> None: + compiled = compile_ok(ir_version=version) + generated = api.generate_python(compiled.distribution) + assert isinstance(generated, api.Generated) + sources = tuple( + api.SourceDocument(artifact.path, artifact.text) + for artifact in generated.artifacts + ) + assert compile_ok(sources, ir_version=version) == compiled + + def test_specs_distribution_is_rejected(self) -> None: + specs = v3_codec.decode_versioned_distribution( + (V3_FIXTURES / "mck-distributions-0011-specs.json").read_text() + ) + assert isinstance(specs, v3_dist.Specs) + result = api.generate_python(specs) + assert isinstance(result, api.GenerationFailure) + assert result.diagnostics[0].code == codes.UNSUPPORTED_DISTRIBUTION + + def test_unsupported_value_is_a_compiler_diagnostic(self) -> None: + library = v3_codec.decode_versioned_distribution( + (V3_FIXTURES / "elm-simpleValueTree.json").read_text() + ) + result = api.generate_python(library) + assert isinstance(result, api.GenerationFailure) + assert result.diagnostics[0].code.startswith("PY") + + +class TestNativeCompatibility: + def test_reports_the_installed_native_compiler(self) -> None: + info = api.native_compiler() + assert info.version == api.EXPECTED_NATIVE_VERSION + assert info.bridge_api == api.EXPECTED_BRIDGE_API + assert info.compiler_version + + def test_rejects_a_mismatched_native_version( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from morphir_tools.python import _bridge + + monkeypatch.setattr(_bridge, "EXPECTED_NATIVE_VERSION", "9.9.9") + _bridge.load.cache_clear() + try: + with pytest.raises(api.NativeCompilerUnavailable, match=r"9\.9\.9"): + api.compile_sources(SOURCES, package="acme/example") + finally: + monkeypatch.undo() + _bridge.load.cache_clear() + + def test_rejects_a_mismatched_bridge_revision( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from morphir_tools.python import _bridge + + monkeypatch.setattr(_bridge, "EXPECTED_BRIDGE_API", 999) + _bridge.load.cache_clear() + try: + with pytest.raises(api.NativeCompilerUnavailable, match="999"): + api.native_compiler() + finally: + monkeypatch.undo() + _bridge.load.cache_clear() + + def test_internal_errors_raise(self, monkeypatch: pytest.MonkeyPatch) -> None: + from morphir_tools.python import _bridge + + class Broken: + @staticmethod + def compile(request: str) -> str: + return json.dumps( + {"success": False, "diagnostics": [], "internalError": "boom"} + ) + + monkeypatch.setattr(_bridge, "module", lambda: Broken) + with pytest.raises(api.NativeCompilerError, match="boom"): + api.compile_sources(SOURCES, package="acme/example") diff --git a/tests/tools/test_python_cli.py b/tests/tools/test_python_cli.py new file mode 100644 index 00000000..a9989374 --- /dev/null +++ b/tests/tools/test_python_cli.py @@ -0,0 +1,477 @@ +"""Tests for ``morphir-python compile`` and ``generate`` (capability tools/python-codegen).""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest +from typer.testing import CliRunner, Result + +from morphir.ir import diagnostics as d +from morphir.ir.v3 import codec as v3_codec +from morphir.ir.v3 import distribution as v3_dist +from morphir.ir.v4 import codec as v4_codec +from morphir_tools import cli +from morphir_tools.cli import app +from morphir_tools.python import diagnostics as codes + +runner = CliRunner() +V3_FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "ir" / "v3" + +MODELS = """\ +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + +type Decision = Pending | Approved + +type Point = tuple[float, float] +""" + +RULES = """\ +from __future__ import annotations +from collections.abc import Callable +from .models import Decision, Point + +def choose(flag: bool, first: int, second: int) -> int: + if flag: + return first + else: + return second + +def pick(flag: bool, first: Decision, second: Decision) -> Decision: + return first if flag else second + +def pair(left: float, right: float) -> Point: + return (left, right) + +def apply_twice(step: Callable[[int], int], value: int) -> int: + return step(step(value)) + +def twice_identity(value: int) -> int: + return apply_twice(lambda item: item, value) +""" + + +def library_at(path: Path) -> v3_dist.Library: + library = v3_codec.decode_versioned_distribution(path.read_text()) + assert isinstance(library, v3_dist.Library) + return library + + +def run(*args: str | Path) -> Result: + return runner.invoke(app, [str(a) for a in args]) + + +@pytest.fixture +def source(tmp_path: Path) -> Path: + root = tmp_path / "src" + (root / "acme").mkdir(parents=True) + (root / "acme" / "models.py").write_text(MODELS) + (root / "acme" / "rules.py").write_text(RULES) + return root + + +def compile_to(source: Path, output: Path, *extra: str) -> Path: + result = run( + "compile", source, "--package", "acme/example", "--output", output, *extra + ) + assert result.exit_code == 0, result.output + return output + + +def tree(root: Path) -> dict[str, str]: + return { + path.relative_to(root).as_posix(): path.read_text() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +class TestCompile: + def test_directory_defaults_to_v3_and_inspects( + self, source: Path, tmp_path: Path + ) -> None: + output = compile_to(source, tmp_path / "model.json") + library = library_at(output) + modules = { + "/".join("-".join(n) for n in path): module.access.value + for path, module in library.definition.modules.items() + } + assert modules == {"acme/models": "Public", "acme/rules": "Public"} + inspected = run("ir", "inspect", output) + assert inspected.exit_code == 0 + assert "format: 3.0.0" in inspected.output + assert "package: acme/example" in inspected.output + + def test_v4_reports_draft_status(self, source: Path, tmp_path: Path) -> None: + result = run( + "compile", + source, + "--package", + "acme/example", + "--ir-version", + "v4", + "--output", + tmp_path / "model.json", + ) + assert result.exit_code == 0, result.output + assert v4_codec.DRAFT_NOTICE in result.output + inspected = run("ir", "inspect", tmp_path / "model.json") + assert "format: 4.0.0" in inspected.output + + def test_repeated_expose_selects_modules( + self, source: Path, tmp_path: Path + ) -> None: + output = compile_to(source, tmp_path / "model.json", "--expose", "acme/models") + library = library_at(output) + access = sorted( + module.access.value for module in library.definition.modules.values() + ) + assert access == ["Private", "Public"] + + def test_single_file_uses_its_parent_as_root(self, tmp_path: Path) -> None: + file = tmp_path / "rules.py" + file.write_text("def keep(value: int) -> int:\n return value\n") + output = compile_to(file, tmp_path / "model.json") + library = library_at(output) + assert [ + "/".join("-".join(n) for n in path) for path in library.definition.modules + ] == ["rules"] + + def test_failure_keeps_existing_output_and_reports_location( + self, tmp_path: Path + ) -> None: + file = tmp_path / "broken.py" + file.write_text("def broken(:\n") + output = tmp_path / "model.json" + output.write_text("previous") + result = run("compile", file, "--package", "acme/x", "--output", output) + assert result.exit_code == 1 + assert f"error: {codes.PY_SYNTAX} at broken.py:1:" in result.output + assert output.read_text() == "previous" + + def test_unsupported_construct_fails(self, tmp_path: Path) -> None: + file = tmp_path / "side_effect.py" + file.write_text("open('owned.txt', 'w').write('x')\n") + result = run( + "compile", file, "--package", "acme/x", "--output", tmp_path / "m.json" + ) + assert result.exit_code == 1 + assert codes.PY_UNSUPPORTED in result.output + assert not (tmp_path / "m.json").exists() + assert not (tmp_path / "owned.txt").exists() + + def test_empty_directory_is_an_error(self, tmp_path: Path) -> None: + (tmp_path / "empty").mkdir() + result = run( + "compile", tmp_path / "empty", "--package", "acme/x", "--output", "x.json" + ) + assert result.exit_code == 1 + assert "no Python source files" in result.output + + def test_symlinked_directories_are_not_followed( + self, source: Path, tmp_path: Path + ) -> None: + outside = tmp_path / "outside" + outside.mkdir() + (outside / "extra.py").write_text("def other(v: int) -> int:\n return v\n") + (source / "linked").symlink_to(outside, target_is_directory=True) + output = compile_to(source, tmp_path / "model.json") + library = library_at(output) + assert len(library.definition.modules) == 2 + + def test_unreadable_source_is_reported(self, tmp_path: Path) -> None: + result = run( + "compile", tmp_path / "absent.py", "--package", "a/b", "--output", "x.json" + ) + assert result.exit_code == 1 + assert "cannot read" in result.output + + +class TestGenerate: + @pytest.mark.parametrize("version", ["v3", "v4"]) + def test_generates_nested_importable_modules( + self, source: Path, tmp_path: Path, version: str + ) -> None: + model = compile_to(source, tmp_path / "m.json", "--ir-version", version) + out = tmp_path / "generated" + result = run("generate", model, "--output-dir", out) + assert result.exit_code == 0, result.output + assert sorted(tree(out)) == ["acme/models.py", "acme/rules.py"] + check = subprocess.run( + [ + sys.executable, + "-I", + "-c", + "import sys; sys.path.insert(0, sys.argv[1]);" + "from acme import rules, models;" + "print(rules.choose(True, 1, 2), rules.choose(False, 1, 2)," + " rules.pair(1.0, 2.0), rules.twice_identity(7)," + " rules.pick(False, models.Pending(), models.Approved(2.5)))", + str(out), + ], + capture_output=True, + text=True, + check=True, + ) + assert check.stdout.strip() == ("1 2 (1.0, 2.0) 7 Approved(amount=2.5)") + + @pytest.mark.parametrize("version", ["v3", "v4"]) + def test_round_trip_recompiles_to_equal_ir( + self, source: Path, tmp_path: Path, version: str + ) -> None: + first = compile_to( + source, + tmp_path / "first.json", + "--ir-version", + version, + "--expose", + "acme/models", + ) + out = tmp_path / "generated" + assert run("generate", first, "--output-dir", out).exit_code == 0 + second = compile_to( + out, + tmp_path / "second.json", + "--ir-version", + version, + "--expose", + "acme/models", + ) + codec = v3_codec if version == "v3" else v4_codec + assert codec.decode_versioned_distribution( + first.read_text() + ) == codec.decode_versioned_distribution(second.read_text()) + + def test_refuses_a_nonempty_directory(self, source: Path, tmp_path: Path) -> None: + model = compile_to(source, tmp_path / "m.json") + out = tmp_path / "generated" + out.mkdir() + (out / "keep.txt").write_text("mine") + result = run("generate", model, "--output-dir", out) + assert result.exit_code == 1 + assert "not empty" in result.output + assert tree(out) == {"keep.txt": "mine"} + + def test_accepts_an_empty_directory(self, source: Path, tmp_path: Path) -> None: + model = compile_to(source, tmp_path / "m.json") + out = tmp_path / "generated" + out.mkdir() + assert run("generate", model, "--output-dir", out).exit_code == 0 + assert sorted(tree(out)) == ["acme/models.py", "acme/rules.py"] + + def test_refuses_a_symlinked_output_directory( + self, source: Path, tmp_path: Path + ) -> None: + model = compile_to(source, tmp_path / "m.json") + real = tmp_path / "real" + real.mkdir() + (tmp_path / "link").symlink_to(real, target_is_directory=True) + result = run("generate", model, "--output-dir", tmp_path / "link") + assert result.exit_code == 1 + assert "symbolic link" in result.output + + def test_injected_write_failure_publishes_nothing( + self, source: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + model = compile_to(source, tmp_path / "m.json") + out = tmp_path / "generated" + calls: list[Path] = [] + original = Path.write_text + + def failing(self: Path, data: str, *args: object, **kwargs: object) -> int: + calls.append(self) + if len(calls) == 2: + raise OSError(28, "No space left on device") + return original(self, data, *args, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(Path, "write_text", failing) + result = run("generate", model, "--output-dir", out) + monkeypatch.undo() + assert result.exit_code == 1 + assert "No space left on device" in result.output + assert "Traceback" not in result.output + assert not out.exists() + assert [p.name for p in tmp_path.iterdir() if p.name.startswith(".")] == [] + + def test_late_module_rejection_writes_nothing( + self, source: Path, tmp_path: Path + ) -> None: + model = json.loads(compile_to(source, tmp_path / "m.json").read_text()) + elm = json.loads((V3_FIXTURES / "elm-simpleValueTree.json").read_text()) + ((_, late),) = elm["distribution"][3]["modules"] + # "zzz" sorts after every supported module, so it is generated last. + model["distribution"][3]["modules"].append([[["zzz"]], late]) + mixed = tmp_path / "mixed.json" + mixed.write_text(json.dumps(model)) + out = tmp_path / "generated" + result = run("generate", mixed, "--output-dir", out) + assert result.exit_code == 1 + assert "error: PY" in result.output + assert not out.exists() + + @pytest.mark.parametrize( + ("document", "code"), + [ + ({"distribution": []}, d.MISSING_FORMAT_VERSION), + ( + {"formatVersion": "three", "distribution": []}, + d.INVALID_FORMAT_VERSION_SYNTAX, + ), + ( + {"formatVersion": 9, "distribution": []}, + d.UNSUPPORTED_FORMAT_VERSION_MAJOR, + ), + ], + ) + def test_version_errors_use_the_library_codes( + self, tmp_path: Path, document: object, code: str + ) -> None: + model = tmp_path / "bad.json" + model.write_text(json.dumps(document)) + result = run("generate", model, "--output-dir", tmp_path / "out") + assert result.exit_code == 1 + assert f"error: {code}" in result.output + assert not (tmp_path / "out").exists() + + def test_specs_distribution_is_rejected(self, tmp_path: Path) -> None: + specs = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "ir" + / "v3" + / "mck-distributions-0011-specs.json" + ) + result = run("generate", specs, "--output-dir", tmp_path / "out") + assert result.exit_code == 1 + assert codes.UNSUPPORTED_DISTRIBUTION in result.output + + +class TestArtifactPaths: + @pytest.mark.parametrize( + "paths", + [ + ("/etc/passwd.py",), + ("../escape.py",), + ("acme/../../escape.py",), + ("C:/windows.py",), + ("back\\slash.py",), + ("",), + ("acme//double.py",), + ("a.py", "a.py"), + ("Rules.py", "rules.py"), + ("acme.py", "acme.py/inner.py"), + ], + ) + def test_invalid_artifact_sets_are_rejected(self, paths: tuple[str, ...]) -> None: + with pytest.raises(cli.OutputError): + cli.validate_artifact_paths(paths) + + def test_valid_artifact_set_is_accepted(self) -> None: + cli.validate_artifact_paths(("acme/models.py", "acme/rules.py", "top.py")) + + +class TestIndependence: + def test_ir_commands_do_not_load_the_compiler(self, tmp_path: Path) -> None: + fixture = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "ir" + / "v3" + / "elm-simpleValueTree.json" + ) + script = textwrap.dedent( + """ + import sys + from typer.testing import CliRunner + from morphir_tools.cli import app + runner = CliRunner() + assert runner.invoke(app, ["ir", "inspect", sys.argv[1]]).exit_code == 0 + out = sys.argv[2] + assert runner.invoke( + app, ["ir", "convert", sys.argv[1], "--to", "v4", "-o", out] + ).exit_code == 0 + loaded = [m for m in sys.modules if m.startswith("morphir_python_native")] + assert loaded == [], loaded + assert "morphir_tools.python._api" not in sys.modules + """ + ) + subprocess.run( + [sys.executable, "-c", script, str(fixture), str(tmp_path / "v4.json")], + check=True, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + ) + + +class TestOutputPermissions: + """Published output follows the user's umask, like an ordinary write.""" + + def test_generated_tree_and_compiled_file_are_not_private( + self, source: Path, tmp_path: Path + ) -> None: + previous = os.umask(0o022) + try: + model = compile_to(source, tmp_path / "m.json") + out = tmp_path / "generated" + assert run("generate", model, "--output-dir", out).exit_code == 0 + finally: + os.umask(previous) + assert model.stat().st_mode & 0o777 == 0o644 + assert out.stat().st_mode & 0o777 == 0o755 + assert (out / "acme").stat().st_mode & 0o777 == 0o755 + assert (out / "acme" / "rules.py").stat().st_mode & 0o777 == 0o644 + + def test_an_existing_empty_directory_keeps_its_mode( + self, source: Path, tmp_path: Path + ) -> None: + model = compile_to(source, tmp_path / "m.json") + out = tmp_path / "generated" + out.mkdir() + out.chmod(0o750) + assert run("generate", model, "--output-dir", out).exit_code == 0 + assert out.stat().st_mode & 0o777 == 0o750 + + +class TestCleanErrors: + def test_a_non_utf8_source_is_reported_without_a_traceback( + self, tmp_path: Path + ) -> None: + file = tmp_path / "latin.py" + file.write_bytes("x = 'caf\xe9'\n".encode("latin-1")) + result = run("compile", file, "--package", "a/b", "--output", tmp_path / "o") + assert result.exit_code == 1 + assert "not UTF-8" in result.output + assert result.exception is None or isinstance(result.exception, SystemExit) + + def test_internal_compiler_errors_are_reported_without_a_traceback( + self, source: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from morphir_tools.python import _bridge + + class Broken: + @staticmethod + def compile(request: str) -> str: + return json.dumps( + {"success": False, "diagnostics": [], "internalError": "boom"} + ) + + monkeypatch.setattr(_bridge, "module", lambda: Broken) + result = run( + "compile", source, "--package", "a/b", "--output", tmp_path / "o.json" + ) + assert result.exit_code == 1 + assert "internal compiler error: boom" in result.output + assert isinstance(result.exception, SystemExit) diff --git a/tests/tools/test_python_crosscodec.py b/tests/tools/test_python_crosscodec.py new file mode 100644 index 00000000..f72095c0 --- /dev/null +++ b/tests/tools/test_python_crosscodec.py @@ -0,0 +1,196 @@ +"""The Rust compiler and the Python codecs agree on the supported mapping.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import TYPE_CHECKING + +from morphir.ir import diagnostics as d +from morphir.ir import format_version as fv +from morphir.ir.migrate import Err, v3_to_v4, v4_to_v3 +from morphir.ir.v3 import distribution as v3_dist +from morphir.ir.v4 import codec as v4_codec +from morphir.ir.v4 import distribution as v4_dist +from morphir_python_native import _native +from morphir_tools import python as api +from morphir_tools.python import diagnostics as codes + +if TYPE_CHECKING: + from collections.abc import Mapping + + import pytest + +NUMBERS = """\ +from __future__ import annotations +type Pair = tuple[tuple[int, int], float] + +def nest(a: int, b: int, c: float) -> Pair: + return ((a, b), c) + +def big() -> int: + return 123456789012345678901234567890 +""" + +SAMPLE = Path(__file__).resolve().parents[1] / "fixtures" / "python" / "loans" +UPSTREAM_V4 = Path(__file__).resolve().parents[1] / "fixtures" / "ir" / "v4" + + +def sample() -> list[api.SourceDocument]: + return [ + api.SourceDocument(p.relative_to(SAMPLE).as_posix(), p.read_text()) + for p in sorted(SAMPLE.rglob("*.py")) + ] + + +def compiled( + sources: list[api.SourceDocument], version: api.IrVersion +) -> api.CompiledV3 | api.CompiledV4: + result = api.compile_sources(sources, package="acme/example", ir_version=version) + assert not isinstance(result, api.CompileFailure), result + return result + + +def texts(result: api.GenerateResult) -> dict[str, str]: + assert isinstance(result, api.Generated), result + return {artifact.path: artifact.text for artifact in result.artifacts} + + +class TestNumbersAndTuples: + def test_v4_keeps_arbitrary_integers_and_nested_tuples(self) -> None: + source = [api.SourceDocument("numbers.py", NUMBERS)] + first = compiled(source, api.IrVersion.V4) + encoded = fv.dump_json( + v4_codec.encode_versioned_distribution(first.distribution) # type: ignore[arg-type] + ) + assert "123456789012345678901234567890" in encoded + generated = texts(api.generate_python(first.distribution)) + assert "return 123456789012345678901234567890" in generated["numbers.py"] + assert "type Pair = tuple[tuple[int, int], float]" in generated["numbers.py"] + again = compiled( + [api.SourceDocument("numbers.py", generated["numbers.py"])], + api.IrVersion.V4, + ) + assert again == first + + def test_v3_rejects_integers_beyond_its_range(self) -> None: + result = api.compile_sources( + [api.SourceDocument("numbers.py", NUMBERS)], package="acme/example" + ) + assert isinstance(result, api.CompileFailure) + assert result.diagnostics[0].code == codes.PY_UNSUPPORTED + assert "64-bit" in result.diagnostics[0].message + + +class TestIndependentlyAuthoredIr: + def test_python_encoded_v4_generates_like_rust_encoded_v4(self) -> None: + # The compiler writes compact v4; generate_python sends the Python + # codec's expanded spelling of the same model. + direct = compiled(sample(), api.IrVersion.V4) + native = json.loads( + json.loads( + _native.compile( + json.dumps( + { + "package": "acme/example", + "sources": [ + {"path": d.path, "text": d.text} for d in sample() + ], + "exposedModules": None, + "irVersion": "4", + } + ) + ) + )["ir"] + ) + expanded = v4_codec.encode_versioned_distribution( + direct.distribution # type: ignore[arg-type] + ) + assert native != expanded + assert v4_codec.decode_versioned_distribution(native) == direct.distribution + from_compact = api.generate_python( + v4_codec.decode_versioned_distribution(json.dumps(native)) + ) + assert texts(from_compact) == texts(api.generate_python(direct.distribution)) + + def test_migrated_typed_v3_values_are_rejected_explicitly(self) -> None: + # v3_to_v4 keeps inferred types as value attributes, which the compiler + # cannot yet carry into Python; it says so instead of dropping them. + classic = compiled(sample(), api.IrVersion.V3) + assert isinstance(classic.distribution, v3_dist.Library) + result = api.generate_python(v3_to_v4(classic.distribution).value) + assert isinstance(result, api.GenerationFailure) + assert result.diagnostics[0].code == codes.PY_UNSUPPORTED + assert "attributes" in result.diagnostics[0].message + + def test_downgrading_compiler_v4_is_refused_explicitly(self) -> None: + # v3 members need documentation strings that compiler v4 output lacks. + draft = compiled(sample(), api.IrVersion.V4) + assert isinstance(draft.distribution, v4_dist.LibraryDistribution) + result = v4_to_v3(draft.distribution) + assert isinstance(result, Err) + found = {item.code for item in result.diagnostics} + assert d.UNSUPPORTED_DOCUMENTATION_DOWNGRADE in found + assert found <= { + d.UNSUPPORTED_DOCUMENTATION_DOWNGRADE, + d.MISSING_INFERRED_TYPE, + } + + def test_an_upstream_example_outside_the_subset_is_diagnosed(self) -> None: + upstream = v4_codec.decode_versioned_distribution( + (UPSTREAM_V4 / "finos-books-and-records-example.json").read_text() + ) + result = api.generate_python(upstream) + assert isinstance(result, api.GenerationFailure) + assert result.diagnostics[0].code.startswith("PY") + + +class TestNoSilentDropping: + def test_documentation_is_rejected_not_dropped(self) -> None: + draft = compiled(sample(), api.IrVersion.V4) + document = v4_codec.encode_versioned_distribution( + draft.distribution # type: ignore[arg-type] + ) + modules = document["distribution"]["Library"]["def"]["modules"] + types = next(iter(modules.values()))["Public"]["types"] + name, definition = next(iter(types.items())) + access, body = next(iter(definition.items())) + types[name] = {access: {"doc": "Documented in IR only.", **body}} + documented = v4_codec.decode_versioned_distribution(json.dumps(document)) + result = api.generate_python(documented) + assert isinstance(result, api.GenerationFailure), result + assert result.diagnostics[0].code == codes.PY_UNSUPPORTED + + +class TestWarnings: + def test_warnings_stay_with_a_successful_model( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from morphir_tools.python import _bridge + + real = _bridge.module() + + class Warns: + @staticmethod + def compile(request: str) -> str: + output: Mapping[str, object] = json.loads(real.compile(request)) + return json.dumps( + { + **output, + "diagnostics": [ + { + "severity": "warning", + "code": codes.PY_PARSE_STAGE, + "message": "parse-stage output is unavailable", + "location": None, + } + ], + } + ) + + monkeypatch.setattr(_bridge, "module", lambda: Warns) + result = api.compile_sources(sample(), package="acme/example") + assert isinstance(result, api.CompiledV3) + (warning,) = result.warnings + assert warning.severity is api.Severity.WARNING + assert warning.code == codes.PY_PARSE_STAGE diff --git a/tests/tools/test_python_docs.py b/tests/tools/test_python_docs.py new file mode 100644 index 00000000..ce629c3b --- /dev/null +++ b/tests/tools/test_python_docs.py @@ -0,0 +1,61 @@ +"""The Python compiler guide's examples run against the public API.""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +GUIDE = Path(__file__).resolve().parents[2] / "docs" / "python-compiler.md" + + +def python_blocks() -> list[str]: + return re.findall(r"```python\n(.*?)```", GUIDE.read_text(), flags=re.DOTALL) + + +def test_the_guide_has_an_api_example() -> None: + assert any("compile_sources" in block for block in python_blocks()) + + +def test_api_examples_run_in_a_fresh_interpreter(tmp_path: Path) -> None: + for block in python_blocks(): + completed = subprocess.run( + [sys.executable, "-I", "-c", block], + capture_output=True, + text=True, + cwd=tmp_path, + check=False, + ) + assert completed.returncode == 0, completed.stderr + assert completed.stdout.split() == ["acme/models.py", "acme/rules.py"] + assert list(tmp_path.iterdir()) == [] + + +def test_api_examples_pass_both_type_checkers(tmp_path: Path) -> None: + root = Path(__file__).resolve().parents[2] + for index, block in enumerate(python_blocks()): + example = tmp_path / f"example_{index}.py" + example.write_text(block) + for command in ( + [ + sys.executable, + "-m", + "mypy", + "--strict", + "--no-incremental", + str(example), + ], + [ + sys.executable, + "-m", + "pyright", + "--pythonpath", + sys.executable, + str(example), + ], + ): + completed = subprocess.run( + command, capture_output=True, text=True, cwd=root, check=False + ) + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/tests/unit/ir/test_v4_codec.py b/tests/unit/ir/test_v4_codec.py index fd720cad..f5effd14 100644 --- a/tests/unit/ir/test_v4_codec.py +++ b/tests/unit/ir/test_v4_codec.py @@ -263,7 +263,8 @@ def test_paths_and_fqnames(self) -> None: assert fail_codes(codec.decode_fqname, "morphir/SDK.list.map") == ( d.INVALID_NAME, ) - assert fail_codes(codec.decode_name, ["legacy"]) == (d.INVALID_JSON,) + assert codec.decode_name(["legacy"]) == n("legacy") + assert fail_codes(codec.decode_name, 7) == (d.INVALID_JSON,) class TestRoundTrip: @@ -363,10 +364,8 @@ def test_unknown_node(self) -> None: assert err.codes == (d.UNKNOWN_NODE,) assert err.diagnostics[0].location == ("Intersection",) - def test_compact_spelling_is_not_guessed(self) -> None: - assert fail_codes(codec.decode_type, "morphir/SDK:basics#int") == ( - d.INVALID_JSON, - ) + def test_ambiguous_spelling_is_not_guessed(self) -> None: + assert fail_codes(codec.decode_type, "morphir/SDK:basics") == (d.INVALID_NAME,) assert fail_codes(codec.decode_type, {"Unit": {}, "Tuple": {}}) == ( d.UNKNOWN_NODE, ) diff --git a/tests/unit/ir/test_v4_compact.py b/tests/unit/ir/test_v4_compact.py new file mode 100644 index 00000000..4480bdae --- /dev/null +++ b/tests/unit/ir/test_v4_compact.py @@ -0,0 +1,1546 @@ +"""Tests for the compact v4 spellings that the draft v4 reader accepts. + +The finos/morphir v4 JSON profile at the pinned commit says that readers MUST +accept every spelling the v4 JSON Schema marks as valid. Each case gives a +compact (or legacy) spelling and the expanded spelling that the writer emits. +The reader must give the same model for both, and the writer must give the +expanded spelling back. Kit case ids name the finos/morphir case a test follows. +""" + +from __future__ import annotations + +import json +import math +from decimal import Decimal +from pathlib import Path as FilePath +from typing import Any + +import pytest + +from morphir.ir import diagnostics as d +from morphir.ir import v3 +from morphir.ir.diagnostics import DiagnosticError +from morphir.ir.format_version import parse_json +from morphir.ir.v3 import codec as v3_codec +from morphir.ir.v4 import codec, module +from morphir.ir.v4 import document as doc +from morphir.ir.v4 import literal as lit +from morphir.ir.v4 import type as t +from morphir.ir.v4 import type_def as td +from morphir.ir.v4 import type_spec as ts +from morphir.ir.v4 import value as v +from morphir.ir.v4.access_controlled import Private, Public +from morphir.ir.v4.distribution import ( + ApplicationDistribution, + EntryPointKind, + LibraryDistribution, +) +from morphir.ir.v4.documented import Documented +from morphir.ir.v4.fqname import FQName +from morphir.ir.v4.name import Name +from morphir.ir.v4.path import Path + +FIXTURES = FilePath(__file__).resolve().parents[2] / "fixtures" / "ir" +COMPILER = FIXTURES / "python-compiler" +UPSTREAM = FIXTURES / "v4" + +Json = Any + + +def n(*words: str) -> Name: + return Name(words) + + +def p(*names: Name) -> Path: + return Path(names) + + +SDK = p(n("morphir"), n("s", "d", "k")) +TA = t.TypeAttributes() +VA = v.ValueAttributes() + + +def sdk(mod: str, local: str) -> FQName: + return FQName(SDK, p(n(mod)), n(*local.split("-"))) + + +INT = t.Reference(TA, sdk("basics", "int")) +FLOAT = t.Reference(TA, sdk("basics", "float")) +BOOL = t.Reference(TA, sdk("basics", "bool")) +STRING = t.Reference(TA, sdk("string", "string")) + + +def fail_codes(fn: Any, *args: Any) -> tuple[str, ...]: + with pytest.raises(DiagnosticError) as info: + fn(*args) + return info.value.codes + + +def fail_location(fn: Any, *args: Any) -> tuple[str | int, ...]: + with pytest.raises(DiagnosticError) as info: + fn(*args) + return info.value.diagnostics[0].location + + +# --- Expanded spellings, as the writer emits them --- + + +def x_ref(fqname: str, *args: Json) -> Json: + body: dict[str, Json] = {"fqname": fqname} + if args: + body["args"] = list(args) + return {"Reference": body} + + +def x_var(name: str) -> Json: + return {"Variable": {"name": name}} + + +def x_lit(kind: str, value: Json) -> Json: + return {"Literal": {"literal": {kind: value}}} + + +X_INT = x_ref("morphir/SDK:basics#int") +X_STRING = x_ref("morphir/SDK:string#string") +X_WILD: Json = {"WildcardPattern": {}} +X_AS_X = {"AsPattern": {"pattern": X_WILD, "name": "x"}} +SDK_ARRAY = [["morphir"], ["s", "d", "k"]] + + +class TestNames: + """Kit names-0001 to names-0008: legacy word arrays are valid input.""" + + @pytest.mark.parametrize( + ("legacy", "words"), + [ + (["value", "in", "u", "s", "d"], ("value", "in", "u", "s", "d")), + (["a"], ("a",)), + (["item", "2"], ("item", "2")), + (["f", "r", "2052", "a"], ("f", "r", "2052", "a")), + ], + ) + def test_legacy_name_array(self, legacy: list[str], words: tuple[str, ...]) -> None: + name = codec.decode_name(legacy) + assert name == n(*words) + assert codec.decode_name(codec.encode_name(name)) == name + + @pytest.mark.parametrize( + "bad", [[], ["Value"], ["value-in"], [1], ["a", None], [["a"]]] + ) + def test_invalid_legacy_name_array(self, bad: list[object]) -> None: + assert fail_codes(codec.decode_name, bad) == (d.INVALID_NAME,) + + def test_name_must_be_a_string_or_array(self) -> None: + assert fail_codes(codec.decode_name, 7) == (d.INVALID_JSON,) + + def test_legacy_path_arrays(self) -> None: + assert codec.decode_path(SDK_ARRAY) == SDK + assert codec.decode_path(["morphir", "SDK"]) == SDK + assert fail_codes(codec.decode_path, []) == (d.INVALID_NAME,) + + def test_legacy_fqname_arrays(self) -> None: + expected = sdk("list", "map") + assert codec.decode_fqname([SDK_ARRAY, [["list"]], ["map"]]) == expected + assert codec.decode_fqname(["morphir/SDK", "list", "map"]) == expected + assert fail_codes(codec.decode_fqname, ["morphir/SDK", "list"]) == ( + d.INVALID_JSON, + ) + assert fail_codes(codec.decode_fqname, 3) == (d.INVALID_JSON,) + + +TYPE_CASES: list[tuple[str, Json, Json]] = [ + ("variable-bare", "a", x_var("a")), + ("variable-wrapped-name", {"Variable": "a"}, x_var("a")), + ("variable-wrapped-legacy", {"Variable": ["a"]}, x_var("a")), + ("variable-expanded", {"Variable": {"attributes": {}, "name": "a"}}, x_var("a")), + ("reference-bare", "morphir/SDK:basics#int", X_INT), + ("reference-wrapped", {"Reference": "morphir/SDK:basics#int"}, X_INT), + ( + "reference-empty-args", + {"Reference": {"fqname": "morphir/SDK:basics#int", "args": []}}, + X_INT, + ), + ( + "reference-attributes", + {"Reference": {"attributes": {}, "fqname": "morphir/SDK:basics#int"}}, + X_INT, + ), + ( + "reference-array", + {"Reference": ["morphir/SDK:list#list", "a"]}, + x_ref("morphir/SDK:list#list", x_var("a")), + ), + ( + "reference-array-two-args", + { + "Reference": [ + "morphir/SDK:dict#dict", + "morphir/SDK:string#string", + "morphir/SDK:basics#int", + ] + }, + x_ref("morphir/SDK:dict#dict", X_STRING, X_INT), + ), + ( + "reference-array-legacy-fqname", + {"Reference": [[SDK_ARRAY, [["list"]], ["list"]], "a"]}, + x_ref("morphir/SDK:list#list", x_var("a")), + ), + ( + "reference-expanded-object-args", + {"Reference": {"fqname": "morphir/SDK:list#list", "args": ["a"]}}, + x_ref("morphir/SDK:list#list", x_var("a")), + ), + ( + "tuple-bare", + ["morphir/SDK:basics#int", "morphir/SDK:string#string"], + {"Tuple": {"elements": [X_INT, X_STRING]}}, + ), + ("tuple-bare-empty", [], {"Tuple": {"elements": []}}), + ( + "tuple-wrapped", + {"Tuple": ["morphir/SDK:basics#int", "a"]}, + {"Tuple": {"elements": [X_INT, x_var("a")]}}, + ), + ( + "tuple-expanded", + {"Tuple": {"attributes": {}, "elements": ["a"]}}, + {"Tuple": {"elements": [x_var("a")]}}, + ), + ( + "record", + { + "Record": { + "fields": { + "name": "morphir/SDK:string#string", + "age": "morphir/SDK:basics#int", + } + } + }, + {"Record": {"fields": {"name": X_STRING, "age": X_INT}}}, + ), + ( + "extensible-record", + { + "ExtensibleRecord": { + "attributes": {}, + "variable": "r", + "fields": {"email": "morphir/SDK:string#string"}, + } + }, + {"ExtensibleRecord": {"variable": "r", "fields": {"email": X_STRING}}}, + ), + ( + "function", + { + "Function": { + "parameterType": "morphir/SDK:basics#int", + "returnType": "morphir/SDK:string#string", + } + }, + {"Function": {"parameterType": X_INT, "returnType": X_STRING}}, + ), + ("unit-expanded", {"Unit": {"attributes": {}}}, {"Unit": {}}), + ( + "empty-constraints", + {"Variable": {"attributes": {"constraints": {}}, "name": "a"}}, + x_var("a"), + ), +] + + +class TestTypeSpellings: + @pytest.mark.parametrize( + ("compact", "expanded"), + [(c, e) for _, c, e in TYPE_CASES], + ids=[i for i, _, _ in TYPE_CASES], + ) + def test_compact_reads_as_expanded(self, compact: Json, expanded: Json) -> None: + decoded = codec.decode_type(compact) + assert decoded == codec.decode_type(expanded) + assert codec.encode_type(decoded) == expanded + + def test_compact_model(self) -> None: + assert codec.decode_type("morphir/SDK:basics#int") == INT + assert codec.decode_type("comparable") == t.Variable(TA, n("comparable")) + assert codec.decode_type(["a", "morphir/SDK:basics#float"]) == t.Tuple( + TA, [t.Variable(TA, n("a")), FLOAT] + ) + + @pytest.mark.parametrize( + ("bad", "code"), + [ + # A bare string at type position is an FQName or a lowercase variable. + ("Int", d.INVALID_NAME), + ("value-in-USD", d.INVALID_NAME), + ("1a", d.INVALID_NAME), + ("morphir/SDK:basics", d.INVALID_NAME), + (42, d.INVALID_JSON), + (True, d.INVALID_JSON), + (None, d.INVALID_JSON), + # A Reference array is an FQName and at least one argument. + ({"Reference": ["morphir/SDK:list#list"]}, d.INVALID_JSON), + ({"Reference": ["morphir/SDK", "list", "list"]}, d.INVALID_NAME), + ({"Reference": 4}, d.INVALID_JSON), + ({"Variable": 4}, d.INVALID_JSON), + ({"Tuple": "a"}, d.INVALID_JSON), + # v3 tagged arrays and type-level holes are not v4 type expressions. + (["Reference", {}, "morphir/SDK:basics#int"], d.UNKNOWN_NODE), + ({"Draft": {}}, d.UNKNOWN_NODE), + # Legacy-window spellings are not valid in the pinned schema. + ({"Record": {"name": "morphir/SDK:string#string"}}, d.UNKNOWN_MEMBER), + ({"Function": {"arg": "a", "result": "a"}}, d.UNKNOWN_MEMBER), + ({"Variable": {"attrs": {}, "name": "a"}}, d.UNKNOWN_MEMBER), + # Kit types-0012. + ( + {"Variable": {"attributes": {"constraints": 5}, "name": "a"}}, + d.INVALID_JSON, + ), + ( + {"Variable": {"attributes": {"colour": "red"}, "name": "a"}}, + d.UNKNOWN_MEMBER, + ), + ( + {"Variable": {"attributes": {"constraints": {"x": 1}}, "name": "a"}}, + d.UNKNOWN_MEMBER, + ), + ], + ) + def test_invalid_spellings(self, bad: Json, code: str) -> None: + assert fail_codes(codec.decode_type, bad) == (code,) + + def test_failure_is_located(self) -> None: + bad = {"Record": {"fields": {"street": "Street"}}} + assert fail_location(codec.decode_type, bad) == ("Record", "fields", "street") + + +LITERAL_CASES: list[tuple[str, Json, lit.Literal]] = [ + ("integer-expanded", {"IntegerLiteral": {"value": 42}}, lit.IntegerLiteral(42)), + ("whole-number", {"WholeNumberLiteral": {"value": 42}}, lit.IntegerLiteral(42)), + ("bool-expanded", {"BoolLiteral": {"value": True}}, lit.BoolLiteral(True)), + ("char-expanded", {"CharLiteral": {"value": "a"}}, lit.CharLiteral("a")), + ("astral-char", {"CharLiteral": "\U0001f600"}, lit.CharLiteral("\U0001f600")), + ("string-expanded", {"StringLiteral": {"value": "x"}}, lit.StringLiteral("x")), + ("float-expanded", {"FloatLiteral": {"value": 4.0}}, lit.FloatLiteral(4.0)), + # Kit values-0046 and values-0047: unknown members beside value are ignored. + ( + "float-unknown-member", + {"FloatLiteral": {"future": True, "value": 4.0}}, + lit.FloatLiteral(4.0), + ), + ( + "integer-unknown-member", + {"IntegerLiteral": {"future": True, "value": 42}}, + lit.IntegerLiteral(42), + ), + ( + "decimal-expanded", + {"DecimalLiteral": {"value": "10.50"}}, + lit.DecimalLiteral(Decimal("10.50")), + ), + # Kit patterns-and-literals-0005: a document has no {"value": ..} spelling. + ( + "document-value-member", + {"DocumentLiteral": {"value": 1}}, + lit.DocumentLiteral(doc.DocObject({"value": doc.DocInt(1)})), + ), + # Kit patterns-and-literals-0019 and 0020: arbitrary precision. + ( + "big-integer", + {"IntegerLiteral": 18446744073709551616}, + lit.IntegerLiteral(18446744073709551616), + ), + ( + "big-negative-integer", + {"IntegerLiteral": -9223372036854775809}, + lit.IntegerLiteral(-9223372036854775809), + ), +] + + +class TestLiteralSpellings: + @pytest.mark.parametrize( + ("spelling", "expected"), + [(s, e) for _, s, e in LITERAL_CASES], + ids=[i for i, _, _ in LITERAL_CASES], + ) + def test_literal_spelling(self, spelling: Json, expected: lit.Literal) -> None: + assert codec.decode_literal(spelling) == expected + + @pytest.mark.parametrize("lexeme", ["-0.00", "1e-7", ".5", "12.", "+3", "10.50"]) + def test_decimal_lexemes(self, lexeme: str) -> None: + decoded = codec.decode_literal({"DecimalLiteral": lexeme}) + assert decoded == lit.DecimalLiteral(Decimal(lexeme)) + + @pytest.mark.parametrize( + "bad", + [ + # Kit patterns-and-literals-0016. + {"DecimalLiteral": "ten"}, + {"DecimalLiteral": ""}, + {"DecimalLiteral": "1_000"}, + {"DecimalLiteral": "NaN"}, + {"DecimalLiteral": " 1"}, + {"DecimalLiteral": "1\n"}, + {"DecimalLiteral": 10.5}, + {"IntegerLiteral": {}}, + {"IntegerLiteral": {"value": 1.5}}, + {"FloatLiteral": math.inf}, + {"FloatLiteral": {"value": math.nan}}, + {"FloatLiteral": 10**400}, + {"CharLiteral": {"value": "ab"}}, + ], + ) + def test_invalid_literals(self, bad: Json) -> None: + assert fail_codes(codec.decode_literal, bad) == (d.INVALID_LITERAL,) + + +VALUE_CASES: list[tuple[str, Json, Json]] = [ + # Decision 0009: bare scalars and arrays at value position. + ("bare-integer", 42, x_lit("IntegerLiteral", 42)), + ("bare-negative-integer", -7, x_lit("IntegerLiteral", -7)), + ("bare-bool", True, x_lit("BoolLiteral", True)), + ("bare-float", 4.0, x_lit("FloatLiteral", 4.0)), + ("bare-variable", "x", x_var("x")), + ("bare-legacy-free-name", "value-in-USD", x_var("value-in-USD")), + ( + "bare-reference", + "morphir/SDK:basics#add", + {"Reference": {"fqname": "morphir/SDK:basics#add"}}, + ), + ( + "bare-list", + [1, 2, 3], + {"List": {"items": [x_lit("IntegerLiteral", i) for i in (1, 2, 3)]}}, + ), + ("bare-empty-list", [], {"List": {"items": []}}), + ( + "bare-list-of-names", + ["x", "y"], + {"List": {"items": [x_var("x"), x_var("y")]}}, + ), + # Literal wrapper forms (kit values-0025). + ("literal-integer", {"Literal": 42}, x_lit("IntegerLiteral", 42)), + ("literal-float", {"Literal": 4.5}, x_lit("FloatLiteral", 4.5)), + ("literal-bool", {"Literal": False}, x_lit("BoolLiteral", False)), + ("literal-string", {"Literal": "a"}, x_lit("StringLiteral", "a")), + ("literal-typed", {"Literal": {"IntegerLiteral": 42}}, x_lit("IntegerLiteral", 42)), + ( + "literal-typed-value", + {"Literal": {"IntegerLiteral": {"value": 42}}}, + x_lit("IntegerLiteral", 42), + ), + ( + "literal-whole-number", + {"Literal": {"WholeNumberLiteral": 42}}, + x_lit("IntegerLiteral", 42), + ), + ( + "literal-expanded-direct", + {"Literal": {"attributes": {}, "literal": 42}}, + x_lit("IntegerLiteral", 42), + ), + ( + "literal-expanded-string", + {"Literal": {"literal": "a"}}, + x_lit("StringLiteral", "a"), + ), + ( + "literal-expanded-typed", + {"Literal": {"attributes": {}, "literal": {"IntegerLiteral": 42}}}, + x_lit("IntegerLiteral", 42), + ), + ( + "literal-document", + {"Literal": {"DocumentLiteral": {"a": [1, None]}}}, + x_lit("DocumentLiteral", {"a": [1, None]}), + ), + # Constructor, Variable, Reference and FieldFunction compact payloads. + ( + "constructor", + {"Constructor": "morphir/SDK:maybe#just"}, + {"Constructor": {"fqname": "morphir/SDK:maybe#just"}}, + ), + ( + "constructor-legacy-fqname", + {"Constructor": [SDK_ARRAY, [["maybe"]], ["just"]]}, + {"Constructor": {"fqname": "morphir/SDK:maybe#just"}}, + ), + ( + "constructor-expanded", + {"Constructor": {"attributes": {}, "fqname": "morphir/SDK:maybe#just"}}, + {"Constructor": {"fqname": "morphir/SDK:maybe#just"}}, + ), + ("variable", {"Variable": "x"}, x_var("x")), + ("variable-legacy", {"Variable": ["my", "x"]}, x_var("my-x")), + ("variable-expanded", {"Variable": {"attributes": {}, "name": "x"}}, x_var("x")), + ( + "reference", + {"Reference": "morphir/SDK:basics#add"}, + {"Reference": {"fqname": "morphir/SDK:basics#add"}}, + ), + ( + "reference-legacy", + {"Reference": ["morphir/SDK", "basics", "add"]}, + {"Reference": {"fqname": "morphir/SDK:basics#add"}}, + ), + ("field-function", {"FieldFunction": "name"}, {"FieldFunction": {"name": "name"}}), + ( + "field-function-expanded", + {"FieldFunction": {"attributes": {}, "name": "name"}}, + {"FieldFunction": {"name": "name"}}, + ), + # Tuple and List wrappers (kit values-0029, values-0031). + ( + "tuple", + {"Tuple": [{"Variable": "x"}, 1]}, + {"Tuple": {"elements": [x_var("x"), x_lit("IntegerLiteral", 1)]}}, + ), + ( + "tuple-object", + {"Tuple": {"elements": ["x", 1]}}, + {"Tuple": {"elements": [x_var("x"), x_lit("IntegerLiteral", 1)]}}, + ), + ( + "list", + {"List": [1, 2]}, + {"List": {"items": [x_lit("IntegerLiteral", 1), x_lit("IntegerLiteral", 2)]}}, + ), + ( + "list-object", + {"List": {"attributes": {}, "items": [1]}}, + {"List": {"items": [x_lit("IntegerLiteral", 1)]}}, + ), + # Expanded nodes whose children are compact. + ( + "if-then-else", + {"IfThenElse": {"condition": True, "then": 1, "else": 2}}, + { + "IfThenElse": { + "condition": x_lit("BoolLiteral", True), + "then": x_lit("IntegerLiteral", 1), + "else": x_lit("IntegerLiteral", 2), + } + }, + ), + ( + "apply", + {"Apply": {"function": "morphir/SDK:basics#negate", "argument": 1}}, + { + "Apply": { + "function": {"Reference": {"fqname": "morphir/SDK:basics#negate"}}, + "argument": x_lit("IntegerLiteral", 1), + } + }, + ), + ( + "field", + {"Field": {"target": "record", "name": "field-name"}}, + {"Field": {"target": x_var("record"), "name": "field-name"}}, + ), + ( + "record", + {"Record": {"fields": {"name": "x", "age": 25}}}, + { + "Record": { + "fields": {"name": x_var("x"), "age": x_lit("IntegerLiteral", 25)} + } + }, + ), + ( + "update-record", + {"UpdateRecord": {"target": "record", "fields": {"name": {"Literal": "new"}}}}, + { + "UpdateRecord": { + "target": x_var("record"), + "fields": {"name": x_lit("StringLiteral", "new")}, + } + }, + ), + ( + "lambda", + {"Lambda": {"pattern": X_AS_X, "body": "x"}}, + {"Lambda": {"pattern": X_AS_X, "body": x_var("x")}}, + ), + ( + "let-definition", + { + "LetDefinition": { + "name": "x", + "definition": { + "ExpressionBody": { + "inputTypes": {}, + "outputType": "morphir/SDK:basics#int", + "body": 1, + } + }, + "in": "x", + } + }, + { + "LetDefinition": { + "name": "x", + "definition": { + "ExpressionBody": { + "inputTypes": {}, + "outputType": X_INT, + "body": x_lit("IntegerLiteral", 1), + } + }, + "in": x_var("x"), + } + }, + ), + ( + "let-recursion", + { + "LetRecursion": { + "definitions": { + "f": { + "ExpressionBody": { + "inputTypes": {"a": "a"}, + "outputType": "a", + "body": "f", + } + } + }, + "in": "f", + } + }, + { + "LetRecursion": { + "definitions": { + "f": { + "ExpressionBody": { + "inputTypes": {"a": x_var("a")}, + "outputType": x_var("a"), + "body": x_var("f"), + } + } + }, + "in": x_var("f"), + } + }, + ), + ( + "destructure", + {"Destructure": {"pattern": [X_AS_X, X_WILD], "value": "pair", "in": "x"}}, + { + "Destructure": { + "pattern": {"TuplePattern": {"patterns": [X_AS_X, X_WILD]}}, + "value": x_var("pair"), + "in": x_var("x"), + } + }, + ), + ( + "pattern-match", + { + "PatternMatch": { + "value": "x", + "cases": [ + {"pattern": {"LiteralPattern": 0}, "body": True}, + {"pattern": X_WILD, "body": False}, + ], + } + }, + { + "PatternMatch": { + "value": x_var("x"), + "cases": [ + { + "pattern": { + "LiteralPattern": {"literal": {"IntegerLiteral": 0}} + }, + "body": x_lit("BoolLiteral", True), + }, + {"pattern": X_WILD, "body": x_lit("BoolLiteral", False)}, + ], + } + }, + ), + ( + "hole", + { + "Hole": { + "reason": {"UnresolvedReference": {"target": "my-org/project:m#gone"}}, + "expectedType": "morphir/SDK:string#string", + } + }, + { + "Hole": { + "reason": {"UnresolvedReference": {"target": "my-org/project:m#gone"}}, + "expectedType": X_STRING, + } + }, + ), + ("unit-expanded", {"Unit": {"attributes": {}}}, {"Unit": {}}), +] + + +class TestValueSpellings: + @pytest.mark.parametrize( + ("compact", "expanded"), + [(c, e) for _, c, e in VALUE_CASES], + ids=[i for i, _, _ in VALUE_CASES], + ) + def test_compact_reads_as_expanded(self, compact: Json, expanded: Json) -> None: + decoded = codec.decode_value(compact) + assert decoded == codec.decode_value(expanded) + assert codec.encode_value(decoded) == expanded + + def test_compact_model(self) -> None: + assert codec.decode_value(42) == v.LiteralValue(VA, lit.IntegerLiteral(42)) + assert codec.decode_value("x") == v.Variable(VA, n("x")) + assert codec.decode_value({"Literal": "a"}) == v.LiteralValue( + VA, lit.StringLiteral("a") + ) + + @pytest.mark.parametrize( + ("bad", "code"), + [ + # Decision 0009, rule 3: a bare string is a name, never a string literal. + ("hello world", d.INVALID_NAME), + ("value-in-Usd", d.INVALID_NAME), + ("a:b", d.INVALID_NAME), + (None, d.INVALID_JSON), + ({"Literal": None}, d.INVALID_LITERAL), + ({"Literal": [1]}, d.INVALID_LITERAL), + ({"Literal": {"attributes": {}}}, d.MISSING_MEMBER), + ({"Literal": {"literal": 1, "extra": 2}}, d.UNKNOWN_MEMBER), + ({"Literal": {"literal": None}}, d.INVALID_LITERAL), + ({"Tuple": "x"}, d.INVALID_JSON), + ({"List": 1}, d.INVALID_JSON), + ({"Constructor": 1}, d.INVALID_JSON), + ({"Constructor": ["morphir/SDK", "maybe"]}, d.INVALID_JSON), + ({"FieldFunction": 1}, d.INVALID_JSON), + ({"Record": {"name": "x"}}, d.UNKNOWN_MEMBER), + ( + {"IfThenElse": {"condition": True, "thenBranch": 1, "elseBranch": 2}}, + d.UNKNOWN_MEMBER, + ), + # Kit values-0042 and a v3 tagged array. + ({"Native": {"fqname": "morphir/SDK:basics#add"}}, d.UNKNOWN_NODE), + (["Variable", {}, ["x"]], d.UNKNOWN_NODE), + ({"IntegerLiteral": 42}, d.UNKNOWN_NODE), + ], + ) + def test_invalid_spellings(self, bad: Json, code: str) -> None: + assert fail_codes(codec.decode_value, bad) == (code,) + + def test_documented_let_definition_has_no_slot(self) -> None: + bad = { + "LetDefinition": { + "name": "x", + "definition": { + "doc": "one", + "ExpressionBody": { + "inputTypes": {}, + "outputType": "a", + "body": 1, + }, + }, + "in": "x", + } + } + assert fail_codes(codec.decode_value, bad) == (d.UNKNOWN_MEMBER,) + assert fail_location(codec.decode_value, bad) == ( + "LetDefinition", + "definition", + "doc", + ) + + +class TestNumbersByLexeme: + """Decision 0009, rule 2: a bare number's kind follows its lexeme.""" + + @pytest.mark.parametrize( + ("text", "expected"), + [ + ("0", lit.IntegerLiteral(0)), + ("-7", lit.IntegerLiteral(-7)), + ("0.0", lit.FloatLiteral(0.0)), + ("4.0", lit.FloatLiteral(4.0)), + ("1e3", lit.FloatLiteral(1000.0)), + ("4E0", lit.FloatLiteral(4.0)), + ("-0.5", lit.FloatLiteral(-0.5)), + ( + "123456789012345678901234567890", + lit.IntegerLiteral(int("123456789012345678901234567890")), + ), + ], + ) + def test_bare_number_in_a_document(self, text: str, expected: lit.Literal) -> None: + body = {"ExpressionBody": {"inputTypes": {}, "outputType": "a", "body": "@"}} + root = { + "formatVersion": 4, + "distribution": { + "Library": { + "packageName": "a", + "def": { + "modules": { + "m": {"Public": {"values": {"v": {"Public": body}}}} + } + }, + } + }, + } + document = json.dumps(root).replace('"@"', text) + dist = codec.decode_versioned_distribution(document) + assert isinstance(dist, LibraryDistribution) + mod = dist.definition.modules[p(n("m"))].value + defn = mod.values[n("v")].value.value + assert isinstance(defn, v.Definition) + assert defn.body == v.LiteralValue(VA, expected) + + def test_json_parser_keeps_the_lexeme_kind(self) -> None: + parsed = parse_json("[0, 0.0, 1e3, 18446744073709551616]") + assert isinstance(parsed, list) + assert [type(x) for x in parsed] == [int, float, float, int] + assert parsed[3] == 18446744073709551616 + + def test_out_of_range_float_is_refused(self) -> None: + assert fail_codes(codec.decode_value, json.loads("1e400")) == ( + d.INVALID_LITERAL, + ) + assert fail_codes(codec.decode_value, {"Literal": math.nan}) == ( + d.INVALID_LITERAL, + ) + + +PATTERN_CASES: list[tuple[str, Json, Json]] = [ + ( + "tuple-bare", + [X_WILD, X_AS_X], + {"TuplePattern": {"patterns": [X_WILD, X_AS_X]}}, + ), + ( + "tuple-wrapped", + {"TuplePattern": [X_WILD]}, + {"TuplePattern": {"patterns": [X_WILD]}}, + ), + ( + "tuple-expanded", + { + "TuplePattern": { + "attributes": {}, + "patterns": [ + { + "AsPattern": { + "attributes": {}, + "pattern": {"WildcardPattern": {"attributes": {}}}, + "name": "x", + } + } + ], + } + }, + {"TuplePattern": {"patterns": [X_AS_X]}}, + ), + ( + "literal-bare-integer", + {"LiteralPattern": 42}, + {"LiteralPattern": {"literal": {"IntegerLiteral": 42}}}, + ), + ( + "literal-bare-string", + {"LiteralPattern": "a"}, + {"LiteralPattern": {"literal": {"StringLiteral": "a"}}}, + ), + ( + "literal-bare-float", + {"LiteralPattern": 0.5}, + {"LiteralPattern": {"literal": {"FloatLiteral": 0.5}}}, + ), + ( + "literal-typed", + {"LiteralPattern": {"CharLiteral": "c"}}, + {"LiteralPattern": {"literal": {"CharLiteral": "c"}}}, + ), + ( + "literal-expanded-direct", + {"LiteralPattern": {"attributes": {}, "literal": True}}, + {"LiteralPattern": {"literal": {"BoolLiteral": True}}}, + ), + ( + "nullary-expanded", + {"EmptyListPattern": {"attributes": {}}}, + {"EmptyListPattern": {}}, + ), + ( + "as-legacy-name", + {"AsPattern": {"pattern": X_WILD, "name": ["x"]}}, + X_AS_X, + ), + ( + "constructor-legacy-fqname", + { + "ConstructorPattern": { + "fqname": ["morphir/SDK", "maybe", "just"], + "patterns": [[X_WILD, X_WILD]], + } + }, + { + "ConstructorPattern": { + "fqname": "morphir/SDK:maybe#just", + "patterns": [{"TuplePattern": {"patterns": [X_WILD, X_WILD]}}], + } + }, + ), +] + + +class TestPatternSpellings: + @pytest.mark.parametrize( + ("compact", "expanded"), + [(c, e) for _, c, e in PATTERN_CASES], + ids=[i for i, _, _ in PATTERN_CASES], + ) + def test_compact_reads_as_expanded(self, compact: Json, expanded: Json) -> None: + decoded = codec.decode_pattern(compact) + assert decoded == codec.decode_pattern(expanded) + assert codec.encode_pattern(decoded) == expanded + + @pytest.mark.parametrize( + ("bad", "code"), + [ + # Kit patterns-and-literals-0007. + ({"LiteralPattern": {"DocumentLiteral": {"name": "A"}}}, d.INVALID_LITERAL), + ( + {"LiteralPattern": {"literal": {"DocumentLiteral": 1}}}, + d.INVALID_LITERAL, + ), + ({"LiteralPattern": None}, d.INVALID_LITERAL), + # The schema has no bare scalar or bare literal at pattern position. + ("x", d.INVALID_JSON), + (42, d.INVALID_JSON), + ({"IntegerLiteral": 42}, d.UNKNOWN_NODE), + (["WildcardPattern", {}], d.UNKNOWN_NODE), + ({"TuplePattern": "x"}, d.INVALID_JSON), + ], + ) + def test_invalid_spellings(self, bad: Json, code: str) -> None: + assert fail_codes(codec.decode_pattern, bad) == (code,) + + +CTORS: ts.Constructors = { + n("just"): [(n("value"), t.Variable(TA, n("a")))], + n("nothing"): [], +} +ALIAS_DEF = td.TypeAliasDefinition([], STRING) +ALIAS_JSON = {"typeParams": [], "typeExp": "morphir/SDK:string#string"} +BODY = v.Definition([], INT, v.Variable(VA, n("x"))) +BODY_JSON = {"inputTypes": {}, "outputType": "morphir/SDK:basics#int", "body": "x"} + + +class TestDefinitionSpellings: + def test_constructor_spellings(self) -> None: + # Kit definitions-0003 and the schema's legacy Constructors array. + compact = {"just": [["value", "a"]], "nothing": []} + legacy = [["just", [[["value"], "a"]]], ["nothing", []]] + assert codec._decode_constructors(compact, ()) == CTORS + assert codec._decode_constructors(legacy, ()) == CTORS + assert list(codec._decode_constructors(legacy, ())) == [n("just"), n("nothing")] + + @pytest.mark.parametrize( + ("bad", "code"), + [ + ([["just", []], ["just", []]], d.DUPLICATE_MEMBER), + ([["just"]], d.INVALID_JSON), + ([["just", [["value"]]]], d.INVALID_JSON), + (["just"], d.INVALID_JSON), + (7, d.INVALID_JSON), + ], + ) + def test_invalid_constructors(self, bad: Json, code: str) -> None: + assert fail_codes(codec._decode_constructors, bad, ()) == (code,) + + @pytest.mark.parametrize( + ("spelling", "expected"), + [ + ( + ["TypeAliasDefinition", ["a"], "a"], + td.TypeAliasDefinition([n("a")], t.Variable(TA, n("a"))), + ), + ( + [ + "CustomTypeDefinition", + ["a"], + { + "access": "Public", + "value": {"just": [["value", "a"]], "nothing": []}, + }, + ], + td.CustomTypeDefinition([n("a")], Public(CTORS)), + ), + ( + [ + "CustomTypeDefinition", + ["a"], + {"private": {"just": [["value", "a"]], "nothing": []}}, + ], + td.CustomTypeDefinition([n("a")], Private(CTORS)), + ), + ( + { + "CustomTypeDefinition": { + "typeParams": [["a"]], + "constructors": [["just", [["value", "a"]]], ["nothing", []]], + } + }, + td.CustomTypeDefinition([n("a")], Public(CTORS)), + ), + ], + ) + def test_type_definition_spellings( + self, spelling: Json, expected: td.Definition + ) -> None: + assert codec.decode_type_definition(spelling) == expected + + @pytest.mark.parametrize( + ("spelling", "expected"), + [ + # Kit definitions-0002. + (["OpaqueTypeSpecification", []], ts.OpaqueTypeSpecification([])), + ( + [ + "TypeAliasSpecification", + ["a"], + {"Reference": ["morphir/SDK:list#list", "a"]}, + ], + ts.TypeAliasSpecification( + [n("a")], + t.Reference(TA, sdk("list", "list"), [t.Variable(TA, n("a"))]), + ), + ), + ( + [ + "CustomTypeSpecification", + ["a"], + {"just": [["value", "a"]], "nothing": []}, + ], + ts.CustomTypeSpecification([n("a")], CTORS), + ), + ( + [ + "DerivedTypeSpecification", + [], + { + "baseType": "morphir/SDK:string#string", + "fromBaseType": "my/pkg:m#from-string", + "toBaseType": ["my/pkg", "m", "to-string"], + }, + ], + ts.DerivedTypeSpecification( + [], + ts.DerivedTypeSpecificationDetails( + STRING, + FQName(p(n("my"), n("pkg")), p(n("m")), n("from", "string")), + FQName(p(n("my"), n("pkg")), p(n("m")), n("to", "string")), + ), + ), + ), + ], + ) + def test_legacy_type_specification_arrays( + self, spelling: Json, expected: ts.Specification + ) -> None: + assert codec.decode_type_specification(spelling) == expected + + @pytest.mark.parametrize( + ("bad", "code"), + [ + (["OpaqueTypeSpecification"], d.INVALID_JSON), + (["TypeAliasSpecification", [], "a", "extra"], d.INVALID_JSON), + (["Nope", []], d.UNKNOWN_NODE), + ([], d.UNKNOWN_NODE), + (["DerivedTypeSpecification", [], {"baseType": "a"}], d.MISSING_MEMBER), + ], + ) + def test_invalid_legacy_specification_arrays(self, bad: Json, code: str) -> None: + assert fail_codes(codec.decode_type_specification, bad) == (code,) + + @pytest.mark.parametrize( + "bad", + [ + ["TypeAliasDefinition", []], + ["IncompleteTypeDefinition", [], {"Draft": {}}], + ["CustomTypeDefinition", [], {"access": "Public", "value": {}, "x": 1}], + ], + ) + def test_invalid_legacy_definition_arrays(self, bad: Json) -> None: + with pytest.raises(DiagnosticError): + codec.decode_type_definition(bad) + + def test_value_specification_spellings(self) -> None: + expected = v.Specification([(n("a"), INT), (n("b"), INT)], INT) + canonical = { + "inputs": {"a": "morphir/SDK:basics#int", "b": "morphir/SDK:basics#int"}, + "output": "morphir/SDK:basics#int", + } + # Kit definitions-0004: the legacy inputs array. + legacy = { + "inputs": [ + ["a", "morphir/SDK:basics#int"], + [["b"], "morphir/SDK:basics#int"], + ], + "output": "morphir/SDK:basics#int", + } + assert codec.decode_value_specification(canonical) == expected + assert codec.decode_value_specification(legacy) == expected + # Kit definitions-0029: inputs may be omitted. + no_inputs = v.Specification([], INT) + assert ( + codec.decode_value_specification({"output": "morphir/SDK:basics#int"}) + == no_inputs + ) + assert fail_codes( + codec.decode_value_specification, {"inputs": [["a"]], "output": "a"} + ) == (d.INVALID_JSON,) + repeated = {"inputs": [["a", "a"], ["a", "a"]], "output": "a"} + assert fail_codes(codec.decode_value_specification, repeated) == ( + d.DUPLICATE_MEMBER, + ) + assert fail_location(codec.decode_value_specification, repeated) == ( + "inputs", + 1, + ) + + def test_module_specification_doc_spellings(self) -> None: + # Kit definitions-0010: doc flattened beside the members, or nested. + spec = v.Specification([(n("a"), INT)], INT) + members = { + "inputs": {"a": "morphir/SDK:basics#int"}, + "output": "morphir/SDK:basics#int", + } + flat = {"types": {}, "values": {"add": {"doc": "Adds", **members}}} + nested = {"types": {}, "values": {"add": {"doc": "Adds", "value": members}}} + expected = module.Specification(values={n("add"): Documented("Adds", spec)}) + assert codec.decode_module_specification(flat) == expected + assert codec.decode_module_specification(nested) == expected + + def test_type_specification_doc_is_flattened(self) -> None: + flat = {"types": {"t": {"doc": "A type", "OpaqueTypeSpecification": {}}}} + expected = module.Specification( + types={n("t"): Documented("A type", ts.OpaqueTypeSpecification([]))} + ) + assert codec.decode_module_specification(flat) == expected + + @pytest.mark.parametrize( + "entry", + [ + # Kit definitions-0006 (canonical) and definitions-0001. + {"Public": {"doc": "D", "TypeAliasDefinition": ALIAS_JSON}}, + {"access": "Public", "doc": "D", "TypeAliasDefinition": ALIAS_JSON}, + { + "access": "Public", + "doc": "D", + "value": {"TypeAliasDefinition": ALIAS_JSON}, + }, + { + "access": "Public", + "value": {"doc": "D", "TypeAliasDefinition": ALIAS_JSON}, + }, + {"Public": {"doc": "D", "value": {"TypeAliasDefinition": ALIAS_JSON}}}, + {"pub": {"doc": "D", "TypeAliasDefinition": ALIAS_JSON}}, + {"public": {"doc": "D", "TypeAliasDefinition": ALIAS_JSON}}, + {"access": "pub", "doc": "D", "TypeAliasDefinition": ALIAS_JSON}, + { + "access": "public", + "doc": "D", + "value": ["TypeAliasDefinition", [], "morphir/SDK:string#string"], + }, + ], + ) + def test_public_type_definition_entries(self, entry: Json) -> None: + mod = codec.decode_module_definition({"types": {"t": entry}}) + assert mod.types == {n("t"): Public(Documented("D", ALIAS_DEF))} + + @pytest.mark.parametrize( + "entry", + [ + # Kit definitions-0017. + {"Private": {"TypeAliasDefinition": ALIAS_JSON}}, + {"private": {"TypeAliasDefinition": ALIAS_JSON}}, + {"access": "Private", "TypeAliasDefinition": ALIAS_JSON}, + {"access": "private", "value": {"TypeAliasDefinition": ALIAS_JSON}}, + ], + ) + def test_private_type_definition_entries(self, entry: Json) -> None: + mod = codec.decode_module_definition({"types": {"t": entry}}) + assert mod.types == {n("t"): Private(Documented(None, ALIAS_DEF))} + + @pytest.mark.parametrize( + ("entry", "expected"), + [ + # Kit definitions-0018 and definitions-0019. + ( + {"Public": {"doc": "Adds", "ExpressionBody": BODY_JSON}}, + Public(Documented("Adds", BODY)), + ), + ( + {"access": "Public", "doc": "Adds", "ExpressionBody": BODY_JSON}, + Public(Documented("Adds", BODY)), + ), + ( + {"access": "Private", "ExpressionBody": BODY_JSON}, + Private(Documented(None, BODY)), + ), + ( + {"private": {"ExpressionBody": BODY_JSON}}, + Private(Documented(None, BODY)), + ), + ], + ) + def test_value_definition_entries(self, entry: Json, expected: object) -> None: + mod = codec.decode_module_definition({"values": {"v": entry}}) + assert mod.values == {n("v"): expected} + + @pytest.mark.parametrize( + ("entry", "code", "location"), + [ + # Kit definitions-0031. + ( + {"priv": {"TypeAliasDefinition": ALIAS_JSON}}, + d.UNKNOWN_NODE, + ("types", "t"), + ), + ( + {"access": "priv", "TypeAliasDefinition": ALIAS_JSON}, + d.UNKNOWN_NODE, + ("types", "t", "access"), + ), + ( + {"access": 1, "TypeAliasDefinition": ALIAS_JSON}, + d.UNKNOWN_NODE, + ("types", "t", "access"), + ), + # Kit definitions-0028. + ( + {"Public": {"doc": ["a", "b"], "TypeAliasDefinition": ALIAS_JSON}}, + d.INVALID_JSON, + ("types", "t", "Public", "doc"), + ), + # The legacy-window wrapper without doc is not valid in the schema. + ( + {"Public": {"value": {"TypeAliasDefinition": ALIAS_JSON}}}, + d.UNKNOWN_NODE, + ("types", "t", "Public", "value"), + ), + # Documentation given twice. + ( + { + "Public": { + "doc": "a", + "value": {"doc": "b", "TypeAliasDefinition": ALIAS_JSON}, + } + }, + d.DUPLICATE_MEMBER, + ("types", "t", "Public", "value", "doc"), + ), + ({"access": "Public"}, d.MISSING_MEMBER, ("types", "t")), + ({"Public": {"doc": "a"}}, d.UNKNOWN_NODE, ("types", "t", "Public")), + # Kit definitions-0023. + ( + {"Public": {"TypeAliasDefinition": {"annotations": [], **ALIAS_JSON}}}, + d.UNKNOWN_MEMBER, + ("types", "t", "Public", "TypeAliasDefinition", "annotations"), + ), + ], + ) + def test_invalid_definition_entries( + self, entry: Json, code: str, location: tuple[str | int, ...] + ) -> None: + bad = {"types": {"t": entry}} + assert fail_codes(codec.decode_module_definition, bad) == (code,) + assert fail_location(codec.decode_module_definition, bad) == location + + @pytest.mark.parametrize( + "entry", + [ + {"Public": {"types": {}, "values": {}}}, + {"pub": {"types": {}, "values": {}}}, + {"access": "Public", "value": {"types": {}, "values": {}}}, + {"access": "Public", "types": {}, "values": {}}, + {"access": "public", "doc": "M", "types": {}}, + ], + ) + def test_package_module_entries(self, entry: Json) -> None: + decoded = codec.decode_package_definition({"modules": {"m": entry}}) + mod = decoded.modules[p(n("m"))] + assert isinstance(mod, Public) + assert mod.value.types == {} + + def test_flattened_module_keeps_doc(self) -> None: + entry = {"access": "Private", "doc": "M", "values": {}} + decoded = codec.decode_package_definition({"modules": {"m": entry}}) + assert decoded.modules[p(n("m"))] == Private(module.Definition(doc="M")) + + def test_flattened_module_rejects_unknown_members(self) -> None: + entry = {"access": "Public", "value": {}, "types": {}} + bad = {"modules": {"m": entry}} + assert fail_codes(codec.decode_package_definition, bad) == (d.UNKNOWN_MEMBER,) + + def test_distribution_legacy_names(self) -> None: + document = { + "formatVersion": 4, + "distribution": { + "Application": { + "packageName": [["acme"], ["example"]], + "entryPoints": { + "start": { + "target": [["acme", "example"], "main", "run"], + "kind": "main", + } + }, + } + }, + } + dist = codec.decode_versioned_distribution(document) + assert isinstance(dist, ApplicationDistribution) + assert dist.package.name == p(n("acme"), n("example")) + entry = dist.entry_points[n("start")] + assert entry.target == FQName( + p(n("acme"), n("example")), p(n("main")), n("run") + ) + assert entry.kind is EntryPointKind.Main + + +def load(path: FilePath) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def library(path: FilePath) -> LibraryDistribution: + dist = codec.decode_versioned_distribution(path.read_text(encoding="utf-8")) + assert isinstance(dist, LibraryDistribution) + return dist + + +def module_of(dist: LibraryDistribution, name: str) -> module.Definition: + entry = dist.definition.modules[p(n(name))] + assert isinstance(entry, Public) + return entry.value + + +def body_of(mod: module.Definition, name: str) -> v.Value: + entry = mod.values[n(*name.split("-"))] + assert isinstance(entry, Public) + defn = entry.value.value + assert isinstance(defn, v.Definition) + return defn.body + + +ACME = p(n("acme"), n("example")) + + +def acme(local: str) -> t.Type: + return t.Reference(TA, FQName(ACME, p(n("models")), n(*local.split("-")))) + + +def float_lit(x: float) -> v.Value: + return v.LiteralValue(VA, lit.FloatLiteral(x)) + + +class TestCompilerOutput: + """The compact output of morphir-python-binding 0.4.0 / morphir-core 0.2.0.""" + + def test_modules_fixture(self) -> None: + dist = library(COMPILER / "modules-v4.json") + assert dist.package.name == ACME + assert list(dist.definition.modules) == [p(n("models")), p(n("rules"))] + + models = module_of(dist, "models") + assert list(models.types) == [ + n("address"), + n("application"), + n("decision"), + n("labeled", "point"), + n("point"), + ] + address = models.types[n("address")] + assert address == Public( + Documented( + None, + td.TypeAliasDefinition( + [], + t.Record( + TA, + [t.Field(n("street"), STRING), t.Field(n("zip", "code"), INT)], + ), + ), + ) + ) + decision = models.types[n("decision")].value.value + assert isinstance(decision, td.CustomTypeDefinition) + assert isinstance(decision.constructors, Public) + ctors = decision.constructors.value + assert list(ctors) == [n("pending"), n("approved")] + assert ctors[n("pending")] == [] + assert ctors[n("approved")] == [ + (n("amount"), FLOAT), + (n("address"), acme("address")), + ] + point = models.types[n("point")].value.value + assert point == td.TypeAliasDefinition([], t.Tuple(TA, [FLOAT, FLOAT])) + + located = models.values[n("located")].value.value + assert isinstance(located, v.Definition) + assert [(name, tpe) for name, _, tpe in located.input_types] == [ + (n("flag"), BOOL), + (n("location"), acme("point")), + ] + assert located.output_type == acme("labeled-point") + assert located.body == v.IfThenElse( + VA, + v.Variable(VA, n("flag")), + v.Tuple( + VA, + [ + v.Variable(VA, n("location")), + v.LiteralValue(VA, lit.StringLiteral("known")), + ], + ), + v.Tuple( + VA, + [ + v.Tuple(VA, [float_lit(0.0), float_lit(0.0)]), + v.LiteralValue(VA, lit.StringLiteral("unknown")), + ], + ), + ) + rules = module_of(dist, "rules") + assert list(rules.values) == [n("origin"), n("select", "decision")] + assert body_of(rules, "origin") == v.Tuple(VA, [float_lit(0.0), float_lit(0.0)]) + + def test_adt_fixture(self) -> None: + dist = library(COMPILER / "adt-v4.json") + models = module_of(dist, "models") + assert list(models.values) == [ + n("choose"), + n("classify"), + n("expression"), + n("guarded"), + n("located"), + n("make", "point"), + n("nested"), + ] + less_than = v.Reference(VA, sdk("basics", "less-than")) + classify = body_of(models, "classify") + assert isinstance(classify, v.IfThenElse) + assert classify.condition == v.Apply( + VA, + v.Apply(VA, less_than, v.Variable(VA, n("value"))), + v.LiteralValue(VA, lit.IntegerLiteral(0)), + ) + assert classify.then_branch == v.LiteralValue(VA, lit.StringLiteral("negative")) + assert isinstance(classify.else_branch, v.IfThenElse) + expression = body_of(models, "expression") + assert expression == v.IfThenElse( + VA, v.Variable(VA, n("flag")), float_lit(-1.5), float_lit(2.5) + ) + assert body_of(models, "make-point") == v.Tuple( + VA, [v.Variable(VA, n("x")), v.Variable(VA, n("y"))] + ) + + @pytest.mark.parametrize("name", ["modules-v4.json", "adt-v4.json"]) + def test_fixture_round_trips_through_the_expanded_writer(self, name: str) -> None: + dist = library(COMPILER / name) + encoded = codec.encode_versioned_distribution(dist) + assert codec.decode_versioned_distribution(json.dumps(encoded)) == dist + assert encoded != load(COMPILER / name) + + def test_v3_twin_decodes(self) -> None: + text = (COMPILER / "modules-v3.json").read_text(encoding="utf-8") + dist = v3_codec.decode_versioned_distribution(text) + assert isinstance(dist, v3.Library) + assert [list(m) for m in dist.definition.modules] == [ + [("models",)], + [("rules",)], + ] + + +class TestUpstreamExamples: + """The finos/morphir v4 examples at the pinned commit.""" + + def test_books_and_records(self) -> None: + dist = library(UPSTREAM / "finos-books-and-records-example.json") + mod = module_of_path(dist, p(n("books", "and", "records"))) + assert mod.doc is not None and mod.doc.startswith("Books and Records") + book = mod.types[n("book")].value.value + model = p(n("morphir"), n("reference"), n("model")) + records = p(n("books", "and", "records")) + assert book == td.TypeAliasDefinition( + [], + t.Reference( + TA, + sdk("dict", "dict"), + [ + t.Reference(TA, FQName(model, records, n("product", "i", "d"))), + t.Reference(TA, FQName(model, records, n("deal"))), + ], + ), + ) + deal_event = mod.types[n("deal", "event")].value.value + assert isinstance(deal_event, td.CustomTypeDefinition) + assert next(iter(deal_event.constructors.value)) == n("deal", "opened") + basics = dist.dependencies[SDK].modules[p(n("basics"))] + assert basics.types[n("int")] == Documented( + None, ts.OpaqueTypeSpecification([]) + ) + + def test_complete_example_hits_the_name_model_gap(self) -> None: + text = (UPSTREAM / "finos-complete-example.json").read_text(encoding="utf-8") + with pytest.raises(DiagnosticError) as info: + codec.decode_versioned_distribution(text) + assert info.value.codes == (d.INVALID_NAME,) + assert info.value.diagnostics[0].location == ( + "distribution", + "Library", + "def", + "modules", + "u-s/f-r-2052-a/data-tables", + ) + + def test_complete_example_decodes_once_the_name_is_representable(self) -> None: + text = (UPSTREAM / "finos-complete-example.json").read_text(encoding="utf-8") + dist = library_from_text(text.replace("u-s/f-r-2052-a", "US/FR-2052-a")) + mod = module_of_path( + dist, p(n("u", "s"), n("f", "r", "2052", "a"), n("data", "tables")) + ) + assert mod.doc == "Data tables module for regulatory reporting" + total = mod.values[n("calculate", "total")] + assert isinstance(total, Public) + defn = total.value.value + assert isinstance(defn, v.Definition) + assert defn.body == float_lit(0.0) + listing = dist.dependencies[SDK].modules[p(n("list"))] + map_spec = listing.values[n("map")].value + assert [name for name, _ in map_spec.inputs] == [n("f"), n("list")] + + +def library_from_text(text: str) -> LibraryDistribution: + dist = codec.decode_versioned_distribution(text) + assert isinstance(dist, LibraryDistribution) + return dist + + +def module_of_path(dist: LibraryDistribution, path: Path) -> module.Definition: + entry = dist.definition.modules[path] + assert isinstance(entry, Public) + return entry.value + + +@pytest.mark.parametrize("text", [":#int", "morphir/sdk:#int", ":basics#int"]) +def test_fqname_strings_need_package_and_module_paths(text: str) -> None: + # The schema's FQNameString pattern requires a segment in each path. + with pytest.raises(DiagnosticError): + codec.decode_fqname(text) + with pytest.raises(DiagnosticError): + codec.decode_type(text) + + +def test_a_decimal_beyond_the_decimal_range_is_a_diagnostic() -> None: + # The lexeme is valid, but Python's Decimal cannot represent the exponent. + with pytest.raises(DiagnosticError) as error: + codec.decode_literal({"DecimalLiteral": "1e999999999999999999999999999"}) + assert error.value.codes == (d.INVALID_LITERAL,) diff --git a/tests/unit/test_ci_changes.py b/tests/unit/test_ci_changes.py new file mode 100644 index 00000000..5496af5d --- /dev/null +++ b/tests/unit/test_ci_changes.py @@ -0,0 +1,121 @@ +"""Change selection and the aggregate gate for CI.""" + +from __future__ import annotations + +import pytest + +from scripts import ci_changes as c + +ALL = frozenset(c.CATEGORIES) + + +@pytest.mark.parametrize( + ("paths", "expected"), + [ + (["README.md", "docs/templates/x.html", "openspec/changes/a/tasks.md"], set()), + (["docs/python-compiler.md"], {"python"}), + (["packages/morphir-python-native/README.md"], {"python", "native"}), + ( + ["rust/crates/morphir-python-native/src/lib.rs"], + {"python", "native", "rust"}, + ), + (["rust/crates/morphir-python-binding/src/lib.rs"], ALL), + (["rust/Cargo.lock"], ALL), + (["packages/morphir/src/morphir/ir/json.py"], {"python", "native"}), + (["uv.lock"], {"python", "native"}), + (["rust/extension.toml"], {"bundle"}), + (["tests/fixtures/python/loans/acme/rules.py"], {"python", "native", "bundle"}), + (["scripts/extension_bundle.py"], {"python", "bundle"}), + ([".gitignore"], {"python", "native"}), + ([".github/workflows/ci.yml"], ALL), + ([".config/mise/tasks/test"], ALL), + (["something/new.txt"], ALL), + ], +) +def test_selection(paths: list[str], expected: set[str]) -> None: + assert c.select(paths).categories == expected + + +def test_documentation_only_changes_run_no_jobs() -> None: + jobs = c.select(["README.md"]).jobs() + assert not any(jobs.values()) + + +def test_native_only_changes_skip_rust_and_bundle() -> None: + jobs = c.select(["packages/morphir-python-native/pyproject.toml"]).jobs() + assert jobs["native-wheel"] and jobs["test"] and jobs["build"] + assert not jobs["rust"] and not jobs["bundle"] + + +def test_bundle_only_changes_build_the_wheel_the_bundle_check_uses() -> None: + jobs = c.select(["rust/extension.toml"]).jobs() + assert jobs["bundle"] and jobs["native-wheel"] + assert not jobs["test"] and not jobs["rust"] + + +def test_full_runs_select_everything() -> None: + assert c.select([], full=True).categories == ALL + assert all(c.select([], full=True).jobs().values()) + + +def needs(**results: str) -> dict[str, dict[str, str]]: + return {"changes": {"result": "success"}} | { + job.replace("_", "-"): {"result": result} for job, result in results.items() + } + + +def test_aggregate_passes_selected_success_and_unselected_skip() -> None: + selected = c.select(["rust/extension.toml"]).jobs() + results = needs(**{job.replace("-", "_"): "skipped" for job in selected}) + results["bundle"] = {"result": "success"} + results["native-wheel"] = {"result": "success"} + assert c.aggregate(selected, results) == [] + + +@pytest.mark.parametrize("outcome", ["failure", "cancelled", "skipped"]) +def test_aggregate_fails_selected_jobs_that_did_not_succeed(outcome: str) -> None: + selected = c.select([], full=True).jobs() + results = needs(**{job.replace("-", "_"): "success" for job in selected}) + results["test"] = {"result": outcome} + assert c.aggregate(selected, results) == [f"test: selected but {outcome}"] + + +def test_aggregate_fails_missing_results() -> None: + selected = c.select([], full=True).jobs() + results = needs(**{job.replace("-", "_"): "success" for job in selected}) + del results["bundle"] + assert c.aggregate(selected, results) == ["bundle: no result"] + + +def test_aggregate_fails_unselected_jobs_that_ran_or_failed() -> None: + selected = c.select(["README.md"]).jobs() + results = needs(**{job.replace("-", "_"): "skipped" for job in selected}) + results["rust"] = {"result": "failure"} + assert c.aggregate(selected, results) == ["rust: not selected but failure"] + + +def test_aggregate_fails_when_selection_failed() -> None: + results = needs() + results["changes"] = {"result": "failure"} + assert c.aggregate({}, results) == ["changes: failure"] + + +def test_select_command_prints_outputs(capsys: pytest.CaptureFixture[str]) -> None: + c.main(["select", "rust/extension.toml"]) + lines = dict(line.split("=", 1) for line in capsys.readouterr().out.splitlines()) + assert lines["bundle"] == "true" + assert lines["python"] == "false" + + +def test_aggregate_command_exits_non_zero(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + c.main( + [ + "aggregate", + "--selected", + '{"lint": true}', + "--needs", + '{"changes": {"result": "success"}, "lint": {"result": "cancelled"}}', + ] + ) + assert "lint: selected but cancelled" in capsys.readouterr().out diff --git a/tests/unit/test_native_qualification.py b/tests/unit/test_native_qualification.py new file mode 100644 index 00000000..3e3623ad --- /dev/null +++ b/tests/unit/test_native_qualification.py @@ -0,0 +1,113 @@ +"""Refusal cases of the native distribution qualification script.""" + +from __future__ import annotations + +import os +import zipfile +from typing import TYPE_CHECKING + +import pytest + +from scripts import qualify_native as q + +if TYPE_CHECKING: + from pathlib import Path + +VERSION = q.crate_version() +STEM = f"morphir_python_native-{VERSION}" +METADATA = ( + "Metadata-Version: 2.4\nName: morphir-python-native\nVersion: {version}\n" + "License-Expression: Apache-2.0\nRequires-Python: >=3.14\n{extra}\n" +) + + +def make_wheel( + tmp_path: Path, + *, + tag: str = "cp314-abi3-manylinux_2_28_x86_64", + version: str = VERSION, + extra: str = "", + omit: str = "", + purelib: bool = False, +) -> Path: + path = tmp_path / f"morphir_python_native-{version}-{tag}.whl" + stem = f"morphir_python_native-{version}" + files = { + "morphir_python_native/__init__.py": "", + "morphir_python_native/_native.abi3.so": "", + "morphir_python_native/_native.pyi": "", + "morphir_python_native/py.typed": "", + f"{stem}.dist-info/licenses/LICENSE": "Apache", + f"{stem}.dist-info/METADATA": METADATA.format(version=version, extra=extra), + f"{stem}.dist-info/WHEEL": f"Root-Is-Purelib: {str(purelib).lower()}\n", + } + with zipfile.ZipFile(path, "w") as archive: + for name, text in files.items(): + if name != omit: + archive.writestr(name, text) + return path + + +def test_versions_agree_across_crate_pin_and_constant() -> None: + q.check_versions(VERSION) + assert q.tools_pin() == VERSION + + +def test_accepts_a_complete_wheel(tmp_path: Path) -> None: + assert q.validate_wheel(make_wheel(tmp_path))["version"] == VERSION + + +@pytest.mark.parametrize( + "options", + [ + {"tag": "cp314-cp314-manylinux_2_28_x86_64"}, + {"tag": "py3-none-any"}, + {"version": "9.9.9"}, + {"extra": "Requires-Dist: typer"}, + {"omit": "morphir_python_native/_native.pyi"}, + {"omit": "morphir_python_native/py.typed"}, + {"omit": "morphir_python_native/_native.abi3.so"}, + {"omit": f"{STEM}.dist-info/licenses/LICENSE"}, + {"purelib": True}, + ], +) +def test_refuses_incomplete_or_mismatched_wheels( + tmp_path: Path, options: dict[str, object] +) -> None: + with pytest.raises(q.QualificationError): + q.validate_wheel(make_wheel(tmp_path, **options)) # type: ignore[arg-type] + + +def test_refuses_an_sdist_without_rust_inputs() -> None: + names = [f"{STEM}/{name}" for name in q.REQUIRED_SDIST_FILES] + q.validate_sdist_names(names, VERSION) + with pytest.raises(q.QualificationError, match=r"Cargo\.lock"): + q.validate_sdist_names( + [n for n in names if not n.endswith("Cargo.lock")], VERSION + ) + with pytest.raises(q.QualificationError, match="unsafe"): + q.validate_sdist_names([*names, f"{STEM}/../escape"], VERSION) + + +def test_rust_free_path_excludes_cargo(tmp_path: Path) -> None: + path = q.rust_free_path(tmp_path / "venv") + import shutil + + assert shutil.which("cargo", path=path) is None + + +def test_venv_python_names_the_interpreter_on_each_system(tmp_path: Path) -> None: + assert q.venv_python(tmp_path, "nt") == tmp_path / "Scripts" / "python.exe" + assert q.venv_python(tmp_path, "posix") == tmp_path / "bin" / "python" + + +def test_rust_free_path_drops_directories_holding_only_rustup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + tools = tmp_path / "tools" + tools.mkdir() + rustup = tools / "rustup" + rustup.write_text("#!/bin/sh\n") + rustup.chmod(0o755) + monkeypatch.setenv("PATH", f"{tools}{os.pathsep}/usr/bin") + assert str(tools) not in q.rust_free_path(tmp_path / "venv").split(os.pathsep) diff --git a/tests/unit/test_release_candidates.py b/tests/unit/test_release_candidates.py new file mode 100644 index 00000000..12049c2b --- /dev/null +++ b/tests/unit/test_release_candidates.py @@ -0,0 +1,119 @@ +"""Release tags and exact candidate sets for native and extension releases.""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING + +import pytest + +from scripts import release_candidates as r + +if TYPE_CHECKING: + from pathlib import Path + +VERSIONS = { + "morphir": "0.0.1", + "python": "0.0.2", + "native": "0.1.0", + "extension": "0.4.0", +} + + +@pytest.mark.parametrize( + ("tag", "kind"), + [ + ("morphir-v0.0.1", "morphir"), + ("python-v0.0.2", "python"), + ("native-v0.1.0", "native"), + ("extension/python/v0.4.0", "extension"), + ], +) +def test_tags_name_their_release(tag: str, kind: str) -> None: + assert r.classify(tag, VERSIONS).kind == kind + + +@pytest.mark.parametrize( + "tag", + ["native-v0.2.0", "extension/python/v0.3.0", "v0.1.0", "extension/rust/v1.0.0"], +) +def test_mismatched_or_unknown_tags_are_refused(tag: str) -> None: + with pytest.raises(r.CandidateError): + r.classify(tag, VERSIONS) + + +def test_the_source_versions_are_readable() -> None: + assert set(r.source_versions()) == set(VERSIONS) + + +def native_set(directory: Path, version: str = "0.1.0") -> None: + stem = f"morphir_python_native-{version}" + for tag in r.NATIVE_PLATFORMS: + (directory / f"{stem}-cp314-abi3-{tag}.whl").write_bytes(tag.encode()) + (directory / f"{stem}.tar.gz").write_bytes(b"sdist") + + +def test_complete_native_candidates_are_accepted(tmp_path: Path) -> None: + native_set(tmp_path) + assert len(r.native_candidates(tmp_path, "0.1.0")) == 6 + + +@pytest.mark.parametrize("change", ["missing", "extra", "version"]) +def test_incomplete_native_candidates_are_refused(tmp_path: Path, change: str) -> None: + native_set(tmp_path) + if change == "missing": + next(tmp_path.glob("*win_amd64.whl")).unlink() + elif change == "extra": + (tmp_path / "morphir_python_native-0.1.0-cp314-cp314-linux_x86_64.whl").touch() + else: + next(tmp_path.glob("*.tar.gz")).rename( + tmp_path / "morphir_python_native-0.0.9.tar.gz" + ) + with pytest.raises(r.CandidateError): + r.native_candidates(tmp_path, "0.1.0") + + +def bundle(directory: Path, *, commit: str = "a" * 40, version: str = "0.4.0") -> None: + artifact = f"morphir-python-binding-{version}.wasm" + wasm = b"\0asm" + digest = hashlib.sha256(wasm).hexdigest() + (directory / artifact).write_bytes(wasm) + (directory / f"{artifact}.sha256").write_text(f"{digest} {artifact}\n") + (directory / "release.json").write_text( + json.dumps( + {"version": version, "gitCommit": commit, "artifacts": [{"sha256": digest}]} + ) + ) + + +def test_a_bundle_from_the_tagged_commit_is_accepted(tmp_path: Path) -> None: + bundle(tmp_path) + assert len(r.bundle_candidates(tmp_path, "0.4.0", "a" * 40)) == 3 + + +@pytest.mark.parametrize("options", [{"commit": "b" * 40}, {"version": "0.3.0"}]) +def test_mismatched_bundles_are_refused( + tmp_path: Path, options: dict[str, str] +) -> None: + bundle(tmp_path, **options) + with pytest.raises((r.CandidateError, KeyError, OSError)): + r.bundle_candidates(tmp_path, "0.4.0", "a" * 40) + + +def test_a_bundle_without_provenance_is_refused(tmp_path: Path) -> None: + bundle(tmp_path) + descriptor = json.loads((tmp_path / "release.json").read_text()) + del descriptor["gitCommit"] + (tmp_path / "release.json").write_text(json.dumps(descriptor)) + with pytest.raises(r.CandidateError, match="built from None"): + r.bundle_candidates(tmp_path, "0.4.0", "a" * 40) + + +def test_the_command_writes_a_receipt(tmp_path: Path) -> None: + native_set(tmp_path) + version = r.source_versions()["native"] + if version != "0.1.0": + pytest.skip("fixture names the current native version") + r.main(["native", str(tmp_path), "--tag", f"native-v{version}"]) + assert len((tmp_path / "SHA256SUMS").read_text().splitlines()) == 6 diff --git a/tests/unit/test_release_process.py b/tests/unit/test_release_process.py index b764bd4f..e2f9bf92 100644 --- a/tests/unit/test_release_process.py +++ b/tests/unit/test_release_process.py @@ -102,3 +102,19 @@ def test_archives_reject_the_other_packages_namespace( ) with pytest.raises(ValueError, match="Unexpected bundled namespace"): validate_archive(package, path) + + +def test_cli_release_requires_the_complete_published_native_release() -> None: + from scripts.check_distributions import ( + native_candidates_for, + require_published_native, + ) + + complete = set(native_candidates_for("0.1.0")) + require_published_native("0.1.0", complete | {"extra.whl"}) + with pytest.raises(ValueError, match="win_amd64"): + require_published_native( + "0.1.0", {name for name in complete if "win_amd64" not in name} + ) + with pytest.raises(ValueError, match="before this CLI"): + require_published_native("0.1.0", set()) diff --git a/uv.lock b/uv.lock index 84da297b..d5981cfa 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,7 @@ requires-python = ">=3.14" members = [ "morphir", "morphir-python", + "morphir-python-native", "morphir-workspace", ] @@ -204,6 +205,7 @@ version = "0.0.2" source = { editable = "packages/morphir-tools" } dependencies = [ { name = "morphir" }, + { name = "morphir-python-native" }, { name = "pygments" }, { name = "typer" }, ] @@ -211,10 +213,15 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "morphir", editable = "packages/morphir" }, + { name = "morphir-python-native", editable = "packages/morphir-python-native" }, { name = "pygments", specifier = ">=2.20.0" }, { name = "typer", specifier = ">=0.15" }, ] +[[package]] +name = "morphir-python-native" +source = { editable = "packages/morphir-python-native" } + [[package]] name = "morphir-workspace" version = "0.0.1"