diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml new file mode 100644 index 0000000..15144b2 --- /dev/null +++ b/.github/workflows/build-linux.yml @@ -0,0 +1,65 @@ +name: Build Linux package + +# Called by the Release and Prerelease workflows: the Linux ARM64 `.deb` +# (Snapdragon X laptops running Ubuntu) with its update signature, uploaded +# as the `linux-deb` artifact for them to publish. + +on: + workflow_call: + inputs: + beta-build: + description: Stamp the next beta version for this build number (prereleases). + type: string + default: "" + secrets: + TAURI_SIGNING_PRIVATE_KEY: + required: true + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: + required: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + # Keep in sync with rust-toolchain.toml. + RUST_TOOLCHAIN: "1.99.0" + +jobs: + deb: + name: Package (Linux ARM64) + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Tauri system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libwebkit2gtk-4.1-dev libayatana-appindicator3-dev librsvg2-dev patchelf + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 # master + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - run: bun install --frozen-lockfile + + - name: Stamp beta version + if: inputs.beta-build != '' + env: + BUILD: ${{ inputs.beta-build }} + run: bun scripts/version.ts beta "$BUILD" + + - name: Build the package (update signature) + env: + TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} + run: bun run build:release --bundles deb + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: linux-deb + path: | + target/release/bundle/deb/*.deb + target/release/bundle/deb/*.deb.sig + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad34934..224879b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: with: bun-version-file: package.json - name: Pinned GenieX installer is published - run: bun scripts/fetch-geniex.ts --check + run: bun scripts/check-geniex.ts # The real target, built only for `main` and pull requests to it (release # PRs); `dev` relies on the checks above. @@ -103,3 +103,27 @@ jobs: - run: bun install --frozen-lockfile - name: Build the app (no installer) run: bun run tauri build --no-bundle + + build-linux-arm64: + name: Build (Linux ARM64) + if: github.ref == 'refs/heads/main' || github.base_ref == 'main' + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Tauri system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libwebkit2gtk-4.1-dev libayatana-appindicator3-dev librsvg2-dev patchelf + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 # master + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - run: bun install --frozen-lockfile + # The Rust job runs on x86_64; the app ships for ARM64. + - name: Test on ARM64 + run: mkdir -p dist && cargo test --workspace --locked + - name: Build the package + run: bun run tauri build --bundles deb diff --git a/.github/workflows/geniex-bump.yml b/.github/workflows/geniex-bump.yml index c40dd2e..f3f3e8e 100644 --- a/.github/workflows/geniex-bump.yml +++ b/.github/workflows/geniex-bump.yml @@ -2,7 +2,7 @@ name: GenieX bump # Daily: when Qualcomm publishes a new stable GenieX, verify the installer # (SHA-256, then install it and run `geniex version`) and open a PR to `dev` -# that bundles it in Calcine's installer. +# that pins it: Calcine installs that version on first launch. on: schedule: @@ -53,10 +53,10 @@ jobs: base: dev branch: chore/geniex-${{ steps.bump.outputs.version }} add-paths: runtime/geniex.json - commit-message: "chore(runtime): bundle GenieX ${{ steps.bump.outputs.version }}" - title: "chore(runtime): bundle GenieX ${{ steps.bump.outputs.version }}" + commit-message: "chore(runtime): pin GenieX ${{ steps.bump.outputs.version }}" + title: "chore(runtime): pin GenieX ${{ steps.bump.outputs.version }}" body: | - Qualcomm published GenieX ${{ steps.bump.outputs.version }}. This pins it in `runtime/geniex.json`, so the next Calcine installers bundle it. + Qualcomm published GenieX ${{ steps.bump.outputs.version }}. This pins it in `runtime/geniex.json`, so the next Calcine release installs it on first launch. Checked by this workflow: the installer matches the official manifest (size and SHA-256), installs silently on Windows ARM64, and `geniex version` reports ${{ steps.bump.outputs.version }}. diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index aae4f90..9abf3c8 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -1,8 +1,9 @@ name: Prerelease # On demand (Actions > Prerelease > Run workflow, on `dev`): publish an -# installable beta, Windows ARM64 NSIS installer with GenieX bundled, as a -# GitHub pre-release, and point the Beta update channel at it. Pushes to +# installable beta, Windows ARM64 NSIS installer (Calcine only) and Linux +# ARM64 package, as a GitHub pre-release, and point the Beta update channel +# at it. Pushes to # `dev` don't build installers; only `main` releases do. on: @@ -21,8 +22,18 @@ env: RUST_TOOLCHAIN: "1.99.0" jobs: + linux: + name: Linux + uses: ./.github/workflows/build-linux.yml + with: + beta-build: ${{ github.run_number }} + secrets: + TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} + beta: - name: Beta (Windows ARM64) + name: Beta + needs: linux runs-on: windows-11-arm permissions: contents: write @@ -44,20 +55,25 @@ jobs: id: version run: echo "version=$(bun scripts/version.ts beta ${{ github.run_number }})" >> "$GITHUB_OUTPUT" - - name: Build installer (GenieX bundled, update signature) + - name: Build installer (update signature) env: TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} run: bun run build:release + - name: Collect the Linux package + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: linux-deb + path: linux-release - name: Collect artifacts env: VERSION: ${{ steps.version.outputs.version }} run: | mkdir -p dist-release - cp target/release/bundle/nsis/*-setup.exe dist-release/ + cp target/release/bundle/nsis/*-setup.exe linux-release/*.deb dist-release/ printf 'Beta from `dev` (%s).\n' "${GITHUB_SHA::7}" > notes.md - bun scripts/updater-manifest.ts "$VERSION" latest-beta.json notes.md + bun scripts/updater-manifest.ts "$VERSION" latest-beta.json notes.md linux-release cd dist-release && sha256sum -- * > SHA256SUMS - name: Publish pre-release @@ -70,9 +86,12 @@ jobs: --prerelease \ --target "$GITHUB_SHA" \ --title "Calcine $VERSION" \ - --notes "Automated beta from \`dev\` (${GITHUB_SHA::7}), bundling GenieX $geniex. + --notes "Automated beta from \`dev\` (${GITHUB_SHA::7}). Installs GenieX $geniex on first launch. + + - Windows 11 on Snapdragon: the \`-setup.exe\`. This build isn't code-signed yet, so Windows SmartScreen will warn before installing. + - Linux on ARM64 (preview): the \`.deb\` (\`sudo apt install ./.deb\`). - This build isn't code-signed yet, so Windows SmartScreen will warn before installing." + Checksums in \`SHA256SUMS\`. [Code signing policy](https://eraflo.github.io/Calcine/#code-signing) · [Privacy policy](https://github.com/eraflo/Calcine/blob/main/PRIVACY.md)" # The Beta channel (Settings > Calcine updates) reads latest-beta.json # from a fixed release. It's a pre-release so it never becomes "latest". diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 346cf7e..e73bfdb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release # When `main` carries a version without a tag yet (after merging the release # PR prepared by `bun scripts/release.ts`), build and publish it: -# signed installer with GenieX bundled, updater manifest, checksums, SBOM and -# build provenance. See docs/RELEASING.md. +# signed Windows installer (Calcine only), Linux package, updater manifest, +# checksums, SBOM and build provenance. See docs/RELEASING.md. on: push: @@ -52,10 +52,19 @@ jobs: echo "release=true" >> "$GITHUB_OUTPUT" fi - release: - name: Release v${{ needs.check.outputs.version }} (Windows ARM64) + linux: + name: Linux needs: check if: needs.check.outputs.release == 'true' + uses: ./.github/workflows/build-linux.yml + secrets: + TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} + + release: + name: Release v${{ needs.check.outputs.version }} + needs: [check, linux] + if: needs.check.outputs.release == 'true' runs-on: windows-11-arm permissions: contents: write @@ -79,7 +88,7 @@ jobs: - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: bun install --frozen-lockfile - - name: Build installer (GenieX bundled, update signature) + - name: Build installer (update signature) run: bun run build:release - name: Collect the installer @@ -89,6 +98,12 @@ jobs: installer="$(ls target/release/bundle/nsis/*-setup.exe)" cp "$installer" dist-release/ echo "path=dist-release/$(basename "$installer")" >> "$GITHUB_OUTPUT" + - name: Collect the Linux package + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: linux-deb + path: linux-release + - run: cp linux-release/*.deb dist-release/ # Authenticode signing through the SignPath Foundation, once the project # is approved: set the repository variables SIGNPATH_ORGANIZATION_ID and @@ -122,8 +137,8 @@ jobs: run: | awk -v v="## $VERSION " 'index($0, v) == 1 {found=1; next} found && /^## / {exit} found' CHANGELOG.md > notes.md [ -s notes.md ] || echo "Calcine $VERSION" > notes.md - bun scripts/updater-manifest.ts "$VERSION" dist-release/latest.json notes.md - cd dist-release && sha256sum -- *.exe > SHA256SUMS + bun scripts/updater-manifest.ts "$VERSION" dist-release/latest.json notes.md linux-release + cd dist-release && sha256sum -- *.exe *.deb > SHA256SUMS - name: SBOM (CycloneDX) uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 @@ -137,12 +152,16 @@ jobs: - name: Build provenance uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: - subject-path: ${{ steps.installer.outputs.path }} + subject-path: | + ${{ steps.installer.outputs.path }} + dist-release/*.deb - name: Publish the release env: GH_TOKEN: ${{ github.token }} run: | + # SignPath asks for the code signing policy on download pages. + printf '\n---\n\nWindows 11 on Snapdragon: the `-setup.exe`. Linux on ARM64 (preview): the `.deb`. Checksums in `SHA256SUMS`.\n\n[Code signing policy](https://eraflo.github.io/Calcine/#code-signing) · [Privacy policy](https://github.com/eraflo/Calcine/blob/main/PRIVACY.md)\n' >> notes.md gh release create "v$VERSION" dist-release/* \ --target "$GITHUB_SHA" \ --title "Calcine $VERSION" \ diff --git a/.gitignore b/.gitignore index 5fe17b5..d6cb3b8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,6 @@ # --- Tauri --- src-tauri/gen/schemas/ src-tauri/WixTools/ -# GenieX installer fetched at build time (pinned in runtime/geniex.json) -src-tauri/resources/geniex/ # Downloaded by scripts/geniex-bump.ts to test a new GenieX /geniex-cli-setup.exe diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..68567c1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,50 @@ +# Changelog + +## 1.0.0 (2026-10-09) + +The first stable release: a desktop app and a secure, OpenAI-compatible local API for +Qualcomm GenieX on Snapdragon, with the NPU first. + +### Models and chat + +- Manage GenieX models from the app: Qualcomm AI Hub catalog for your chipset, Hugging Face + search with precisions, sizes and memory checks, links from ModelScope and Docker Hub, local + import, remove and clean (9155f29, 9f13225, 57e37c3) +- Streaming chat with reasoning, sampling presets, advanced options, images, voice, model + preloading and Markdown export (0a1f0d8, 8dd9155, b007ce3, 666e228, 732574c) +- Structured output (`response_format`) and tool calls (`tools`, `tool_choice`) on every + model, the NPU included: Calcine checks each reply and has the model fix it (60f1198, 4060676) +- Long conversations on the NPU: context counted with each model's tokenizer, the oldest + messages forgotten in steps that keep GenieX's cache useful, and summarized (c72d6d9, 527801d) +- Speculative decoding in the Chat and the Benchmark (cd33afa) + +### API + +- Secured gateway on `127.0.0.1:18181`: per-app keys, Host and Origin checks, body checks, a + request queue and a log without prompts (0a1f0d8) +- Ollama-compatible API, optionally on port 11434 (c07b3a3) +- Other devices on the local network, over HTTPS with network keys and private addresses only, + off by default (a2ce233) +- `calcine-cli`: the same API without the window, on the `PATH` (b43a42d, 8be7a41) +- Setup snippets for LangChain, Continue and Open WebUI (914e8ea) + +### Hardware + +- Live NPU, GPU, CPU and memory load, and a self-test per compute unit (9f13225) +- Benchmark page with Qualcomm's `geniex-bench` (44b0746) +- Live power and energy per token on Snapdragon X (878b22a) + +### Install and updates + +- GenieX downloaded from Qualcomm on first launch and checked against a pinned SHA-256; update, + roll back and repair it from the app (fe27313, acb05b2) +- Signed Calcine updates on Stable and Beta channels, start with the session, configurable API + port and allowed web origins (6223b34) +- Linux on ARM64, as a preview: a `.deb` package, GenieX's Linux archive, `calcine-cli geniex` + for machines without a screen (c9f88cc) +- English and French, light and dark themes, command palette and tray (44a3a1f, 9f13225) + +### Fixes + +- The network port is free again before it listens anew after a settings change (eb8f0af) +- The chat header stays inside a narrow chat column (71fcc06) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66b5085..a998121 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,10 +32,12 @@ Common scopes: `app`, `ui`, `chat`, `models`, `discover`, `server`, `gateway`, ## Local setup -Requirements: Windows 11 ARM64 on a Snapdragon device (for real inference), -[rustup](https://rustup.rs) (the pinned toolchain in `rust-toolchain.toml` is -installed automatically), the Visual Studio C++ build tools, and -[bun](https://bun.sh). +Requirements: a Snapdragon device for real inference, running Windows 11 ARM64 +(with the Visual Studio C++ build tools) or Linux ARM64 (with +[Tauri's system dependencies](https://v2.tauri.app/start/prerequisites/#linux) +and `ocl-icd-libopencl1`), [rustup](https://rustup.rs) (the pinned toolchain in +`rust-toolchain.toml` is installed automatically) and [bun](https://bun.sh). +The mock (`bun run app:mock`) runs anywhere. ```bash bun install diff --git a/Cargo.lock b/Cargo.lock index f50f9e5..4461556 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -62,6 +76,45 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -292,6 +345,12 @@ dependencies = [ "tower-service", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -316,7 +375,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -325,6 +384,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -464,7 +532,7 @@ dependencies = [ [[package]] name = "calcine" -version = "0.1.0" +version = "1.0.0" dependencies = [ "calcine-core", "calcine-gateway", @@ -489,11 +557,12 @@ dependencies = [ "tracing", "tracing-subscriber", "win32job", + "windows-sys 0.61.2", ] [[package]] name = "calcine-core" -version = "0.1.0" +version = "1.0.0" dependencies = [ "async-trait", "serde", @@ -506,7 +575,7 @@ dependencies = [ [[package]] name = "calcine-gateway" -version = "0.1.0" +version = "1.0.0" dependencies = [ "axum", "bytes", @@ -515,6 +584,8 @@ dependencies = [ "futures-util", "getrandom 0.4.3", "hex", + "hyper-util", + "rcgen", "reqwest", "serde", "serde_json", @@ -522,22 +593,25 @@ dependencies = [ "specta", "thiserror 2.0.21", "tokio", + "tokio-rustls", "tokio-stream", "tracing", ] [[package]] name = "calcine-geniex" -version = "0.1.0" +version = "1.0.0" dependencies = [ "async-trait", "calcine-core", "futures-util", "hex", + "libc", "reqwest", "serde", "serde_json", "sha2 0.11.0", + "tokenizers", "tokio", "tracing", "which", @@ -545,7 +619,7 @@ dependencies = [ [[package]] name = "calcine-hub" -version = "0.1.0" +version = "1.0.0" dependencies = [ "async-trait", "calcine-core", @@ -558,7 +632,7 @@ dependencies = [ [[package]] name = "calcine-hw" -version = "0.1.0" +version = "1.0.0" dependencies = [ "async-trait", "calcine-core", @@ -572,7 +646,7 @@ dependencies = [ [[package]] name = "calcine-mock" -version = "0.1.0" +version = "1.0.0" dependencies = [ "async-trait", "axum", @@ -626,6 +700,15 @@ dependencies = [ "toml 1.1.7+spec-1.1.0", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.6.0" @@ -697,6 +780,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -808,6 +906,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" @@ -862,6 +979,22 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" @@ -882,6 +1015,20 @@ dependencies = [ "darling_macro 0.24.1", ] +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + [[package]] name = "darling_core" version = "0.23.0" @@ -908,6 +1055,17 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.23.0" @@ -930,6 +1088,21 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "dbus" version = "0.9.12" @@ -972,6 +1145,20 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -992,6 +1179,37 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1161,6 +1379,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + [[package]] name = "embed-resource" version = "3.0.12" @@ -1235,6 +1459,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "event-listener" version = "5.4.2" @@ -1255,6 +1485,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -2161,6 +2402,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2466,6 +2716,22 @@ dependencies = [ "uuid", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "markup5ever" version = "0.39.0" @@ -2513,6 +2779,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minisign-verify" version = "0.2.5" @@ -2550,6 +2822,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "muda" version = "0.20.0" @@ -2636,6 +2930,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "notify-rust" version = "4.18.2" @@ -2668,12 +2972,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2948,6 +3271,15 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3113,6 +3445,22 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3420,6 +3768,51 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3585,6 +3978,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.5" @@ -3604,6 +4006,7 @@ version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -4200,12 +4603,30 @@ dependencies = [ "specta", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "string_cache" version = "0.9.0" @@ -4294,6 +4715,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -4957,6 +5389,39 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tokenizers" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.21", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.53.2" @@ -5288,12 +5753,27 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.9.0" @@ -6236,6 +6716,24 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + [[package]] name = "xattr" version = "1.6.1" @@ -6246,6 +6744,16 @@ dependencies = [ "rustix", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -6266,7 +6774,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] @@ -6377,7 +6885,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3cbad14..bb1ddb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/*", "src-tauri"] [workspace.package] -version = "0.1.0" +version = "1.0.0" edition = "2024" rust-version = "1.99" license = "MIT" @@ -55,6 +55,14 @@ hex = "0.4" getrandom = "0.4" win32job = "2" windows-sys = "0.61" +libc = "0.2" +# The local network port: TLS with ring (no cmake or nasm needed), a +# self-signed certificate, HTTP/1.1 and HTTP/2 over it. +rcgen = { version = "0.14", default-features = false, features = ["crypto", "pem", "ring"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } +hyper-util = { version = "0.1", features = ["server-auto", "service", "tokio"] } +# Pure Rust (no Oniguruma or C++): counts tokens with a model's tokenizer.json. +tokenizers = { version = "0.23.2", default-features = false, features = ["fancy-regex"] } [workspace.lints.rust] # Denied everywhere; the only exception is calcine-hw's Windows performance diff --git a/PRIVACY.md b/PRIVACY.md new file mode 100644 index 0000000..98d5986 --- /dev/null +++ b/PRIVACY.md @@ -0,0 +1,67 @@ +# Privacy policy + +Calcine runs language models on your PC. Your prompts, replies, images, +recordings and conversations never leave it. Calcine has no account, no +telemetry, no analytics and no crash reporting. + +Calcine does connect to the internet for the things listed below. Each +connection is a normal HTTPS request: the service sees your IP address, the +address requested and a user agent, and nothing else from Calcine. + +## Connections Calcine makes + +| When | Where | What for | +|---|---|---| +| At startup, if **Check for updates automatically** is on (the default) | `github.com` (Calcine's releases) | See whether a newer Calcine is available | +| At startup, if **Check for updates automatically** is on (the default) | `qaihub-public-assets.s3.us-west-2.amazonaws.com` (Qualcomm) | See whether a newer GenieX is available | +| When you install, update or repair GenieX (also offered on first launch, or with `calcine-cli geniex install`) | `qaihub-public-assets.s3.us-west-2.amazonaws.com` (Qualcomm) | Download the official GenieX installer (an archive on Linux), checked against its published SHA-256 | +| When you install a Calcine update | `github.com` | Download the update, checked against Calcine's signature | +| When you browse **Discover** or change the chipset on **Hardware** | `huggingface.co`, `qaihub-public-assets.s3.us-west-2.amazonaws.com` (Qualcomm AI Hub) | Search models, read their precisions and sizes, list the chipsets they're built for | +| When you download a model | Hugging Face, ModelScope, Docker Hub or Qualcomm AI Hub, depending on the model | GenieX downloads the model you asked for | +| When you open **Benchmark** or download its tool | `api.github.com`, `github.com` (GenieX's releases) | Find and download Qualcomm's `geniex-bench`, checked against its published SHA-256 | +| When you click a link | Your web browser opens GitHub, Hugging Face or Qualcomm AI Hub | Release notes and model pages | + +Turn off **Settings › Calcine updates › Check for updates automatically** and +Calcine connects only when you ask it to: updates are then checked when you +click **Check now**. + +Calcine starts GenieX with `--skip-update`, so GenieX doesn't check for updates +by itself. + +These services have their own privacy policies: +[GitHub](https://docs.github.com/site-policy/privacy-policies/github-general-privacy-statement), +[Hugging Face](https://huggingface.co/privacy), +[Qualcomm](https://www.qualcomm.com/site/privacy) (its files are served by +[Amazon Web Services](https://aws.amazon.com/privacy/)). + +## Data kept on your PC + +- **Conversations and chat settings**: in Calcine's web storage, on this PC. +- **API keys**: only a SHA-256 fingerprint of each key, in `api-keys.json`. +- **Request log**: who called which model, when, how fast and how many tokens, + never the prompts or replies. Kept in memory and lost when Calcine closes. +- **Settings and benchmark results**: JSON files in Calcine's data folder. +- **Models**: in GenieX's model folder. + +Calcine's data folder is `%APPDATA%\com.eraflo.calcine` (settings, keys, +benchmarks, the network certificate) and `%LOCALAPPDATA%\com.eraflo.calcine` +(web storage, downloaded installers and tools). Uninstalling Calcine can remove +them. On Linux both are `~/.local/share/com.eraflo.calcine` (or under +`$XDG_DATA_HOME`), and GenieX lives in `~/.local/share/geniex-cli`. + +## The local API + +Calcine's API listens on `127.0.0.1`, so other machines can't reach it. +Apps on this PC need an API key unless you turn that off, and web pages are +refused unless you allow their origin. + +If you turn on **Settings › Local network** (off by default), your other +devices can use the API too, over HTTPS, with keys you allow on the network, +from private addresses only. Their requests are handled like local ones: run +on this PC, logged without prompts, never sent elsewhere. See the +[security model](docs/security-model.md). + +## Contact + +Questions about this policy: open an issue at +https://github.com/eraflo/Calcine/issues. diff --git a/README.md b/README.md index 17d252d..e745896 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ chat with them on the Hexagon NPU, and give every app on your PC a secure, OpenA [![CI](https://github.com/eraflo/Calcine/actions/workflows/ci.yml/badge.svg)](https://github.com/eraflo/Calcine/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/license-MIT-2dd4bf)](LICENSE) ![Windows 11 ARM64](https://img.shields.io/badge/Windows%2011-ARM64-a78bfa) +![Linux ARM64 preview](https://img.shields.io/badge/Linux-ARM64%20preview-a78bfa) [**Website**](https://eraflo.github.io/Calcine/) · [**Download**](https://github.com/eraflo/Calcine/releases) · @@ -44,10 +45,29 @@ every web page you visit. Hugging Face, ModelScope or Docker Hub link, or import a folder. - **See the NPU work.** Live NPU, GPU, CPU and memory load, drivers, and a self-test that compares speed on each compute unit. +- **Benchmark properly.** Qualcomm's `geniex-bench`, downloaded and verified for you: + time to first token, prompt and generation speed per unit, with warmup, repetitions, + history and CSV export. +- **Measure energy.** On Snapdragon X, the chip's own energy metering shows live power, and + energy per token on each compute unit and power mode, so you can pick the most efficient. +- **Structured output and tools on the NPU.** `response_format` (JSON or a JSON Schema) and + `tool_choice` work on every model: Calcine asks for JSON or for the call, checks the reply + against the schema and has the model fix it if needed. Agents can rely on them. +- **Long conversations on the NPU.** AI Hub models have a fixed context window (4096 tokens + for most). Calcine counts it with the model's own tokenizer, shows how full it is, and lets + the model forget the oldest messages instead of failing, keeping a summary it writes itself. + It cuts in steps that keep GenieX's cache useful: the next reply starts in under 0.1 s + instead of re-reading the whole conversation. +- **Works with Ollama apps.** Ollama's API is translated too, and can answer on Ollama's + port, 11434, for apps that only speak Ollama. - **OpenAI-compatible API.** On `127.0.0.1:18181`, GenieX's default port, with per-app keys, a request queue and a request log that never records prompts. -- **Always up to date.** GenieX ships in the installer and can be updated or rolled back - from the app. Calcine updates itself with signed packages. +- **Share with your other devices.** Optional, over HTTPS with a certificate made on this PC, + for keys you allow on the network and private networks only. +- **Without the window.** `calcine-cli` serves the same API from a terminal or a server, + with the app's keys, settings and models, on Windows and on Linux. +- **Always up to date.** Calcine installs GenieX on first launch, and can update or roll it + back. Calcine updates itself with signed packages. - **English and French**, light and dark themes, a command palette (Ctrl K) and a tray icon to keep the API running in the background. @@ -72,14 +92,17 @@ every web page you visit. ## Get started -1. Download the installer from [Releases](https://github.com/eraflo/Calcine/releases) - (Windows 11 on Snapdragon, ARM64). It also installs GenieX. -2. Open Calcine and download a model: the welcome screen suggests a few that fit your PC. -3. Chat with it, or create an API key in **Server › API keys** and connect an app. +1. Download the installer from [Releases](https://github.com/eraflo/Calcine/releases): + the `-setup.exe` for Windows 11 on Snapdragon, or the `.deb` for Linux on ARM64 + (`sudo apt install ./Calcine__arm64.deb`). +2. Open Calcine: on first launch it downloads GenieX, Qualcomm's runtime, from Qualcomm + and checks it before installing. +3. Download a model: the welcome screen suggests a few that fit your PC. +4. Chat with it, or create an API key in **Server › API keys** and connect an app. > [!NOTE] -> Calcine is in beta and its installers aren't code-signed yet, so Windows SmartScreen asks -> for confirmation before installing (**More info › Run anyway**). Releases list SHA-256 +> Calcine's installers aren't code-signed yet, so Windows SmartScreen asks for +> confirmation before installing (**More info › Run anyway**). Releases list SHA-256 > checksums, and stable releases also carry an SBOM and a build provenance attestation > (`gh attestation verify -R eraflo/Calcine`). @@ -111,6 +134,23 @@ Clients already written for `geniex serve` keep working on the same port. For th can't send a key, **Require an API key** can be turned off: keyless callers can then only run models. +### Without the window + +`calcine-cli` serves the same API from a terminal, with the app's settings, keys and +models. The installer puts it on your `PATH` (`calcine-cli path remove` takes it off; +the Linux package installs it in `/usr/bin`). Quit Calcine first (also from the tray), +since only one of them can answer on a port, or give `calcine-cli` another one with +`--port`. On a machine without a screen, `calcine-cli geniex install` installs GenieX +the way the welcome screen does. + +```powershell +calcine-cli keys create "My script" # prints the key, once +calcine-cli serve # until Ctrl+C +calcine-cli serve --network # also other devices, over HTTPS +calcine-cli geniex install # GenieX, checked like on first launch +calcine-cli geniex chipset # which chipset models are downloaded for +``` + ## How it works ```mermaid @@ -128,15 +168,20 @@ flowchart LR The gateway listens on loopback only, rejects DNS rebinding and browser origins you haven't allowed, checks per-app keys (stored as SHA-256 fingerprints, scoped to inference or management), and refuses local file paths and internal URLs in request bodies. GenieX -itself is never exposed, and a Windows Job Object stops it with Calcine, even after a -crash. Details in [docs/security-model.md](docs/security-model.md). +itself is never exposed, and it stops with Calcine, even after a crash (a Job Object on +Windows, a parent-death signal on Linux). Details in [docs/security-model.md](docs/security-model.md). ## Requirements -- A Snapdragon PC running Windows 11 on Arm (Linux on ARM64 is planned) +- A Snapdragon PC running Windows 11 on Arm, or Linux on ARM64 (Ubuntu 24.04 or later) - Free memory and disk space for the models you use: Calcine shows what fits before you download -GenieX is installed by the Calcine installer; nothing else is needed. +Calcine installs GenieX on first launch; nothing else is needed. + +On Linux, GenieX runs llama.cpp (GGUF) models on the CPU, and on the Hexagon NPU when the +kernel exposes the compute DSP (FastRPC). Qualcomm AI Hub builds its Linux NPU models for +Dragonwing boards (IQ-9075, QCS6490): pick your chipset in **Hardware › Chipset** if +GenieX can't detect it. The live NPU load and power readings are Windows-only for now. ## Development @@ -152,11 +197,30 @@ See [CONTRIBUTING.md](CONTRIBUTING.md), [docs/architecture.md](docs/architecture [docs/RELEASING.md](docs/RELEASING.md). The website lives in [`site/`](site) and is published to GitHub Pages from `main`. +## Privacy + +No account, no telemetry: prompts and conversations stay on your PC. Calcine connects to +the internet to check for updates (which you can turn off), and to download GenieX, models +and tools when you ask. Every connection is listed in [PRIVACY.md](PRIVACY.md). + +## Code signing policy + +Calcine has applied to the SignPath Foundation's free code signing program for open-source +projects. Once accepted: Free code signing provided by [SignPath.io](https://about.signpath.io/), certificate by [SignPath Foundation](https://signpath.org/). + +- Committers and reviewers: [eraflo](https://github.com/eraflo) +- Approvers: [eraflo](https://github.com/eraflo) + +Installers are built by GitHub Actions from this repository +([`release.yml`](.github/workflows/release.yml)), and each release is signed only after a +manual approval. They contain Calcine only: GenieX is downloaded from Qualcomm on first +launch. Privacy policy: [PRIVACY.md](PRIVACY.md). + ## License -[MIT](LICENSE). Calcine redistributes GenieX under its BSD 3-Clause license, see -[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). Calcine is an independent project, not -affiliated with Qualcomm. +[MIT](LICENSE). Calcine doesn't redistribute GenieX: it downloads Qualcomm's official +installer (an archive on Linux) on your PC. See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). Calcine is an +independent project, not affiliated with Qualcomm. Screenshots show Calcine's demo mode (`bun run app:mock`), with sample data shaped like a Snapdragon X Elite laptop. diff --git a/SECURITY.md b/SECURITY.md index fba4bc0..d4569ee 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,8 +1,10 @@ # Security policy Calcine exposes a local HTTP API (`127.0.0.1:18181`) that runs models on your -machine. We take issues that could let a web page, another program, or a remote -host reach that API, read local files, or tamper with updates seriously. +machine, and optionally the same API to your other devices over HTTPS (port +18443, off by default). We take issues that could let a web page, another +program, or a remote host reach that API without a valid key, read local files, +or tamper with updates seriously. ## Supported versions diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index ceb8eb1..e9e2ed5 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,7 +1,6 @@ # Third-party notices -Calcine is licensed under the MIT License (see [LICENSE](LICENSE)). Calcine installers -redistribute the following third-party software, unmodified. +Calcine is licensed under the MIT License (see [LICENSE](LICENSE)). `crates/calcine-hub/src/quant.rs` follows the precision naming rules of GenieX's model manager (`sdk/model-manager/crates/core/src/manifest_builder.rs`, @@ -11,14 +10,17 @@ BSD 3-Clause, license below) so that Calcine names precisions exactly like ## GenieX CLI - **Project**: https://github.com/qualcomm/GenieX -- **Distributed as**: the official `geniex-cli-setup-windows-arm64-.exe` - installer, downloaded from Qualcomm's release bucket and verified by SHA-256 - at build time. The bundled version is pinned in `runtime/geniex.json`. +- **Not redistributed**: Calcine's installer doesn't contain GenieX. On first + launch, and when you update it, Calcine downloads Qualcomm's official + `geniex-cli-setup-windows-arm64-.exe` from Qualcomm's release bucket, + checks it against the SHA-256 pinned in `runtime/geniex.json` (or the release + manifest, for updates), and runs it. GenieX includes its own runtimes + (Qualcomm AI Engine Direct / QAIRT and llama.cpp); see the + [GenieX NOTICE](https://github.com/qualcomm/GenieX/blob/main/NOTICE) for their + licenses. +- **geniex-bench**: downloaded on demand from GenieX's GitHub release, not + redistributed either. - **License**: BSD 3-Clause -- **Bundled components**: the GenieX installer includes its own runtimes - (Qualcomm AI Engine Direct / QAIRT and llama.cpp) and their dependencies. See - the [GenieX NOTICE](https://github.com/qualcomm/GenieX/blob/main/NOTICE) for - their respective licenses. ``` BSD 3-Clause License diff --git a/crates/calcine-core/Cargo.toml b/crates/calcine-core/Cargo.toml index b660439..7cfcd94 100644 --- a/crates/calcine-core/Cargo.toml +++ b/crates/calcine-core/Cargo.toml @@ -12,13 +12,13 @@ publish = false [dependencies] async-trait.workspace = true serde.workspace = true +serde_json.workspace = true specta.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = ["rt", "sync", "time"] } +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } tokio-util.workspace = true [dev-dependencies] -serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt", "time"] } [lints] diff --git a/crates/calcine-core/src/bench/energy.rs b/crates/calcine-core/src/bench/energy.rs new file mode 100644 index 0000000..ced0a36 --- /dev/null +++ b/crates/calcine-core/src/bench/energy.rs @@ -0,0 +1,294 @@ +//! Energy profiles: a model's speed and energy use on each compute unit, in +//! each power mode. +//! +//! Measured on a Snapdragon X laptop on mains power with GenieX 0.8, the +//! power modes changed neither speed nor energy noticeably, whether set per +//! request or for the server: the compute unit is what makes the +//! difference. +//! +//! Measured through `geniex serve` rather than `geniex-bench`, so each +//! window is exact: the model is loaded in the power mode first, the system +//! is measured idle with it loaded, then the energy meter is read right +//! before and after each generation. What generating costs is the energy +//! above that idle draw, per generated token (the prompt included). + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use specta::Type; + +use super::{BenchMeasure, BenchStat}; +use crate::hardware::{EnergyReading, HardwareProbe}; +use crate::jobs::JobCtx; +use crate::models::{ComputeUnit, Runtime}; +use crate::runtime::InferenceServer; +use crate::{Error, Result}; + +/// GenieX's power modes, fastest first. +pub const POWER_MODES: [&str; 8] = [ + "burst", + "sustained_high_performance", + "high_performance", + "balanced", + "low_balanced", + "high_power_saver", + "power_saver", + "low_power_saver", +]; + +/// After loading, before measuring idle: lets the clocks and whatever the +/// load stirred up settle. +const SETTLE: Duration = Duration::from_secs(3); +/// How long idle is measured. +const IDLE: Duration = Duration::from_secs(4); + +/// Long enough to keep generating; each repetition starts differently so +/// GenieX reads the whole prompt again instead of reusing its cache. +const PROMPT: &str = "Write a long, detailed story about a lighthouse keeper who finds a \ + message in a bottle. Describe the coast, the weather and the people of the \ + village. Keep writing until you are told to stop."; + +/// Which model to profile, on which units, in which power modes. Every unit +/// is measured in every mode. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct EnergyRequest { + pub model: String, + pub runtime: Runtime, + /// The NPU for AI Hub models; llama.cpp models can use any unit. + pub units: Vec, + pub power_modes: Vec, + pub generated_tokens: u32, + pub repetitions: u32, +} + +impl EnergyRequest { + pub fn validate(&self) -> std::result::Result<(), String> { + if self.model.trim().is_empty() { + return Err("choose a model".into()); + } + if self.units.is_empty() { + return Err("choose at least one compute unit".into()); + } + if self.power_modes.is_empty() { + return Err("choose at least one power mode".into()); + } + if let Some(mode) = self + .power_modes + .iter() + .find(|mode| !POWER_MODES.contains(&mode.as_str())) + { + return Err(format!("unknown power mode {mode}")); + } + if self.runtime == Runtime::Qairt && self.units.iter().any(|unit| *unit != ComputeUnit::Npu) + { + return Err("AI Hub models run on the NPU".into()); + } + if !(64..=2048).contains(&self.generated_tokens) { + return Err("generate between 64 and 2048 tokens".into()); + } + if !(1..=10).contains(&self.repetitions) { + return Err("run between 1 and 10 repetitions".into()); + } + Ok(()) + } +} + +/// What energy a power mode used. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct EnergyMeasure { + /// The whole system, idle with the model loaded, just before. + pub idle_watts: f64, + /// The whole system while generating (median). + pub active_watts: f64, + /// Energy above idle per generated token, the prompt included (median): + /// what generating costs. + pub joules_per_token: f64, + /// Energy of the whole system per generated token (median): what the + /// battery sees, the screen and everything else included. + #[serde(default)] + pub system_joules_per_token: f64, +} + +/// One generation's numbers. +#[derive(Debug, Clone, Copy)] +struct Run { + watts: f64, + joules_per_token: f64, + system_joules_per_token: f64, + tokens: f64, + prompt_tokens: f64, + prompt_ms: f64, + prefill_tps: f64, + decode_tps: f64, +} + +/// Measure one power mode. Errors when the device has no energy metering. +pub(crate) async fn measure_mode( + server: &dyn InferenceServer, + hardware: &dyn HardwareProbe, + request: &EnergyRequest, + unit: ComputeUnit, + mode: &str, + ctx: &JobCtx, +) -> Result { + let completion = |max_tokens: u32, run: u32| { + let mut body = json!({ + "model": request.model, + "messages": [{ "role": "user", "content": format!("Story {run}. {PROMPT}") }], + "max_tokens": max_tokens, + "temperature": 0.7, + "enable_think": false, + "power_mode": mode, + }); + if request.runtime == Runtime::LlamaCpp { + body["compute"] = json!(unit); + } + body + }; + // Load the model in this power mode, then measure it idle. + server.complete(&completion(1, 0)).await?; + pause(SETTLE, ctx).await?; + let start = energy(hardware).await?; + pause(IDLE, ctx).await?; + let idle_watts = start + .watts_until(&energy(hardware).await?) + .ok_or_else(|| Error::InvalidInput("the energy meter didn't move".into()))?; + + let mut runs = Vec::new(); + for run in 1..=request.repetitions { + if ctx.is_cancelled() { + return Err(Error::Cancelled); + } + let before = energy(hardware).await?; + let reply = server + .complete(&completion(request.generated_tokens, run)) + .await?; + let after = energy(hardware).await?; + runs.push(read_run(&reply, &before, &after, idle_watts)?); + } + Ok(measure(&runs, idle_watts)) +} + +async fn energy(hardware: &dyn HardwareProbe) -> Result { + hardware.energy().await?.ok_or(Error::NotImplemented( + "energy measurements need a device with energy metering", + )) +} + +async fn pause(duration: Duration, ctx: &JobCtx) -> Result<()> { + tokio::select! { + () = tokio::time::sleep(duration) => Ok(()), + () = ctx.cancelled() => Err(Error::Cancelled), + } +} + +fn read_run( + reply: &Value, + before: &EnergyReading, + after: &EnergyReading, + idle_watts: f64, +) -> Result { + let number = |pointer: &str| reply.pointer(pointer).and_then(Value::as_f64); + let tokens = number("/usage/completion_tokens").unwrap_or(0.0); + let watts = before + .watts_until(after) + .ok_or_else(|| Error::InvalidInput("the energy meter didn't move".into()))?; + if tokens < 1.0 { + return Err(Error::InvalidInput("the model generated nothing".into())); + } + let seconds = after.seconds - before.seconds; + Ok(Run { + watts, + joules_per_token: (watts - idle_watts).max(0.0) * seconds / tokens, + system_joules_per_token: watts * seconds / tokens, + tokens, + prompt_tokens: number("/usage/prompt_tokens").unwrap_or(0.0), + prompt_ms: number("/timings/prompt_ms").unwrap_or(0.0), + prefill_tps: number("/timings/prompt_per_second").unwrap_or(0.0), + decode_tps: number("/timings/predicted_per_second").unwrap_or(0.0), + }) +} + +fn measure(runs: &[Run], idle_watts: f64) -> BenchMeasure { + let stat = |pick: fn(&Run) -> f64| BenchStat::of(&runs.iter().map(pick).collect::>()); + BenchMeasure { + ttft_ms: stat(|run| run.prompt_ms), + prefill_tps: stat(|run| run.prefill_tps), + decode_tps: stat(|run| run.decode_tps), + generated_tokens: stat(|run| run.tokens).median, + prompt_tokens: stat(|run| run.prompt_tokens).median, + geniex_version: String::new(), + energy: Some(EnergyMeasure { + idle_watts, + active_watts: stat(|run| run.watts).median, + joules_per_token: stat(|run| run.joules_per_token).median, + system_joules_per_token: stat(|run| run.system_joules_per_token).median, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request() -> EnergyRequest { + EnergyRequest { + model: "qualcomm/Qwen3-4B".into(), + runtime: Runtime::Qairt, + units: vec![ComputeUnit::Npu], + power_modes: vec!["burst".into(), "power_saver".into()], + generated_tokens: 256, + repetitions: 2, + } + } + + #[test] + fn validates_requests() { + assert!(request().validate().is_ok()); + let unknown = EnergyRequest { + power_modes: vec!["turbo".into()], + ..request() + }; + assert_eq!(unknown.validate(), Err("unknown power mode turbo".into())); + let gpu = EnergyRequest { + units: vec![ComputeUnit::Npu, ComputeUnit::Gpu], + ..request() + }; + assert!(gpu.validate().is_err()); + let none = EnergyRequest { + power_modes: vec![], + ..request() + }; + assert!(none.validate().is_err()); + } + + #[test] + fn energy_per_token_is_above_idle() { + let reply = json!({ + "usage": { "completion_tokens": 200, "prompt_tokens": 60 }, + "timings": { "prompt_ms": 80.0, "prompt_per_second": 750.0, "predicted_per_second": 50.0 }, + }); + let before = EnergyReading { + joules: 100.0, + seconds: 10.0, + }; + // 4 s at 22 W, 15 W idle: 28 J above idle for 200 tokens. + let after = EnergyReading { + joules: 188.0, + seconds: 14.0, + }; + let run = read_run(&reply, &before, &after, 15.0).unwrap(); + assert!((run.watts - 22.0).abs() < 1e-9); + assert!((run.joules_per_token - 0.14).abs() < 1e-9); + assert!((run.system_joules_per_token - 0.44).abs() < 1e-9); + let measure = measure(&[run, run], 15.0); + assert_eq!(measure.decode_tps.median, 50.0); + assert_eq!(measure.energy.unwrap().active_watts, 22.0); + + let empty = json!({ "usage": { "completion_tokens": 0 } }); + assert!(read_run(&empty, &before, &after, 15.0).is_err()); + } +} diff --git a/crates/calcine-core/src/bench/mod.rs b/crates/calcine-core/src/bench/mod.rs new file mode 100644 index 0000000..95ed147 --- /dev/null +++ b/crates/calcine-core/src/bench/mod.rs @@ -0,0 +1,350 @@ +//! Benchmarks with `geniex-bench`: time to first token, prefill and decode +//! speed of a model on each compute unit, measured with warmup and +//! repetitions on a fixed random prompt, and kept as history. + +mod energy; + +pub(crate) use energy::measure_mode; +pub use energy::{EnergyMeasure, EnergyRequest, POWER_MODES}; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use specta::Type; + +use crate::Result; +use crate::jobs::JobCtx; +use crate::models::{ComputeUnit, Runtime}; + +/// Whether the benchmark tool is ready. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct BenchTool { + /// Version installed for Calcine, if any. + pub installed: Option, + /// The version matching the installed GenieX, the one to install. + pub wanted: Option, + /// Download size of `wanted`, when known. + pub download_bytes: Option, +} + +impl BenchTool { + pub fn is_ready(&self) -> bool { + self.installed.is_some() && self.installed == self.wanted + } +} + +/// What to measure. One run per compute unit. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct BenchRequest { + /// GenieX id, `org/model:precision`. + pub model: String, + pub runtime: Runtime, + pub units: Vec, + /// Prompt length, in random tokens. + pub prompt_tokens: u32, + /// Tokens generated per repetition. + pub generated_tokens: u32, + /// Measured repetitions, after one warmup. + pub repetitions: u32, + /// HTP power mode (`burst`, `balanced`, …). + pub power_mode: String, + /// Speculative decoding method (`ngram-cache`, `draft-simple`, …), + /// llama.cpp only. + #[serde(default)] + pub spec_type: Option, + /// Draft model for the `draft-*` methods (GenieX id). + #[serde(default)] + pub draft_model: Option, +} + +/// Speculative decoding methods `geniex-bench` and GenieX accept. +pub const SPEC_TYPES: &[&str] = &[ + "ngram-cache", + "ngram-simple", + "ngram-map-k", + "ngram-map-k4v", + "ngram-mod", + "draft-simple", + "draft-eagle3", + "draft-mtp", +]; + +impl BenchRequest { + /// Why this request can't run, if it can't. + pub fn validate(&self) -> std::result::Result<(), String> { + if self.model.trim().is_empty() { + return Err("choose a model".into()); + } + if self.units.is_empty() { + return Err("choose at least one compute unit".into()); + } + if !(16..=8192).contains(&self.prompt_tokens) { + return Err("prompt length must be between 16 and 8192 tokens".into()); + } + if !(8..=2048).contains(&self.generated_tokens) { + return Err("generate between 8 and 2048 tokens".into()); + } + if !(1..=20).contains(&self.repetitions) { + return Err("run between 1 and 20 repetitions".into()); + } + if let Some(spec) = &self.spec_type { + if !SPEC_TYPES.contains(&spec.as_str()) { + return Err(format!("unknown speculative decoding method {spec}")); + } + if self.runtime != Runtime::LlamaCpp { + return Err("speculative decoding needs a llama.cpp model".into()); + } + if spec.starts_with("draft-") && self.draft_model.as_deref().is_none_or(str::is_empty) { + return Err("choose a draft model".into()); + } + } + Ok(()) + } +} + +/// A measured value across repetitions. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct BenchStat { + pub median: f64, + pub min: f64, + pub max: f64, + pub stdev: f64, +} + +impl BenchStat { + /// Median, extremes and standard deviation of `values`. + pub fn of(values: &[f64]) -> Self { + if values.is_empty() { + return Self { + median: 0.0, + min: 0.0, + max: 0.0, + stdev: 0.0, + }; + } + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + let middle = sorted.len() / 2; + let median = if sorted.len().is_multiple_of(2) { + f64::midpoint(sorted[middle - 1], sorted[middle]) + } else { + sorted[middle] + }; + #[allow(clippy::cast_precision_loss)] + let count = values.len() as f64; + let mean = values.iter().sum::() / count; + let variance = values + .iter() + .map(|value| (value - mean).powi(2)) + .sum::() + / count; + Self { + median, + min: sorted[0], + max: sorted[sorted.len() - 1], + stdev: variance.sqrt(), + } + } +} + +/// What one compute unit measured. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct BenchMeasure { + pub ttft_ms: BenchStat, + pub prefill_tps: BenchStat, + pub decode_tps: BenchStat, + /// Median tokens actually generated (a model may stop early). + pub generated_tokens: f64, + /// Prompt tokens processed. QAIRT pads them to a multiple of 128. + pub prompt_tokens: f64, + pub geniex_version: String, + /// Power and energy, for energy profiles. + #[serde(default)] + pub energy: Option, +} + +/// What measured a result. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +pub enum BenchSource { + /// Qualcomm's `geniex-bench`, on a random prompt. + #[default] + GeniexBench, + /// An energy profile, through `geniex serve`. + EnergyProfile, +} + +/// One compute unit's result in the history. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct BenchResult { + pub id: String, + /// Results measured together share a session. + pub session_id: String, + pub started_at_ms: u64, + pub model: String, + pub runtime: Runtime, + pub unit: ComputeUnit, + pub prompt_tokens: u32, + pub generated_tokens: u32, + pub repetitions: u32, + pub power_mode: String, + #[serde(default)] + pub spec_type: Option, + #[serde(default)] + pub draft_model: Option, + #[serde(default)] + pub source: BenchSource, + /// `None` when the run failed. + pub measure: Option, + pub error: Option, +} + +/// Runs `geniex-bench` and keeps the results. +#[async_trait] +pub trait Benchmarker: Send + Sync { + /// Installed and wanted tool versions. + async fn tool(&self) -> Result; + + /// Download, verify and unpack the tool for the installed GenieX. + async fn install_tool(&self, ctx: JobCtx) -> Result<()>; + + /// Measure one compute unit. Cancellable. + async fn measure( + &self, + request: &BenchRequest, + unit: ComputeUnit, + ctx: &JobCtx, + ) -> Result; + + /// Past results, newest first. + fn history(&self) -> Vec; + + /// Add results to the history. + fn record(&self, results: Vec) -> Result<()>; + + /// Forget results by id. + fn forget(&self, ids: &[String]) -> Result<()>; +} + +/// Results kept in the history. +pub const KEPT_RESULTS: usize = 500; + +/// Newest first, at most [`KEPT_RESULTS`]. +pub fn merge_history(mut history: Vec, results: Vec) -> Vec { + let mut merged = results; + merged.append(&mut history); + merged.sort_by_key(|result| std::cmp::Reverse(result.started_at_ms)); + merged.truncate(KEPT_RESULTS); + merged +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request() -> BenchRequest { + BenchRequest { + model: "qualcomm/Qwen3-0.6B:W4A16".into(), + runtime: Runtime::Qairt, + units: vec![ComputeUnit::Npu], + prompt_tokens: 512, + generated_tokens: 128, + repetitions: 5, + power_mode: "burst".into(), + spec_type: None, + draft_model: None, + } + } + + #[test] + fn validates_requests() { + assert!(request().validate().is_ok()); + assert!( + BenchRequest { + units: vec![], + ..request() + } + .validate() + .is_err() + ); + assert!( + BenchRequest { + repetitions: 0, + ..request() + } + .validate() + .is_err() + ); + assert!( + BenchRequest { + prompt_tokens: 1, + ..request() + } + .validate() + .is_err() + ); + let ngram = BenchRequest { + runtime: Runtime::LlamaCpp, + spec_type: Some("ngram-cache".into()), + ..request() + }; + assert!(ngram.validate().is_ok()); + assert!( + BenchRequest { + runtime: Runtime::Qairt, + ..ngram.clone() + } + .validate() + .is_err() + ); + let draft = BenchRequest { + spec_type: Some("draft-simple".into()), + ..ngram.clone() + }; + assert!(draft.validate().is_err()); + assert!( + BenchRequest { + draft_model: Some("a/b:Q4_0".into()), + ..draft + } + .validate() + .is_ok() + ); + assert!( + BenchRequest { + spec_type: Some("bogus".into()), + ..ngram + } + .validate() + .is_err() + ); + } + + #[test] + fn keeps_the_newest_results() { + let result = |id: &str, at: u64| BenchResult { + id: id.into(), + session_id: "s".into(), + started_at_ms: at, + model: "m".into(), + runtime: Runtime::Qairt, + unit: ComputeUnit::Npu, + prompt_tokens: 512, + generated_tokens: 128, + repetitions: 5, + power_mode: "burst".into(), + spec_type: None, + draft_model: None, + source: BenchSource::GeniexBench, + measure: None, + error: None, + }; + let merged = merge_history(vec![result("old", 1)], vec![result("new", 2)]); + assert_eq!(merged[0].id, "new"); + assert_eq!(merged.len(), 2); + } +} diff --git a/crates/calcine-core/src/context/mod.rs b/crates/calcine-core/src/context/mod.rs new file mode 100644 index 0000000..7626ba5 --- /dev/null +++ b/crates/calcine-core/src/context/mod.rs @@ -0,0 +1,180 @@ +//! How much of a model's context window a conversation takes, and which +//! messages to forget when it doesn't fit. +//! +//! AI Hub models have their window compiled in (4096 tokens for most), and +//! GenieX refuses a prompt that doesn't fit. Counting the prompt first lets +//! Calcine show how full a conversation is and, when asked, forget the +//! oldest messages instead of failing. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use specta::Type; + +use crate::Result; + +/// Tokens a chat request takes. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct TokenCount { + /// Each message, in order, with its share of the chat template. + pub per_message: Vec, + /// The rest of the prompt: tools, the start of the reply, the default + /// system prompt GenieX adds when there's none. + pub overhead: u32, + /// Counted with the model's tokenizer (within a few tokens of what + /// GenieX counts) rather than estimated from the text's length. + pub exact: bool, +} + +impl TokenCount { + pub fn total(&self) -> u32 { + self.per_message.iter().sum::() + self.overhead + } +} + +/// How full a conversation is. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct ContextUsage { + pub tokens: TokenCount, + /// The model's context window, when known. + pub window: Option, +} + +/// Counts tokens with each model's own tokenizer. +#[async_trait] +pub trait ContextMeter: Send + Sync { + /// The window compiled into an AI Hub model. `None` for llama.cpp models, + /// whose window is a server option. + async fn compiled_window(&self, model: &str) -> Result>; + + /// Tokens a chat completion request (`messages`, `tools`, `enable_think`) + /// takes. + async fn count(&self, model: &str, request: &Value) -> Result; +} + +/// Room left for the reply: what the request asks for, at most half the +/// window, 512 tokens when it doesn't say. +pub fn reply_reserve(request: &Value, window: u32) -> u32 { + let asked = ["max_completion_tokens", "max_tokens"] + .iter() + .find_map(|key| request.get(*key).and_then(Value::as_u64)) + .map_or(512, |tokens| u32::try_from(tokens).unwrap_or(u32::MAX)); + asked.min(window / 2) +} + +/// Where the kept conversation starts so the prompt fits `budget` tokens: +/// leading system messages stay, the oldest exchanges go first, and the kept +/// part starts at a user message. `Some(0)` when everything fits, `None` +/// when even the last exchange doesn't. +pub fn first_kept(messages: &[Value], count: &TokenCount, budget: u32) -> Option { + if count.total() <= budget { + return Some(0); + } + let role = |index: usize| messages[index].get("role").and_then(Value::as_str); + let leading = (0..messages.len()) + .take_while(|&index| matches!(role(index), Some("system" | "developer"))) + .count(); + let tokens = |index: usize| count.per_message.get(index).copied().unwrap_or(0); + let fixed = count.overhead + (0..leading).map(tokens).sum::(); + let mut kept: u32 = (leading..messages.len()).map(tokens).sum(); + for start in leading..messages.len() { + if start > leading { + kept -= tokens(start - 1); + } + if role(start) == Some("user") && fixed + kept <= budget { + return Some(start); + } + } + None +} + +/// Remove what [`first_kept`] leaves out: how many messages were forgotten, +/// or `None` when even the last exchange doesn't fit. +pub fn forget_oldest(messages: &mut Vec, count: &TokenCount, budget: u32) -> Option { + let start = first_kept(messages, count, budget)?; + let leading = messages + .iter() + .take_while(|message| { + matches!( + message.get("role").and_then(Value::as_str), + Some("system" | "developer") + ) + }) + .count(); + if start <= leading { + return Some(0); + } + messages.drain(leading..start); + Some(start - leading) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn conversation() -> (Vec, TokenCount) { + let messages = vec![ + json!({ "role": "system", "content": "Be brief." }), + json!({ "role": "user", "content": "one" }), + json!({ "role": "assistant", "content": "", "tool_calls": [] }), + json!({ "role": "tool", "content": "result" }), + json!({ "role": "assistant", "content": "two" }), + json!({ "role": "user", "content": "three" }), + json!({ "role": "assistant", "content": "four" }), + json!({ "role": "user", "content": "five" }), + ]; + let count = TokenCount { + per_message: vec![10, 100, 20, 50, 30, 100, 100, 100], + overhead: 20, + exact: true, + }; + (messages, count) + } + + #[test] + fn keeps_everything_that_fits() { + let (messages, count) = conversation(); + assert_eq!(count.total(), 530); + assert_eq!(first_kept(&messages, &count, 530), Some(0)); + } + + #[test] + fn forgets_whole_exchanges_from_the_oldest() { + let (messages, count) = conversation(); + // Without the first exchange (and its tool call): 30 + 300. + assert_eq!(first_kept(&messages, &count, 529), Some(5)); + assert_eq!(first_kept(&messages, &count, 330), Some(5)); + // Only the last question. + assert_eq!(first_kept(&messages, &count, 329), Some(7)); + assert_eq!(first_kept(&messages, &count, 130), Some(7)); + assert_eq!(first_kept(&messages, &count, 129), None); + } + + #[test] + fn removes_the_forgotten_messages() { + let (mut messages, count) = conversation(); + assert_eq!(forget_oldest(&mut messages, &count, 330), Some(4)); + assert_eq!(messages.len(), 4); + assert_eq!(messages[0]["role"], "system"); + assert_eq!(messages[1]["content"], "three"); + + let (mut messages, count) = conversation(); + assert_eq!(forget_oldest(&mut messages, &count, 1000), Some(0)); + assert_eq!(messages.len(), 8); + assert_eq!(forget_oldest(&mut messages, &count, 10), None); + } + + #[test] + fn reserves_room_for_the_reply() { + assert_eq!(reply_reserve(&json!({}), 4096), 512); + assert_eq!(reply_reserve(&json!({ "max_tokens": 100 }), 4096), 100); + assert_eq!( + reply_reserve(&json!({ "max_completion_tokens": 9000 }), 4096), + 2048 + ); + } +} diff --git a/crates/calcine-core/src/hardware/mod.rs b/crates/calcine-core/src/hardware/mod.rs index 04cdf88..770499d 100644 --- a/crates/calcine-core/src/hardware/mod.rs +++ b/crates/calcine-core/src/hardware/mod.rs @@ -4,4 +4,7 @@ mod probe; mod types; pub use probe::HardwareProbe; -pub use types::{Accelerator, DiskSpace, HardwareInfo, HardwareUsage, MemoryInfo, Processor}; +pub use types::{ + Accelerator, DiskSpace, EnergyReading, HardwareInfo, HardwareUsage, MemoryInfo, PowerDraw, + Processor, +}; diff --git a/crates/calcine-core/src/hardware/probe.rs b/crates/calcine-core/src/hardware/probe.rs index aed2519..97d1042 100644 --- a/crates/calcine-core/src/hardware/probe.rs +++ b/crates/calcine-core/src/hardware/probe.rs @@ -2,7 +2,7 @@ use std::path::Path; use async_trait::async_trait; -use super::types::{HardwareInfo, HardwareUsage}; +use super::types::{EnergyReading, HardwareInfo, HardwareUsage}; use crate::Result; /// The device's compute units, memory and storage. @@ -15,4 +15,10 @@ pub trait HardwareProbe: Send + Sync { /// Current load. Percentages are measured since the previous call, so /// the first call may read 0. async fn usage(&self) -> Result; + + /// Energy the system has used so far. `None` on devices without energy + /// metering. + async fn energy(&self) -> Result> { + Ok(None) + } } diff --git a/crates/calcine-core/src/hardware/types.rs b/crates/calcine-core/src/hardware/types.rs index 473c99b..6f89874 100644 --- a/crates/calcine-core/src/hardware/types.rs +++ b/crates/calcine-core/src/hardware/types.rs @@ -48,6 +48,35 @@ pub struct HardwareUsage { /// Hexagon NPU, 0-100. `None` when the OS doesn't report it. pub npu_percent: Option, pub memory: MemoryInfo, + /// Power drawn, on devices with energy metering. + pub power: Option, +} + +/// Power drawn since the previous sample, in watts. Snapdragon X meters the +/// CPU clusters, the GPU and the whole system, not the NPU on its own. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct PowerDraw { + pub system_watts: f32, + pub cpu_watts: Option, + pub gpu_watts: Option, +} + +/// Energy the whole system has used since its meter started. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct EnergyReading { + pub joules: f64, + /// When it was measured, in seconds since the meter started. + pub seconds: f64, +} + +impl EnergyReading { + /// Average power between `self` and a later reading. `None` when no time + /// passed. + pub fn watts_until(&self, later: &Self) -> Option { + let seconds = later.seconds - self.seconds; + (seconds > 0.0).then(|| (later.joules - self.joules) / seconds) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] diff --git a/crates/calcine-core/src/jobs/manager.rs b/crates/calcine-core/src/jobs/manager.rs index 02fc8d0..a54e1f1 100644 --- a/crates/calcine-core/src/jobs/manager.rs +++ b/crates/calcine-core/src/jobs/manager.rs @@ -151,10 +151,15 @@ impl JobManager { let Some(entry) = jobs.get_mut(&id) else { return; }; + // A new phase or step is always sent; byte counts are throttled. + let moved_on = entry.job.progress.is_none_or(|previous| { + previous.phase != progress.phase || previous.step != progress.step + }); entry.job.progress = Some(progress); - let due = entry - .last_progress_event - .is_none_or(|last| last.elapsed() >= PROGRESS_THROTTLE); + let due = moved_on + || entry + .last_progress_event + .is_none_or(|last| last.elapsed() >= PROGRESS_THROTTLE); if !due { return; } @@ -254,6 +259,7 @@ mod tests { total_bytes: Some(10), bytes_per_second: None, phase: None, + step: None, }); Ok(()) }); diff --git a/crates/calcine-core/src/jobs/mod.rs b/crates/calcine-core/src/jobs/mod.rs index 379e766..12c3242 100644 --- a/crates/calcine-core/src/jobs/mod.rs +++ b/crates/calcine-core/src/jobs/mod.rs @@ -5,4 +5,4 @@ mod manager; mod types; pub use manager::{JobCtx, JobManager}; -pub use types::{Job, JobId, JobKind, JobPhase, JobProgress, JobState}; +pub use types::{Job, JobId, JobKind, JobPhase, JobProgress, JobState, JobStep}; diff --git a/crates/calcine-core/src/jobs/types.rs b/crates/calcine-core/src/jobs/types.rs index 01d66aa..a045ac7 100644 --- a/crates/calcine-core/src/jobs/types.rs +++ b/crates/calcine-core/src/jobs/types.rs @@ -17,6 +17,12 @@ pub enum JobKind { InstallRuntime { version: String }, /// Downloading and installing a newer Calcine. UpdateApp { version: String }, + /// Downloading the benchmark tool (`geniex-bench`). + InstallBench { version: String }, + /// Benchmarking a model on one or more compute units. + Benchmark { model: String }, + /// Measuring a model's speed and energy in several power modes. + EnergyProfile { model: String }, } /// What a multi-step job is doing right now. @@ -26,6 +32,17 @@ pub enum JobPhase { Downloading, Verifying, Installing, + /// Running one benchmark measurement. + Measuring, +} + +/// Where a job made of several steps is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct JobStep { + /// 1-based. + pub current: u32, + pub total: u32, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] @@ -45,6 +62,8 @@ pub struct JobProgress { pub bytes_per_second: Option, /// Set by jobs with several steps (installing GenieX). pub phase: Option, + /// Set by jobs that repeat a step (one benchmark per compute unit). + pub step: Option, } /// A snapshot of one job, as sent to the UI. diff --git a/crates/calcine-core/src/lib.rs b/crates/calcine-core/src/lib.rs index 83032de..4344498 100644 --- a/crates/calcine-core/src/lib.rs +++ b/crates/calcine-core/src/lib.rs @@ -14,7 +14,11 @@ //! | [`runtime`] | `RuntimeInfo` | `RuntimeManager` | //! | [`hardware`] | `HardwareInfo`, `HardwareUsage`… | `HardwareProbe` | //! | [`jobs`] | `Job`, `JobState`… | — (`JobManager` runs them) | +//! | [`bench`] | `BenchRequest`, `BenchResult`… | `Benchmarker` | +//! | [`context`] | `TokenCount`, `ContextUsage` | `ContextMeter` | +pub mod bench; +pub mod context; pub mod error; pub mod hardware; pub mod jobs; diff --git a/crates/calcine-core/src/runtime/mod.rs b/crates/calcine-core/src/runtime/mod.rs index 69f0490..dd8d177 100644 --- a/crates/calcine-core/src/runtime/mod.rs +++ b/crates/calcine-core/src/runtime/mod.rs @@ -7,7 +7,7 @@ mod types; mod update; pub use manager::RuntimeManager; -pub use server::{InferenceServer, ServerState}; +pub use server::{InferenceServer, ServerOptions, ServerState}; pub use types::RuntimeInfo; pub use update::{ CachedInstaller, InstallSource, InstallerAsset, ReleaseChannel, RuntimeInstaller, diff --git a/crates/calcine-core/src/runtime/server.rs b/crates/calcine-core/src/runtime/server.rs index 6ddc405..8b59e56 100644 --- a/crates/calcine-core/src/runtime/server.rs +++ b/crates/calcine-core/src/runtime/server.rs @@ -28,6 +28,51 @@ pub enum ServerState { }, } +/// Defaults `geniex serve` starts with. Changing them restarts the server. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase", default)] +pub struct ServerOptions { + /// Unload the model after this many idle seconds (`--keepalive`). + pub keepalive_secs: u32, + /// Context window of llama.cpp models, in tokens (`--nctx`). AI Hub + /// models have theirs compiled in. + pub context_size: u32, +} + +impl ServerOptions { + pub const MIN_KEEPALIVE_SECS: u32 = 30; + pub const MIN_CONTEXT_SIZE: u32 = 512; + pub const MAX_CONTEXT_SIZE: u32 = 131_072; + + /// Why these options can't be used, if they can't. + pub fn validate(&self) -> std::result::Result<(), String> { + if self.keepalive_secs < Self::MIN_KEEPALIVE_SECS { + return Err(format!( + "keep the model loaded for at least {} seconds", + Self::MIN_KEEPALIVE_SECS + )); + } + if !(Self::MIN_CONTEXT_SIZE..=Self::MAX_CONTEXT_SIZE).contains(&self.context_size) { + return Err(format!( + "choose a context window between {} and {} tokens", + Self::MIN_CONTEXT_SIZE, + Self::MAX_CONTEXT_SIZE + )); + } + Ok(()) + } +} + +impl Default for ServerOptions { + /// GenieX's own defaults. + fn default() -> Self { + Self { + keepalive_secs: 300, + context_size: 4096, + } + } +} + /// Runs the OpenAI-compatible inference server on demand. #[async_trait] pub trait InferenceServer: Send + Sync { @@ -43,4 +88,41 @@ pub trait InferenceServer: Send + Sync { /// Recent output lines of the server process, oldest first. fn logs(&self) -> Vec; + + /// Options the server starts with. + fn options(&self) -> ServerOptions; + + /// Change the options. A running server restarts with them. + async fn set_options(&self, options: ServerOptions) -> Result<()>; + + /// Run a chat completion (not streamed) on the running server, bypassing + /// the gateway. Used to measure it. + async fn complete(&self, request: &serde_json::Value) -> Result; +} + +#[cfg(test)] +mod tests { + use super::ServerOptions; + + #[test] + fn validates_ranges() { + assert!(ServerOptions::default().validate().is_ok()); + let short = ServerOptions { + keepalive_secs: 5, + ..ServerOptions::default() + }; + assert!(short.validate().is_err()); + let huge = ServerOptions { + context_size: 1 << 20, + ..ServerOptions::default() + }; + assert!(huge.validate().is_err()); + } + + #[test] + fn missing_fields_take_defaults() { + let options: ServerOptions = serde_json::from_str(r#"{"keepaliveSecs": 60}"#).unwrap(); + assert_eq!(options.context_size, 4096); + assert_eq!(options.keepalive_secs, 60); + } } diff --git a/crates/calcine-core/src/services.rs b/crates/calcine-core/src/services.rs index 04cb91f..a2f567e 100644 --- a/crates/calcine-core/src/services.rs +++ b/crates/calcine-core/src/services.rs @@ -5,9 +5,13 @@ use serde::{Deserialize, Serialize}; use specta::Type; use crate::Result; +use crate::bench::{ + BenchRequest, BenchResult, BenchSource, Benchmarker, EnergyRequest, measure_mode, +}; +use crate::context::{ContextMeter, ContextUsage}; use crate::hardware::HardwareProbe; -use crate::jobs::{JobId, JobKind, JobManager, JobState}; -use crate::models::{ModelCatalog, ModelDirectory, ModelKey, ModelStore, PullRequest}; +use crate::jobs::{JobCtx, JobId, JobKind, JobManager, JobPhase, JobProgress, JobState, JobStep}; +use crate::models::{ComputeUnit, ModelCatalog, ModelDirectory, ModelKey, ModelStore, PullRequest}; use crate::runtime::{ InferenceServer, InstallSource, RuntimeInstaller, RuntimeManager, ServerState, }; @@ -36,6 +40,10 @@ pub struct Services { pub hardware: Arc, /// Installs and updates GenieX itself. pub installer: Arc, + /// `geniex-bench` and the benchmark history. + pub bench: Arc, + /// Context windows and token counts. + pub context: Arc, pub jobs: JobManager, } @@ -97,6 +105,132 @@ impl Services { }) } + /// Download the benchmark tool, or return the download already running. + pub fn start_bench_install(&self, version: String) -> JobId { + if let Some(job) = self.running_job(|kind| matches!(kind, JobKind::InstallBench { .. })) { + return job; + } + let bench = self.bench.clone(); + self.jobs + .spawn(JobKind::InstallBench { version }, move |ctx| async move { + bench.install_tool(ctx).await + }) + } + + /// Benchmark a model on each requested compute unit, one after the + /// other, then save the results. `geniex serve` is paused meanwhile so + /// it doesn't share the hardware; a failed unit is recorded with its + /// error and the others still run. + pub fn start_benchmark(&self, request: BenchRequest) -> Result { + request.validate().map_err(crate::Error::InvalidInput)?; + self.ensure_no_benchmark()?; + let bench = self.bench.clone(); + let server = self.server.clone(); + let kind = JobKind::Benchmark { + model: request.model.clone(), + }; + Ok(self.jobs.spawn(kind, move |ctx| async move { + let was_running = matches!( + *server.state().borrow(), + ServerState::Ready { .. } | ServerState::Starting + ); + server.stop().await?; + let outcome = measure_all(bench.as_ref(), &request, &ctx).await; + if was_running { + let _ = server.ensure_running().await; + } + let results = outcome?; + let all_failed = results.iter().all(|result| result.measure.is_none()); + let first_error = results.iter().find_map(|result| result.error.clone()); + bench.record(results)?; + match first_error { + Some(message) if all_failed => Err(crate::Error::Command { + command: "geniex-bench".into(), + message, + }), + _ => Ok(()), + } + })) + } + + /// Measure a model's speed and energy in each requested power mode, + /// through `geniex serve` (started if needed), then save the results. + /// Other requests to Calcine's API meanwhile would skew them. + pub fn start_energy_profile(&self, request: EnergyRequest) -> Result { + request.validate().map_err(crate::Error::InvalidInput)?; + self.ensure_no_benchmark()?; + let bench = self.bench.clone(); + let server = self.server.clone(); + let hardware = self.hardware.clone(); + let runtime = self.runtime.clone(); + let kind = JobKind::EnergyProfile { + model: request.model.clone(), + }; + Ok(self.jobs.spawn(kind, move |ctx| async move { + let version = runtime + .info() + .await + .map(|info| info.cli_version) + .unwrap_or_default(); + let results = + profile_all(server.as_ref(), hardware.as_ref(), &request, &version, &ctx).await?; + let all_failed = results.iter().all(|result| result.measure.is_none()); + let first_error = results.iter().find_map(|result| result.error.clone()); + bench.record(results)?; + match first_error { + Some(message) if all_failed => Err(crate::Error::Command { + command: "geniex serve".into(), + message, + }), + _ => Ok(()), + } + })) + } + + fn ensure_no_benchmark(&self) -> Result<()> { + let running = self.running_job(|kind| { + matches!( + kind, + JobKind::Benchmark { .. } | JobKind::EnergyProfile { .. } + ) + }); + match running { + Some(_) => Err(crate::Error::InvalidInput( + "a benchmark is already running".into(), + )), + None => Ok(()), + } + } + + fn running_job(&self, matches: impl Fn(&JobKind) -> bool) -> Option { + self.jobs + .list() + .into_iter() + .find(|job| matches(&job.kind) && job.state == JobState::Running) + .map(|job| job.id) + } + + /// A model's context window: compiled into AI Hub models, the server + /// option for llama.cpp ones. + pub async fn context_window(&self, model: &str) -> Result { + Ok(match self.context.compiled_window(model).await? { + Some(window) => window, + None => self.server.options().context_size, + }) + } + + /// How much of `model`'s context window a chat request takes. + pub async fn context_usage( + &self, + model: &str, + request: &serde_json::Value, + ) -> Result { + Ok(ContextUsage { + tokens: self.context.count(model, request).await?, + window: Some(self.context_window(model).await?), + }) + } + /// Delete models or precisions. Stops `geniex serve` first: Windows /// can't delete the files of a loaded model. pub async fn remove_models(&self, keys: &[ModelKey]) -> Result<()> { @@ -111,6 +245,122 @@ impl Services { } } +/// One measurement per unit and power mode. Stops early when cancelled, or when the +/// device can't measure energy at all. +async fn profile_all( + server: &dyn InferenceServer, + hardware: &dyn HardwareProbe, + request: &EnergyRequest, + version: &str, + ctx: &JobCtx, +) -> Result> { + let session_id = format!("{:x}-{}", now_ms(), ctx.id()); + let pairs: Vec<(ComputeUnit, &String)> = request + .units + .iter() + .flat_map(|unit| request.power_modes.iter().map(move |mode| (*unit, mode))) + .collect(); + let total = u32::try_from(pairs.len()).unwrap_or(u32::MAX); + let mut results = Vec::with_capacity(pairs.len()); + for (index, (unit, mode)) in pairs.into_iter().enumerate() { + ctx.report(JobProgress { + phase: Some(JobPhase::Measuring), + step: Some(JobStep { + current: u32::try_from(index + 1).unwrap_or(u32::MAX), + total, + }), + ..JobProgress::default() + }); + let started_at_ms = now_ms(); + let (measure, error) = match measure_mode(server, hardware, request, unit, mode, ctx).await + { + Ok(mut measure) => { + version.clone_into(&mut measure.geniex_version); + (Some(measure), None) + } + Err(err @ (crate::Error::Cancelled | crate::Error::NotImplemented(_))) => { + return Err(err); + } + Err(err) => (None, Some(err.to_string())), + }; + results.push(BenchResult { + id: format!("{session_id}-{index}"), + session_id: session_id.clone(), + started_at_ms, + model: request.model.clone(), + runtime: request.runtime, + unit, + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + prompt_tokens: measure + .as_ref() + .map_or(0, |measure| measure.prompt_tokens as u32), + generated_tokens: request.generated_tokens, + repetitions: request.repetitions, + power_mode: mode.clone(), + spec_type: None, + draft_model: None, + source: BenchSource::EnergyProfile, + measure, + error, + }); + } + Ok(results) +} + +/// One measurement per compute unit. Stops early only when cancelled. +async fn measure_all( + bench: &dyn Benchmarker, + request: &BenchRequest, + ctx: &JobCtx, +) -> Result> { + let session_id = format!("{:x}-{}", now_ms(), ctx.id()); + let total = u32::try_from(request.units.len()).unwrap_or(u32::MAX); + let mut results = Vec::with_capacity(request.units.len()); + for (index, unit) in request.units.iter().enumerate() { + let current = u32::try_from(index + 1).unwrap_or(u32::MAX); + ctx.report(JobProgress { + phase: Some(JobPhase::Measuring), + step: Some(JobStep { current, total }), + ..JobProgress::default() + }); + let started_at_ms = now_ms(); + let outcome = bench.measure(request, *unit, ctx).await; + if matches!(outcome, Err(crate::Error::Cancelled)) { + return Err(crate::Error::Cancelled); + } + let (measure, error) = match outcome { + Ok(measure) => (Some(measure), None), + Err(err) => (None, Some(err.to_string())), + }; + results.push(BenchResult { + id: format!("{session_id}-{index}"), + session_id: session_id.clone(), + started_at_ms, + model: request.model.clone(), + runtime: request.runtime, + unit: *unit, + prompt_tokens: request.prompt_tokens, + generated_tokens: request.generated_tokens, + repetitions: request.repetitions, + power_mode: request.power_mode.clone(), + spec_type: request.spec_type.clone(), + draft_model: request.draft_model.clone(), + source: BenchSource::GeniexBench, + measure, + error, + }); + } + Ok(results) +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }) +} + impl fmt::Debug for Services { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Services") diff --git a/crates/calcine-gateway/Cargo.toml b/crates/calcine-gateway/Cargo.toml index d64dc67..5bd6500 100644 --- a/crates/calcine-gateway/Cargo.toml +++ b/crates/calcine-gateway/Cargo.toml @@ -16,6 +16,8 @@ calcine-core.workspace = true futures-util.workspace = true getrandom.workspace = true hex.workspace = true +hyper-util.workspace = true +rcgen.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true @@ -23,12 +25,13 @@ sha2.workspace = true specta.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["net", "sync", "time", "macros", "rt"] } +tokio-rustls.workspace = true tokio-stream.workspace = true tracing.workspace = true [dev-dependencies] calcine-mock.workspace = true -reqwest = { workspace = true, features = ["json"] } +reqwest = { workspace = true, features = ["json", "native-tls"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [lints] diff --git a/crates/calcine-gateway/src/caller.rs b/crates/calcine-gateway/src/caller.rs index 824e9a7..df37428 100644 --- a/crates/calcine-gateway/src/caller.rs +++ b/crates/calcine-gateway/src/caller.rs @@ -10,12 +10,14 @@ pub enum Caller { App(ApiKeyInfo), /// No key, accepted because keys aren't required (compatibility mode). Anonymous, + /// No key, on the Ollama port: Ollama clients can't send one. + OllamaClient, } impl Caller { pub fn can_infer(&self) -> bool { match self { - Self::Calcine | Self::Anonymous => true, + Self::Calcine | Self::Anonymous | Self::OllamaClient => true, Self::App(key) => key.scopes.contains(&KeyScope::Inference), } } @@ -24,7 +26,7 @@ impl Caller { match self { Self::Calcine => true, Self::App(key) => key.scopes.contains(&KeyScope::Manage), - Self::Anonymous => false, + Self::Anonymous | Self::OllamaClient => false, } } @@ -33,7 +35,7 @@ impl Caller { match self { Self::Calcine => true, Self::App(key) => key.allow_local_files, - Self::Anonymous => false, + Self::Anonymous | Self::OllamaClient => false, } } @@ -43,6 +45,7 @@ impl Caller { Self::Calcine => "Calcine".into(), Self::App(key) => key.name.clone(), Self::Anonymous => "No key".into(), + Self::OllamaClient => "Ollama app".into(), } } } diff --git a/crates/calcine-gateway/src/context.rs b/crates/calcine-gateway/src/context.rs new file mode 100644 index 0000000..723a8d8 --- /dev/null +++ b/crates/calcine-gateway/src/context.rs @@ -0,0 +1,95 @@ +//! Fitting a conversation into the model's context window. +//! +//! GenieX refuses a prompt longer than the window (4096 tokens for most AI +//! Hub models). With `"truncation": "auto"` (OpenAI's name for it), and +//! always for Ollama clients as Ollama does, Calcine forgets the oldest +//! exchanges instead, keeping the system prompt and room for the reply. + +use axum::http::{HeaderValue, StatusCode}; +use axum::response::Response; +use calcine_core::context::{forget_oldest, reply_reserve}; +use serde_json::{Value, json}; + +use crate::state::AppState; + +/// How many messages were left out to fit, on the response. +pub const FORGOTTEN_HEADER: &str = "x-calcine-forgotten-messages"; + +/// Whether the request asks to forget what doesn't fit. Removes the field, +/// which GenieX doesn't know. +pub fn take_truncation(request: &mut Value) -> bool { + request + .as_object_mut() + .and_then(|object| object.remove("truncation")) + .is_some_and(|truncation| truncation == "auto") +} + +/// Forget the oldest messages that don't fit: how many were forgotten, or +/// why the conversation can't fit. Counting problems (an unknown model, an +/// unreadable tokenizer) leave the request as it is, for GenieX to judge. +pub async fn fit(app: &AppState, request: &mut Value) -> Result { + let Some(model) = request + .get("model") + .and_then(Value::as_str) + .map(str::to_owned) + else { + return Ok(0); + }; + let services = &app.services; + let (Ok(window), Ok(count)) = ( + services.context_window(&model).await, + services.context.count(&model, request).await, + ) else { + return Ok(0); + }; + let reserve = reply_reserve(request, window); + let Some(messages) = request.get_mut("messages").and_then(Value::as_array_mut) else { + return Ok(0); + }; + forget_oldest(messages, &count, window.saturating_sub(reserve)).ok_or_else(|| { + format!( + "the last message doesn't fit in {model}'s context window of {window} tokens, with \ + {reserve} kept for the reply: shorten it, or ask for a shorter reply with max_tokens" + ) + }) +} + +/// Say on the response how many messages were forgotten. +pub fn mark(mut response: Response, forgotten: usize) -> Response { + if forgotten > 0 { + response + .headers_mut() + .insert(FORGOTTEN_HEADER, HeaderValue::from(forgotten)); + } + response +} + +/// The prompt doesn't fit, in the OpenAI shape. +pub fn too_long(message: &str) -> Response { + use axum::Json; + use axum::response::IntoResponse; + ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": { + "message": message, + "type": "invalid_request_error", + "code": "context_length_exceeded", + } })), + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn takes_the_truncation_field() { + let mut request = json!({ "model": "m", "truncation": "auto" }); + assert!(take_truncation(&mut request)); + assert_eq!(request, json!({ "model": "m" })); + let mut request = json!({ "truncation": "disabled" }); + assert!(!take_truncation(&mut request)); + assert!(!take_truncation(&mut json!({}))); + } +} diff --git a/crates/calcine-gateway/src/error.rs b/crates/calcine-gateway/src/error.rs index 90cebc9..faa7474 100644 --- a/crates/calcine-gateway/src/error.rs +++ b/crates/calcine-gateway/src/error.rs @@ -16,6 +16,11 @@ pub fn api_error(status: StatusCode, kind: &str, message: &str) -> Response { /// GenieX answers `{"error": "…"}`; rewrap it, keeping the status code. pub fn upstream_error(status: StatusCode, body: &[u8]) -> Response { let message = upstream_message(body); + if is_context_error(body) { + return crate::context::too_long(&format!( + "{message}: send fewer messages, or add \"truncation\": \"auto\" to forget the oldest ones" + )); + } let kind = if status.is_client_error() { "invalid_request_error" } else { @@ -24,6 +29,13 @@ pub fn upstream_error(status: StatusCode, body: &[u8]) -> Response { api_error(status, kind, &message) } +/// GenieX's "the prompt doesn't fit the window" error. +pub fn is_context_error(body: &[u8]) -> bool { + serde_json::from_slice::(body).is_ok_and(|error| { + error.pointer("/error/code").and_then(Value::as_str) == Some("context_length_exceeded") + }) +} + pub fn upstream_message(body: &[u8]) -> String { serde_json::from_slice::(body) .ok() diff --git a/crates/calcine-gateway/src/keys.rs b/crates/calcine-gateway/src/keys.rs index 867305b..b0351e1 100644 --- a/crates/calcine-gateway/src/keys.rs +++ b/crates/calcine-gateway/src/keys.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::SystemTime; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -36,6 +37,10 @@ pub struct ApiKeyInfo { /// May send local file paths and URLs (images, audio, grammars) for /// GenieX to read. Off by default: it lets the app read any file. pub allow_local_files: bool, + /// May be used from other devices, on the local network port. Off by + /// default: most apps run on this PC. + #[serde(default)] + pub network: bool, pub created_at_ms: u64, pub last_used_at_ms: Option, } @@ -47,6 +52,8 @@ pub struct NewApiKey { pub name: String, pub scopes: Vec, pub allow_local_files: bool, + #[serde(default)] + pub network: bool, } /// A freshly created key. `token` is only ever returned here. @@ -83,6 +90,9 @@ pub enum KeyError { pub struct KeyStore { path: Option, keys: Mutex>, + /// When the file was last read or written by this store. Another process + /// (the app and `calcine-cli`) may change it: it's read again then. + seen: Mutex>, last_persist_ms: Mutex, internal_token: String, } @@ -100,13 +110,16 @@ impl KeyStore { Err(err) if err.kind() == std::io::ErrorKind::NotFound => Vec::new(), Err(err) => return Err(err.into()), }; - Ok(Self::with_keys(Some(path), keys)) + let store = Self::with_keys(Some(path), keys); + *store.seen() = store.version(); + Ok(store) } fn with_keys(path: Option, keys: Vec) -> Self { Self { path, keys: Mutex::new(keys), + seen: Mutex::new(None), last_persist_ms: Mutex::new(0), internal_token: random_token(), } @@ -118,10 +131,12 @@ impl KeyStore { } pub fn list(&self) -> Vec { + self.sync(); self.lock().iter().map(|key| key.info.clone()).collect() } pub fn create(&self, request: NewApiKey) -> Result { + self.sync(); let name = request.name.trim(); if name.is_empty() { return Err(KeyError::MissingName); @@ -139,6 +154,7 @@ impl KeyStore { preview: format!("{}…", &token[..TOKEN_PREFIX.len() + 4]), scopes, allow_local_files: request.allow_local_files, + network: request.network, created_at_ms: now_ms(), last_used_at_ms: None, }; @@ -152,6 +168,7 @@ impl KeyStore { /// Returns `false` if no key has this id. pub fn revoke(&self, id: &str) -> Result { + self.sync(); let removed = { let mut keys = self.lock(); let before = keys.len(); @@ -164,11 +181,30 @@ impl KeyStore { Ok(removed) } + /// Let a key be used from other devices, or not. `None` when there's + /// no such key. + pub fn set_network(&self, id: &str, network: bool) -> Result, KeyError> { + self.sync(); + let changed = { + let mut keys = self.lock(); + keys.iter_mut().find(|key| key.info.id == id).map(|key| { + key.info.network = network; + key.info.clone() + }) + }; + if changed.is_some() { + self.persist()?; + } + Ok(changed) + } + /// Who is calling with `token`, if anyone. pub fn authenticate(&self, token: &str) -> Option { if constant_time_eq(token.as_bytes(), self.internal_token.as_bytes()) { return Some(Caller::Calcine); } + // A key created or revoked by the other program counts at once. + self.sync(); let digest = hash(token); let now = now_ms(); let caller = { @@ -208,14 +244,77 @@ impl KeyStore { let partial = path.with_extension("json.partial"); std::fs::write(&partial, json)?; std::fs::rename(partial, path)?; + *self.seen() = self.version(); Ok(()) } + fn version(&self) -> Option { + let meta = std::fs::metadata(self.path.as_ref()?).ok()?; + Some(FileVersion { + modified: meta.modified().ok(), + len: meta.len(), + inode: inode(&meta), + }) + } + + fn seen(&self) -> MutexGuard<'_, Option> { + self.seen.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Read the file again when another program changed it. + fn sync(&self) { + let Some(path) = &self.path else { + return; + }; + let version = self.version(); + if version == *self.seen() { + return; + } + let keys = match std::fs::read(path) { + Ok(bytes) => serde_json::from_slice::>(&bytes), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(err) => { + tracing::warn!(%err, "couldn't read API keys again"); + return; + } + }; + match keys { + Ok(keys) => { + *self.lock() = keys; + *self.seen() = version; + } + Err(err) => { + tracing::warn!(%err, "API keys file is unreadable, keeping the keys in memory"); + } + } + } + fn lock(&self) -> MutexGuard<'_, Vec> { self.keys.lock().unwrap_or_else(PoisonError::into_inner) } } +/// Tells one version of the keys file from another. Linux timestamps are +/// coarse (a few milliseconds), so the size and the inode count too: each +/// save writes a new file and renames it in place. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct FileVersion { + modified: Option, + len: u64, + inode: u64, +} + +#[cfg(unix)] +fn inode(meta: &std::fs::Metadata) -> u64 { + use std::os::unix::fs::MetadataExt; + meta.ino() +} + +#[cfg(not(unix))] +fn inode(_meta: &std::fs::Metadata) -> u64 { + 0 +} + fn random_token() -> String { format!("{TOKEN_PREFIX}{}", hex::encode(random_bytes::<20>())) } @@ -243,6 +342,7 @@ mod tests { name: name.into(), scopes: vec![KeyScope::Inference], allow_local_files: false, + network: false, } } @@ -294,6 +394,21 @@ mod tests { assert!(matches!(store.create(no_scope), Err(KeyError::NoScope))); } + #[test] + fn sees_keys_changed_by_another_program() { + let dir = + std::env::temp_dir().join(format!("calcine-keys-{}", hex::encode(random_bytes::<4>()))); + let path = dir.join("api-keys.json"); + let app = KeyStore::load(path.clone()).unwrap(); + let cli = KeyStore::load(path).unwrap(); + let created = cli.create(new_key("Script")).unwrap(); + assert!(app.authenticate(&created.token).is_some()); + assert_eq!(app.list().len(), 1); + assert!(cli.revoke(&created.key.id).unwrap()); + assert!(app.authenticate(&created.token).is_none()); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn keys_survive_a_reload_without_storing_secrets() { let dir = diff --git a/crates/calcine-gateway/src/lib.rs b/crates/calcine-gateway/src/lib.rs index ffb1798..71c792a 100644 --- a/crates/calcine-gateway/src/lib.rs +++ b/crates/calcine-gateway/src/lib.rs @@ -6,16 +6,23 @@ //! //! - `/v1/*`: OpenAI-compatible API (models, chat, completions, …) //! - `/calcine/v1/*`: management (download, remove, jobs) for keys with `manage` +//! - `response_format` (JSON, JSON Schema) on every model: see [`structured`] +//! - `/api/*`: Ollama's API, translated (see [`ollama`]), also on port 11434 +//! when turned on mod caller; +mod context; mod error; pub mod keys; pub mod log; +pub mod network; +pub mod ollama; mod proxy; mod routes; pub mod security; pub mod settings; mod state; +pub mod structured; use std::path::PathBuf; use std::sync::{Arc, PoisonError, RwLock}; @@ -28,9 +35,10 @@ use tokio::sync::{broadcast, oneshot, watch}; pub use crate::keys::{ApiKeyInfo, CreatedApiKey, KeyError, KeyScope, KeyStore, NewApiKey}; pub use crate::log::RequestEntry; use crate::log::RequestLog; +use crate::security::Listener; pub use crate::settings::GatewaySettings; use crate::state::AppState; -pub use crate::state::GatewayStatus; +pub use crate::state::{GatewayStatus, NetworkStatus}; /// Browser origins of Calcine's own webview (production and `tauri dev`). pub const CALCINE_ORIGINS: &[&str] = &[ @@ -46,6 +54,9 @@ pub struct GatewayOptions { pub settings_path: Option, /// Origins always allowed (Calcine's own UI). pub builtin_origins: Vec, + /// Where the network port's certificate is kept (`None`: no network + /// port). + pub network_dir: Option, } #[derive(Debug)] @@ -56,6 +67,7 @@ pub struct Gateway { settings: Arc>, settings_path: Option, builtin_origins: Vec, + network_dir: Option, status: watch::Sender, running: tokio::sync::Mutex>, } @@ -66,12 +78,22 @@ struct Running { shutdown: oneshot::Sender<()>, /// Mirrors inference-server state into the status for this port. watcher: tokio::task::AbortHandle, + /// Stops the Ollama port, when it's on. + ollama: Option>, + /// Stops the network port, when it's on. + network: Option, } impl Running { fn shut_down(self) { self.watcher.abort(); let _ = self.shutdown.send(()); + if let Some(ollama) = self.ollama { + let _ = ollama.send(()); + } + if let Some(network) = self.network { + network.stop(); + } } } @@ -90,6 +112,10 @@ impl Gateway { active_requests: 0, queued_requests: 0, require_api_key: settings.require_api_key, + ollama_url: None, + ollama_error: None, + network: None, + network_error: None, }; Self { services, @@ -98,11 +124,23 @@ impl Gateway { settings: Arc::new(RwLock::new(settings)), settings_path: options.settings_path, builtin_origins: options.builtin_origins, + network_dir: options.network_dir, status: watch::Sender::new(status), running: tokio::sync::Mutex::new(None), } } + /// Change settings for this run only, without saving them. Call before + /// [`Self::start`]. + pub fn override_settings(&self, change: impl FnOnce(&mut GatewaySettings)) { + change( + &mut self + .settings + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + } + /// Start listening. Errors (port taken) are also reported in [`Self::status`]. pub async fn start(&self) -> Result<(), String> { let mut running = self.running.lock().await; @@ -119,12 +157,12 @@ impl Gateway { return Err(message); } }; - *running = Some(self.serve(listener)?); + *running = Some(self.serve(listener).await?); Ok(()) } - /// Serve the API on `listener`. - fn serve(&self, listener: tokio::net::TcpListener) -> Result { + /// Serve the API on `listener`, and on the Ollama port when it's on. + async fn serve(&self, listener: tokio::net::TcpListener) -> Result { let port = listener.local_addr().map_err(|err| err.to_string())?.port(); let app = Arc::new(AppState::new( @@ -143,18 +181,17 @@ impl Gateway { }); app.refresh_status(); - let (shutdown, stop) = oneshot::channel::<()>(); - let router = routes::router(app.clone()); - tokio::spawn(async move { - let served = axum::serve(listener, router) - .with_graceful_shutdown(async { - let _ = stop.await; - }) - .await; - if let Err(err) = served { - tracing::error!(%err, "gateway stopped unexpectedly"); - } - }); + let shutdown = spawn_server(listener, routes::router(app.clone(), Listener::Main)); + let ollama = if self.settings().ollama_port_enabled { + self.serve_ollama(&app).await + } else { + None + }; + let network = if self.settings().network_enabled { + self.serve_network(&app) + } else { + None + }; // Mirror inference-server transitions into the gateway status. let watcher = app.clone(); @@ -171,14 +208,194 @@ impl Gateway { app, shutdown, watcher, + ollama, + network, + }) + } + + /// Listen on Ollama's port with the same state (queue, log) as the main + /// port. A taken port is reported in the status, not as an error. + async fn serve_ollama(&self, app: &Arc) -> Option> { + let wanted = self.settings().ollama_port; + let listener = match tokio::net::TcpListener::bind(("127.0.0.1", wanted)).await { + Ok(listener) => listener, + Err(err) => { + let message = if err.kind() == std::io::ErrorKind::AddrInUse { + format!( + "Port {wanted} is already used, probably by Ollama. Quit Ollama to let Calcine answer Ollama apps there." + ) + } else { + format!("Couldn't listen on 127.0.0.1:{wanted}: {err}") + }; + self.status.send_modify(|status| { + status.ollama_url = None; + status.ollama_error = Some(message); + }); + return None; + } + }; + let port = listener.local_addr().map_or(wanted, |addr| addr.port()); + let stop = spawn_server( + listener, + routes::router(app.clone(), Listener::Ollama { port }), + ); + self.status.send_modify(|status| { + status.ollama_url = Some(format!("http://127.0.0.1:{port}")); + status.ollama_error = None; + }); + tracing::info!(port, "Ollama-compatible port listening"); + Some(stop) + } + + /// Listen on the network port over HTTPS, with the same state as the + /// main port. Problems (no certificate, port taken) go to the status. + fn serve_network(&self, app: &Arc) -> Option { + let fail = |message: String| { + tracing::warn!(%message, "network port not listening"); + self.status.send_modify(|status| { + status.network = None; + status.network_error = Some(message); + }); + None + }; + let Some(dir) = &self.network_dir else { + return fail("there's no folder to keep the certificate in".into()); + }; + let names = network::local_names(); + let identity = match network::tls::Identity::load_or_create(dir, &names) { + Ok(identity) => identity, + Err(message) => return fail(message), + }; + let acceptor = match identity.acceptor() { + Ok(acceptor) => acceptor, + Err(message) => return fail(message), + }; + let wanted = self.settings().network_port; + // Bound synchronously so a taken port shows at once. + let listener = match std::net::TcpListener::bind(("0.0.0.0", wanted)).and_then(|listener| { + listener.set_nonblocking(true)?; + tokio::net::TcpListener::from_std(listener) + }) { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::AddrInUse => { + return fail(format!( + "Port {wanted} is already used by another program. Choose another one." + )); + } + Err(err) => return fail(format!("Couldn't listen on port {wanted}: {err}")), + }; + let port = listener.local_addr().map_or(wanted, |addr| addr.port()); + let serving = network::spawn( + listener, + acceptor, + routes::router(app.clone(), Listener::Network { port }), + ); + let host = network::local_address() + .map(|address| address.to_string()) + .or_else(|| names.first().cloned()) + .unwrap_or_else(|| "localhost".into()); + self.status.send_modify(|status| { + status.network = Some(NetworkStatus { + base_url: format!("https://{host}:{port}/v1"), + host_name: names.first().cloned(), + fingerprint: identity.fingerprint(), + certificate_path: network::tls::Identity::pem_path(dir).display().to_string(), + }); + status.network_error = None; + }); + tracing::info!(port, "network port listening"); + Some(serving) + } + + /// Turn the network port on or off, on `port`, for the addresses in + /// `allowed` (empty: private networks). Takes effect immediately. + pub async fn set_network( + &self, + enabled: bool, + port: u16, + allowed: Vec, + ) -> Result<(), String> { + let allowed: Vec = allowed + .iter() + .map(|range| range.trim().to_owned()) + .filter(|range| !range.is_empty()) + .collect(); + for range in &allowed { + network::allow::Range::parse(range)?; + } + if port == 0 { + return Err("choose a port between 1 and 65535".into()); + } + self.update_settings(|settings| { + settings.network_enabled = enabled; + settings.network_port = port; + settings.network_allowed = allowed; }) + .map_err(|err| err.to_string())?; + self.restart_network().await; + Ok(()) + } + + /// Make a new certificate (other devices must trust it again), and use + /// it right away. + pub async fn renew_certificate(&self) -> Result<(), String> { + let dir = self + .network_dir + .as_ref() + .ok_or("there's no folder to keep the certificate in")?; + network::tls::Identity::create(dir, &network::local_names())?; + self.restart_network().await; + Ok(()) + } + + async fn restart_network(&self) { + let mut running = self.running.lock().await; + let Some(current) = running.as_mut() else { + return; + }; + if let Some(serving) = current.network.take() { + serving.stopped().await; + } + self.status.send_modify(|status| { + status.network = None; + status.network_error = None; + }); + if self.settings().network_enabled { + let app = current.app.clone(); + current.network = self.serve_network(&app); + } + } + + /// Answer Ollama apps on port 11434, or stop. Takes effect immediately. + pub async fn set_ollama_port(&self, enabled: bool) -> Result<(), String> { + self.update_settings(|settings| settings.ollama_port_enabled = enabled) + .map_err(|err| err.to_string())?; + let mut running = self.running.lock().await; + let Some(current) = running.as_mut() else { + return Ok(()); + }; + if let Some(stop) = current.ollama.take() { + let _ = stop.send(()); + } + self.status.send_modify(|status| { + status.ollama_url = None; + status.ollama_error = None; + }); + if enabled { + let app = current.app.clone(); + current.ollama = self.serve_ollama(&app).await; + } + Ok(()) } pub async fn stop(&self) { if let Some(running) = self.running.lock().await.take() { running.shut_down(); } - self.status.send_modify(|status| status.listening = false); + self.status.send_modify(|status| { + status.listening = false; + status.ollama_url = None; + }); } pub fn status(&self) -> GatewayStatus { @@ -227,7 +444,7 @@ impl Gateway { } self.update_settings(|settings| settings.port = port) .map_err(|err| err.to_string())?; - *running = Some(self.serve(listener)?); + *running = Some(self.serve(listener).await?); Ok(()) } @@ -292,6 +509,22 @@ impl Gateway { } } +/// Serve `router` on `listener` until the returned sender fires. +fn spawn_server(listener: tokio::net::TcpListener, router: axum::Router) -> oneshot::Sender<()> { + let (shutdown, stop) = oneshot::channel::<()>(); + tokio::spawn(async move { + let served = axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stop.await; + }) + .await; + if let Err(err) = served { + tracing::error!(%err, "gateway stopped unexpectedly"); + } + }); + shutdown +} + async fn bind(port: u16) -> Result { tokio::net::TcpListener::bind(("127.0.0.1", port)) .await diff --git a/crates/calcine-gateway/src/network/allow.rs b/crates/calcine-gateway/src/network/allow.rs new file mode 100644 index 0000000..2e503b3 --- /dev/null +++ b/crates/calcine-gateway/src/network/allow.rs @@ -0,0 +1,135 @@ +//! Which addresses may connect over the network: CIDR ranges, the private +//! ones by default. + +use std::net::IpAddr; + +/// Home and office networks, carrier-grade NAT (Tailscale), link-local, and +/// this machine itself. +pub const PRIVATE: &[&str] = &[ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "100.64.0.0/10", + "169.254.0.0/16", + "127.0.0.0/8", + "fc00::/7", + "fe80::/10", + "::1/128", +]; + +/// An address range: `192.168.1.0/24`, `fd00::/8`, or one address. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Range { + network: IpAddr, + prefix: u8, +} + +impl Range { + pub fn parse(text: &str) -> Result { + let text = text.trim(); + let invalid = || format!("{text} isn't an address or a range like 192.168.1.0/24"); + let (address, prefix) = match text.split_once('/') { + Some((address, prefix)) => (address, Some(prefix)), + None => (text, None), + }; + let network: IpAddr = address.parse().map_err(|_| invalid())?; + let max = if network.is_ipv4() { 32 } else { 128 }; + let prefix = match prefix { + Some(prefix) => prefix.parse::().map_err(|_| invalid())?, + None => max, + }; + if prefix > max { + return Err(invalid()); + } + Ok(Self { network, prefix }) + } + + pub fn contains(&self, address: IpAddr) -> bool { + // An IPv4 client on a dual-stack socket shows as `::ffff:a.b.c.d`. + let address = match address { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(address, IpAddr::V4), + IpAddr::V4(_) => address, + }; + match (self.network, address) { + (IpAddr::V4(network), IpAddr::V4(address)) => { + let mask = u32::MAX + .checked_shl(32 - u32::from(self.prefix)) + .unwrap_or(0); + u32::from(network) & mask == u32::from(address) & mask + } + (IpAddr::V6(network), IpAddr::V6(address)) => { + let mask = u128::MAX + .checked_shl(128 - u32::from(self.prefix)) + .unwrap_or(0); + u128::from(network) & mask == u128::from(address) & mask + } + _ => false, + } + } +} + +/// The ranges in `allowed`, or the private ones when it's empty. Invalid +/// entries are refused when saving, so here they're skipped. +pub fn ranges(allowed: &[String]) -> Vec { + if allowed.is_empty() { + return PRIVATE + .iter() + .filter_map(|range| Range::parse(range).ok()) + .collect(); + } + allowed + .iter() + .filter_map(|range| Range::parse(range).ok()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ip(text: &str) -> IpAddr { + text.parse().unwrap() + } + + #[test] + fn parses_ranges_and_single_addresses() { + assert!(Range::parse("192.168.1.0/24").is_ok()); + assert!(Range::parse("fd00::/8").is_ok()); + assert!(Range::parse("10.1.2.3").unwrap().contains(ip("10.1.2.3"))); + assert!(Range::parse("10.0.0.0/33").is_err()); + assert!(Range::parse("example.com").is_err()); + assert!(Range::parse("10.0.0.0/x").is_err()); + } + + #[test] + fn matches_addresses() { + let home = Range::parse("192.168.1.0/24").unwrap(); + assert!(home.contains(ip("192.168.1.42"))); + assert!(!home.contains(ip("192.168.2.42"))); + assert!(!home.contains(ip("fe80::1"))); + // IPv4 clients seen through a dual-stack socket. + assert!(home.contains(ip("::ffff:192.168.1.42"))); + let everything = Range::parse("0.0.0.0/0").unwrap(); + assert!(everything.contains(ip("8.8.8.8"))); + } + + #[test] + fn defaults_to_private_networks() { + let private = ranges(&[]); + for address in [ + "192.168.0.10", + "10.1.1.1", + "100.101.102.103", + "127.0.0.1", + "fd12::1", + ] { + assert!( + private.iter().any(|range| range.contains(ip(address))), + "{address}" + ); + } + assert!(!private.iter().any(|range| range.contains(ip("8.8.8.8")))); + let only = ranges(&["10.0.0.0/8".into()]); + assert!(!only.iter().any(|range| range.contains(ip("127.0.0.1")))); + } +} diff --git a/crates/calcine-gateway/src/network/limiter.rs b/crates/calcine-gateway/src/network/limiter.rs new file mode 100644 index 0000000..7d46bbf --- /dev/null +++ b/crates/calcine-gateway/src/network/limiter.rs @@ -0,0 +1,120 @@ +//! Slows down key guessing over the network: an address that sends too many +//! invalid keys is turned away for a while. + +use std::collections::HashMap; +use std::net::IpAddr; +use std::sync::{Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +/// Invalid keys allowed per address within `WINDOW`. +const MAX_FAILURES: u32 = 10; +const WINDOW: Duration = Duration::from_secs(60); +/// How long an address is turned away after that. +const BLOCKED_FOR: Duration = Duration::from_secs(300); + +#[derive(Debug, Default)] +pub struct Limiter { + addresses: Mutex>, +} + +#[derive(Debug, Clone, Copy)] +struct Record { + failures: u32, + since: Instant, + blocked_until: Option, +} + +impl Limiter { + /// How long `address` is still turned away, if it is. + pub fn blocked(&self, address: IpAddr, now: Instant) -> Option { + let addresses = self + .addresses + .lock() + .unwrap_or_else(PoisonError::into_inner); + addresses + .get(&address)? + .blocked_until + .filter(|until| *until > now) + .map(|until| until - now) + } + + /// Count an invalid key from `address`. + pub fn failed(&self, address: IpAddr, now: Instant) { + let mut addresses = self + .addresses + .lock() + .unwrap_or_else(PoisonError::into_inner); + // Forget what's old, so the map doesn't grow forever. + addresses.retain(|_, record| { + now.duration_since(record.since) < WINDOW + || record.blocked_until.is_some_and(|until| until > now) + }); + let record = addresses.entry(address).or_insert(Record { + failures: 0, + since: now, + blocked_until: None, + }); + if now.duration_since(record.since) >= WINDOW { + *record = Record { + failures: 0, + since: now, + blocked_until: None, + }; + } + record.failures += 1; + if record.failures >= MAX_FAILURES { + record.blocked_until = Some(now + BLOCKED_FOR); + tracing::warn!(%address, "too many invalid API keys from the network, blocking"); + } + } + + /// A valid key clears the count. + pub fn succeeded(&self, address: IpAddr) { + self.addresses + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&address); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blocks_after_too_many_failures() { + let limiter = Limiter::default(); + let address: IpAddr = "192.168.1.9".parse().unwrap(); + let start = Instant::now(); + for _ in 0..MAX_FAILURES - 1 { + limiter.failed(address, start); + } + assert_eq!(limiter.blocked(address, start), None); + limiter.failed(address, start); + assert_eq!(limiter.blocked(address, start), Some(BLOCKED_FOR)); + assert_eq!(limiter.blocked(address, start + BLOCKED_FOR), None); + // Another address isn't affected. + assert_eq!( + limiter.blocked("192.168.1.10".parse().unwrap(), start), + None + ); + } + + #[test] + fn old_failures_and_successes_reset_the_count() { + let limiter = Limiter::default(); + let address: IpAddr = "10.0.0.2".parse().unwrap(); + let start = Instant::now(); + for _ in 0..MAX_FAILURES - 1 { + limiter.failed(address, start); + } + limiter.failed(address, start + WINDOW); + assert_eq!(limiter.blocked(address, start + WINDOW), None); + for _ in 0..MAX_FAILURES - 2 { + limiter.failed(address, start + WINDOW); + } + limiter.succeeded(address); + limiter.failed(address, start + WINDOW); + assert_eq!(limiter.blocked(address, start + WINDOW), None); + } +} diff --git a/crates/calcine-gateway/src/network/mod.rs b/crates/calcine-gateway/src/network/mod.rs new file mode 100644 index 0000000..78f5ee1 --- /dev/null +++ b/crates/calcine-gateway/src/network/mod.rs @@ -0,0 +1,124 @@ +//! The local network port: the same API, for other devices, over HTTPS +//! only, with keys allowed on the network only, from allowed addresses. +//! Off by default. +//! +//! - `tls`: the self-signed certificate and the TLS acceptor +//! - `allow`: which addresses may connect +//! - `limiter`: addresses sending too many invalid keys are turned away + +pub mod allow; +pub mod limiter; +pub mod tls; + +use std::net::{IpAddr, SocketAddr}; + +use axum::Extension; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::conn::auto; +use hyper_util::service::TowerToHyperService; +use tokio::sync::{oneshot, watch}; +use tokio_rustls::TlsAcceptor; + +/// The network port's default: HTTPS, next to the loopback API's 18181. +pub const DEFAULT_PORT: u16 = 18443; + +/// The address a network request came from, set per connection. +#[derive(Debug, Clone, Copy)] +pub struct Peer(pub SocketAddr); + +/// The running network port. +#[derive(Debug)] +pub struct Serving { + shutdown: oneshot::Sender<()>, + task: tokio::task::JoinHandle<()>, +} + +impl Serving { + /// Stop listening and close the connections still open. + pub fn stop(self) { + let _ = self.shutdown.send(()); + } + + /// Stop, and wait until the port is free to listen on again: Linux + /// refuses a second listener while the first one is still open. + pub async fn stopped(self) { + let _ = self.shutdown.send(()); + let _ = self.task.await; + } +} + +/// Serve `router` over TLS on `listener` until [`Serving::stop`], which also +/// closes the connections still open. Each request carries its connection's +/// [`Peer`]. +pub fn spawn( + listener: tokio::net::TcpListener, + acceptor: TlsAcceptor, + router: axum::Router, +) -> Serving { + let (shutdown, mut stop) = oneshot::channel::<()>(); + let (closing, _) = watch::channel(false); + let task = tokio::spawn(async move { + loop { + let accepted = tokio::select! { + accepted = listener.accept() => accepted, + _ = &mut stop => break, + }; + let Ok((stream, address)) = accepted else { + continue; + }; + let acceptor = acceptor.clone(); + let router = router.clone().layer(Extension(Peer(address))); + let mut closed = closing.subscribe(); + tokio::spawn(async move { + let serve = async { + // Clients that don't complete TLS (port scans, wrong + // scheme) are just dropped. + let Ok(stream) = acceptor.accept(stream).await else { + return; + }; + let served = auto::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), TowerToHyperService::new(router)) + .await; + if let Err(err) = served { + tracing::debug!(%err, %address, "network connection ended"); + } + }; + // Turning the port off cuts kept-alive connections too. + tokio::select! { + () = serve => {} + _ = closed.changed() => {} + } + }); + } + let _ = closing.send(true); + }); + Serving { shutdown, task } +} + +/// The names other devices can reach this PC by: its name, and the address +/// it uses on the local network. +pub fn local_names() -> Vec { + let mut names = Vec::new(); + if let Some(name) = std::env::var("COMPUTERNAME") + .or_else(|_| std::env::var("HOSTNAME")) + .ok() + .filter(|name| !name.is_empty()) + { + names.push(name.to_lowercase()); + // Windows answers `name.local` over mDNS. + names.push(format!("{}.local", name.to_lowercase())); + } + if let Some(address) = local_address() { + names.push(address.to_string()); + } + names +} + +/// The address this PC uses to reach the network. Connecting a UDP socket +/// sends nothing: it only asks the system which interface it would use. +pub fn local_address() -> Option { + let socket = std::net::UdpSocket::bind(("0.0.0.0", 0)).ok()?; + socket.connect(("192.0.2.1", 9)).ok()?; + let address = socket.local_addr().ok()?.ip(); + (!address.is_unspecified() && !address.is_loopback()).then_some(address) +} diff --git a/crates/calcine-gateway/src/network/tls.rs b/crates/calcine-gateway/src/network/tls.rs new file mode 100644 index 0000000..5175e64 --- /dev/null +++ b/crates/calcine-gateway/src/network/tls.rs @@ -0,0 +1,130 @@ +//! The certificate the network port presents: self-signed, made on this PC +//! the first time, kept in Calcine's data folder. Clients trust it by its +//! SHA-256 fingerprint (pinning) or by importing `certificate.pem`. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use sha2::{Digest, Sha256}; +use tokio_rustls::TlsAcceptor; +use tokio_rustls::rustls::ServerConfig; +use tokio_rustls::rustls::crypto::ring; +use tokio_rustls::rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; + +/// A certificate and its private key. +pub struct Identity { + certificate: Vec, + key: Vec, +} + +impl std::fmt::Debug for Identity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Identity") + .field("fingerprint", &self.fingerprint()) + .finish_non_exhaustive() + } +} + +const CERTIFICATE: &str = "certificate.der"; +const KEY: &str = "key.der"; +/// The certificate again, in the format clients import. +const CERTIFICATE_PEM: &str = "certificate.pem"; + +impl Identity { + /// The certificate in `dir`, made first if there's none. + pub fn load_or_create(dir: &Path, names: &[String]) -> Result { + let certificate = std::fs::read(dir.join(CERTIFICATE)); + let key = std::fs::read(dir.join(KEY)); + if let (Ok(certificate), Ok(key)) = (certificate, key) { + return Ok(Self { certificate, key }); + } + Self::create(dir, names) + } + + /// A new certificate for `names` (host names and addresses), replacing + /// the one in `dir`. + pub fn create(dir: &Path, names: &[String]) -> Result { + let failed = |err: &dyn std::fmt::Display| format!("couldn't make a certificate: {err}"); + let mut names = names.to_vec(); + names.push("localhost".into()); + names.dedup(); + let key = rcgen::KeyPair::generate().map_err(|err| failed(&err))?; + let mut params = rcgen::CertificateParams::new(names).map_err(|err| failed(&err))?; + params + .distinguished_name + .push(rcgen::DnType::CommonName, "Calcine local network API"); + // Valid from the start of this year for five years. + let year = current_year(); + params.not_before = rcgen::date_time_ymd(year, 1, 1); + params.not_after = rcgen::date_time_ymd(year + 5, 1, 1); + let certificate = params.self_signed(&key).map_err(|err| failed(&err))?; + + std::fs::create_dir_all(dir).map_err(|err| failed(&err))?; + let write = |name: &str, bytes: &[u8]| { + std::fs::write(dir.join(name), bytes).map_err(|err| failed(&err)) + }; + write(CERTIFICATE, certificate.der())?; + write(KEY, &key.serialize_der())?; + write(CERTIFICATE_PEM, certificate.pem().as_bytes())?; + Ok(Self { + certificate: certificate.der().to_vec(), + key: key.serialize_der(), + }) + } + + /// SHA-256 of the certificate, `AB:CD:…`, as browsers and `openssl` show it. + pub fn fingerprint(&self) -> String { + Sha256::digest(&self.certificate) + .iter() + .map(|byte| format!("{byte:02X}")) + .collect::>() + .join(":") + } + + /// Where clients find the certificate to import. + pub fn pem_path(dir: &Path) -> PathBuf { + dir.join(CERTIFICATE_PEM) + } + + /// TLS 1.2 and 1.3, HTTP/2 or HTTP/1.1. + pub fn acceptor(&self) -> Result { + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(self.key.clone())); + let mut config = ServerConfig::builder_with_provider(Arc::new(ring::default_provider())) + .with_safe_default_protocol_versions() + .map_err(|err| err.to_string())? + .with_no_client_auth() + .with_single_cert(vec![CertificateDer::from(self.certificate.clone())], key) + .map_err(|err| format!("the certificate can't be used: {err}"))?; + config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + Ok(TlsAcceptor::from(Arc::new(config))) + } +} + +fn current_year() -> i32 { + let days = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs() / 86_400); + // Close enough: the certificate starts on January 1st anyway. + 1970 + i32::try_from(days * 400 / 146_097).unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn makes_a_certificate_once_and_keeps_it() { + let dir = std::env::temp_dir().join(format!("calcine-tls-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let first = + Identity::load_or_create(&dir, &["calcine-pc".into(), "192.168.1.5".into()]).unwrap(); + assert_eq!(first.fingerprint().len(), 32 * 3 - 1); + assert!(Identity::pem_path(&dir).is_file()); + assert!(first.acceptor().is_ok()); + let again = Identity::load_or_create(&dir, &[]).unwrap(); + assert_eq!(again.fingerprint(), first.fingerprint()); + let new = Identity::create(&dir, &[]).unwrap(); + assert_ne!(new.fingerprint(), first.fingerprint()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/calcine-gateway/src/ollama.rs b/crates/calcine-gateway/src/ollama.rs new file mode 100644 index 0000000..2471f41 --- /dev/null +++ b/crates/calcine-gateway/src/ollama.rs @@ -0,0 +1,630 @@ +//! Ollama's API (`/api/*`) translated to and from GenieX's OpenAI API, for +//! apps that only speak Ollama. +//! +//! Requests become streamed `/v1/chat/completions` calls; the OpenAI +//! server-sent events are turned back into Ollama's JSON lines, or into one +//! JSON object when the client asked for `"stream": false`. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde_json::{Map, Value, json}; + +/// Ollama's default port, where Ollama clients look first. +pub const OLLAMA_PORT: u16 = 11434; + +/// Version reported by `/api/version`. Some clients refuse old servers. +pub const COMPAT_VERSION: &str = "0.12.0"; + +/// `/api/chat` answers with `message`, `/api/generate` with `response`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + Chat, + Generate, +} + +/// An Ollama request, ready for GenieX. +#[derive(Debug, Clone, PartialEq)] +pub struct Translated { + /// OpenAI chat completion body, always streamed. + pub body: Value, + pub model: String, + /// Whether the client wants JSON lines (Ollama streams by default). + pub stream: bool, +} + +/// Ollama names a model `name:tag`; `:latest` is what clients add when a +/// name has no tag, and GenieX ids don't have it. +pub fn model_id(name: &str) -> String { + name.strip_suffix(":latest").unwrap_or(name).to_owned() +} + +/// `/api/chat` → `/v1/chat/completions`. +pub fn chat_request(body: &Value) -> Result { + let model = required_model(body)?; + let messages = body + .get("messages") + .and_then(Value::as_array) + .ok_or("messages is required")? + .iter() + .map(message) + .collect::, _>>()?; + let mut request = base_request(&model, body); + request.insert("messages".into(), Value::Array(messages)); + if let Some(tools) = body.get("tools").filter(|tools| tools.is_array()) { + request.insert("tools".into(), tools.clone()); + } + Ok(Translated { + body: Value::Object(request), + model, + stream: wants_stream(body), + }) +} + +/// `/api/generate` → `/v1/chat/completions`, with `system` and `prompt` +/// as messages. An empty prompt only loads the model (see [`is_load_only`]). +pub fn generate_request(body: &Value) -> Result { + let model = required_model(body)?; + let mut messages = Vec::new(); + if let Some(system) = body.get("system").and_then(Value::as_str) + && !system.is_empty() + { + messages.push(json!({ "role": "system", "content": system })); + } + let user = json!({ + "role": "user", + "content": body.get("prompt").and_then(Value::as_str).unwrap_or(""), + "images": body.get("images").cloned().unwrap_or(Value::Null), + }); + messages.push(message(&user)?); + let mut request = base_request(&model, body); + request.insert("messages".into(), Value::Array(messages)); + Ok(Translated { + body: Value::Object(request), + model, + stream: wants_stream(body), + }) +} + +/// Ollama loads a model when `/api/generate` gets an empty prompt. +pub fn is_load_only(body: &Value) -> bool { + body.get("prompt") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + && body + .get("images") + .and_then(Value::as_array) + .is_none_or(Vec::is_empty) +} + +fn required_model(body: &Value) -> Result { + body.get("model") + .and_then(Value::as_str) + .filter(|model| !model.is_empty()) + .map(model_id) + .ok_or_else(|| "model is required".to_owned()) +} + +fn wants_stream(body: &Value) -> bool { + body.get("stream").and_then(Value::as_bool).unwrap_or(true) +} + +/// Fields shared by chat and generate: model, options, thinking, format. +fn base_request(model: &str, body: &Value) -> Map { + let mut request = Map::new(); + request.insert("model".into(), json!(model)); + request.insert("stream".into(), json!(true)); + request.insert("stream_options".into(), json!({ "include_usage": true })); + // Keep the chain of thought out of the answer, in `reasoning_content`. + request.insert("reasoning_format".into(), json!("deepseek")); + if let Some(think) = body.get("think") { + // `true`, `false`, or a level (`"high"`, …) that GenieX doesn't have. + request.insert("enable_think".into(), json!(think.as_bool() != Some(false))); + } + match body.get("format") { + Some(Value::String(format)) if format == "json" => { + request.insert("response_format".into(), json!({ "type": "json_object" })); + } + Some(schema @ Value::Object(_)) => { + request.insert( + "response_format".into(), + json!({ "type": "json_schema", "json_schema": { "name": "response", "schema": schema } }), + ); + } + _ => {} + } + if let Some(options) = body.get("options").and_then(Value::as_object) { + for (from, to) in OPTIONS { + if let Some(value) = options.get(*from).filter(|value| !value.is_null()) { + request.insert((*to).into(), value.clone()); + } + } + // Ollama uses -1 (infinite) and -2 (fill the context) for no limit. + if let Some(limit) = options.get("num_predict").and_then(Value::as_i64) + && limit > 0 + { + request.insert("max_tokens".into(), json!(limit)); + } + } + request +} + +/// Ollama `options` with an OpenAI or GenieX equivalent. `num_ctx` isn't +/// here: the context window is a server option in Calcine. +const OPTIONS: &[(&str, &str)] = &[ + ("temperature", "temperature"), + ("top_p", "top_p"), + ("top_k", "top_k"), + ("min_p", "min_p"), + ("seed", "seed"), + ("stop", "stop"), + ("repeat_penalty", "repetition_penalty"), + ("presence_penalty", "presence_penalty"), + ("frequency_penalty", "frequency_penalty"), +]; + +/// One Ollama message as an OpenAI message: images become `image_url` +/// parts, tool calls get the OpenAI shape. +fn message(message: &Value) -> Result { + let role = message + .get("role") + .and_then(Value::as_str) + .ok_or("every message needs a role")?; + let text = message.get("content").and_then(Value::as_str).unwrap_or(""); + let images: Vec<&str> = message + .get("images") + .and_then(Value::as_array) + .map(|images| images.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + let content = if images.is_empty() { + json!(text) + } else { + let mut parts: Vec = images + .iter() + .map(|image| json!({ "type": "image_url", "image_url": { "url": data_url(image) } })) + .collect(); + if !text.is_empty() { + parts.push(json!({ "type": "text", "text": text })); + } + Value::Array(parts) + }; + let mut out = Map::new(); + out.insert("role".into(), json!(role)); + out.insert("content".into(), content); + if let Some(name) = message.get("tool_name").and_then(Value::as_str) { + out.insert("name".into(), json!(name)); + } + if let Some(calls) = message.get("tool_calls").and_then(Value::as_array) { + let calls: Vec = calls + .iter() + .enumerate() + .map(|(index, call)| { + let function = call.get("function").cloned().unwrap_or(Value::Null); + let arguments = match function.get("arguments") { + Some(Value::String(text)) => text.clone(), + Some(value) => value.to_string(), + None => "{}".to_owned(), + }; + json!({ + "id": format!("call_{index}"), + "type": "function", + "function": { "name": function.get("name").cloned().unwrap_or(Value::Null), "arguments": arguments }, + }) + }) + .collect(); + out.insert("tool_calls".into(), Value::Array(calls)); + } + Ok(Value::Object(out)) +} + +/// Ollama sends images as bare base64; OpenAI wants a data URL. The type +/// is read from the first bytes. +fn data_url(base64: &str) -> String { + if base64.starts_with("data:") { + return base64.to_owned(); + } + let mime = match base64.get(..4) { + Some("iVBO") => "image/png", + Some("R0lG") => "image/gif", + Some("UklG") => "image/webp", + _ => "image/jpeg", + }; + format!("data:{mime};base64,{base64}") +} + +/// Builds Ollama's answer from GenieX's server-sent events. +#[derive(Debug)] +pub struct Answer { + model: String, + mode: Mode, + pending: String, + content: String, + thinking: String, + tool_calls: Vec, + done_reason: Option, + prompt_tokens: Option, + completion_tokens: Option, + prompt_ms: Option, + predicted_ms: Option, + tokens_per_second: Option, + error: Option, +} + +#[derive(Debug, Default)] +struct ToolCall { + name: String, + arguments: String, +} + +impl Answer { + pub fn new(model: &str, mode: Mode) -> Self { + Self { + model: model.to_owned(), + mode, + pending: String::new(), + content: String::new(), + thinking: String::new(), + tool_calls: Vec::new(), + done_reason: None, + prompt_tokens: None, + completion_tokens: None, + prompt_ms: None, + predicted_ms: None, + tokens_per_second: None, + error: None, + } + } + + /// Read more of the event stream; returns the text pieces to stream to + /// the client, as Ollama chunks. + pub fn feed(&mut self, bytes: &[u8]) -> Vec { + self.pending.push_str(&String::from_utf8_lossy(bytes)); + let mut chunks = Vec::new(); + while let Some(newline) = self.pending.find('\n') { + let line: String = self.pending.drain(..=newline).collect(); + let Some(data) = line.trim_end().strip_prefix("data:") else { + continue; + }; + if let Some(chunk) = self.event(data.trim()) { + chunks.push(chunk); + } + } + chunks + } + + fn event(&mut self, data: &str) -> Option { + let event: Value = serde_json::from_str(data).ok()?; + if let Some(error) = event.get("error") { + self.error = Some(match error { + Value::String(message) => message.clone(), + other => other + .get("message") + .and_then(Value::as_str) + .unwrap_or("generation failed") + .to_owned(), + }); + return None; + } + if let Some(usage) = event.get("usage") { + self.prompt_tokens = usage.get("prompt_tokens").and_then(Value::as_u64); + self.completion_tokens = usage.get("completion_tokens").and_then(Value::as_u64); + } + if let Some(timings) = event.get("timings") { + self.prompt_ms = timings.get("prompt_ms").and_then(Value::as_f64); + self.predicted_ms = timings.get("predicted_ms").and_then(Value::as_f64); + self.tokens_per_second = timings.get("predicted_per_second").and_then(Value::as_f64); + } + let choice = event.pointer("/choices/0")?; + if let Some(reason) = choice.get("finish_reason").and_then(Value::as_str) { + self.done_reason = Some( + if reason == "tool_calls" { + "stop" + } else { + reason + } + .to_owned(), + ); + } + let delta = choice.get("delta")?; + for call in delta + .get("tool_calls") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let index = call + .get("index") + .and_then(Value::as_u64) + .and_then(|index| usize::try_from(index).ok()) + .unwrap_or(self.tool_calls.len()); + if self.tool_calls.len() <= index { + self.tool_calls.resize_with(index + 1, ToolCall::default); + } + let slot = &mut self.tool_calls[index]; + if let Some(name) = call.pointer("/function/name").and_then(Value::as_str) { + slot.name.push_str(name); + } + if let Some(arguments) = call.pointer("/function/arguments").and_then(Value::as_str) { + slot.arguments.push_str(arguments); + } + } + let content = delta.get("content").and_then(Value::as_str).unwrap_or(""); + let thinking = delta + .get("reasoning_content") + .and_then(Value::as_str) + .unwrap_or(""); + if content.is_empty() && thinking.is_empty() { + return None; + } + self.content.push_str(content); + self.thinking.push_str(thinking); + Some(self.chunk(content, thinking, false)) + } + + /// Why generation failed, if GenieX said so inside the stream. + pub fn error(&self) -> Option<&str> { + self.error.as_deref() + } + + pub fn prompt_tokens(&self) -> Option { + self.prompt_tokens + } + + pub fn completion_tokens(&self) -> Option { + self.completion_tokens + } + + pub fn tokens_per_second(&self) -> Option { + self.tokens_per_second + } + + /// The last line of a stream, with statistics. + pub fn done(&self, total_ms: u64) -> Value { + self.finish(total_ms, "", "") + } + + /// The whole answer in one object, for `"stream": false`. + pub fn whole(&self, total_ms: u64) -> Value { + self.finish(total_ms, &self.content, &self.thinking) + } + + fn finish(&self, total_ms: u64, content: &str, thinking: &str) -> Value { + let mut chunk = self.chunk(content, thinking, true); + let object = chunk.as_object_mut().expect("chunks are objects"); + if !self.tool_calls.is_empty() && self.mode == Mode::Chat { + let calls: Vec = self + .tool_calls + .iter() + .map(|call| { + let arguments = serde_json::from_str::(&call.arguments) + .unwrap_or_else(|_| json!({})); + json!({ "function": { "name": call.name, "arguments": arguments } }) + }) + .collect(); + object["message"]["tool_calls"] = Value::Array(calls); + } + let millis_to_nanos = |ms: f64| { + std::time::Duration::try_from_secs_f64(ms / 1000.0).map_or(0, |duration| { + u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX) + }) + }; + object.insert( + "done_reason".into(), + json!(self.done_reason.as_deref().unwrap_or("stop")), + ); + object.insert("total_duration".into(), json!(total_ms * 1_000_000)); + object.insert("load_duration".into(), json!(0)); + object.insert( + "prompt_eval_count".into(), + json!(self.prompt_tokens.unwrap_or(0)), + ); + object.insert( + "prompt_eval_duration".into(), + json!(self.prompt_ms.map_or(0, millis_to_nanos)), + ); + object.insert( + "eval_count".into(), + json!(self.completion_tokens.unwrap_or(0)), + ); + object.insert( + "eval_duration".into(), + json!(self.predicted_ms.map_or(0, millis_to_nanos)), + ); + chunk + } + + fn chunk(&self, content: &str, thinking: &str, done: bool) -> Value { + let mut chunk = json!({ "model": self.model, "created_at": now_rfc3339() }); + match self.mode { + Mode::Chat => { + let mut message = json!({ "role": "assistant", "content": content }); + if !thinking.is_empty() { + message["thinking"] = json!(thinking); + } + chunk["message"] = message; + } + Mode::Generate => { + chunk["response"] = json!(content); + if !thinking.is_empty() { + chunk["thinking"] = json!(thinking); + } + } + } + chunk["done"] = json!(done); + chunk + } +} + +/// The reply to `/api/generate` with an empty prompt, once the model is loaded. +pub fn loaded(model: &str) -> Value { + json!({ + "model": model, + "created_at": now_rfc3339(), + "response": "", + "done": true, + "done_reason": "load", + }) +} + +/// Current UTC time as RFC 3339, like Ollama's `created_at`. +pub fn now_rfc3339() -> String { + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + rfc3339(since_epoch.as_secs(), since_epoch.subsec_nanos()) +} + +/// `secs` since the Unix epoch as `YYYY-MM-DDTHH:MM:SS.nnnnnnnnnZ`. +fn rfc3339(secs: u64, nanos: u32) -> String { + let days = i64::try_from(secs / 86_400).unwrap_or(0); + let rest = secs % 86_400; + // Howard Hinnant's days-to-civil algorithm. + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{nanos:09}Z", + rest / 3600, + rest % 3600 / 60, + rest % 60 + ) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn translates_a_chat_request() { + let translated = chat_request(&json!({ + "model": "qualcomm/Qwen3-4B:latest", + "messages": [ + { "role": "system", "content": "Be brief." }, + { "role": "user", "content": "What is this?", "images": ["iVBORw0KGgo="] }, + ], + "options": { "temperature": 0.2, "num_predict": 64, "repeat_penalty": 1.1, "num_ctx": 8192 }, + "think": false, + "format": "json", + })) + .unwrap(); + assert_eq!(translated.model, "qualcomm/Qwen3-4B"); + assert!(translated.stream); + let body = translated.body; + assert_eq!(body["stream"], true); + assert_eq!(body["temperature"], 0.2); + assert_eq!(body["max_tokens"], 64); + assert_eq!(body["repetition_penalty"], 1.1); + assert!(body.get("num_ctx").is_none()); + assert_eq!(body["enable_think"], false); + assert_eq!(body["response_format"]["type"], "json_object"); + assert_eq!( + body["messages"][0], + json!({ "role": "system", "content": "Be brief." }) + ); + assert_eq!( + body["messages"][1]["content"][0]["image_url"]["url"], + "data:image/png;base64,iVBORw0KGgo=" + ); + assert_eq!(body["messages"][1]["content"][1]["text"], "What is this?"); + } + + #[test] + fn no_limit_and_no_think_leave_geniex_defaults() { + let body = chat_request(&json!({ + "model": "m", + "messages": [], + "options": { "num_predict": -1 }, + "stream": false, + })) + .unwrap(); + assert!(!body.stream); + assert!(body.body.get("max_tokens").is_none()); + assert!(body.body.get("enable_think").is_none()); + } + + #[test] + fn translates_generate_and_detects_loading() { + let body = + generate_request(&json!({ "model": "m", "system": "S", "prompt": "P" })).unwrap(); + assert_eq!(body.body["messages"][0]["content"], "S"); + assert_eq!(body.body["messages"][1]["content"], "P"); + assert!(is_load_only(&json!({ "model": "m" }))); + assert!(is_load_only(&json!({ "model": "m", "prompt": "" }))); + assert!(!is_load_only(&json!({ "model": "m", "prompt": "Hi" }))); + } + + #[test] + fn requires_a_model() { + assert!(chat_request(&json!({ "messages": [] })).is_err()); + } + + #[test] + fn turns_events_into_ollama_chunks() { + let mut answer = Answer::new("m", Mode::Chat); + let chunks = answer.feed( + b"data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"Hmm\"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"Hel", + ); + assert_eq!(chunks.len(), 1); + assert_eq!(chunks[0]["message"]["thinking"], "Hmm"); + let chunks = answer.feed( + b"lo\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":2},\"timings\":{\"predicted_ms\":50.0}}\n\ndata: [DONE]\n\n", + ); + assert_eq!(chunks[0]["message"]["content"], "Hello"); + assert_eq!(chunks[0]["done"], false); + + let done = answer.done(120); + assert_eq!(done["done"], true); + assert_eq!(done["message"]["content"], ""); + assert_eq!(done["done_reason"], "stop"); + assert_eq!(done["eval_count"], 2); + assert_eq!(done["eval_duration"], 50_000_000); + assert_eq!(done["total_duration"], 120_000_000); + + let whole = answer.whole(120); + assert_eq!(whole["message"]["content"], "Hello"); + assert_eq!(whole["message"]["thinking"], "Hmm"); + } + + #[test] + fn collects_tool_calls_and_errors() { + let mut answer = Answer::new("m", Mode::Chat); + answer.feed(b"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"name\":\"weather\",\"arguments\":\"{\\\"city\\\":\"}}]}}]}\n"); + answer.feed(b"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Paris\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n"); + let done = answer.whole(1); + assert_eq!( + done["message"]["tool_calls"][0], + json!({ "function": { "name": "weather", "arguments": { "city": "Paris" } } }) + ); + + let mut failing = Answer::new("m", Mode::Generate); + failing.feed(b"data:{\"error\":{\"message\":\"model not found\"}}\n"); + assert_eq!(failing.error(), Some("model not found")); + } + + #[test] + fn generate_answers_with_response() { + let mut answer = Answer::new("m", Mode::Generate); + let chunks = answer.feed(b"data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n"); + assert_eq!(chunks[0]["response"], "Hi"); + assert!(chunks[0].get("message").is_none()); + } + + #[test] + fn formats_rfc3339() { + assert_eq!(rfc3339(0, 0), "1970-01-01T00:00:00.000000000Z"); + assert_eq!(rfc3339(1_791_504_000, 5), "2026-10-09T00:00:00.000000005Z"); + assert_eq!(rfc3339(951_782_400, 0), "2000-02-29T00:00:00.000000000Z"); + } +} diff --git a/crates/calcine-gateway/src/proxy.rs b/crates/calcine-gateway/src/proxy.rs index 0bb0382..24665bc 100644 --- a/crates/calcine-gateway/src/proxy.rs +++ b/crates/calcine-gateway/src/proxy.rs @@ -12,28 +12,51 @@ use tokio::sync::{OwnedSemaphorePermit, mpsc}; use tokio_stream::wrappers::ReceiverStream; use crate::caller::Caller; -use crate::error::{api_error, upstream_error, upstream_message}; +use crate::error::{api_error, is_context_error, upstream_error, upstream_message}; use crate::log::{RequestEntry, RequestLog, SseUsage, Usage}; use crate::security::sanitize; use crate::state::{ActiveRequest, AppState}; /// One request in flight: where it's logged, when it started, and its turn /// in the queue (released when this is dropped). -struct Exchange { - log: Arc, - id: u32, - started: Instant, +pub(crate) struct Exchange { + pub log: Arc, + pub id: u32, + pub started: Instant, _turn: (OwnedSemaphorePermit, ActiveRequest), } impl Exchange { - fn fail(&self, status: StatusCode, message: &str) -> Response { + /// Log a failure. The caller answers in its own error format. + pub fn log_failure(&self, status: StatusCode, message: &str) { self.log.update(self.id, |entry| { entry.status = Some(status.as_u16()); entry.duration_ms = Some(elapsed_ms(self.started)); entry.error = Some(message.to_owned()); }); - api_error(status, "server_error", message) + } +} + +/// Why a request didn't reach GenieX. +#[derive(Debug)] +pub(crate) struct Refused { + pub status: StatusCode, + pub kind: &'static str, + pub message: String, +} + +impl Refused { + fn new(status: StatusCode, kind: &'static str, message: impl Into) -> Self { + Self { + status, + kind, + message: message.into(), + } + } + + /// In the OpenAI error format. + pub fn into_response(self) -> Response { + api_error(self.status, self.kind, &self.message) } } @@ -43,13 +66,6 @@ pub async fn inference( path: &'static str, body: Bytes, ) -> Response { - if !caller.can_infer() { - return api_error( - StatusCode::FORBIDDEN, - "permission_error", - "this API key can't run models", - ); - } let json: Value = match serde_json::from_slice(&body) { Ok(json) => json, Err(err) => { @@ -60,8 +76,41 @@ pub async fn inference( ); } }; - if let Err(reason) = sanitize::check_body(&json, caller.allows_local_files()) { - return api_error(StatusCode::FORBIDDEN, "permission_error", &reason); + let stream = json.get("stream").and_then(Value::as_bool).unwrap_or(false); + let (exchange, response) = match send(&app, &caller, path, path, &json, body).await { + Ok(sent) => sent, + Err(refused) => return refused.into_response(), + }; + if stream && response.status().is_success() { + streamed(exchange, response) + } else { + buffered(exchange, response).await + } +} + +/// Check, log and queue a request, then send `body` (the JSON `json`) to +/// GenieX at `upstream_path`. The request log shows `logged_path`. +pub(crate) async fn send( + app: &Arc, + caller: &Caller, + logged_path: &'static str, + upstream_path: &'static str, + json: &Value, + body: Bytes, +) -> Result<(Exchange, reqwest::Response), Refused> { + if !caller.can_infer() { + return Err(Refused::new( + StatusCode::FORBIDDEN, + "permission_error", + "this API key can't run models", + )); + } + if let Err(reason) = sanitize::check_body(json, caller.allows_local_files()) { + return Err(Refused::new( + StatusCode::FORBIDDEN, + "permission_error", + reason, + )); } let model = json.get("model").and_then(Value::as_str).map(str::to_owned); @@ -69,7 +118,7 @@ pub async fn inference( let id = app.log.start(RequestEntry::new( caller.label(), "POST", - path, + logged_path, model, stream, )); @@ -78,11 +127,11 @@ pub async fn inference( // GenieX runs one inference at a time: wait in line. app.enter_queue(); let Ok(permit) = app.queue.clone().acquire_owned().await else { - return api_error( + return Err(Refused::new( StatusCode::SERVICE_UNAVAILABLE, "server_error", "the gateway is shutting down", - ); + )); }; app.leave_queue_and_start(); let exchange = Exchange { @@ -95,36 +144,83 @@ pub async fn inference( let upstream = match app.services.server.ensure_running().await { Ok(url) => url, Err(err) => { - return exchange.fail( + let message = format!("GenieX couldn't start: {err}"); + exchange.log_failure(StatusCode::SERVICE_UNAVAILABLE, &message); + return Err(Refused::new( StatusCode::SERVICE_UNAVAILABLE, - &format!("GenieX couldn't start: {err}"), - ); + "server_error", + message, + )); } }; - let response = match app - .http - .post(format!("{upstream}{path}")) - .header(header::CONTENT_TYPE, "application/json") - .body(body) - .send() - .await - { - Ok(response) => response, + let post = |body: Bytes| { + app.http + .post(format!("{upstream}{upstream_path}")) + .header(header::CONTENT_TYPE, "application/json") + .body(body) + .send() + }; + match post(body.clone()).await { + Ok(response) if response.status() == StatusCode::BAD_REQUEST => { + let bytes = response.bytes().await.unwrap_or_default(); + if !is_context_error(&bytes) { + return Ok((exchange, rebuilt(StatusCode::BAD_REQUEST, bytes))); + } + // GenieX 0.8 counts the conversation still in its cache against + // the window when the new one doesn't continue it (a regenerated + // reply, another client): a prompt that fits is refused. A + // one-token request in between empties the cache. + clear_cache(app, &upstream, json).await; + match post(body).await { + Ok(response) => Ok((exchange, response)), + Err(_) => Ok((exchange, rebuilt(StatusCode::BAD_REQUEST, bytes))), + } + } + Ok(response) => Ok((exchange, response)), Err(err) => { - return exchange.fail( + let message = format!("couldn't reach GenieX: {err}"); + exchange.log_failure(StatusCode::BAD_GATEWAY, &message); + Err(Refused::new( StatusCode::BAD_GATEWAY, - &format!("couldn't reach GenieX: {err}"), - ); + "server_error", + message, + )) } - }; - - if stream && response.status().is_success() { - streamed(exchange, response) - } else { - buffered(exchange, response).await } } +/// A response already read, to pass on as if it hadn't been. +fn rebuilt(status: StatusCode, body: Bytes) -> reqwest::Response { + let mut response = axum::http::Response::new(body); + *response.status_mut() = status; + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); + reqwest::Response::from(response) +} + +/// Run a one-token request so GenieX drops the conversation it caches. +async fn clear_cache(app: &AppState, upstream: &str, request: &Value) { + let Some(model) = request.get("model") else { + return; + }; + let reset = serde_json::json!({ + "model": model, + "messages": [{ "role": "user", "content": "." }], + "max_tokens": 1, + "enable_think": false, + }); + let _ = app + .http + .post(format!("{upstream}/v1/chat/completions")) + .header(header::CONTENT_TYPE, "application/json") + .body(reset.to_string()) + .send() + .await; + tracing::info!("cleared GenieX's cached conversation to retry a prompt it refused"); +} + /// Non-streaming responses and errors: read fully, log, pass on. async fn buffered(exchange: Exchange, response: reqwest::Response) -> Response { let status = response.status(); @@ -193,6 +289,6 @@ fn streamed(exchange: Exchange, response: reqwest::Response) -> Response { .into_response() } -fn elapsed_ms(since: Instant) -> u64 { +pub(crate) fn elapsed_ms(since: Instant) -> u64 { u64::try_from(since.elapsed().as_millis()).unwrap_or(u64::MAX) } diff --git a/crates/calcine-gateway/src/routes/mod.rs b/crates/calcine-gateway/src/routes/mod.rs index 66d2237..2da5bf5 100644 --- a/crates/calcine-gateway/src/routes/mod.rs +++ b/crates/calcine-gateway/src/routes/mod.rs @@ -1,24 +1,34 @@ mod manage; +mod ollama; mod openai; use std::sync::Arc; -use axum::Router; use axum::extract::DefaultBodyLimit; use axum::http::StatusCode; use axum::middleware; +use axum::routing::get; +use axum::{Extension, Router}; use crate::error::api_error; -use crate::security; +use crate::security::{self, Listener}; use crate::state::AppState; /// Base64 images and audio can be large. const BODY_LIMIT: usize = 64 * 1024 * 1024; -pub fn router(app: Arc) -> Router { - Router::new() +/// The API served on `listener`. The Ollama port also answers `/` like +/// Ollama, so clients recognize it. +pub fn router(app: Arc, listener: Listener) -> Router { + let routes = Router::new() .merge(openai::router()) .merge(manage::router()) + .merge(ollama::router()); + let routes = match listener { + Listener::Main | Listener::Network { .. } => routes, + Listener::Ollama { .. } => routes.route("/", get(ollama::root)), + }; + routes .fallback(|| async { api_error( StatusCode::NOT_FOUND, @@ -27,6 +37,8 @@ pub fn router(app: Arc) -> Router { ) }) .layer(middleware::from_fn_with_state(app.clone(), security::guard)) + // Outside the guard, so it knows which socket the request came in on. + .layer(Extension(listener)) .layer(DefaultBodyLimit::max(BODY_LIMIT)) .with_state(app) } diff --git a/crates/calcine-gateway/src/routes/ollama.rs b/crates/calcine-gateway/src/routes/ollama.rs new file mode 100644 index 0000000..f662588 --- /dev/null +++ b/crates/calcine-gateway/src/routes/ollama.rs @@ -0,0 +1,436 @@ +//! Ollama's API under `/api`, for apps that only speak Ollama. See +//! [`crate::ollama`] for the translation. + +use std::sync::Arc; + +use axum::body::{Body, Bytes}; +use axum::extract::State; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{any, get, post}; +use axum::{Extension, Json, Router}; +use calcine_core::models::{LocalModel, ModelType, Runtime}; +use futures_util::StreamExt; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + +use crate::caller::Caller; +use crate::error::upstream_message; +use crate::ollama::{self, Answer, Mode}; +use crate::proxy::{self, Exchange, elapsed_ms}; +use crate::state::AppState; +use crate::{context, structured}; + +pub fn router() -> Router> { + Router::new() + .route("/api/version", get(version)) + .route("/api/tags", get(tags)) + .route("/api/ps", get(running)) + .route("/api/show", post(show)) + .route("/api/chat", post(chat)) + .route("/api/generate", post(generate)) + .route("/api/{*rest}", any(unsupported)) +} + +/// What Ollama answers on `/`; clients check it to find a server. +pub async fn root() -> &'static str { + "Ollama is running" +} + +async fn version() -> Json { + Json(json!({ "version": ollama::COMPAT_VERSION })) +} + +fn error(status: StatusCode, message: &str) -> Response { + (status, Json(json!({ "error": message }))).into_response() +} + +async fn unsupported(uri: axum::http::Uri) -> Response { + error( + StatusCode::NOT_IMPLEMENTED, + &format!( + "Calcine doesn't support {}: download and remove models in the Calcine app", + uri.path() + ), + ) +} + +async fn local_models(app: &AppState, caller: &Caller) -> Result, Box> { + if !caller.can_infer() { + return Err(Box::new(error( + StatusCode::FORBIDDEN, + "this API key can't list models", + ))); + } + app.services + .models + .list() + .await + .map_err(|err| Box::new(error(StatusCode::SERVICE_UNAVAILABLE, &err.to_string()))) +} + +/// One entry per precision, named like GenieX ids: `name:precision`. +fn tag_entries(model: &LocalModel) -> Vec { + let precisions: Vec> = if model.precisions.is_empty() { + vec![None] + } else { + model.precisions.iter().map(|p| Some(p.as_str())).collect() + }; + precisions + .into_iter() + .map(|precision| { + let name = + precision.map_or_else(|| model.name.clone(), |p| format!("{}:{p}", model.name)); + json!({ + "name": name, + "model": name, + "modified_at": ollama::now_rfc3339(), + "size": model.size_bytes, + "digest": hex::encode(Sha256::digest(name.as_bytes())), + "details": details(model, precision), + }) + }) + .collect() +} + +fn details(model: &LocalModel, precision: Option<&str>) -> Value { + json!({ + "parent_model": "", + "format": match model.runtime { + Runtime::LlamaCpp => "gguf", + Runtime::Qairt => "qairt", + Runtime::Unknown => "unknown", + }, + "family": model.name.split('/').next().unwrap_or(""), + "families": Value::Null, + "parameter_size": "", + "quantization_level": precision.unwrap_or(""), + }) +} + +async fn tags(State(app): State>, Extension(caller): Extension) -> Response { + match local_models(&app, &caller).await { + Ok(models) => { + let entries: Vec = models.iter().flat_map(tag_entries).collect(); + Json(json!({ "models": entries })).into_response() + } + Err(response) => *response, + } +} + +/// GenieX doesn't say which model it holds in memory. +async fn running() -> Json { + Json(json!({ "models": [] })) +} + +async fn show( + State(app): State>, + Extension(caller): Extension, + body: Bytes, +) -> Response { + let Ok(body) = serde_json::from_slice::(&body) else { + return error(StatusCode::BAD_REQUEST, "body isn't valid JSON"); + }; + let Some(name) = body + .get("model") + .or_else(|| body.get("name")) + .and_then(Value::as_str) + .map(ollama::model_id) + else { + return error(StatusCode::BAD_REQUEST, "model is required"); + }; + let models = match local_models(&app, &caller).await { + Ok(models) => models, + Err(response) => return *response, + }; + let (base, precision) = match name.split_once(':') { + Some((base, precision)) => (base, Some(precision)), + None => (name.as_str(), None), + }; + let Some(model) = models.iter().find(|model| { + model.name == base && precision.is_none_or(|p| model.precisions.iter().any(|m| m == p)) + }) else { + return error(StatusCode::NOT_FOUND, &format!("model '{name}' not found")); + }; + let mut capabilities = vec!["completion"]; + if model.model_type == ModelType::Vlm { + capabilities.push("vision"); + } + Json(json!({ + "modelfile": "", + "parameters": "", + "template": "", + "details": details(model, precision.or(model.precisions.first().map(String::as_str))), + "model_info": {}, + "capabilities": capabilities, + "modified_at": ollama::now_rfc3339(), + })) + .into_response() +} + +async fn chat( + State(app): State>, + Extension(caller): Extension, + body: Bytes, +) -> Response { + answer(app, caller, Mode::Chat, &body).await +} + +async fn generate( + State(app): State>, + Extension(caller): Extension, + body: Bytes, +) -> Response { + answer(app, caller, Mode::Generate, &body).await +} + +async fn answer(app: Arc, caller: Caller, mode: Mode, body: &[u8]) -> Response { + let Ok(json) = serde_json::from_slice::(body) else { + return error(StatusCode::BAD_REQUEST, "body isn't valid JSON"); + }; + if mode == Mode::Generate && ollama::is_load_only(&json) { + return load(app, caller, &json).await; + } + let translated = match mode { + Mode::Chat => ollama::chat_request(&json), + Mode::Generate => ollama::generate_request(&json), + }; + let mut translated = match translated { + Ok(translated) => translated, + Err(message) => return error(StatusCode::BAD_REQUEST, &message), + }; + let path = match mode { + Mode::Chat => "/api/chat", + Mode::Generate => "/api/generate", + }; + // Ollama forgets the oldest messages that don't fit: so does Calcine. + if caller.can_infer() + && let Err(message) = context::fit(&app, &mut translated.body).await + { + return error(StatusCode::BAD_REQUEST, &message); + } + // `format` (JSON or a schema): checked by Calcine, GenieX ignores it. + if let Some(format) = structured::requested(&translated.body) { + return structured_answer(&app, &caller, &translated, format, mode, path).await; + } + let upstream = Bytes::from(translated.body.to_string()); + let (exchange, response) = match proxy::send( + &app, + &caller, + path, + "/v1/chat/completions", + &translated.body, + upstream, + ) + .await + { + Ok(sent) => sent, + Err(refused) => return error(refused.status, &refused.message), + }; + exchange + .log + .update(exchange.id, |entry| entry.stream = translated.stream); + + let status = response.status(); + if !status.is_success() { + let bytes = response.bytes().await.unwrap_or_default(); + let message = upstream_message(&bytes); + exchange.log_failure(status, &message); + return error(status, &message); + } + let answer = Answer::new(&translated.model, mode); + if translated.stream { + stream(exchange, response, answer) + } else { + whole(exchange, response, answer).await + } +} + +/// A structured answer, as Ollama would give it: the JSON is the content. +async fn structured_answer( + app: &Arc, + caller: &Caller, + translated: &ollama::Translated, + format: structured::Format, + mode: Mode, + path: &'static str, +) -> Response { + let started = std::time::Instant::now(); + let completion = match structured::generate( + app, + caller, + &translated.body, + &structured::Goal::Json(format), + path, + translated.stream, + ) + .await + { + Ok(completion) => completion, + Err(failure) => return error(failure.status, &failure.message), + }; + // Replay the completion as one event, so Ollama's shape is built once. + let message = completion + .pointer("/choices/0/message") + .cloned() + .unwrap_or_default(); + let event = json!({ + "choices": [{ + "delta": { "content": message["content"], "reasoning_content": message["reasoning_content"] }, + "finish_reason": "stop", + }], + "usage": completion["usage"], + "timings": completion["timings"], + }); + let mut answer = Answer::new(&translated.model, mode); + let pieces = answer.feed(format!("data: {event}\n").as_bytes()); + let total_ms = elapsed_ms(started); + if translated.stream { + let mut body: Vec = Vec::new(); + for piece in pieces.iter().chain(std::iter::once(&answer.done(total_ms))) { + body.extend_from_slice(&line(piece)); + } + ([(header::CONTENT_TYPE, "application/x-ndjson")], body).into_response() + } else { + Json(answer.whole(total_ms)).into_response() + } +} + +/// `/api/generate` with no prompt loads the model (or, with `keep_alive: 0`, +/// asks to unload it, which GenieX does by itself after the unload delay). +async fn load(app: Arc, caller: Caller, json: &Value) -> Response { + let Some(model) = json + .get("model") + .and_then(Value::as_str) + .map(ollama::model_id) + else { + return error(StatusCode::BAD_REQUEST, "model is required"); + }; + if json.get("keep_alive").and_then(Value::as_i64) == Some(0) { + let mut reply = ollama::loaded(&model); + reply["done_reason"] = json!("unload"); + return Json(reply).into_response(); + } + let body = json!({ + "model": model, + "messages": [{ "role": "user", "content": "Hi" }], + "max_tokens": 1, + "enable_think": false, + "stream": false, + }); + let upstream = Bytes::from(body.to_string()); + let (exchange, response) = match proxy::send( + &app, + &caller, + "/api/generate", + "/v1/chat/completions", + &body, + upstream, + ) + .await + { + Ok(sent) => sent, + Err(refused) => return error(refused.status, &refused.message), + }; + let status = response.status(); + let bytes = response.bytes().await.unwrap_or_default(); + if !status.is_success() { + let message = upstream_message(&bytes); + exchange.log_failure(status, &message); + return error(status, &message); + } + exchange.log.update(exchange.id, |entry| { + entry.status = Some(status.as_u16()); + entry.duration_ms = Some(elapsed_ms(exchange.started)); + }); + Json(ollama::loaded(&model)).into_response() +} + +/// JSON lines, one per piece of text, then a last line with statistics. +fn stream(exchange: Exchange, response: reqwest::Response, mut answer: Answer) -> Response { + let (tx, rx) = mpsc::channel::>(64); + tokio::spawn(async move { + let mut upstream = response.bytes_stream(); + let mut first_token_ms = None; + let mut broken = None; + 'read: while let Some(chunk) = upstream.next().await { + let bytes = match chunk { + Ok(bytes) => bytes, + Err(err) => { + broken = Some(format!("stream from GenieX broke: {err}")); + break; + } + }; + for piece in answer.feed(&bytes) { + first_token_ms.get_or_insert_with(|| elapsed_ms(exchange.started)); + if tx.send(Ok(line(&piece))).await.is_err() { + broken = Some("client disconnected".to_owned()); + break 'read; + } + } + } + let error = broken.or_else(|| answer.error().map(str::to_owned)); + let last = match &error { + Some(message) => json!({ "error": message }), + None => answer.done(elapsed_ms(exchange.started)), + }; + let _ = tx.send(Ok(line(&last))).await; + finish_log(&exchange, &answer, first_token_ms, error); + }); + ( + [(header::CONTENT_TYPE, "application/x-ndjson")], + Body::from_stream(ReceiverStream::new(rx)), + ) + .into_response() +} + +/// The whole answer in one object (`"stream": false`). +async fn whole(exchange: Exchange, response: reqwest::Response, mut answer: Answer) -> Response { + let mut upstream = response.bytes_stream(); + let mut first_token_ms = None; + let mut broken = None; + while let Some(chunk) = upstream.next().await { + match chunk { + Ok(bytes) => { + if !answer.feed(&bytes).is_empty() { + first_token_ms.get_or_insert_with(|| elapsed_ms(exchange.started)); + } + } + Err(err) => { + broken = Some(format!("stream from GenieX broke: {err}")); + break; + } + } + } + let error_message = broken.or_else(|| answer.error().map(str::to_owned)); + finish_log(&exchange, &answer, first_token_ms, error_message.clone()); + match error_message { + Some(message) => error(StatusCode::INTERNAL_SERVER_ERROR, &message), + None => Json(answer.whole(elapsed_ms(exchange.started))).into_response(), + } +} + +fn finish_log( + exchange: &Exchange, + answer: &Answer, + first_token_ms: Option, + error: Option, +) { + exchange.log.update(exchange.id, |entry| { + entry.status = Some(200); + entry.duration_ms = Some(elapsed_ms(exchange.started)); + entry.first_token_ms = first_token_ms; + entry.prompt_tokens = answer.prompt_tokens(); + entry.completion_tokens = answer.completion_tokens(); + entry.tokens_per_second = answer.tokens_per_second(); + entry.error = error; + }); +} + +fn line(value: &Value) -> Bytes { + let mut bytes = value.to_string().into_bytes(); + bytes.push(b'\n'); + Bytes::from(bytes) +} diff --git a/crates/calcine-gateway/src/routes/openai.rs b/crates/calcine-gateway/src/routes/openai.rs index 79c11df..a120e92 100644 --- a/crates/calcine-gateway/src/routes/openai.rs +++ b/crates/calcine-gateway/src/routes/openai.rs @@ -13,8 +13,9 @@ use serde_json::{Value, json}; use crate::caller::Caller; use crate::error::api_error; -use crate::proxy; use crate::state::AppState; +use crate::structured::{self, tools}; +use crate::{context, proxy}; pub fn router() -> Router> { Router::new() @@ -37,7 +38,39 @@ async fn chat_completions( Extension(caller): Extension, body: Bytes, ) -> Response { - proxy::inference(app, caller, "/v1/chat/completions", body).await + let Ok(mut json) = serde_json::from_slice::(&body) else { + return proxy::inference(app, caller, "/v1/chat/completions", body).await; + }; + if !caller.can_infer() { + return proxy::inference(app, caller, "/v1/chat/completions", body).await; + } + // GenieX ignores `tool_choice` and `response_format`: Calcine enforces them. + let refused = tools::refused(&json); + if refused { + json = tools::without_tools(&json); + } + let changed = refused || json.get("truncation").is_some(); + let forgotten = if context::take_truncation(&mut json) { + match context::fit(&app, &mut json).await { + Ok(forgotten) => forgotten, + Err(message) => return context::too_long(&message), + } + } else { + 0 + }; + let response = match structured::goal(&json) { + Ok(Some(goal)) => structured::complete(app, caller, json, goal).await, + Ok(None) => { + let body = if changed { + Bytes::from(json.to_string()) + } else { + body + }; + proxy::inference(app, caller, "/v1/chat/completions", body).await + } + Err(message) => api_error(StatusCode::BAD_REQUEST, "invalid_request_error", &message), + }; + context::mark(response, forgotten) } async fn completions( diff --git a/crates/calcine-gateway/src/security/mod.rs b/crates/calcine-gateway/src/security/mod.rs index e4dde09..58652dd 100644 --- a/crates/calcine-gateway/src/security/mod.rs +++ b/crates/calcine-gateway/src/security/mod.rs @@ -6,12 +6,20 @@ //! use the API even without a CORS preflight. //! 3. A valid API key (`Authorization: Bearer calcine_…`), unless keys are //! turned off, in which case callers are anonymous with limited rights. +//! On the Ollama port, callers without a key get the same limited rights: +//! Ollama clients can't send one. +//! +//! The local network port replaces the `Host` check with the client's +//! address (allowed ranges, too many invalid keys), always needs a key +//! allowed on the network, and never lets it reach local files. //! //! Bodies are then checked per route (see [`sanitize`]). pub mod sanitize; +use std::net::IpAddr; use std::sync::Arc; +use std::time::Instant; use axum::extract::{Request, State}; use axum::http::{HeaderMap, HeaderValue, Method, StatusCode, header}; @@ -20,19 +28,50 @@ use axum::response::{IntoResponse, Response}; use crate::caller::Caller; use crate::error::api_error; +use crate::network::{Peer, allow}; use crate::state::AppState; /// Paths that answer without a key (liveness only, no data). -const PUBLIC_PATHS: &[&str] = &["/v1", "/v1/"]; +const PUBLIC_PATHS: &[&str] = &["/", "/v1", "/v1/", "/api/version"]; + +/// Which socket a request came in on. Set per listener, never from headers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Listener { + /// The main port (18181 by default). + Main, + /// Ollama's port, for apps that only speak Ollama. + Ollama { port: u16 }, + /// The local network port, over HTTPS, for other devices. + Network { port: u16 }, +} pub async fn guard(State(app): State>, mut request: Request, next: Next) -> Response { - if !host_allowed(request.headers(), app.port) { - return api_error( - StatusCode::BAD_REQUEST, - "invalid_request_error", - "unexpected Host header", - ); - } + let listener = request + .extensions() + .get::() + .copied() + .unwrap_or(Listener::Main); + let peer = match listener { + // Other devices: HTTPS, any host name, from allowed addresses only. + Listener::Network { .. } => match admit(&app, &request) { + Ok(peer) => Some(peer), + Err(refused) => return *refused, + }, + Listener::Main | Listener::Ollama { .. } => { + let port = match listener { + Listener::Ollama { port } => port, + _ => app.port, + }; + if !host_allowed(request.headers(), port) { + return api_error( + StatusCode::BAD_REQUEST, + "invalid_request_error", + "unexpected Host header", + ); + } + None + } + }; let origin = request .headers() @@ -56,39 +95,108 @@ pub async fn guard(State(app): State>, mut request: Request, next: } if !PUBLIC_PATHS.contains(&request.uri().path()) { - let caller = match bearer_token(request.headers()) { - Some(token) => match app.keys.authenticate(token) { - Some(caller) => caller, - None => { - return with_cors( - api_error( - StatusCode::UNAUTHORIZED, - "authentication_error", - "invalid API key", - ), - origin.as_deref(), - ); - } - }, - None if !app.settings().require_api_key => Caller::Anonymous, - None => { - return with_cors( - api_error( - StatusCode::UNAUTHORIZED, - "authentication_error", - "missing API key. Create one in Calcine (Server → API keys) and send it as \ - `Authorization: Bearer `.", - ), - origin.as_deref(), - ); + match identify(&app, listener, request.headers(), peer) { + Ok(caller) => { + request.extensions_mut().insert(caller); } - }; - request.extensions_mut().insert(caller); + Err(refused) => return with_cors(*refused, origin.as_deref()), + } } with_cors(next.run(request).await, origin.as_deref()) } +/// The address of a network request, when it may connect at all. +fn admit(app: &AppState, request: &Request) -> Result> { + let Some(Peer(peer)) = request.extensions().get::().copied() else { + return Err(Box::new(api_error( + StatusCode::FORBIDDEN, + "permission_error", + "unknown client address", + ))); + }; + let address = peer.ip(); + let allowed = allow::ranges(&app.settings().network_allowed) + .iter() + .any(|range| range.contains(address)); + if !allowed { + return Err(Box::new(api_error( + StatusCode::FORBIDDEN, + "permission_error", + &format!( + "Calcine doesn't accept connections from {address}. Allow it in Calcine \ + (Settings → Local API → Local network)." + ), + ))); + } + if let Some(wait) = app.limiter.blocked(address, Instant::now()) { + return Err(Box::new(api_error( + StatusCode::TOO_MANY_REQUESTS, + "rate_limit_error", + &format!( + "too many invalid API keys from {address}: try again in {} minutes", + wait.as_secs().div_ceil(60) + ), + ))); + } + Ok(address) +} + +/// Who is calling, or why they're refused. Over the network a key is always +/// needed, it must be allowed there, and it never reaches local files. +fn identify( + app: &AppState, + listener: Listener, + headers: &HeaderMap, + peer: Option, +) -> Result> { + let network = matches!(listener, Listener::Network { .. }); + let invalid = || { + if let Some(address) = peer { + app.limiter.failed(address, Instant::now()); + } + Box::new(api_error( + StatusCode::UNAUTHORIZED, + "authentication_error", + "invalid API key", + )) + }; + let Some(token) = bearer_token(headers) else { + return match listener { + Listener::Ollama { .. } => Ok(Caller::OllamaClient), + Listener::Main if !app.settings().require_api_key => Ok(Caller::Anonymous), + _ => Err(Box::new(api_error( + StatusCode::UNAUTHORIZED, + "authentication_error", + "missing API key. Create one in Calcine (Server → API keys) and send it as \ + `Authorization: Bearer `.", + ))), + }; + }; + match app.keys.authenticate(token) { + // Calcine's own token is for its window on this PC. + Some(Caller::Calcine) if network => Err(invalid()), + Some(Caller::App(mut key)) if network => { + if let Some(address) = peer { + app.limiter.succeeded(address); + } + if !key.network { + return Err(Box::new(api_error( + StatusCode::FORBIDDEN, + "permission_error", + "this API key can't be used from other devices. Allow it in Calcine \ + (Server → API keys).", + ))); + } + // A path from another device would name a file on this PC. + key.allow_local_files = false; + Ok(Caller::App(key)) + } + Some(caller) => Ok(caller), + None => Err(invalid()), + } +} + /// A browser origin: `http(s)://host[:port]`, nothing after. pub(crate) fn is_origin(value: &str) -> bool { let Some(rest) = value @@ -160,6 +268,10 @@ fn with_cors(mut response: Response, origin: Option<&str>) -> Response { header::ACCESS_CONTROL_ALLOW_METHODS, HeaderValue::from_static("GET, POST, DELETE, OPTIONS"), ); + headers.insert( + header::ACCESS_CONTROL_EXPOSE_HEADERS, + HeaderValue::from_static(crate::context::FORGOTTEN_HEADER), + ); headers.insert( header::ACCESS_CONTROL_MAX_AGE, HeaderValue::from_static("600"), diff --git a/crates/calcine-gateway/src/settings.rs b/crates/calcine-gateway/src/settings.rs index 6795ef9..720750d 100644 --- a/crates/calcine-gateway/src/settings.rs +++ b/crates/calcine-gateway/src/settings.rs @@ -17,6 +17,19 @@ pub struct GatewaySettings { pub require_api_key: bool, /// Extra browser origins allowed to call the API (e.g. a local web UI). pub allowed_origins: Vec, + /// Also answer on Ollama's port, where callers need no key (inference + /// only), for apps that only speak Ollama. + pub ollama_port_enabled: bool, + /// Ollama's port, 11434. Configurable for tests. + pub ollama_port: u16, + /// Also answer other devices on the local network, over HTTPS, with + /// keys allowed on the network. + pub network_enabled: bool, + /// The network port, 18443 by default. + pub network_port: u16, + /// Who may connect over the network: addresses or ranges like + /// `192.168.1.0/24`. Empty means private networks. + pub network_allowed: Vec, } impl Default for GatewaySettings { @@ -25,6 +38,11 @@ impl Default for GatewaySettings { port: DEFAULT_PORT, require_api_key: true, allowed_origins: Vec::new(), + ollama_port_enabled: false, + ollama_port: crate::ollama::OLLAMA_PORT, + network_enabled: false, + network_port: crate::network::DEFAULT_PORT, + network_allowed: Vec::new(), } } } diff --git a/crates/calcine-gateway/src/state.rs b/crates/calcine-gateway/src/state.rs index e807ad1..2913070 100644 --- a/crates/calcine-gateway/src/state.rs +++ b/crates/calcine-gateway/src/state.rs @@ -11,6 +11,7 @@ use tokio::sync::{Semaphore, watch}; use crate::keys::KeyStore; use crate::log::RequestLog; +use crate::network::limiter::Limiter; use crate::settings::GatewaySettings; /// What the Server page shows. @@ -26,6 +27,28 @@ pub struct GatewayStatus { pub active_requests: u32, pub queued_requests: u32, pub require_api_key: bool, + /// Where Ollama apps connect, when the Ollama port is on and listening. + pub ollama_url: Option, + /// Why the Ollama port isn't listening (taken by Ollama itself, ...). + pub ollama_error: Option, + /// Where other devices connect, when the network port is listening. + pub network: Option, + /// Why the network port isn't listening. + pub network_error: Option, +} + +/// The network port, as other devices see it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Type)] +#[serde(rename_all = "camelCase")] +pub struct NetworkStatus { + /// Base URL for other devices, e.g. `https://192.168.1.20:18443/v1`. + pub base_url: String, + /// This PC's name on the network, for clients that prefer it. + pub host_name: Option, + /// SHA-256 of the certificate, for clients to pin. + pub fingerprint: String, + /// The certificate as a `.pem` file, for clients to import. + pub certificate_path: String, } #[derive(Debug)] @@ -37,6 +60,8 @@ pub struct AppState { /// GenieX runs one inference at a time; waiting here makes the queue visible. pub queue: Arc, pub port: u16, + /// Turns away network addresses that send too many invalid keys. + pub limiter: Limiter, settings: Arc>, builtin_origins: Vec, active: AtomicU32, @@ -61,6 +86,7 @@ impl AppState { http: reqwest::Client::new(), queue: Arc::new(Semaphore::new(1)), port, + limiter: Limiter::default(), settings, builtin_origins, active: AtomicU32::new(0), diff --git a/crates/calcine-gateway/src/structured/mod.rs b/crates/calcine-gateway/src/structured/mod.rs new file mode 100644 index 0000000..0873512 --- /dev/null +++ b/crates/calcine-gateway/src/structured/mod.rs @@ -0,0 +1,593 @@ +//! Replies of a required shape, for every model, the NPU included: +//! structured output (`response_format`) and forced tool calls +//! (`tool_choice`). +//! +//! GenieX 0.8 ignores both in API requests, so Calcine does it: it asks for +//! JSON (with the schema, if any) or for a tool call, checks the reply, and +//! asks the model to fix it when it doesn't match, a few times at most. The +//! client gets the usual OpenAI shape, streamed or not. + +mod schema; +pub mod tools; + +use std::sync::Arc; + +use axum::body::Bytes; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use serde_json::{Map, Value, json}; + +use crate::caller::Caller; +use crate::error::{api_error, upstream_message}; +use crate::proxy::{self, elapsed_ms}; +use crate::state::AppState; + +pub use schema::{extract, filled, validate}; + +/// Tries after the first answer. +const RETRIES: usize = 2; + +/// Introduces the filled-in example in instructions (the mock backend looks +/// for it). The example's strings are ``. +pub const SHAPE: &str = "Use this shape, replacing each with a real value:"; + +/// The JSON the client asked for. +#[derive(Debug, Clone, PartialEq)] +pub enum Format { + /// `{"type": "json_object"}`: any JSON object. + Object, + /// `{"type": "json_schema", "json_schema": {"schema": …}}`. + Schema { name: Option, schema: Value }, +} + +/// What a reply must be. +#[derive(Debug, Clone, PartialEq)] +pub enum Goal { + /// JSON content in this format. + Json(Format), + /// A call to one of the tools. + Tool(tools::Forced), +} + +/// The structured format a chat request asks for, if any. +pub fn requested(body: &Value) -> Option { + let format = body.get("response_format")?; + match format.get("type").and_then(Value::as_str)? { + "json_object" => Some(Format::Object), + "json_schema" => { + let spec = format.get("json_schema")?; + Some(Format::Schema { + name: spec.get("name").and_then(Value::as_str).map(str::to_owned), + schema: spec.get("schema").cloned().unwrap_or(json!({})), + }) + } + _ => None, + } +} + +/// What a chat request requires of the reply, if anything: a forced tool +/// call comes first. An error when `tool_choice` doesn't make sense. +pub fn goal(body: &Value) -> Result, String> { + if let Some(forced) = tools::forced(body)? { + return Ok(Some(Goal::Tool(forced))); + } + Ok(requested(body).map(Goal::Json)) +} + +/// The instruction added to the system prompt. +pub fn instruction(format: &Format) -> String { + let base = "Reply with a single valid JSON value and nothing else: no explanation, \ + no Markdown, no code fence."; + match format { + Format::Object => format!("{base} The value must be a JSON object."), + // Small models follow a filled-in example better than a schema. + Format::Schema { name, schema } => format!( + "{base} {SHAPE}\n{}\n\nIt must match this JSON Schema{}:\n{}", + schema::example(schema), + name.as_ref() + .map(|name| format!(" (\"{name}\")")) + .unwrap_or_default(), + serde_json::to_string_pretty(schema).unwrap_or_default() + ), + } +} + +/// Why `value` isn't acceptable for `format`, if it isn't. +pub fn check(value: &Value, format: &Format) -> Result<(), String> { + match format { + Format::Object if !value.is_object() => Err("the reply must be a JSON object".into()), + Format::Object => Ok(()), + Format::Schema { schema, .. } => { + validate(value, schema).and_then(|()| filled(value, schema)) + } + } +} + +impl Goal { + /// The request GenieX gets: the instruction in the system prompt, not + /// streamed, without what GenieX would ignore anyway. + pub fn upstream_request(&self, body: &Value) -> Value { + let mut request = body.clone(); + let Some(object) = request.as_object_mut() else { + return request; + }; + for key in [ + "response_format", + "tool_choice", + "stream_options", + "parallel_tool_calls", + ] { + object.remove(key); + } + object.insert("stream".into(), json!(false)); + // Reasoning goes to `reasoning_content`, out of the reply. + object.insert("reasoning_format".into(), json!("deepseek")); + let instruction = match self { + Self::Json(format) => instruction(format), + // The call is asked for as JSON, in the instruction. + Self::Tool(forced) => { + object.remove("tools"); + tools::instruction(forced) + } + }; + let mut messages = object + .get_mut("messages") + .and_then(Value::as_array_mut) + .map(std::mem::take) + .unwrap_or_default(); + match messages.first_mut() { + Some(first) if first.get("role").and_then(Value::as_str) == Some("system") => { + let current = first.get("content").and_then(Value::as_str).unwrap_or(""); + first["content"] = json!(format!("{current}\n\n{instruction}").trim().to_owned()); + } + _ => messages.insert(0, json!({ "role": "system", "content": instruction })), + } + object.insert("messages".into(), Value::Array(messages)); + request + } + + /// The reply's message as the client gets it, or why it doesn't do. + fn accept(&self, message: &Value) -> Result { + match self { + Self::Json(format) => { + let text = message.get("content").and_then(Value::as_str).unwrap_or(""); + let value = extract(text).ok_or_else(|| "the reply isn't JSON".to_owned())?; + check(&value, format)?; + let mut accepted = message.clone(); + accepted["content"] = json!(value.to_string()); + if let Some(object) = accepted.as_object_mut() { + object.remove("tool_calls"); + } + Ok(accepted) + } + Self::Tool(forced) => tools::check(message, forced), + } + } + + /// What to ask after showing the model what's wrong. + const fn fix_hint(&self) -> &'static str { + match self { + Self::Json(_) => "Reply again with only the corrected JSON.", + Self::Tool(_) => "Reply again with only the corrected JSON call.", + } + } + + const fn failure(&self) -> &'static str { + match self { + Self::Json(_) => "the model didn't produce valid JSON", + Self::Tool(_) => "the model didn't call a tool as required", + } + } +} + +/// How the client wants the answer delivered. +#[derive(Debug, Clone, Copy)] +struct Delivery { + stream: bool, + include_usage: bool, +} + +/// Why the reply couldn't be had, to answer in the client's error format. +#[derive(Debug)] +pub struct Failure { + pub status: StatusCode, + pub kind: &'static str, + pub message: String, +} + +/// Answer a chat request that sets a goal. +pub async fn complete(app: Arc, caller: Caller, body: Value, goal: Goal) -> Response { + let delivery = Delivery { + stream: body.get("stream").and_then(Value::as_bool).unwrap_or(false), + include_usage: body + .pointer("/stream_options/include_usage") + .and_then(Value::as_bool) + .unwrap_or(false), + }; + match generate( + &app, + &caller, + &body, + &goal, + "/v1/chat/completions", + delivery.stream, + ) + .await + { + Ok(answer) => deliver(&answer, delivery), + Err(failure) => api_error(failure.status, failure.kind, &failure.message), + } +} + +/// Ask GenieX until the reply meets `goal`: the completion, with the +/// checked message. The request log shows `logged_path`. +pub async fn generate( + app: &Arc, + caller: &Caller, + body: &Value, + goal: &Goal, + logged_path: &'static str, + client_streams: bool, +) -> Result { + let mut request = goal.upstream_request(body); + let bytes = Bytes::from(request.to_string()); + let (exchange, response) = proxy::send( + app, + caller, + logged_path, + "/v1/chat/completions", + &request, + bytes, + ) + .await + .map_err(|refused| Failure { + status: refused.status, + kind: refused.kind, + message: refused.message, + })?; + exchange + .log + .update(exchange.id, |entry| entry.stream = client_streams); + + let mut response = Some(response); + let mut usage = (0_u64, 0_u64); + let mut last_error = String::new(); + for attempt in 0..=RETRIES { + let reply = match response.take() { + Some(first) => Ok(first), + None => retry(app, &request).await, + }; + let completion = read_completion(reply).await.map_err(|(status, message)| { + exchange.log_failure(status, &message); + Failure { + status, + kind: "server_error", + message, + } + })?; + usage.0 += token_count(&completion, "prompt_tokens"); + usage.1 += token_count(&completion, "completion_tokens"); + let message = completion + .pointer("/choices/0/message") + .cloned() + .unwrap_or_default(); + match goal.accept(&message) { + Ok(accepted) => { + exchange.log.update(exchange.id, |entry| { + entry.status = Some(200); + entry.duration_ms = Some(elapsed_ms(exchange.started)); + entry.prompt_tokens = Some(usage.0); + entry.completion_tokens = Some(usage.1); + entry.tokens_per_second = completion + .pointer("/timings/predicted_per_second") + .and_then(Value::as_f64); + }); + return Ok(finished(completion, accepted, usage)); + } + Err(reason) => { + tracing::info!(attempt, %reason, "the reply didn't meet the goal, asking again"); + ask_to_fix(&mut request, &message, &reason, goal.fix_hint()); + last_error = reason; + } + } + } + let message = format!( + "{} after {} tries: {last_error}", + goal.failure(), + RETRIES + 1 + ); + exchange.log_failure(StatusCode::UNPROCESSABLE_ENTITY, &message); + Err(Failure { + status: StatusCode::UNPROCESSABLE_ENTITY, + kind: "invalid_response", + message, + }) +} + +/// The completion GenieX returned, or the error to answer with. +async fn read_completion( + reply: Result, +) -> Result { + let reply = reply.map_err(|message| (StatusCode::BAD_GATEWAY, message))?; + let status = reply.status(); + let bytes = reply.bytes().await.unwrap_or_default(); + if !status.is_success() { + return Err((status, upstream_message(&bytes))); + } + Ok(serde_json::from_slice(&bytes).unwrap_or_default()) +} + +fn token_count(completion: &Value, field: &str) -> u64 { + completion + .get("usage") + .and_then(|usage| usage.get(field)) + .and_then(Value::as_u64) + .unwrap_or(0) +} + +/// Show the model its reply and what's wrong with it. +fn ask_to_fix(request: &mut Value, message: &Value, reason: &str, hint: &str) { + let mut reply = json!({ + "role": "assistant", + "content": message.get("content").and_then(Value::as_str).unwrap_or(""), + }); + if let Some(calls) = message + .get("tool_calls") + .filter(|calls| calls.as_array().is_some_and(|calls| !calls.is_empty())) + { + reply["tool_calls"] = calls.clone(); + } + if let Some(messages) = request["messages"].as_array_mut() { + messages.push(reply); + messages.push(json!({ + "role": "user", + "content": format!("That reply isn't valid: {reason}. {hint}"), + })); + } +} + +fn deliver(answer: &Value, delivery: Delivery) -> Response { + if delivery.stream { + as_stream(answer, delivery.include_usage) + } else { + ( + [(header::CONTENT_TYPE, "application/json")], + answer.to_string(), + ) + .into_response() + } +} + +/// Ask again, within the same queue turn. +async fn retry(app: &AppState, request: &Value) -> Result { + let upstream = app + .services + .server + .ensure_running() + .await + .map_err(|err| format!("GenieX couldn't start: {err}"))?; + app.http + .post(format!("{upstream}/v1/chat/completions")) + .header(header::CONTENT_TYPE, "application/json") + .body(request.to_string()) + .send() + .await + .map_err(|err| format!("couldn't reach GenieX: {err}")) +} + +/// The last completion, with the checked message and the usage of every +/// try. +fn finished(mut completion: Value, message: Value, usage: (u64, u64)) -> Value { + let calls_tools = message + .get("tool_calls") + .and_then(Value::as_array) + .is_some_and(|calls| !calls.is_empty()); + if let Some(choice) = completion.pointer_mut("/choices/0") { + choice["message"] = message; + choice["finish_reason"] = json!(if calls_tools { "tool_calls" } else { "stop" }); + } + completion["usage"] = json!({ + "prompt_tokens": usage.0, + "completion_tokens": usage.1, + "total_tokens": usage.0 + usage.1, + }); + completion +} + +/// The answer as server-sent events, for clients that asked for a stream. +fn as_stream(answer: &Value, include_usage: bool) -> Response { + let base = |choices: Value| { + let mut chunk = Map::new(); + for key in ["id", "created", "model"] { + if let Some(value) = answer.get(key) { + chunk.insert(key.into(), value.clone()); + } + } + chunk.insert("object".into(), json!("chat.completion.chunk")); + chunk.insert("choices".into(), choices); + Value::Object(chunk) + }; + let mut events = vec![ + base(json!([{ "index": 0, "delta": delta(answer), "finish_reason": null }])), + base(json!([{ + "index": 0, + "delta": {}, + "finish_reason": answer.pointer("/choices/0/finish_reason").cloned().unwrap_or(json!("stop")), + }])), + ]; + if include_usage { + let mut last = base(json!([])); + last["usage"] = answer["usage"].clone(); + if let Some(timings) = answer.get("timings") { + last["timings"] = timings.clone(); + } + events.push(last); + } + let mut body = String::new(); + for event in &events { + body.push_str("data: "); + body.push_str(&event.to_string()); + body.push_str("\n\n"); + } + body.push_str("data: [DONE]\n\n"); + ( + [ + (header::CONTENT_TYPE, "text/event-stream"), + (header::CACHE_CONTROL, "no-cache"), + ], + body, + ) + .into_response() +} + +/// The whole message as one delta; tool calls get their stream index. +fn delta(answer: &Value) -> Value { + let message = answer + .pointer("/choices/0/message") + .cloned() + .unwrap_or_default(); + let mut delta = json!({ "role": "assistant", "content": message["content"] }); + if let Some(reasoning) = message + .get("reasoning_content") + .filter(|value| value.is_string()) + { + delta["reasoning_content"] = reasoning.clone(); + } + if let Some(calls) = message.get("tool_calls").and_then(Value::as_array) { + let calls: Vec = calls + .iter() + .enumerate() + .map(|(index, call)| { + let mut call = call.clone(); + call["index"] = json!(index); + call + }) + .collect(); + delta["tool_calls"] = Value::Array(calls); + } + delta +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_requested_format() { + assert_eq!( + requested(&json!({ "response_format": { "type": "json_object" } })), + Some(Format::Object) + ); + let schema = json!({ "type": "object" }); + assert_eq!( + requested( + &json!({ "response_format": { "type": "json_schema", "json_schema": { "name": "p", "schema": schema } } }) + ), + Some(Format::Schema { + name: Some("p".into()), + schema + }) + ); + assert_eq!( + requested(&json!({ "response_format": { "type": "text" } })), + None + ); + assert_eq!(requested(&json!({})), None); + } + + #[test] + fn a_forced_call_comes_before_the_format() { + let body = json!({ + "tools": [{ "type": "function", "function": { "name": "f" } }], + "tool_choice": "required", + "response_format": { "type": "json_object" }, + }); + assert!(matches!(goal(&body), Ok(Some(Goal::Tool(_))))); + assert!(matches!( + goal(&json!({ "response_format": { "type": "json_object" } })), + Ok(Some(Goal::Json(Format::Object))) + )); + assert_eq!(goal(&json!({})), Ok(None)); + } + + #[test] + fn adds_the_instruction_to_the_system_prompt() { + let body = json!({ + "model": "m", + "stream": true, + "stream_options": { "include_usage": true }, + "response_format": { "type": "json_object" }, + "messages": [{ "role": "system", "content": "Be brief." }, { "role": "user", "content": "Hi" }], + }); + let request = Goal::Json(Format::Object).upstream_request(&body); + assert_eq!(request["stream"], false); + assert!(request.get("response_format").is_none()); + let system = request["messages"][0]["content"].as_str().unwrap(); + assert!(system.starts_with("Be brief.\n\nReply with a single valid JSON value")); + assert_eq!(request["messages"][1]["content"], "Hi"); + + let bare = Goal::Json(Format::Schema { + name: None, + schema: json!({ "type": "array" }), + }) + .upstream_request(&json!({ "messages": [{ "role": "user", "content": "Hi" }] })); + assert_eq!(bare["messages"][0]["role"], "system"); + assert!( + bare["messages"][0]["content"] + .as_str() + .unwrap() + .contains("\"array\"") + ); + } + + #[test] + fn offers_only_the_named_tool() { + let body = json!({ + "messages": [{ "role": "user", "content": "Hi" }], + "tools": [ + { "type": "function", "function": { "name": "a" } }, + { "type": "function", "function": { "name": "b" } }, + ], + "tool_choice": { "type": "function", "function": { "name": "b" } }, + }); + let Ok(Some(goal)) = goal(&body) else { + panic!("expected a forced call") + }; + let request = goal.upstream_request(&body); + assert!(request.get("tool_choice").is_none() && request.get("tools").is_none()); + let system = request["messages"][0]["content"].as_str().unwrap(); + assert!(system.starts_with("Call the b tool now")); + assert!(system.contains("- b") && !system.contains("- a")); + } + + #[test] + fn objects_must_be_objects() { + assert!(check(&json!({ "a": 1 }), &Format::Object).is_ok()); + assert!(check(&json!([1]), &Format::Object).is_err()); + } + + #[test] + fn streams_the_checked_answer() { + let completion = json!({ "id": "x", "model": "m", "choices": [{ "message": { "role": "assistant", "content": "junk" } }] }); + let accepted = Goal::Json(Format::Object) + .accept(&json!({ "role": "assistant", "content": "Here: {\"a\": 1}" })) + .unwrap(); + let answer = finished(completion, accepted, (10, 5)); + assert_eq!(answer["choices"][0]["message"]["content"], "{\"a\":1}"); + assert_eq!(answer["choices"][0]["finish_reason"], "stop"); + assert_eq!(answer["usage"]["total_tokens"], 15); + let _ = as_stream(&answer, true); + } + + #[test] + fn streams_tool_calls_with_their_index() { + let call = json!({ "id": "c", "type": "function", "function": { "name": "f", "arguments": "{}" } }); + let answer = finished( + json!({ "choices": [{ "message": {} }] }), + json!({ "role": "assistant", "content": null, "tool_calls": [call] }), + (1, 1), + ); + assert_eq!(answer["choices"][0]["finish_reason"], "tool_calls"); + assert_eq!(delta(&answer)["tool_calls"][0]["index"], 0); + } +} diff --git a/crates/calcine-gateway/src/structured/schema.rs b/crates/calcine-gateway/src/structured/schema.rs new file mode 100644 index 0000000..71f04e7 --- /dev/null +++ b/crates/calcine-gateway/src/structured/schema.rs @@ -0,0 +1,403 @@ +//! Reading JSON out of a model's reply and checking it against a JSON Schema +//! subset: `type`, `enum`, `const`, `properties`, `required`, +//! `additionalProperties`, `items`, `minItems`/`maxItems`, +//! `minLength`/`maxLength`, `minimum`/`maximum`, `anyOf`/`oneOf`. Other +//! keywords are accepted without being checked. + +use serde_json::Value; + +/// The JSON in a reply: the whole text, a fenced block, or the first object +/// or array in it. +pub fn extract(text: &str) -> Option { + let text = text.trim(); + if let Ok(value) = serde_json::from_str(text) { + return Some(value); + } + if let Some(fenced) = fenced_block(text) + && let Ok(value) = serde_json::from_str(fenced.trim()) + { + return Some(value); + } + let start = text.find(['{', '['])?; + let close = if text[start..].starts_with('{') { + '}' + } else { + ']' + }; + let end = text.rfind(close)?; + (end > start) + .then(|| serde_json::from_str(&text[start..=end]).ok()) + .flatten() +} + +fn fenced_block(text: &str) -> Option<&str> { + let start = text.find("```")?; + let after = &text[start + 3..]; + let body_start = after.find('\n')? + 1; + let body = &after[body_start..]; + let end = body.find("```")?; + Some(&body[..end]) +} + +/// A placeholder value with the schema's shape: every property, one item +/// per array, the first allowed value of an enum. Strings are named after +/// what they hold, `""`: small models replace those, while they copy +/// `"..."` as it is. +pub fn example(schema: &Value) -> Value { + named_example(schema, "value") +} + +fn named_example(schema: &Value, label: &str) -> Value { + let Some(schema) = schema.as_object() else { + return Value::Null; + }; + if let Some(first) = schema + .get("enum") + .and_then(Value::as_array) + .and_then(|values| values.first()) + { + return first.clone(); + } + if let Some(constant) = schema.get("const") { + return constant.clone(); + } + if let Some(first) = ["anyOf", "oneOf"] + .iter() + .find_map(|keyword| schema.get(*keyword).and_then(Value::as_array)) + .and_then(|options| options.first()) + { + return named_example(first, label); + } + let kind = match schema.get("type") { + Some(Value::String(kind)) => kind.as_str(), + Some(Value::Array(kinds)) => kinds + .iter() + .filter_map(Value::as_str) + .find(|kind| *kind != "null") + .unwrap_or("null"), + _ if schema.contains_key("properties") => "object", + _ if schema.contains_key("items") => "array", + _ => "string", + }; + match kind { + "object" => Value::Object( + schema + .get("properties") + .and_then(Value::as_object) + .map(|properties| { + properties + .iter() + .map(|(key, property)| (key.clone(), named_example(property, key))) + .collect() + }) + .unwrap_or_default(), + ), + "array" => Value::Array(vec![named_example( + schema.get("items").unwrap_or(&Value::Null), + &format!("{label} item"), + )]), + "integer" | "number" => Value::from(0), + "boolean" => Value::Bool(false), + "null" => Value::Null, + _ => Value::String(format!("<{label}>")), + } +} + +/// `Ok` when `value` matches `schema`, else where and why it doesn't. +pub fn validate(value: &Value, schema: &Value) -> Result<(), String> { + check(value, schema, "$") +} + +/// `Ok` when no string in `value` is a placeholder from `schema`'s example: +/// small models sometimes copy the example as it is. +pub fn filled(value: &Value, schema: &Value) -> Result<(), String> { + let mut placeholders = Vec::new(); + strings(&example(schema), &mut placeholders); + copied(value, &placeholders, "$").map_or(Ok(()), |(path, placeholder)| { + Err(format!( + "{path} is still the placeholder \"{placeholder}\", write a real value" + )) + }) +} + +fn strings(value: &Value, found: &mut Vec) { + match value { + Value::String(text) if text.starts_with('<') => found.push(text.clone()), + Value::Array(items) => items.iter().for_each(|item| strings(item, found)), + Value::Object(fields) => fields.values().for_each(|field| strings(field, found)), + _ => {} + } +} + +fn copied(value: &Value, placeholders: &[String], path: &str) -> Option<(String, String)> { + match value { + Value::String(text) if placeholders.contains(text) => Some((path.to_owned(), text.clone())), + Value::Array(items) => items + .iter() + .enumerate() + .find_map(|(index, item)| copied(item, placeholders, &format!("{path}[{index}]"))), + Value::Object(fields) => fields + .iter() + .find_map(|(key, field)| copied(field, placeholders, &format!("{path}.{key}"))), + _ => None, + } +} + +type Schema = serde_json::Map; + +fn check(value: &Value, schema: &Value, path: &str) -> Result<(), String> { + let Value::Object(schema) = schema else { + // `true` and anything that isn't a schema object allow everything. + return if *schema == Value::Bool(false) { + Err(format!("{path} isn't allowed")) + } else { + Ok(()) + }; + }; + check_shape(value, schema, path)?; + match value { + Value::Object(object) => check_object(object, schema, path), + Value::Array(items) => check_array(items, schema, path), + Value::String(text) => check_length(text.chars().count(), schema, path), + Value::Number(number) => check_range(number.as_f64().unwrap_or(0.0), schema, path), + Value::Null | Value::Bool(_) => Ok(()), + } +} + +/// `anyOf`/`oneOf`, `enum`, `const` and `type`. +fn check_shape(value: &Value, schema: &Schema, path: &str) -> Result<(), String> { + for keyword in ["anyOf", "oneOf"] { + if let Some(options) = schema.get(keyword).and_then(Value::as_array) + && !options + .iter() + .any(|option| check(value, option, path).is_ok()) + { + return Err(format!("{path} matches none of the allowed shapes")); + } + } + if let Some(allowed) = schema.get("enum").and_then(Value::as_array) + && !allowed.contains(value) + { + return Err(format!( + "{path} must be one of {}", + Value::Array(allowed.clone()) + )); + } + if let Some(constant) = schema.get("const") + && constant != value + { + return Err(format!("{path} must be {constant}")); + } + if let Some(kind) = schema.get("type") { + let kinds: Vec<&str> = match kind { + Value::String(kind) => vec![kind.as_str()], + Value::Array(kinds) => kinds.iter().filter_map(Value::as_str).collect(), + _ => Vec::new(), + }; + if !kinds.is_empty() && !kinds.iter().any(|kind| is_type(value, kind)) { + return Err(format!( + "{path} must be {}, not {}", + kinds.join(" or "), + type_of(value) + )); + } + } + Ok(()) +} + +fn check_object( + object: &serde_json::Map, + schema: &Schema, + path: &str, +) -> Result<(), String> { + if let Some(required) = schema.get("required").and_then(Value::as_array) { + for key in required.iter().filter_map(Value::as_str) { + if !object.contains_key(key) { + return Err(format!("{path}.{key} is missing")); + } + } + } + let properties = schema.get("properties").and_then(Value::as_object); + for (key, item) in object { + let item_path = format!("{path}.{key}"); + match properties.and_then(|properties| properties.get(key)) { + Some(property) => check(item, property, &item_path)?, + None => match schema.get("additionalProperties") { + Some(Value::Bool(false)) => { + return Err(format!("{item_path} isn't an expected field")); + } + Some(extra @ Value::Object(_)) => check(item, extra, &item_path)?, + _ => {} + }, + } + } + Ok(()) +} + +fn check_array(items: &[Value], schema: &Schema, path: &str) -> Result<(), String> { + let length = items.len() as u64; + if let Some(min) = schema.get("minItems").and_then(Value::as_u64) + && length < min + { + return Err(format!("{path} needs at least {min} items")); + } + if let Some(max) = schema.get("maxItems").and_then(Value::as_u64) + && length > max + { + return Err(format!("{path} allows at most {max} items")); + } + if let Some(item_schema) = schema.get("items") { + for (index, item) in items.iter().enumerate() { + check(item, item_schema, &format!("{path}[{index}]"))?; + } + } + Ok(()) +} + +fn check_length(length: usize, schema: &Schema, path: &str) -> Result<(), String> { + let length = length as u64; + if let Some(min) = schema.get("minLength").and_then(Value::as_u64) + && length < min + { + return Err(format!("{path} needs at least {min} characters")); + } + if let Some(max) = schema.get("maxLength").and_then(Value::as_u64) + && length > max + { + return Err(format!("{path} allows at most {max} characters")); + } + Ok(()) +} + +fn check_range(number: f64, schema: &Schema, path: &str) -> Result<(), String> { + if let Some(min) = schema.get("minimum").and_then(Value::as_f64) + && number < min + { + return Err(format!("{path} must be at least {min}")); + } + if let Some(max) = schema.get("maximum").and_then(Value::as_f64) + && number > max + { + return Err(format!("{path} must be at most {max}")); + } + Ok(()) +} + +fn is_type(value: &Value, kind: &str) -> bool { + match kind { + "object" => value.is_object(), + "array" => value.is_array(), + "string" => value.is_string(), + "boolean" => value.is_boolean(), + "null" => value.is_null(), + "number" => value.is_number(), + "integer" => { + value.is_i64() + || value.is_u64() + || value.as_f64().is_some_and(|number| number.fract() == 0.0) + } + _ => true, + } +} + +fn type_of(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "a boolean", + Value::Number(_) => "a number", + Value::String(_) => "a string", + Value::Array(_) => "an array", + Value::Object(_) => "an object", + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn extracts_json_from_chatty_replies() { + assert_eq!(extract(r#"{"a": 1}"#), Some(json!({ "a": 1 }))); + assert_eq!( + extract("Sure!\n```json\n{\"a\": [1, 2]}\n```\nAnything else?"), + Some(json!({ "a": [1, 2] })) + ); + assert_eq!( + extract("Here it is: {\"name\": \"Mira\"} hope it helps"), + Some(json!({ "name": "Mira" })) + ); + assert_eq!(extract("[1, 2, 3]"), Some(json!([1, 2, 3]))); + assert_eq!(extract("no json here"), None); + } + + #[test] + fn validates_objects() { + let schema = json!({ + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 1 }, + "age": { "type": "integer", "minimum": 0 }, + "tags": { "type": "array", "items": { "enum": ["a", "b"] } }, + }, + "required": ["name", "age"], + "additionalProperties": false, + }); + assert!( + validate( + &json!({ "name": "Mira", "age": 17, "tags": ["a"] }), + &schema + ) + .is_ok() + ); + assert_eq!( + validate(&json!({ "name": "Mira" }), &schema).unwrap_err(), + "$.age is missing" + ); + assert_eq!( + validate(&json!({ "name": "Mira", "age": "17" }), &schema).unwrap_err(), + "$.age must be integer, not a string" + ); + assert_eq!( + validate(&json!({ "name": "Mira", "age": 17, "extra": 1 }), &schema).unwrap_err(), + "$.extra isn't an expected field" + ); + assert!( + validate( + &json!({ "name": "Mira", "age": 17, "tags": ["c"] }), + &schema + ) + .unwrap_err() + .starts_with("$.tags[0] must be one of") + ); + assert!(validate(&json!({ "name": "Mira", "age": -1 }), &schema).is_err()); + } + + #[test] + fn builds_an_example_with_the_schema_shape() { + let schema = json!({ + "type": "object", + "properties": { + "name": { "type": "string" }, + "age": { "type": "integer" }, + "mood": { "enum": ["happy", "sad"] }, + "tags": { "type": "array", "items": { "type": "string" } }, + "home": { "type": "object", "properties": { "city": { "type": "string" } } }, + }, + }); + assert_eq!( + example(&schema), + json!({ "name": "", "age": 0, "mood": "happy", "tags": [""], "home": { "city": "" } }) + ); + } + + #[test] + fn handles_unions_and_whole_numbers() { + let schema = json!({ "anyOf": [{ "type": "string" }, { "type": "null" }] }); + assert!(validate(&json!(null), &schema).is_ok()); + assert!(validate(&json!(3), &schema).is_err()); + assert!(validate(&json!(3.0), &json!({ "type": "integer" })).is_ok()); + assert!(validate(&json!([1]), &json!({ "type": "array", "minItems": 2 })).is_err()); + } +} diff --git a/crates/calcine-gateway/src/structured/tools.rs b/crates/calcine-gateway/src/structured/tools.rs new file mode 100644 index 0000000..4db3c3a --- /dev/null +++ b/crates/calcine-gateway/src/structured/tools.rs @@ -0,0 +1,355 @@ +//! Forced tool calls (`tool_choice`) for every model, the NPU included. +//! +//! GenieX 0.8 gives the model its `tools` and returns the calls it makes, +//! but ignores `tool_choice`. Calcine enforces it: `none` takes the tools +//! away, `required` or a named function asks for the call as JSON and checks +//! it: a tool from the list, with arguments that match its parameters. +//! +//! The call is asked for as JSON, with a filled-in example, rather than +//! through the model's own tool format: on the NPU, Qwen3 4B skips tools it +//! thinks it doesn't need and Qwen3 0.6B ignores the instruction, while both +//! write the JSON reliably. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde_json::{Value, json}; + +use super::SHAPE; +use super::schema::{example, extract, filled, validate}; + +/// A call the client requires. +#[derive(Debug, Clone, PartialEq)] +pub struct Forced { + /// The function to call, or `None` for any of `tools`. + pub function: Option, + /// The tools the model may call. + pub tools: Vec, +} + +/// Whether the request turns tools off (`tool_choice: "none"`). +pub fn refused(body: &Value) -> bool { + body.get("tool_choice").and_then(Value::as_str) == Some("none") +} + +/// The request without its tools, for `tool_choice: "none"`. +pub fn without_tools(body: &Value) -> Value { + let mut request = body.clone(); + if let Some(object) = request.as_object_mut() { + for key in ["tools", "tool_choice", "parallel_tool_calls"] { + object.remove(key); + } + } + request +} + +/// The call a chat request requires, if any; an error when `tool_choice` +/// names a function that isn't in `tools`. +pub fn forced(body: &Value) -> Result, String> { + let tools: Vec = body + .get("tools") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let function = match body.get("tool_choice") { + Some(Value::String(choice)) if choice == "required" => None, + Some(choice @ Value::Object(_)) => { + let Some(name) = choice + .pointer("/function/name") + .or_else(|| choice.get("name")) + .and_then(Value::as_str) + else { + return Err("tool_choice needs a function name".into()); + }; + if !tools.iter().any(|tool| tool_name(tool) == Some(name)) { + return Err(format!("tool_choice names {name}, which isn't in tools")); + } + Some(name.to_owned()) + } + _ => return Ok(None), + }; + if tools.is_empty() { + return Err("tool_choice requires a call but no tools were given".into()); + } + Ok(Some(Forced { function, tools })) +} + +fn tool_name(tool: &Value) -> Option<&str> { + tool.pointer("/function/name").and_then(Value::as_str) +} + +fn parameters(tool: &Value) -> Value { + tool.pointer("/function/parameters") + .cloned() + .unwrap_or(json!({ "type": "object" })) +} + +/// The instruction added to the system prompt (GenieX gets no `tools`). +pub fn instruction(forced: &Forced) -> String { + let offered = offered(forced); + let lead = match &forced.function { + Some(name) => format!("Call the {name} tool now, even if you could answer without it."), + None => "Call one of the tools below now, even if you could answer without it.".to_owned(), + }; + // The name first, as models write calls. + let shape = offered.first().map_or_else(String::new, |tool| { + format!( + r#"{{"name": {}, "arguments": {}}}"#, + json!(tool_name(tool)), + example(¶meters(tool)) + ) + }); + let list: Vec = offered + .iter() + .map(|tool| { + let description = tool + .pointer("/function/description") + .and_then(Value::as_str) + .map(|text| format!(": {text}")) + .unwrap_or_default(); + format!( + "- {}{description}\n Arguments (JSON Schema): {}", + tool_name(tool).unwrap_or(""), + parameters(tool) + ) + }) + .collect(); + format!( + "{lead} Reply with only the call, as a single valid JSON value and nothing else: no \ + explanation, no Markdown, no code fence. {SHAPE}\n{shape}\n\nThe tools:\n{}", + list.join("\n") + ) +} + +/// The tools the model may call: only the named function, when there is one. +fn offered(forced: &Forced) -> Vec<&Value> { + forced + .tools + .iter() + .filter(|tool| { + forced + .function + .as_deref() + .is_none_or(|name| tool_name(tool) == Some(name)) + }) + .collect() +} + +/// The calls in a reply: GenieX's `tool_calls`, or the JSON in its content +/// (one call, or an array of them). +fn calls_in(message: &Value) -> Vec { + if let Some(calls) = message + .get("tool_calls") + .and_then(Value::as_array) + .filter(|calls| !calls.is_empty()) + { + return calls.clone(); + } + let written = message + .get("content") + .and_then(Value::as_str) + .and_then(extract); + let written = match written { + Some(Value::Array(calls)) => calls, + Some(call @ Value::Object(_)) => vec![call], + _ => return Vec::new(), + }; + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_micros()); + written + .iter() + .filter_map(|call| { + let name = call.get("name")?.as_str()?; + let arguments = match call.get("arguments") { + Some(Value::String(text)) => text.clone(), + Some(value) => value.to_string(), + None => "{}".to_owned(), + }; + Some((name.to_owned(), arguments)) + }) + .enumerate() + .map(|(index, (name, arguments))| { + json!({ + "id": format!("call_{stamp:x}{index}"), + "type": "function", + "function": { "name": name, "arguments": arguments }, + }) + }) + .collect() +} + +/// The reply's message with its checked calls, or why it doesn't do. +pub fn check(message: &Value, forced: &Forced) -> Result { + let calls = calls_in(message); + if calls.is_empty() { + return Err(match &forced.function { + Some(name) => format!("the {name} tool wasn't called"), + None => "no tool was called".to_owned(), + }); + } + for call in &calls { + let name = call + .pointer("/function/name") + .and_then(Value::as_str) + .unwrap_or(""); + if let Some(wanted) = &forced.function + && name != wanted + { + return Err(format!("{name} was called instead of {wanted}")); + } + let tool = forced + .tools + .iter() + .find(|tool| tool_name(tool) == Some(name)) + .ok_or_else(|| format!("{name} isn't one of the tools"))?; + let arguments = match call.pointer("/function/arguments") { + Some(Value::String(text)) if text.trim().is_empty() => json!({}), + Some(Value::String(text)) => serde_json::from_str(text) + .map_err(|_| format!("the arguments of {name} aren't valid JSON"))?, + Some(value) => value.clone(), + None => json!({}), + }; + validate(&arguments, ¶meters(tool)) + .and_then(|()| filled(&arguments, ¶meters(tool))) + .map_err(|reason| format!("{name}: {reason}"))?; + } + let mut checked = json!({ "role": "assistant", "content": null, "tool_calls": calls }); + if let Some(reasoning) = message + .get("reasoning_content") + .filter(|reasoning| reasoning.as_str().is_some_and(|text| !text.is_empty())) + { + checked["reasoning_content"] = reasoning.clone(); + } + Ok(checked) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn weather() -> Value { + json!({ "type": "function", "function": { "name": "get_weather", "parameters": { + "type": "object", + "properties": { "city": { "type": "string" } }, + "required": ["city"], + } } }) + } + + fn time() -> Value { + json!({ "type": "function", "function": { "name": "get_time" } }) + } + + fn call(name: &str, arguments: &str) -> Value { + json!({ "role": "assistant", "content": "", "tool_calls": [ + { "id": "call_1", "type": "function", "function": { "name": name, "arguments": arguments } }, + ] }) + } + + #[test] + fn reads_tool_choice() { + let tools = json!([weather(), time()]); + assert_eq!(forced(&json!({ "tools": tools })), Ok(None)); + assert_eq!( + forced(&json!({ "tools": tools, "tool_choice": "auto" })), + Ok(None) + ); + assert_eq!( + forced(&json!({ "tools": tools, "tool_choice": "required" })) + .unwrap() + .unwrap() + .function, + None + ); + let named = json!({ "tools": tools, "tool_choice": { "type": "function", "function": { "name": "get_time" } } }); + let named = forced(&named).unwrap().unwrap(); + assert_eq!(named.function.as_deref(), Some("get_time")); + assert_eq!(offered(&named), vec![&time()]); + + assert!( + forced(&json!({ "tools": tools, "tool_choice": { "function": { "name": "nope" } } })) + .is_err() + ); + assert!(forced(&json!({ "tool_choice": "required" })).is_err()); + } + + #[test] + fn shows_the_call_to_write() { + let any = Forced { + function: None, + tools: vec![weather(), time()], + }; + let instruction = instruction(&any); + assert!(instruction.contains(r#"{"name": "get_weather", "arguments": {"city":""}}"#)); + assert!(instruction.contains("- get_weather\n Arguments (JSON Schema):")); + assert!(instruction.contains("- get_time")); + } + + #[test] + fn none_takes_the_tools_away() { + let body = json!({ "model": "m", "tools": [weather()], "tool_choice": "none" }); + assert!(refused(&body)); + assert_eq!(without_tools(&body), json!({ "model": "m" })); + assert!(!refused(&json!({ "tool_choice": "auto" }))); + } + + #[test] + fn checks_the_calls() { + let any = Forced { + function: None, + tools: vec![weather(), time()], + }; + let checked = check(&call("get_weather", r#"{"city": "Paris"}"#), &any).unwrap(); + assert_eq!(checked["content"], Value::Null); + assert_eq!(checked["tool_calls"][0]["function"]["name"], "get_weather"); + assert!(check(&call("get_time", ""), &any).is_ok()); + + assert_eq!( + check(&json!({ "content": "Sunny." }), &any), + Err("no tool was called".into()) + ); + assert_eq!( + check(&call("get_weather", "{}"), &any), + Err("get_weather: $.city is missing".into()) + ); + assert_eq!( + check(&call("get_weather", "{city"), &any), + Err("the arguments of get_weather aren't valid JSON".into()) + ); + assert_eq!( + check(&call("get_weather", r#"{"city": ""}"#), &any), + Err( + "get_weather: $.city is still the placeholder \"\", write a real value" + .into() + ) + ); + assert_eq!( + check(&call("search", "{}"), &any), + Err("search isn't one of the tools".into()) + ); + + // Calls written as JSON in the content, one or several. + let written = json!({ "content": "```json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}\n```" }); + let checked = check(&written, &any).unwrap(); + assert_eq!(checked["tool_calls"][0]["type"], "function"); + assert_eq!( + checked["tool_calls"][0]["function"]["arguments"], + "{\"city\":\"Paris\"}" + ); + let several = json!({ "content": r#"[{"name": "get_time", "arguments": {}}, {"name": "get_time"}]"# }); + let checked = check(&several, &any).unwrap(); + assert_eq!(checked["tool_calls"].as_array().unwrap().len(), 2); + assert_ne!( + checked["tool_calls"][0]["id"], + checked["tool_calls"][1]["id"] + ); + + let named = Forced { + function: Some("get_time".into()), + tools: vec![weather(), time()], + }; + assert_eq!( + check(&call("get_weather", r#"{"city": "Paris"}"#), &named), + Err("get_weather was called instead of get_time".into()) + ); + } +} diff --git a/crates/calcine-gateway/tests/gateway.rs b/crates/calcine-gateway/tests/gateway.rs index 2d8f1eb..4c20177 100644 --- a/crates/calcine-gateway/tests/gateway.rs +++ b/crates/calcine-gateway/tests/gateway.rs @@ -39,6 +39,7 @@ async fn start(settings: GatewaySettings) -> Harness { keys: Arc::new(KeyStore::in_memory()), settings_path: Some(settings_path), builtin_origins: CALCINE_ORIGINS.iter().map(|o| (*o).to_owned()).collect(), + network_dir: Some(dir.join("network")), }, ); gateway.start().await.unwrap(); @@ -66,6 +67,7 @@ impl Harness { name: "Test app".into(), scopes: scopes.to_vec(), allow_local_files, + network: false, }) .unwrap() .token @@ -310,6 +312,7 @@ async fn a_taken_port_is_reported() { keys: Arc::new(KeyStore::in_memory()), settings_path: Some(path), builtin_origins: vec![], + network_dir: None, }, ); assert!(gateway.start().await.is_err()); @@ -379,3 +382,582 @@ async fn allowed_origins_are_validated_and_take_effect() { // Allowed origin: past the Origin check, stopped by the missing key. assert_eq!(reply.status(), StatusCode::UNAUTHORIZED); } + +fn ollama_chat(stream: bool) -> Value { + json!({ + "model": "qualcomm/Qwen3-0.6B:latest", + "stream": stream, + "messages": [{ "role": "user", "content": "hello" }], + }) +} + +#[tokio::test(flavor = "multi_thread")] +async fn speaks_ollama_on_the_main_port_with_a_key() { + let h = start(any_port()).await; + let version: Value = h + .http + .get(format!("{}/api/version", h.base)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(version["version"].is_string()); + + let anonymous = h + .http + .get(format!("{}/api/tags", h.base)) + .send() + .await + .unwrap(); + assert_eq!(anonymous.status(), StatusCode::UNAUTHORIZED); + + let token = h.key(&[KeyScope::Inference], false); + let tags: Value = h + .http + .get(format!("{}/api/tags", h.base)) + .bearer_auth(&token) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!( + tags["models"] + .as_array() + .unwrap() + .iter() + .any(|m| m["name"] == "qualcomm/Qwen3-4B:W4A16") + ); + + // Streamed by default, as JSON lines ending with statistics. + let response = h + .http + .post(format!("{}/api/chat", h.base)) + .bearer_auth(&token) + .json(&ollama_chat(true)) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/x-ndjson" + ); + let lines: Vec = response + .text() + .await + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + let (last, pieces) = lines.split_last().unwrap(); + assert_eq!(last["done"], true); + assert!(last["eval_count"].as_u64().is_some_and(|count| count > 0)); + let text: String = pieces + .iter() + .filter_map(|piece| piece["message"]["content"].as_str()) + .collect(); + assert!(text.contains("hello")); + + let whole: Value = h + .http + .post(format!("{}/api/chat", h.base)) + .bearer_auth(&token) + .json(&ollama_chat(false)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(whole["done"], true); + assert!( + whole["message"]["content"] + .as_str() + .unwrap() + .contains("hello") + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ollama_generate_and_unsupported_routes() { + let h = start(any_port()).await; + let token = h.key(&[KeyScope::Inference], false); + let generate: Value = h + .http + .post(format!("{}/api/generate", h.base)) + .bearer_auth(&token) + .json(&json!({ "model": "qualcomm/Qwen3-0.6B", "prompt": "hello", "stream": false })) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(generate["response"].as_str().unwrap().contains("hello")); + + let pull = h + .http + .post(format!("{}/api/pull", h.base)) + .bearer_auth(&token) + .json(&json!({ "model": "x" })) + .send() + .await + .unwrap(); + assert_eq!(pull.status(), StatusCode::NOT_IMPLEMENTED); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_ollama_port_needs_no_key_but_only_runs_models() { + let h = start(GatewaySettings { + ollama_port_enabled: true, + ollama_port: 0, + ..any_port() + }) + .await; + let ollama = h + .gateway + .status() + .ollama_url + .expect("Ollama port listening"); + + let root = h.http.get(&ollama).send().await.unwrap(); + assert_eq!(root.text().await.unwrap(), "Ollama is running"); + + let tags = h + .http + .get(format!("{ollama}/api/tags")) + .send() + .await + .unwrap(); + assert_eq!(tags.status(), StatusCode::OK); + + let chat: Value = h + .http + .post(format!("{ollama}/api/chat")) + .json(&ollama_chat(false)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(chat["done"], true); + let entry = h.gateway.requests().into_iter().next().unwrap(); + assert_eq!(entry.client, "Ollama app"); + assert_eq!(entry.path, "/api/chat"); + + let manage = h + .http + .get(format!("{ollama}/calcine/v1/models")) + .send() + .await + .unwrap(); + assert_eq!(manage.status(), StatusCode::FORBIDDEN); + + // The main port still wants a key. + let main = h + .http + .get(format!("{}/api/tags", h.base)) + .send() + .await + .unwrap(); + assert_eq!(main.status(), StatusCode::UNAUTHORIZED); + + // Browser pages are refused here too. + let page = h + .http + .get(format!("{ollama}/api/tags")) + .header(header::ORIGIN, "https://evil.example") + .send() + .await + .unwrap(); + assert_eq!(page.status(), StatusCode::FORBIDDEN); + + h.gateway.set_ollama_port(false).await.unwrap(); + assert!(h.gateway.status().ollama_url.is_none()); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + h.http + .get(format!("{ollama}/api/tags")) + .timeout(Duration::from_secs(2)) + .send() + .await + .is_err() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn enforces_structured_output() { + let h = start(any_port()).await; + let token = h.key(&[KeyScope::Inference], false); + let schema = |required: &str| { + json!({ + "model": "qualcomm/Qwen3-0.6B", + "messages": [{ "role": "user", "content": "hello" }], + "response_format": { "type": "json_schema", "json_schema": { "name": "r", "schema": { + "type": "object", + "properties": { "reply": { "type": "string" } }, + "required": [required], + } } }, + }) + }; + + let ok: Value = h + .chat(Some(&token), &schema("reply")) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let content: Value = + serde_json::from_str(ok["choices"][0]["message"]["content"].as_str().unwrap()).unwrap(); + assert_eq!(content["reply"], "Mock answer"); + + let never = h + .chat(Some(&token), &schema("missing")) + .send() + .await + .unwrap(); + assert_eq!(never.status(), StatusCode::UNPROCESSABLE_ENTITY); + let error: Value = never.json().await.unwrap(); + assert!( + error["error"]["message"] + .as_str() + .unwrap() + .contains("$.missing is missing") + ); + + // Streamed clients get the checked JSON as server-sent events. + let mut streamed = schema("reply"); + streamed["stream"] = json!(true); + let response = h.chat(Some(&token), &streamed).send().await.unwrap(); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "text/event-stream" + ); + let body = response.text().await.unwrap(); + assert!(body.contains("Mock answer") && body.trim_end().ends_with("data: [DONE]")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ollama_format_is_enforced_too() { + let h = start(any_port()).await; + let token = h.key(&[KeyScope::Inference], false); + let reply: Value = h + .http + .post(format!("{}/api/chat", h.base)) + .bearer_auth(&token) + .json(&json!({ + "model": "qualcomm/Qwen3-0.6B", + "stream": false, + "messages": [{ "role": "user", "content": "hello" }], + "format": { "type": "object", "required": ["reply"] }, + })) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(reply["done"], true); + let content: Value = + serde_json::from_str(reply["message"]["content"].as_str().unwrap()).unwrap(); + assert_eq!(content["reply"], "Mock answer"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn enforces_tool_choice() { + let h = start(any_port()).await; + let token = h.key(&[KeyScope::Inference], false); + let tool = |name: &str, unit: Value| { + json!({ "type": "function", "function": { "name": name, "parameters": { + "type": "object", + "properties": { "city": { "type": "string" }, "unit": unit }, + "required": ["city"], + } } }) + }; + let weather = tool("get_weather", json!({ "type": "string" })); + let time = tool("get_time", json!({ "type": "string" })); + let ask = |tool_choice: Value, tools: Value| { + json!({ + "model": "qualcomm/Qwen3-0.6B", + "messages": [{ "role": "user", "content": "Weather in Paris?" }], + "tools": tools, + "tool_choice": tool_choice, + }) + }; + + // A named function is the only one offered, and its call is checked. + let named = ask( + json!({ "type": "function", "function": { "name": "get_time" } }), + json!([weather, time]), + ); + let reply: Value = h + .chat(Some(&token), &named) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(reply["choices"][0]["finish_reason"], "tool_calls"); + assert_eq!( + reply["choices"][0]["message"]["tool_calls"][0]["function"]["name"], + "get_time" + ); + + // "none" takes the tools away: the model answers in text. + let reply: Value = h + .chat(Some(&token), &ask(json!("none"), json!([weather]))) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(reply["choices"][0]["message"].get("tool_calls").is_none()); + assert!( + reply["choices"][0]["message"]["content"] + .as_str() + .unwrap() + .contains("mock backend") + ); + + // Arguments that never match the parameters: 422, saying why. + let strict = tool("get_weather", json!({ "type": "string", "minLength": 10 })); + let never = h + .chat(Some(&token), &ask(json!("required"), json!([strict]))) + .send() + .await + .unwrap(); + assert_eq!(never.status(), StatusCode::UNPROCESSABLE_ENTITY); + let error: Value = never.json().await.unwrap(); + assert!( + error["error"]["message"] + .as_str() + .unwrap() + .contains("get_weather: $.unit") + ); + + // A function that isn't in the tools is the client's mistake. + let unknown = ask( + json!({ "type": "function", "function": { "name": "search" } }), + json!([weather]), + ); + let response = h.chat(Some(&token), &unknown).send().await.unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + // Streamed clients get the call as server-sent events. + let mut streamed = ask(json!("required"), json!([weather])); + streamed["stream"] = json!(true); + let body = h + .chat(Some(&token), &streamed) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + assert!(body.contains("\"tool_calls\"") && body.contains("\"finish_reason\":\"tool_calls\"")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn forgets_the_oldest_messages_when_asked() { + let h = start(any_port()).await; + let token = h.key(&[KeyScope::Inference], false); + let long = "word ".repeat(1300); + let conversation = json!([ + { "role": "system", "content": "Be brief." }, + { "role": "user", "content": long }, + { "role": "assistant", "content": "OK" }, + { "role": "user", "content": long }, + { "role": "assistant", "content": "OK" }, + { "role": "user", "content": "hello" }, + ]); + let ask = |truncation: Option<&str>| { + let mut body = json!({ "model": "qualcomm/Qwen3-4B", "messages": conversation }); + if let Some(truncation) = truncation { + body["truncation"] = json!(truncation); + } + body + }; + + // Too long for the window: refused, saying how to fix it. + let refused = h.chat(Some(&token), &ask(None)).send().await.unwrap(); + assert_eq!(refused.status(), StatusCode::BAD_REQUEST); + let error: Value = refused.json().await.unwrap(); + assert_eq!(error["error"]["code"], "context_length_exceeded"); + assert!( + error["error"]["message"] + .as_str() + .unwrap() + .contains("truncation") + ); + + // With truncation, the first exchange goes and the reply comes. + let fitted = h + .chat(Some(&token), &ask(Some("auto"))) + .send() + .await + .unwrap(); + assert_eq!(fitted.status(), StatusCode::OK); + assert_eq!(fitted.headers()["x-calcine-forgotten-messages"], "2"); + let reply: Value = fitted.json().await.unwrap(); + assert!(reply["choices"][0]["message"]["content"].is_string()); + + // A last message that can't fit alone. + let huge = json!({ + "model": "qualcomm/Qwen3-4B", + "truncation": "auto", + "messages": [{ "role": "user", "content": "word ".repeat(3000) }], + }); + let response = h.chat(Some(&token), &huge).send().await.unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let error: Value = response.json().await.unwrap(); + assert!( + error["error"]["message"] + .as_str() + .unwrap() + .contains("doesn't fit") + ); + + // Ollama clients get it without asking, as with Ollama. + let ollama = h + .http + .post(format!("{}/api/chat", h.base)) + .bearer_auth(&token) + .json(&json!({ "model": "qualcomm/Qwen3-4B", "stream": false, "messages": conversation })) + .send() + .await + .unwrap(); + assert_eq!(ollama.status(), StatusCode::OK); +} + +/// The port in `https://host:port/v1`. +fn port_of(url: &str) -> u16 { + url.rsplit(':') + .next() + .and_then(|rest| rest.split('/').next()) + .and_then(|port| port.parse().ok()) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn serves_other_devices_over_https_with_network_keys_only() { + let h = start(GatewaySettings { + network_enabled: true, + network_port: 0, + ..any_port() + }) + .await; + let network = h + .gateway + .status() + .network + .expect("the network port listens"); + assert_eq!(network.fingerprint.len(), 95); + let port = port_of(&network.base_url); + // Trust only Calcine's certificate, like a client that imported it. + // Changing the network settings cuts open connections on purpose: a + // fresh one per request keeps the client from reusing a closed one. + let pem = std::fs::read(&network.certificate_path).unwrap(); + let client = Client::builder() + .add_root_certificate(reqwest::Certificate::from_pem(&pem).unwrap()) + .pool_max_idle_per_host(0) + .build() + .unwrap(); + let models = format!("https://localhost:{port}/v1/models"); + let create = |network: bool, allow_local_files: bool| { + h.gateway + .keys() + .create(NewApiKey { + name: "Phone".into(), + scopes: vec![KeyScope::Inference], + allow_local_files, + network, + }) + .unwrap() + .token + }; + + // Plain HTTP isn't spoken there. + assert!( + h.http + .get(format!("http://localhost:{port}/v1/models")) + .send() + .await + .is_err() + ); + + let status = |token: Option| { + let request = client.get(&models); + let request = match token { + Some(token) => request.bearer_auth(token), + None => request, + }; + async move { request.send().await.unwrap().status() } + }; + assert_eq!(status(None).await, StatusCode::UNAUTHORIZED); + assert_eq!( + status(Some(create(false, false))).await, + StatusCode::FORBIDDEN + ); + assert_eq!(status(Some(create(true, false))).await, StatusCode::OK); + // Calcine's own token only works from its window on this PC. + let internal = h.gateway.keys().internal_token().to_owned(); + assert_eq!(status(Some(internal)).await, StatusCode::UNAUTHORIZED); + + // A path from another device would name a file on this PC. + let reaching = client + .post(format!("https://localhost:{port}/v1/chat/completions")) + .bearer_auth(create(true, true)) + .json(&json!({ + "model": "qualcomm/Qwen3-4B", + "messages": [{ "role": "user", "content": [ + { "type": "image_url", "image_url": { "url": "C:/Windows/win.ini" } }, + ] }], + })) + .send() + .await + .unwrap(); + assert_eq!(reaching.status(), StatusCode::FORBIDDEN); + + // Addresses outside the allowed ranges are turned away. + h.gateway + .set_network(true, port, vec!["10.0.0.0/8".into()]) + .await + .unwrap(); + assert_eq!( + status(Some(create(true, false))).await, + StatusCode::FORBIDDEN + ); + assert!( + h.gateway + .set_network(true, port, vec!["not an address".into()]) + .await + .is_err() + ); + + // Too many invalid keys: the address waits. + h.gateway.set_network(true, port, vec![]).await.unwrap(); + for _ in 0..10 { + assert_eq!( + status(Some("calcine_wrong".into())).await, + StatusCode::UNAUTHORIZED + ); + } + assert_eq!( + status(Some(create(true, false))).await, + StatusCode::TOO_MANY_REQUESTS + ); + + // Off: nothing listens there anymore. + h.gateway.set_network(false, port, vec![]).await.unwrap(); + assert!(h.gateway.status().network.is_none()); + assert!(client.get(&models).send().await.is_err()); +} diff --git a/crates/calcine-geniex/Cargo.toml b/crates/calcine-geniex/Cargo.toml index 7d746f7..223b960 100644 --- a/crates/calcine-geniex/Cargo.toml +++ b/crates/calcine-geniex/Cargo.toml @@ -18,10 +18,14 @@ sha2.workspace = true serde.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["fs"] } +tokenizers.workspace = true reqwest = { workspace = true, features = ["native-tls"] } tracing.workspace = true which.workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +libc.workspace = true + [dev-dependencies] reqwest.workspace = true tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread"] } diff --git a/crates/calcine-geniex/src/backend/mod.rs b/crates/calcine-geniex/src/backend/mod.rs index c4805c6..b5db4cf 100644 --- a/crates/calcine-geniex/src/backend/mod.rs +++ b/crates/calcine-geniex/src/backend/mod.rs @@ -5,4 +5,4 @@ mod pull; mod runtime; mod serve; -pub use serve::{GeniexServer, ServeOptions}; +pub use serve::GeniexServer; diff --git a/crates/calcine-geniex/src/backend/pull.rs b/crates/calcine-geniex/src/backend/pull.rs index de5c8ad..0eb458d 100644 --- a/crates/calcine-geniex/src/backend/pull.rs +++ b/crates/calcine-geniex/src/backend/pull.rs @@ -13,6 +13,9 @@ use crate::parse::progress::{FrameSplitter, parse_frame}; /// Non-progress lines kept to explain a failure. const KEPT_LINES: usize = 12; +/// What GenieX says before asking for the chipset. +const NO_CHIPSET: &str = "No chipset configured"; + pub(crate) async fn run(cli: &Cli, request: &PullRequest, ctx: &JobCtx) -> Result<()> { let reference = request.reference.cli_arg(); let mut args = vec!["pull", reference.as_str()]; @@ -55,6 +58,19 @@ pub(crate) async fn run(cli: &Cli, request: &PullRequest, ctx: &JobCtx) -> Resul if status.success() { return Ok(()); } + // Without a terminal GenieX can't ask which chipset this is (on Linux + // it often can't tell). It says so on stdout, then fails on stderr. + if stdout_lines + .iter() + .chain(&stderr_lines) + .any(|line| line.contains(NO_CHIPSET)) + { + return Err(Error::InvalidInput( + "GenieX doesn't know which Snapdragon this is. Choose your chipset in \ + Hardware › Chipset (or run calcine-cli geniex chipset), then download again." + .into(), + )); + } let message = [stderr_lines, stdout_lines] .into_iter() .find(|lines| !lines.is_empty()) diff --git a/crates/calcine-geniex/src/backend/serve.rs b/crates/calcine-geniex/src/backend/serve.rs index 9094dd5..159f8de 100644 --- a/crates/calcine-geniex/src/backend/serve.rs +++ b/crates/calcine-geniex/src/backend/serve.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use async_trait::async_trait; -use calcine_core::runtime::{InferenceServer, ServerState}; +use calcine_core::runtime::{InferenceServer, ServerOptions, ServerState}; use calcine_core::{Error, Result}; use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader}; use tokio::sync::{oneshot, watch}; @@ -21,26 +21,11 @@ const LOG_LINES: usize = 500; const STARTUP_TIMEOUT: Duration = Duration::from_secs(60); const HEALTH_INTERVAL: Duration = Duration::from_millis(250); -/// Options passed to `geniex serve`. -#[derive(Debug, Clone)] -pub struct ServeOptions { - /// Unload the model after this many idle seconds (`--keepalive`). - pub keepalive_secs: u32, -} - -impl Default for ServeOptions { - fn default() -> Self { - Self { - keepalive_secs: 300, - } - } -} - /// The supervised `geniex serve` process. #[derive(Debug)] pub struct GeniexServer { geniex: Geniex, - options: ServeOptions, + options: Mutex, running: tokio::sync::Mutex>, state: watch::Sender, logs: Arc>>, @@ -55,10 +40,10 @@ struct Running { } impl GeniexServer { - pub fn new(geniex: Geniex, options: ServeOptions) -> Self { + pub fn new(geniex: Geniex, options: ServerOptions) -> Self { Self { geniex, - options, + options: Mutex::new(options), running: tokio::sync::Mutex::new(None), state: watch::Sender::new(ServerState::Stopped), logs: Arc::new(Mutex::new(VecDeque::with_capacity(LOG_LINES))), @@ -69,9 +54,11 @@ impl GeniexServer { async fn start(&self) -> Result { let port = free_port()?; let host = format!("127.0.0.1:{port}"); - let keepalive = self.options.keepalive_secs.to_string(); + let options = self.options(); + let keepalive = options.keepalive_secs.to_string(); + let context_size = options.context_size.to_string(); // The server is internal: only the gateway talks to it, never a browser. - let args = [ + let mut args = vec![ "serve", "--host", &host, @@ -79,7 +66,12 @@ impl GeniexServer { "http://127.0.0.1", "--keepalive", &keepalive, + "--nctx", + &context_size, ]; + if let Some(unit) = crate::platform::default_compute() { + args.extend(["--compute", unit]); + } let mut child = self.geniex.cli()?.command(&args).spawn()?; tracing::info!(%host, "starting geniex serve"); @@ -192,6 +184,53 @@ impl InferenceServer for GeniexServer { .cloned() .collect() } + + fn options(&self) -> ServerOptions { + *self.options.lock().unwrap_or_else(PoisonError::into_inner) + } + + async fn set_options(&self, options: ServerOptions) -> Result<()> { + options.validate().map_err(Error::InvalidInput)?; + *self.options.lock().unwrap_or_else(PoisonError::into_inner) = options; + let was_running = self + .running + .lock() + .await + .as_ref() + .is_some_and(|r| !r.exited.load(Ordering::SeqCst)); + if was_running { + self.stop().await?; + self.ensure_running().await?; + } + Ok(()) + } + + async fn complete(&self, request: &serde_json::Value) -> Result { + let url = self.ensure_running().await?; + let response = self + .http + .post(format!("{url}/v1/chat/completions")) + .json(request) + .send() + .await + .map_err(|err| Error::Network(format!("couldn't reach GenieX: {err}")))?; + let status = response.status(); + let body: serde_json::Value = response.json().await.map_err(|err| { + Error::Network(format!("GenieX answered in an unexpected way: {err}")) + })?; + if !status.is_success() { + let message = body + .pointer("/error/message") + .or_else(|| body.get("error")) + .and_then(serde_json::Value::as_str) + .unwrap_or("the request failed"); + return Err(Error::Command { + command: "geniex serve".into(), + message: message.to_owned(), + }); + } + Ok(body) + } } /// Wait for the process to exit or for a stop request, and report a crash. diff --git a/crates/calcine-geniex/src/bench/mod.rs b/crates/calcine-geniex/src/bench/mod.rs new file mode 100644 index 0000000..8ba17d7 --- /dev/null +++ b/crates/calcine-geniex/src/bench/mod.rs @@ -0,0 +1,400 @@ +//! `geniex-bench`, Qualcomm's benchmark tool, run for Calcine. +//! +//! - `release`: finding the tool in GenieX's GitHub release, with checksums +//! - `report`: reading its JSON report and its errors +//! +//! The tool is a standalone archive (its own `geniex.dll` and runtimes), +//! downloaded on demand into Calcine's data folder at the version of the +//! installed GenieX. Each measurement runs in its own process: in one +//! process, a failure on one compute unit stops the rest. + +mod release; +mod report; + +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use async_trait::async_trait; +use calcine_core::bench::{ + BenchMeasure, BenchRequest, BenchResult, BenchTool, Benchmarker, merge_history, +}; +use calcine_core::jobs::{JobCtx, JobPhase, JobProgress}; +use calcine_core::models::{ComputeUnit, Runtime}; +use calcine_core::runtime::RuntimeManager; +use calcine_core::{Error, Result}; +use tokio::process::Command; + +use crate::Geniex; +use crate::update::download; + +/// A large model with many repetitions takes minutes; this is a backstop. +const RUN_TIMEOUT: Duration = Duration::from_mins(30); + +/// `bin/geniex-bench` inside an unpacked version, with its libraries in +/// `lib/` on Linux. +const BINARY: &str = if cfg!(windows) { + "geniex-bench.exe" +} else { + "geniex-bench" +}; + +/// Where the tool and the history live. +#[derive(Debug, Clone)] +pub struct BenchConfig { + /// Unpacked tool versions, downloads and run reports. + pub tools_dir: PathBuf, + /// Benchmark history (`benchmarks.json`). + pub history_path: PathBuf, +} + +/// [`Benchmarker`] running the official `geniex-bench`. +#[derive(Debug)] +pub struct GeniexBench { + geniex: Geniex, + config: BenchConfig, + http: reqwest::Client, + history: Mutex>, +} + +impl GeniexBench { + pub fn new(geniex: Geniex, config: BenchConfig) -> Result { + let http = reqwest::Client::builder() + // GitHub's API refuses requests without a user agent. + .user_agent(concat!("Calcine/", env!("CARGO_PKG_VERSION"))) + .build() + .map_err(|err| Error::Network(err.to_string()))?; + let history = load_history(&config.history_path); + Ok(Self { + geniex, + config, + http, + history: Mutex::new(history), + }) + } + + /// The GenieX version the tool must match. + async fn wanted(&self) -> Result { + Ok(self.geniex.info().await?.cli_version) + } + + /// Unpacked versions, by folder name `geniex-bench--arm64-`. + fn installed(&self) -> Vec<(String, PathBuf)> { + let Ok(entries) = std::fs::read_dir(&self.config.tools_dir) else { + return Vec::new(); + }; + let prefix = release::archive_name(""); + let prefix = prefix.trim_end_matches(release::ARCHIVE_EXTENSION); + let mut versions: Vec<(String, PathBuf)> = entries + .filter_map(std::result::Result::ok) + .filter_map(|entry| { + let name = entry.file_name().to_string_lossy().into_owned(); + let version = name.strip_prefix(prefix)?.to_owned(); + let binary = entry.path().join("bin").join(BINARY); + binary.is_file().then_some((version, binary)) + }) + .collect(); + versions.sort_by(|a, b| calcine_core::runtime::compare_versions(&b.0, &a.0)); + versions + } + + /// The tool to run: the one matching GenieX, else the newest. + async fn binary(&self) -> Result { + let installed = self.installed(); + let wanted = self.wanted().await.ok(); + installed + .iter() + .find(|(version, _)| Some(version) == wanted.as_ref()) + .or_else(|| installed.first()) + .map(|(_, binary)| binary.clone()) + .ok_or_else(|| Error::InvalidInput("download the benchmark tool first".into())) + } + + async fn release( + &self, + version: &str, + ) -> Result<(calcine_core::runtime::InstallerAsset, Option)> { + let json: serde_json::Value = self + .http + .get(format!("{}/releases/tags/{version}", release::RELEASES_API)) + .header("Accept", "application/vnd.github+json") + .timeout(Duration::from_secs(20)) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(|err| Error::Network(format!("couldn't read the GenieX release: {err}")))? + .json() + .await + .map_err(|err| Error::Parse(format!("unexpected GenieX release: {err}")))?; + release::find(&json, version) + } + + fn lock(&self) -> MutexGuard<'_, Vec> { + self.history.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn save(&self, history: &[BenchResult]) -> Result<()> { + let path = &self.config.history_path; + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + let json = serde_json::to_vec_pretty(history).map_err(std::io::Error::other)?; + let partial = path.with_extension("json.partial"); + std::fs::write(&partial, json)?; + std::fs::rename(&partial, path)?; + Ok(()) + } +} + +#[async_trait] +impl Benchmarker for GeniexBench { + async fn tool(&self) -> Result { + let wanted = self.wanted().await.ok(); + let installed = self.installed(); + let current = installed + .iter() + .find(|(version, _)| Some(version) == wanted.as_ref()) + .or_else(|| installed.first()) + .map(|(version, _)| version.clone()); + let download_bytes = match &wanted { + Some(version) if current.as_ref() != Some(version) => self + .release(version) + .await + .ok() + .map(|(asset, _)| asset.size), + _ => None, + }; + Ok(BenchTool { + installed: current, + wanted, + download_bytes, + }) + } + + async fn install_tool(&self, ctx: JobCtx) -> Result<()> { + if !cfg!(windows) { + return Err(Error::InvalidInput( + "benchmarks need Windows on Arm for now".into(), + )); + } + let version = self.wanted().await?; + let (asset, checksum_url) = self.release(&version).await?; + // Qualcomm's .sha256 file and GitHub's digest must agree. + if let Some(url) = checksum_url { + let text = self + .http + .get(url) + .timeout(Duration::from_secs(20)) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(|err| Error::Network(format!("couldn't read the checksum: {err}")))? + .text() + .await + .map_err(|err| Error::Network(err.to_string()))?; + if release::checksum_file_hash(&text).as_deref() != Some(asset.sha256.as_str()) { + return Err(Error::InvalidInput(format!( + "the published checksums of {} don't agree; nothing was installed", + asset.name + ))); + } + } + + let tools = &self.config.tools_dir; + let archive = tools.join("downloads").join(&asset.name); + download::fetch(&self.http, &asset, &archive, &ctx).await?; + + ctx.report(JobProgress { + phase: Some(JobPhase::Installing), + ..JobProgress::default() + }); + let folder = asset + .name + .trim_end_matches(release::ARCHIVE_EXTENSION) + .to_owned(); + let staging = tools.join(format!(".staging-{version}")); + let _ = tokio::fs::remove_dir_all(&staging).await; + tokio::fs::create_dir_all(&staging).await?; + extract(&archive, &staging).await?; + let unpacked = staging.join(&folder); + if !unpacked.join("bin").join(BINARY).is_file() { + let _ = tokio::fs::remove_dir_all(&staging).await; + return Err(Error::Parse(format!( + "{} doesn't contain {BINARY}", + asset.name + ))); + } + let target = tools.join(&folder); + let _ = tokio::fs::remove_dir_all(&target).await; + tokio::fs::rename(&unpacked, &target).await?; + let _ = tokio::fs::remove_dir_all(&staging).await; + let _ = tokio::fs::remove_file(&archive).await; + // Only removed when empty: another download may be in there. + if let Some(downloads) = archive.parent() { + let _ = tokio::fs::remove_dir(downloads).await; + } + // Older versions only take space (a few hundred MB each). + for (old, binary) in self.installed() { + if old != version + && let Some(dir) = binary.parent().and_then(Path::parent) + { + let _ = tokio::fs::remove_dir_all(dir).await; + } + } + Ok(()) + } + + async fn measure( + &self, + request: &BenchRequest, + unit: ComputeUnit, + ctx: &JobCtx, + ) -> Result { + let binary = self.binary().await?; + let plugin = match request.runtime { + Runtime::Qairt => "qairt", + Runtime::LlamaCpp => "llama_cpp", + Runtime::Unknown => { + return Err(Error::InvalidInput( + "this model's runtime can't be benchmarked".into(), + )); + } + }; + let runs = self.config.tools_dir.join("runs"); + tokio::fs::create_dir_all(&runs).await?; + let report_path = runs.join(format!("{}-{}.json", ctx.id(), device(unit))); + let _ = tokio::fs::remove_file(&report_path).await; + + let mut command = Command::new(&binary); + command + .args(["--plugin", plugin, "--device", device(unit), "-m"]) + .arg(&request.model) + .args(["--warmup", "1", "-r"]) + .arg(request.repetitions.to_string()) + .arg("-p") + .arg(request.prompt_tokens.to_string()) + .arg("-n") + .arg(request.generated_tokens.to_string()) + .arg("--power-mode") + .arg(&request.power_mode) + .arg("--output-json") + .arg(&report_path) + .args(["--cell-id", "calcine"]); + if let Some(spec) = &request.spec_type { + command.arg("--spec-type").arg(spec); + if let Some(draft) = &request.draft_model { + command.arg("--draft-model").arg(draft); + } + } + if let Some(data_dir) = &self.geniex.config().data_dir { + command.arg("--mm-data-dir").arg(data_dir); + } + command + .env("GENIEX_LOG", "error") + .env("NO_COLOR", "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + // `bin/geniex-bench` loads its libraries from `lib/`. + if let Some(root) = binary.parent().and_then(Path::parent) { + crate::platform::prepare(&mut command, &root.join("lib")); + } + tracing::info!(model = %request.model, ?unit, "starting geniex-bench"); + + let child = command.spawn()?; + let output = tokio::select! { + output = tokio::time::timeout(RUN_TIMEOUT, child.wait_with_output()) => output + .map_err(|_| Error::Timeout { + command: "geniex-bench".into(), + seconds: RUN_TIMEOUT.as_secs(), + })??, + // Dropping the child kills it. + () = ctx.cancelled() => return Err(Error::Cancelled), + }; + let report = tokio::fs::read_to_string(&report_path).await; + let _ = tokio::fs::remove_file(&report_path).await; + match report { + Ok(json) if output.status.success() => report::parse(&json), + _ => { + let text = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + Err(Error::Command { + command: "geniex-bench".into(), + message: report::failure(&text) + .unwrap_or_else(|| format!("stopped without results ({})", output.status)), + }) + } + } + } + + fn history(&self) -> Vec { + self.lock().clone() + } + + fn record(&self, results: Vec) -> Result<()> { + let mut history = self.lock(); + let merged = merge_history(std::mem::take(&mut *history), results); + *history = merged; + self.save(&history) + } + + fn forget(&self, ids: &[String]) -> Result<()> { + let mut history = self.lock(); + history.retain(|result| !ids.contains(&result.id)); + self.save(&history) + } +} + +/// `geniex-bench --device` for a compute unit. +fn device(unit: ComputeUnit) -> &'static str { + match unit { + ComputeUnit::Npu => "npu", + ComputeUnit::Gpu => "gpu", + ComputeUnit::Cpu => "cpu", + ComputeUnit::Hybrid => "hybrid", + } +} + +fn load_history(path: &Path) -> Vec { + std::fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() +} + +/// Unpack with the system's `tar`: Windows' own bsdtar reads zip archives, +/// GNU tar the Linux tarballs; both refuse absolute paths and `..` entries. +async fn extract(archive: &Path, destination: &Path) -> Result<()> { + let tar = if cfg!(windows) { + let system_root = std::env::var_os("SystemRoot").unwrap_or_else(|| "C:\\Windows".into()); + PathBuf::from(system_root).join("System32").join("tar.exe") + } else { + PathBuf::from("tar") + }; + let mut command = Command::new(tar); + command + .arg("-xf") + .arg(archive) + .arg("-C") + .arg(destination) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + #[cfg(windows)] + command.creation_flags(0x0800_0000); // CREATE_NO_WINDOW + let output = command.output().await?; + if output.status.success() { + Ok(()) + } else { + Err(Error::Command { + command: "tar".into(), + message: String::from_utf8_lossy(&output.stderr).trim().to_owned(), + }) + } +} diff --git a/crates/calcine-geniex/src/bench/release.rs b/crates/calcine-geniex/src/bench/release.rs new file mode 100644 index 0000000..748d171 --- /dev/null +++ b/crates/calcine-geniex/src/bench/release.rs @@ -0,0 +1,127 @@ +//! Finding `geniex-bench` in GenieX's GitHub release. Qualcomm attaches it +//! to every release with a `.sha256` file; GitHub also reports each asset's +//! SHA-256 (`digest`). Both must agree. + +use calcine_core::runtime::InstallerAsset; +use calcine_core::{Error, Result}; +use serde_json::Value; + +/// Where GenieX releases are published on GitHub. +pub const RELEASES_API: &str = "https://api.github.com/repos/qualcomm/GenieX"; + +/// Archives are zips on Windows, gzipped tarballs on Linux. +pub const ARCHIVE_EXTENSION: &str = if cfg!(windows) { ".zip" } else { ".tar.gz" }; + +/// The archive for this platform and GenieX version. +pub fn archive_name(version: &str) -> String { + format!( + "geniex-bench-{}-{}-{version}{ARCHIVE_EXTENSION}", + crate::platform::NAME, + crate::platform::ARCH + ) +} + +/// The archive and the URL of its `.sha256` file, from a release's JSON. +pub fn find(release: &Value, version: &str) -> Result<(InstallerAsset, Option)> { + let name = archive_name(version); + let assets = release + .get("assets") + .and_then(Value::as_array) + .ok_or_else(|| Error::Parse("the GenieX release lists no files".into()))?; + let asset = |wanted: &str| { + assets + .iter() + .find(|asset| asset.get("name").and_then(Value::as_str) == Some(wanted)) + }; + let archive = asset(&name).ok_or_else(|| { + Error::InvalidInput(format!( + "GenieX {version} has no benchmark tool for this PC ({name})" + )) + })?; + let sha256 = archive + .get("digest") + .and_then(Value::as_str) + .and_then(|digest| digest.strip_prefix("sha256:")) + .filter(|hex| hex.len() == 64) + .ok_or_else(|| Error::Parse(format!("GitHub gives no checksum for {name}")))? + .to_ascii_lowercase(); + let url = archive + .get("browser_download_url") + .and_then(Value::as_str) + .ok_or_else(|| Error::Parse(format!("GitHub gives no link for {name}")))? + .to_owned(); + let size = archive.get("size").and_then(Value::as_u64).unwrap_or(0); + let checksum_url = asset(&format!("{name}.sha256")) + .and_then(|file| file.get("browser_download_url")) + .and_then(Value::as_str) + .map(str::to_owned); + Ok(( + InstallerAsset { + name, + url, + size, + sha256, + }, + checksum_url, + )) +} + +/// The hash in a `.sha256` file (` `). +pub fn checksum_file_hash(text: &str) -> Option { + text.split_whitespace() + .next() + .filter(|hex| hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit())) + .map(str::to_ascii_lowercase) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + const DIGEST: &str = "f7d3df090edff588e5da8d9cf97ee7aef8f8ad2ae5fe7911aa52340f719e054a"; + + #[test] + fn finds_the_archive_and_its_checksums() { + // This platform's archive, among the others GenieX publishes. + let name = archive_name("v0.8.0"); + let release = json!({ "assets": [ + { "name": "geniex-cli-setup-windows-arm64-v0.8.0.exe", "digest": "sha256:00", "size": 1, "browser_download_url": "x" }, + { "name": "geniex-bench-android-arm64-v0.8.0.tar.gz", "digest": "sha256:00", "size": 1, "browser_download_url": "x" }, + { "name": name, "digest": format!("sha256:{DIGEST}"), "size": 84_772_012, "browser_download_url": "https://github.com/a.zip" }, + { "name": format!("{name}.sha256"), "browser_download_url": "https://github.com/a.zip.sha256" }, + ]}); + let (asset, checksum) = find(&release, "v0.8.0").unwrap(); + assert_eq!(asset.sha256, DIGEST); + assert_eq!(asset.size, 84_772_012); + assert_eq!(asset.url, "https://github.com/a.zip"); + assert_eq!(checksum.as_deref(), Some("https://github.com/a.zip.sha256")); + } + + #[test] + fn needs_a_digest() { + let release = json!({ "assets": [ + { "name": archive_name("v0.8.0"), "size": 1, "browser_download_url": "x" }, + ]}); + assert!(find(&release, "v0.8.0").is_err()); + assert!(find(&json!({ "assets": [] }), "v0.8.0").is_err()); + } + + #[test] + fn names_archives_per_platform() { + let expected = if cfg!(windows) { + "geniex-bench-windows-arm64-v0.8.0.zip" + } else { + "geniex-bench-linux-arm64-v0.8.0.tar.gz" + }; + assert_eq!(archive_name("v0.8.0"), expected); + } + + #[test] + fn reads_sha256_files() { + let text = format!("{DIGEST} geniex-bench-windows-arm64-v0.8.0.zip\n"); + assert_eq!(checksum_file_hash(&text).as_deref(), Some(DIGEST)); + assert_eq!(checksum_file_hash("not a hash"), None); + } +} diff --git a/crates/calcine-geniex/src/bench/report.rs b/crates/calcine-geniex/src/bench/report.rs new file mode 100644 index 0000000..25292e3 --- /dev/null +++ b/crates/calcine-geniex/src/bench/report.rs @@ -0,0 +1,123 @@ +//! Reading what `geniex-bench` writes: the per-run JSON report, and the +//! error it prints when a run fails. + +use calcine_core::bench::{BenchMeasure, BenchStat}; +use calcine_core::{Error, Result}; +use serde_json::Value; + +/// The `agg` section of a report (schema 6), as a measure. +pub fn parse(json: &str) -> Result { + let report: Value = serde_json::from_str(json) + .map_err(|err| Error::Parse(format!("geniex-bench report isn't JSON: {err}")))?; + let agg = report + .get("agg") + .ok_or_else(|| Error::Parse("geniex-bench report has no results".into()))?; + let stat = |name: &str| -> Result { + let value = agg + .get(name) + .ok_or_else(|| Error::Parse(format!("geniex-bench report has no {name}")))?; + let field = |key: &str| value.get(key).and_then(Value::as_f64).unwrap_or(0.0); + Ok(BenchStat { + median: field("median"), + min: field("min"), + max: field("max"), + stdev: field("stdev"), + }) + }; + let median = |name: &str| { + agg.pointer(&format!("/{name}/median")) + .and_then(Value::as_f64) + .unwrap_or(0.0) + }; + Ok(BenchMeasure { + ttft_ms: stat("ttft_ms")?, + prefill_tps: stat("prefill_tps")?, + decode_tps: stat("decode_tps")?, + generated_tokens: median("gen_tokens"), + prompt_tokens: median("prompt_tokens"), + geniex_version: report + .get("geniex_version") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + energy: None, + }) +} + +/// Why a run failed, from its output: the last `ERROR:` line, or the last +/// warning that explains it (llama.cpp failing to load on a unit). +pub fn failure(output: &str) -> Option { + let lines: Vec = output.lines().map(strip_ansi).collect(); + let error = lines.iter().rev().find_map(|line| { + line.trim() + .strip_prefix("ERROR:") + .map(|message| message.trim().to_owned()) + })?; + // `Model loading failed` alone doesn't say why; the warning before does. + let reason = lines.iter().rev().find_map(|line| { + let line = line.trim(); + line.contains("failed to initialize the context").then(|| { + line.rsplit_once("failed to initialize the context:") + .map_or(line, |(_, reason)| reason.trim()) + .to_owned() + }) + }); + Some(match reason { + Some(reason) => format!("{error} ({reason})"), + None => error, + }) +} + +fn strip_ansi(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\u{1b}' { + // Skip `ESC [ ... letter`. + if chars.peek() == Some(&'[') { + chars.next(); + for c in chars.by_ref() { + if c.is_ascii_alphabetic() { + break; + } + } + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_a_real_report() { + let measure = parse(include_str!("../../tests/fixtures/bench_report.json")).unwrap(); + assert!((measure.decode_tps.median - 53.072_619).abs() < 1e-6); + assert!((measure.ttft_ms.min - 36.461).abs() < 1e-6); + assert!((measure.prefill_tps.stdev - 241.864_635).abs() < 1e-6); + assert!((measure.generated_tokens - 32.0).abs() < f64::EPSILON); + assert!((measure.prompt_tokens - 128.0).abs() < f64::EPSILON); + assert_eq!(measure.geniex_version, "v0.8.0"); + } + + #[test] + fn rejects_reports_without_results() { + assert!(parse("{}").is_err()); + assert!(parse("not json").is_err()); + } + + #[test] + fn finds_the_failure_and_its_reason() { + let output = "\u{1b}[33m[ WARN] [plugin.cpp:52] llama_init_from_model: failed to initialize the context: ggml-hex: failed to open session (see log for details)\u{1b}[0m\n\ + ERROR: geniex_llm_create: Model loading failed (code=-100201)\n"; + assert_eq!( + failure(output).unwrap(), + "geniex_llm_create: Model loading failed (code=-100201) (ggml-hex: failed to open session (see log for details))" + ); + assert_eq!(failure("[ok ] cell ttft=1ms"), None); + } +} diff --git a/crates/calcine-geniex/src/cli/discovery.rs b/crates/calcine-geniex/src/cli/discovery.rs index 16bbbd6..5697717 100644 --- a/crates/calcine-geniex/src/cli/discovery.rs +++ b/crates/calcine-geniex/src/cli/discovery.rs @@ -20,18 +20,32 @@ pub fn locate(custom: Option<&Path>) -> Option { .or_else(|| which::which("geniex").ok()) } -/// Where the official GenieX installer puts the executable. +/// Where GenieX is installed, the folder holding the executable: /// -/// Windows: `%LOCALAPPDATA%\GenieX CLI\geniex.exe` (per-user Inno Setup install). -pub fn official_install_path() -> Option { +/// - Windows: `%LOCALAPPDATA%\GenieX CLI`, where the official per-user Inno +/// Setup installer puts it; +/// - Linux: `$XDG_DATA_HOME/geniex-cli` (`~/.local/share/geniex-cli`), where +/// Calcine unpacks the official archive (it has no installer). +pub fn official_install_dir() -> Option { if cfg!(windows) { - std::env::var_os("LOCALAPPDATA") - .map(|dir| PathBuf::from(dir).join("GenieX CLI").join("geniex.exe")) + std::env::var_os("LOCALAPPDATA").map(|dir| PathBuf::from(dir).join("GenieX CLI")) } else { - None + let data_home = std::env::var_os("XDG_DATA_HOME") + .filter(|dir| !dir.is_empty()) + .map(PathBuf::from) + .or_else(|| { + std::env::var_os("HOME") + .map(|home| PathBuf::from(home).join(".local").join("share")) + })?; + Some(data_home.join("geniex-cli")) } } +/// The executable in [`official_install_dir`]. +pub fn official_install_path() -> Option { + official_install_dir().map(|dir| dir.join(format!("geniex{}", std::env::consts::EXE_SUFFIX))) +} + /// GenieX's data directory: `custom`, else `GENIEX_DATADIR`, else /// `~/.cache/geniex` (where `geniex list` reports models are cached). pub fn data_dir(custom: Option<&Path>) -> Option { diff --git a/crates/calcine-geniex/src/cli/runner.rs b/crates/calcine-geniex/src/cli/runner.rs index 486bf58..670b838 100644 --- a/crates/calcine-geniex/src/cli/runner.rs +++ b/crates/calcine-geniex/src/cli/runner.rs @@ -10,10 +10,6 @@ use tokio::process::Command; /// Default limit for short, non-interactive commands (`list`, `version`, …). const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); -/// Hide the console window when spawning from the GUI on Windows. -#[cfg(windows)] -const CREATE_NO_WINDOW: u32 = 0x0800_0000; - /// A resolved GenieX executable plus the global flags Calcine always passes. #[derive(Debug, Clone)] pub struct Cli { @@ -59,8 +55,10 @@ impl Cli { .stdout(Stdio::piped()) .stderr(Stdio::piped()) .kill_on_drop(true); - #[cfg(windows)] - command.creation_flags(CREATE_NO_WINDOW); + // GenieX's libraries sit next to the executable. + if let Some(dir) = self.binary.parent() { + crate::platform::prepare(&mut command, dir); + } command } diff --git a/crates/calcine-geniex/src/context.rs b/crates/calcine-geniex/src/context.rs new file mode 100644 index 0000000..584220f --- /dev/null +++ b/crates/calcine-geniex/src/context.rs @@ -0,0 +1,329 @@ +//! Context windows and token counts for cached models. +//! +//! AI Hub models carry their window in `genie_config.json` and their +//! tokenizer in `tokenizer.json`. The chat template's share is a few +//! constants, measured against the prompt sizes `geniex serve` reports for +//! Qwen3 (ChatML), within a couple of tokens. llama.cpp models keep their +//! tokenizer inside the GGUF file, so their counts are estimated. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; + +use async_trait::async_trait; +use calcine_core::context::{ContextMeter, TokenCount}; +use calcine_core::runtime::RuntimeManager; +use calcine_core::{Error, Result}; +use serde_json::Value; +use tokenizers::Tokenizer; + +use crate::Geniex; + +/// `<|im_start|>` and `<|im_end|>\n` around each message (its role and +/// newline are counted with its text). +const PER_MESSAGE: u32 = 3; +/// `<|im_start|>assistant\n`, where the reply starts. +const REPLY_START: u32 = 3; +/// The empty `` block when thinking is off. +const EMPTY_THINK: u32 = 4; +/// The system prompt GenieX adds when the request has none. +const DEFAULT_SYSTEM: u32 = 13; +/// The template's explanation of how to call tools. +const TOOLS_PREAMBLE: u32 = 90; +/// What an image costs, roughly, for vision models. +const IMAGE: u32 = 256; + +/// Counts tokens with each cached model's tokenizer. +#[derive(Debug)] +pub struct GeniexContext { + geniex: Geniex, + tokenizers: Mutex>>, +} + +impl GeniexContext { + pub fn new(geniex: Geniex) -> Self { + Self { + geniex, + tokenizers: Mutex::new(HashMap::new()), + } + } + + /// The model's folder in the cache, for names like `qualcomm/Qwen3-4B` + /// or `qualcomm/Qwen3-4B:W4A16`. Names come from API clients: anything + /// that could leave the cache is refused. + fn model_dir(&self, model: &str) -> Result> { + let name = model.split(':').next().unwrap_or(model); + let safe = |part: &str| { + !part.is_empty() + && part != "." + && part != ".." + && part + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) + }; + if !name.split('/').all(safe) { + return Err(Error::InvalidInput(format!("{model} isn't a model name"))); + } + Ok(self + .geniex + .models_dir() + .map(|dir| name.split('/').fold(dir, |path, part| path.join(part))) + .filter(|dir| dir.is_dir())) + } + + async fn tokenizer(&self, dir: &Path) -> Option> { + let path = dir.join("tokenizer.json"); + if let Some(loaded) = self + .tokenizers + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&path) + { + return Some(loaded.clone()); + } + if !path.is_file() { + return None; + } + let file = path.clone(); + let loaded = tokio::task::spawn_blocking(move || Tokenizer::from_file(file)) + .await + .ok()? + .inspect_err(|err| tracing::warn!(%err, path = %path.display(), "can't load tokenizer")) + .ok() + .map(Arc::new)?; + self.tokenizers + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(path, loaded.clone()); + Some(loaded) + } +} + +#[async_trait] +impl ContextMeter for GeniexContext { + async fn compiled_window(&self, model: &str) -> Result> { + let Some(dir) = self.model_dir(model)? else { + return Ok(None); + }; + let Ok(text) = tokio::fs::read_to_string(dir.join("genie_config.json")).await else { + return Ok(None); + }; + let config: Value = serde_json::from_str(&text) + .map_err(|err| Error::Parse(format!("genie_config.json: {err}")))?; + Ok(context_size(&config)) + } + + async fn count(&self, model: &str, request: &Value) -> Result { + let tokenizer = match self.model_dir(model)? { + Some(dir) => self.tokenizer(&dir).await, + None => None, + }; + let parts = Parts::of(request); + let texts = parts.texts(); + let counted: Option> = match tokenizer { + Some(tokenizer) => tokio::task::spawn_blocking(move || { + tokenizer.encode_batch(texts, false).ok().map(|encodings| { + encodings + .iter() + .map(|encoding| len(encoding.len())) + .collect() + }) + }) + .await + .map_err(|err| Error::Io(std::io::Error::other(err)))?, + None => None, + }; + Ok(if let Some(lengths) = counted { + parts.count(&lengths, true) + } else { + let lengths: Vec = parts.texts().iter().map(|text| estimate(text)).collect(); + parts.count(&lengths, false) + }) + } +} + +fn len(count: usize) -> u32 { + u32::try_from(count).unwrap_or(u32::MAX) +} + +/// About three characters per token: on the safe side for English and code. +pub fn estimate(text: &str) -> u32 { + len(text.chars().count().div_ceil(3)) +} + +/// `dialog.context.size`, wherever the config keeps it. +fn context_size(config: &Value) -> Option { + match config { + Value::Object(fields) => fields + .get("context") + .and_then(|context| context.get("size")) + .and_then(Value::as_u64) + .and_then(|size| u32::try_from(size).ok()) + .or_else(|| fields.values().find_map(context_size)), + _ => None, + } +} + +/// The texts of a request, to count in one batch. +#[derive(Debug, Default)] +pub struct Parts { + /// One text per message. + messages: Vec, + /// Images in each message. + images: Vec, + tools: Option, + has_system: bool, + thinks: bool, +} + +impl Parts { + pub fn of(request: &Value) -> Self { + let messages = request + .get("messages") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + let mut parts = Self { + tools: request + .get("tools") + .and_then(Value::as_array) + .filter(|tools| !tools.is_empty()) + .map(|tools| Value::Array(tools.clone()).to_string()), + has_system: messages + .first() + .and_then(|message| message.get("role")) + .and_then(Value::as_str) + .is_some_and(|role| matches!(role, "system" | "developer")), + thinks: request + .get("enable_think") + .and_then(Value::as_bool) + .unwrap_or(true), + ..Self::default() + }; + for message in messages { + let (text, images) = message_text(message); + parts.messages.push(text); + parts.images.push(images); + } + parts + } + + /// Messages first, then the tools. + pub fn texts(&self) -> Vec { + let mut texts = self.messages.clone(); + texts.extend(self.tools.clone()); + texts + } + + /// The count, given the length of each of `texts()`. + pub fn count(&self, lengths: &[u32], exact: bool) -> TokenCount { + let at = |index: usize| lengths.get(index).copied().unwrap_or(0); + let per_message = (0..self.messages.len()) + .map(|index| at(index) + PER_MESSAGE + self.images[index] * IMAGE) + .collect(); + let tools = self + .tools + .as_ref() + .map_or(0, |_| TOOLS_PREAMBLE + at(self.messages.len())); + let overhead = REPLY_START + + if self.thinks { 0 } else { EMPTY_THINK } + + if self.has_system { 0 } else { DEFAULT_SYSTEM } + + tools; + TokenCount { + per_message, + overhead, + exact: exact && self.images.iter().all(|&images| images == 0), + } + } +} + +/// A message's text (role, content, tool calls) and how many images it has. +fn message_text(message: &Value) -> (String, u32) { + let mut text = message + .get("role") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + text.push('\n'); + let mut images = 0; + match message.get("content") { + Some(Value::String(content)) => text.push_str(content), + Some(Value::Array(content)) => { + for part in content { + match part.get("type").and_then(Value::as_str) { + Some("text") => { + text.push_str(part.get("text").and_then(Value::as_str).unwrap_or("")); + } + Some("image_url" | "input_audio") => images += 1, + _ => {} + } + } + } + _ => {} + } + if let Some(calls) = message.get("tool_calls").and_then(Value::as_array) { + for call in calls { + text.push_str( + &call + .get("function") + .map(Value::to_string) + .unwrap_or_default(), + ); + } + } + (text, images) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn finds_the_compiled_window() { + let config = + json!({ "dialog": { "version": 1, "context": { "size": 4096, "n-vocab": 151_936 } } }); + assert_eq!(context_size(&config), Some(4096)); + assert_eq!(context_size(&json!({ "dialog": {} })), None); + } + + #[test] + fn adds_the_template_around_messages() { + let request = json!({ + "enable_think": false, + "messages": [{ "role": "user", "content": "Hello" }], + }); + let parts = Parts::of(&request); + // "user\nHello": 3 tokens with Qwen3's tokenizer. GenieX reports 26. + let count = parts.count(&[3], true); + assert_eq!(count.per_message, vec![6]); + assert_eq!(count.total(), 26); + } + + #[test] + fn images_make_it_an_estimate() { + let request = json!({ "messages": [ + { "role": "system", "content": "Be brief." }, + { "role": "user", "content": [ + { "type": "image_url", "image_url": { "url": "data:," } }, + { "type": "text", "text": "What is it?" }, + ] }, + ] }); + let parts = Parts::of(&request); + assert_eq!(parts.texts()[1], "user\nWhat is it?"); + let count = parts.count(&[4, 5], true); + assert_eq!(count.per_message[1], 5 + PER_MESSAGE + IMAGE); + assert_eq!(count.overhead, REPLY_START); + assert!(!count.exact); + } + + #[test] + fn refuses_names_that_leave_the_cache() { + let context = GeniexContext::new(Geniex::default()); + assert!(context.model_dir("../secrets").is_err()); + assert!(context.model_dir("qualcomm/..").is_err()); + assert!(context.model_dir("..\\Windows").is_err()); + assert!(context.model_dir("qualcomm/Qwen3-4B:W4A16").is_ok()); + } +} diff --git a/crates/calcine-geniex/src/lib.rs b/crates/calcine-geniex/src/lib.rs index 0aa7e9f..c9d40e0 100644 --- a/crates/calcine-geniex/src/lib.rs +++ b/crates/calcine-geniex/src/lib.rs @@ -7,11 +7,18 @@ //! - `backend`: the service trait implementations (models, pull, runtime) //! - [`parse`]: output parsers, tested against captures from a real install //! - [`update`]: installing and updating GenieX with its official installer +//! - [`bench`]: benchmarks with Qualcomm's `geniex-bench` +//! - `context`: context windows and token counts mod backend; -pub use backend::{GeniexServer, ServeOptions}; +pub use backend::GeniexServer; +pub mod bench; +pub use bench::{BenchConfig, GeniexBench}; pub mod cli; +mod context; +pub use context::GeniexContext; pub mod parse; +mod platform; pub mod update; pub use update::{BundledInstaller, GeniexInstaller, InstallerConfig}; diff --git a/crates/calcine-geniex/src/parse/progress.rs b/crates/calcine-geniex/src/parse/progress.rs index c85c158..b851208 100644 --- a/crates/calcine-geniex/src/parse/progress.rs +++ b/crates/calcine-geniex/src/parse/progress.rs @@ -64,6 +64,7 @@ pub fn parse_frame(frame: &str) -> Option { total_bytes: (total.bytes > 0).then_some(total.bytes), bytes_per_second, phase: None, + step: None, }) } diff --git a/crates/calcine-geniex/src/platform.rs b/crates/calcine-geniex/src/platform.rs new file mode 100644 index 0000000..680016f --- /dev/null +++ b/crates/calcine-geniex/src/platform.rs @@ -0,0 +1,109 @@ +//! What differs between Windows and Linux when running GenieX's programs. +//! +//! - Release assets are named by platform (`windows`, `linux`); GenieX only +//! ships ARM64 builds. +//! - On Windows, programs run without a console window popping up. +//! - On Linux, GenieX's executables look for their shared libraries in the +//! current directory (their `RUNPATH` is `.` plus build paths), so they +//! get their folder in `LD_LIBRARY_PATH`; and they die with Calcine, the +//! way the Windows Job Object kills them. + +use std::path::Path; + +use tokio::process::Command; + +/// The platform name in GenieX's release assets. +pub const NAME: &str = if cfg!(windows) { "windows" } else { "linux" }; + +/// GenieX only ships ARM64 builds. +pub const ARCH: &str = "arm64"; + +/// The compute unit `geniex serve` uses when a request doesn't name one. +/// GenieX loads llama.cpp models on the Hexagon NPU by default, which fails +/// on Linux machines that don't expose the compute DSP (no FastRPC device): +/// those default to the CPU. Requests can still ask for another unit. +pub fn default_compute() -> Option<&'static str> { + if cfg!(target_os = "linux") + && !Path::new("/dev/fastrpc-cdsp").exists() + && !Path::new("/dev/fastrpc-cdsp-secure").exists() + { + Some("cpu") + } else { + None + } +} + +/// Set up `command` as described above; `libraries` is the folder holding +/// the program's shared libraries. +pub fn prepare(command: &mut Command, libraries: &Path) { + #[cfg(windows)] + { + let _ = libraries; + // CREATE_NO_WINDOW + command.creation_flags(0x0800_0000); + } + #[cfg(unix)] + { + command.env( + "LD_LIBRARY_PATH", + library_path(libraries, std::env::var_os("LD_LIBRARY_PATH")), + ); + #[cfg(target_os = "linux")] + die_with_parent(command); + } +} + +/// `dir` first, then what was already there. +#[cfg(unix)] +fn library_path(dir: &Path, existing: Option) -> std::ffi::OsString { + let mut path = dir.as_os_str().to_owned(); + if let Some(existing) = existing.filter(|existing| !existing.is_empty()) { + path.push(":"); + path.push(existing); + } + path +} + +/// Ask the kernel to kill the child when Calcine exits, even after a crash. +/// +/// The signal fires when the *thread* that spawned the child ends, so spawn +/// from async tasks (runtime workers live as long as Calcine), never from +/// `spawn_blocking`, whose threads end when idle. +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn die_with_parent(command: &mut Command) { + let parent = std::process::id(); + // SAFETY: `prctl` and `getppid` are async-signal-safe, as `pre_exec` + // requires, and touch no memory of the parent. + unsafe { + command.pre_exec(move || { + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) == -1 { + return Err(std::io::Error::last_os_error()); + } + // Calcine died before the line above: don't outlive it. (No + // allocation here, so a raw error.) + if libc::getppid().cast_unsigned() != parent { + return Err(std::io::Error::from_raw_os_error(libc::ESRCH)); + } + Ok(()) + }); + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + #[test] + fn puts_the_program_folder_first() { + assert_eq!(library_path(Path::new("/opt/geniex"), None), "/opt/geniex"); + assert_eq!( + library_path(Path::new("/opt/geniex"), Some("/usr/lib/x".into())), + "/opt/geniex:/usr/lib/x" + ); + assert_eq!( + library_path(Path::new("/opt/geniex"), Some("".into())), + "/opt/geniex" + ); + } +} diff --git a/crates/calcine-geniex/src/update/download.rs b/crates/calcine-geniex/src/update/download.rs index 7ade510..cfae05d 100644 --- a/crates/calcine-geniex/src/update/download.rs +++ b/crates/calcine-geniex/src/update/download.rs @@ -35,9 +35,9 @@ pub async fn fetch( Ok(digest) => { let _ = tokio::fs::remove_file(&partial).await; Err(Error::InvalidInput(format!( - "the downloaded GenieX installer doesn't match the official checksum \ - (expected {}, got {digest}); it wasn't installed", - asset.sha256 + "the downloaded {} doesn't match the official checksum \ + (expected {}, got {digest}); it was deleted", + asset.name, asset.sha256 ))) } Err(err) => { @@ -59,7 +59,7 @@ async fn download( .send() .await .and_then(reqwest::Response::error_for_status) - .map_err(|err| Error::Network(format!("couldn't download GenieX: {err}")))?; + .map_err(|err| Error::Network(format!("couldn't download {}: {err}", asset.name)))?; let total = response.content_length().unwrap_or(asset.size); let mut file = tokio::fs::File::create(path).await?; let mut hasher = Sha256::new(); @@ -72,7 +72,9 @@ async fn download( () = ctx.cancelled() => return Err(Error::Cancelled), }; let Some(chunk) = chunk else { break }; - let chunk = chunk.map_err(|err| Error::Network(format!("GenieX download broke: {err}")))?; + let chunk = chunk.map_err(|err| { + Error::Network(format!("the download of {} broke: {err}", asset.name)) + })?; hasher.update(&chunk); file.write_all(&chunk).await?; done += chunk.len() as u64; @@ -88,6 +90,7 @@ async fn download( total_bytes: Some(total), bytes_per_second: speed, phase: Some(JobPhase::Downloading), + step: None, }); } file.flush().await?; diff --git a/crates/calcine-geniex/src/update/index.rs b/crates/calcine-geniex/src/update/index.rs index e0ff03b..bc9ac15 100644 --- a/crates/calcine-geniex/src/update/index.rs +++ b/crates/calcine-geniex/src/update/index.rs @@ -47,14 +47,22 @@ pub fn latest_tag(index_json: &str, channel: ReleaseChannel) -> Result Result> { +/// The package of `kind` (`cli-installer` on Windows, `cli-archive` on +/// Linux) for `platform`/`arch` in a release manifest, or `None` when that +/// release doesn't ship one. +pub fn release( + manifest_json: &str, + platform: &str, + arch: &str, + kind: &str, +) -> Result> { let manifest: Manifest = serde_json::from_str(manifest_json) .map_err(|err| Error::Parse(format!("unexpected GenieX release manifest: {err}")))?; - let Some(asset) = manifest.assets.into_iter().find(|asset| { - asset.kind == "cli-installer" && asset.platform == platform && asset.arch == arch - }) else { + let Some(asset) = manifest + .assets + .into_iter() + .find(|asset| asset.kind == kind && asset.platform == platform && asset.arch == arch) + else { return Ok(None); }; Ok(Some(RuntimeRelease { @@ -115,7 +123,9 @@ mod tests { #[test] fn finds_the_windows_arm64_installer() { - let release = release(MANIFEST, "windows", "arm64").unwrap().unwrap(); + let release = release(MANIFEST, "windows", "arm64", "cli-installer") + .unwrap() + .unwrap(); assert_eq!(release.version, "v0.8.0"); assert!(!release.prerelease); assert_eq!( @@ -132,7 +142,10 @@ mod tests { #[test] fn no_installer_for_other_platforms() { - assert_eq!(release(MANIFEST, "windows", "x64").unwrap(), None); + assert_eq!( + release(MANIFEST, "windows", "x64", "cli-installer").unwrap(), + None + ); } #[test] diff --git a/crates/calcine-geniex/src/update/mod.rs b/crates/calcine-geniex/src/update/mod.rs index d469468..8916cce 100644 --- a/crates/calcine-geniex/src/update/mod.rs +++ b/crates/calcine-geniex/src/update/mod.rs @@ -1,15 +1,17 @@ -//! Installing and updating GenieX through its official installer. +//! Installing and updating GenieX: its official installer on Windows, its +//! official archive on Linux. //! //! - `index`: the release index and manifests `geniex update` reads //! - `download`: fetching an installer with progress and SHA-256 check //! - `signature`: Authenticode check of the installer //! -//! The installer is per-user Inno Setup: it uninstalls the previous version -//! (killing any running `geniex.exe`) and installs to -//! `%LOCALAPPDATA%\GenieX CLI`. Installers are kept in a cache to roll back -//! or repair without downloading. +//! On Windows the installer is per-user Inno Setup: it uninstalls the +//! previous version (killing any running `geniex.exe`) and installs to +//! `%LOCALAPPDATA%\GenieX CLI`. On Linux the archive is unpacked to +//! `~/.local/share/geniex-cli`, replacing the previous version in one move. +//! Packages are kept in a cache to roll back or repair without downloading. -mod download; +pub(crate) mod download; mod index; mod signature; @@ -25,6 +27,23 @@ use calcine_core::runtime::{ use calcine_core::{Error, Result}; use crate::Geniex; +use crate::cli::discovery; +use crate::platform; + +/// What GenieX ships for this platform: an installer, or an archive. +const PACKAGE_KIND: &str = if cfg!(windows) { + "cli-installer" +} else { + "cli-archive" +}; + +/// Cached packages are named ``. +const CACHE_PREFIX: &str = if cfg!(windows) { + "geniex-cli-setup-" +} else { + "geniex-cli-" +}; +const CACHE_SUFFIX: &str = if cfg!(windows) { ".exe" } else { ".tar.gz" }; /// Where Qualcomm publishes GenieX releases. pub const RELEASE_ENDPOINT: &str = @@ -104,7 +123,7 @@ impl GeniexInstaller { fn cached_path(&self, version: &str) -> PathBuf { self.config .cache_dir - .join(format!("geniex-cli-setup-{version}.exe")) + .join(format!("{CACHE_PREFIX}{version}{CACHE_SUFFIX}")) } /// The installer file for `source`, downloading and verifying it if needed. @@ -150,8 +169,8 @@ impl GeniexInstaller { .filter_map(|entry| { let name = entry.file_name().into_string().ok()?; let version = name - .strip_prefix("geniex-cli-setup-")? - .strip_suffix(".exe")? + .strip_prefix(CACHE_PREFIX)? + .strip_suffix(CACHE_SUFFIX)? .to_owned(); Some((version, entry.path())) }) @@ -167,16 +186,21 @@ impl RuntimeInstaller for GeniexInstaller { let latest = match index::latest_tag(&index_json, channel)? { Some(tag) => { let manifest = self.get_text(&format!("manifest-{tag}.json")).await?; - index::release(&manifest, "windows", "arm64")? + index::release(&manifest, platform::NAME, platform::ARCH, PACKAGE_KIND)? } None => None, }; - let publisher_signed = match self.get_text("windows-signed.txt").await { - Ok(text) => index::publisher_signed(&text), - Err(err) => { - tracing::warn!(%err, "couldn't read windows-signed.txt"); - false + // Code signing only applies to the Windows installer. + let publisher_signed = if cfg!(windows) { + match self.get_text("windows-signed.txt").await { + Ok(text) => index::publisher_signed(&text), + Err(err) => { + tracing::warn!(%err, "couldn't read windows-signed.txt"); + false + } } + } else { + false }; let update_available = latest.as_ref().is_some_and(|latest| { current.as_deref().is_none_or(|current| { @@ -210,7 +234,11 @@ impl RuntimeInstaller for GeniexInstaller { // Past this point the job can't be cancelled: stopping the installer // halfway could leave GenieX half removed. ctx.report(phase(JobPhase::Installing)); - run_installer(&installer).await?; + if cfg!(windows) { + run_installer(&installer).await?; + } else { + unpack(&installer).await?; + } let reported = self.current_version().await.ok_or_else(|| Error::Command { command: "version".into(), @@ -260,6 +288,47 @@ fn phase(phase: JobPhase) -> JobProgress { } } +/// Unpack the Linux archive into the install folder: first beside it, then +/// swapped in, so a failure never leaves half a GenieX. +async fn unpack(archive: &Path) -> Result<()> { + let target = discovery::official_install_dir() + .ok_or_else(|| Error::InvalidInput("can't find the user's data folder".into()))?; + let parent = target + .parent() + .ok_or_else(|| Error::InvalidInput("unexpected install folder".into()))?; + let staging = parent.join(".geniex-cli-new"); + let previous = parent.join(".geniex-cli-old"); + let _ = tokio::fs::remove_dir_all(&staging).await; + tokio::fs::create_dir_all(&staging).await?; + let output = tokio::process::Command::new("tar") + .arg("-xzf") + .arg(archive) + .arg("-C") + .arg(&staging) + // The archive holds one `geniex-cli-linux-arm64-` folder. + .arg("--strip-components=1") + .kill_on_drop(true) + .output() + .await?; + if !output.status.success() || !staging.join("geniex").is_file() { + let _ = tokio::fs::remove_dir_all(&staging).await; + return Err(Error::Command { + command: "tar".into(), + message: format!( + "the GenieX archive didn't unpack: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + }); + } + let _ = tokio::fs::remove_dir_all(&previous).await; + if target.exists() { + tokio::fs::rename(&target, &previous).await?; + } + tokio::fs::rename(&staging, &target).await?; + let _ = tokio::fs::remove_dir_all(&previous).await; + Ok(()) +} + /// Run the Inno Setup installer silently and wait for it. async fn run_installer(installer: &Path) -> Result<()> { let mut command = tokio::process::Command::new(installer); diff --git a/crates/calcine-geniex/tests/context_real.rs b/crates/calcine-geniex/tests/context_real.rs new file mode 100644 index 0000000..d25bcf7 --- /dev/null +++ b/crates/calcine-geniex/tests/context_real.rs @@ -0,0 +1,44 @@ +//! Token counts against a real cached model (skipped without one). + +use calcine_core::context::ContextMeter; +use calcine_geniex::{Geniex, GeniexContext}; +use serde_json::json; + +#[tokio::test] +#[ignore = "needs qualcomm/Qwen3-0.6B in the GenieX cache"] +async fn matches_what_geniex_reports() { + let context = GeniexContext::new(Geniex::default()); + let model = "qualcomm/Qwen3-0.6B"; + assert_eq!(context.compiled_window(model).await.unwrap(), Some(4096)); + // Prompt sizes `geniex serve` reported for these requests. + for (request, reported) in [ + ( + json!({ "enable_think": false, "messages": [{ "role": "user", "content": "Hello" }] }), + 26, + ), + ( + json!({ "enable_think": false, "messages": [{ "role": "user", "content": "Hello world how are you" }] }), + 30, + ), + ( + json!({ "enable_think": false, "messages": [{ "role": "system", "content": "Be brief." }, { "role": "user", "content": "Hello" }] }), + 22, + ), + ( + json!({ "enable_think": false, "messages": [{ "role": "user", "content": "Hello" }, { "role": "assistant", "content": "Hi there" }, { "role": "user", "content": "Bye" }] }), + 40, + ), + ( + json!({ "enable_think": true, "messages": [{ "role": "user", "content": "Hello" }] }), + 22, + ), + ] { + let count = context.count(model, &request).await.unwrap(); + assert!(count.exact); + let total = i64::from(count.total()); + assert!( + (total - reported).abs() <= 2, + "{request}: {total} vs {reported}" + ); + } +} diff --git a/crates/calcine-geniex/tests/fixtures/bench_report.json b/crates/calcine-geniex/tests/fixtures/bench_report.json new file mode 100644 index 0000000..fee13f9 --- /dev/null +++ b/crates/calcine-geniex/tests/fixtures/bench_report.json @@ -0,0 +1,83 @@ +{ + "schema_version": "6", + "cell_id": "test-qairt", + "plugin": "qairt", + "device": "npu", + "device_id": "NPU", + "model_path": "C:\\Users\\me\\.cache\\geniex\\models\\qualcomm/Qwen3-0.6B\\part2_of_2.bin", + "model_size_bytes": 437252096, + "geniex_version": "v0.8.0", + "qairt_version": "2.45", + "llama_cpp_version": "9425611", + "params": { + "warmup": 1, + "repetitions": 2, + "n_prompt": 128, + "n_gen": 32, + "temperature": 0.0, + "top_p": 0.0, + "seed": 42, + "n_ctx": 0, + "n_threads": 0, + "n_ubatch": 0, + "n_gpu_layers": 0 + }, + "runs": [ + { + "run_idx": 0, + "ttft_us": 40397, + "media_us": 0, + "prompt_time_us": 40397, + "decode_time_us": 607513, + "prompt_tokens": 128, + "gen_tokens": 32, + "prefill_tps": 3168.55212, + "decode_tps": 52.67377, + "stop_reason": "length" + }, + { + "run_idx": 1, + "ttft_us": 36461, + "media_us": 0, + "prompt_time_us": 36461, + "decode_time_us": 598450, + "prompt_tokens": 128, + "gen_tokens": 32, + "prefill_tps": 3510.600368, + "decode_tps": 53.471468, + "stop_reason": "length" + } + ], + "agg": { + "ttft_ms": { + "median": 38.429, + "min": 36.461, + "max": 40.397, + "mean": 38.429, + "stdev": 2.783172 + }, + "prefill_tps": { + "median": 3339.576244, + "min": 3168.55212, + "max": 3510.600368, + "mean": 3339.576244, + "stdev": 241.864635 + }, + "decode_tps": { + "median": 53.072619, + "min": 52.67377, + "max": 53.471468, + "mean": 53.072619, + "stdev": 0.564058 + }, + "gen_tokens": { + "median": 32.0 + }, + "prompt_tokens": { + "median": 128.0 + }, + "media_ms": { + "median": 0.0 + } + } +} diff --git a/crates/calcine-geniex/tests/real_geniex.rs b/crates/calcine-geniex/tests/real_geniex.rs index 6e3f767..6957c6a 100644 --- a/crates/calcine-geniex/tests/real_geniex.rs +++ b/crates/calcine-geniex/tests/real_geniex.rs @@ -57,10 +57,10 @@ async fn lists_the_ai_hub_catalog_for_this_device() { #[tokio::test] #[ignore = "needs a local GenieX install"] async fn serve_starts_answers_and_stops() { - use calcine_core::runtime::{InferenceServer, ServerState}; - use calcine_geniex::{GeniexServer, ServeOptions}; + use calcine_core::runtime::{InferenceServer, ServerOptions, ServerState}; + use calcine_geniex::GeniexServer; - let server = GeniexServer::new(Geniex::default(), ServeOptions::default()); + let server = GeniexServer::new(Geniex::default(), ServerOptions::default()); let url = server .ensure_running() .await diff --git a/crates/calcine-hw/src/lib.rs b/crates/calcine-hw/src/lib.rs index b010c50..207ea55 100644 --- a/crates/calcine-hw/src/lib.rs +++ b/crates/calcine-hw/src/lib.rs @@ -2,11 +2,16 @@ //! //! - `snapshot`: what the machine has (CPU, memory, disk, accelerators) //! - `usage`: live load, sampled on demand +//! - `power`: power and energy, on devices with energy metering //! - `system`: CPU, memory and disk, portable (`sysinfo`) //! - `windows`: Hexagon NPU and Adreno GPU, from the driver database (WMI) -//! and the performance counters. Other platforms report no accelerators -//! or accelerator load for now. +//! and the performance counters +//! - `linux`: the Hexagon NPU and Adreno GPU from `/dev` and `/sys`, no load +//! or power yet. Other platforms report no accelerators. +#[cfg(target_os = "linux")] +mod linux; +mod power; mod snapshot; mod system; mod usage; @@ -17,7 +22,7 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use calcine_core::hardware::{HardwareInfo, HardwareProbe, HardwareUsage}; +use calcine_core::hardware::{EnergyReading, HardwareInfo, HardwareProbe, HardwareUsage}; use calcine_core::{Error, Result}; use crate::usage::UsageSampler; @@ -53,6 +58,17 @@ impl HardwareProbe for SystemProbe { }) .await } + + async fn energy(&self) -> Result> { + let sampler = self.sampler.clone(); + blocking(move || { + let mut sampler = sampler + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sampler.get_or_insert_with(UsageSampler::new).energy() + }) + .await + } } async fn blocking(work: impl FnOnce() -> T + Send + 'static) -> Result { diff --git a/crates/calcine-hw/src/linux.rs b/crates/calcine-hw/src/linux.rs new file mode 100644 index 0000000..790108e --- /dev/null +++ b/crates/calcine-hw/src/linux.rs @@ -0,0 +1,145 @@ +//! Linux: the Hexagon NPU and the Adreno GPU, from `/dev` and `/sys`. +//! +//! - NPU: the compute DSP the FastRPC driver exposes (`/dev/fastrpc-cdsp`), +//! or a remote processor named `cdsp`. +//! - GPU: a DRM card driven by `msm` (Adreno, upstream), or the KGSL driver +//! of Qualcomm's own kernels. +//! - SoC: the device tree's `compatible`, e.g. `qcom,x1e80100`. + +use std::path::Path; + +use calcine_core::hardware::Accelerator; +use calcine_core::models::ComputeUnit; + +use crate::snapshot::PlatformDevices; + +pub(crate) fn devices() -> PlatformDevices { + devices_in(Path::new("/")) +} + +fn devices_in(root: &Path) -> PlatformDevices { + let mut accelerators = Vec::new(); + if has_npu(root) { + accelerators.push(Accelerator { + unit: ComputeUnit::Npu, + name: "Qualcomm Hexagon NPU".into(), + driver_version: None, + }); + } + if let Some(name) = gpu(root) { + accelerators.push(Accelerator { + unit: ComputeUnit::Gpu, + name, + driver_version: None, + }); + } + PlatformDevices { + accelerators, + cpu_name: soc(root), + } +} + +fn read(root: &Path, path: &str) -> Option { + std::fs::read_to_string(root.join(path)).ok() +} + +/// Entries of a sysfs class folder, like `/sys/class/drm/*`. +fn entries(root: &Path, class: &str) -> Vec { + std::fs::read_dir(root.join(class)) + .map(|entries| entries.flatten().map(|entry| entry.path()).collect()) + .unwrap_or_default() +} + +fn has_npu(root: &Path) -> bool { + root.join("dev/fastrpc-cdsp").exists() + || root.join("dev/fastrpc-cdsp-secure").exists() + || entries(root, "sys/class/remoteproc") + .iter() + .any(|processor| { + std::fs::read_to_string(processor.join("name")) + .is_ok_and(|name| name.to_ascii_lowercase().contains("cdsp")) + }) +} + +fn gpu(root: &Path) -> Option { + if root.join("sys/class/kgsl/kgsl-3d0").exists() { + let model = read(root, "sys/class/kgsl/kgsl-3d0/gpu_model") + .map(|model| model.trim().to_owned()) + .filter(|model| !model.is_empty()); + return Some(model.map_or_else( + || "Qualcomm Adreno GPU".into(), + |model| format!("Qualcomm {model}"), + )); + } + entries(root, "sys/class/drm") + .iter() + .any(|card| { + std::fs::read_to_string(card.join("device/uevent")) + .is_ok_and(|uevent| uevent.lines().any(|line| line == "DRIVER=msm")) + }) + .then(|| "Qualcomm Adreno GPU".into()) +} + +/// `qcom,x1e80100` → `Qualcomm X1E80100`. Strings are NUL-separated. +fn soc(root: &Path) -> Option { + let compatible = std::fs::read(root.join("sys/firmware/devicetree/base/compatible")).ok()?; + String::from_utf8_lossy(&compatible) + .split('\0') + .find_map(|entry| entry.strip_prefix("qcom,")) + .filter(|chip| !chip.is_empty()) + .map(|chip| format!("Qualcomm {}", chip.to_ascii_uppercase())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tree(files: &[(&str, &[u8])]) -> std::path::PathBuf { + let root = + std::env::temp_dir().join(format!("calcine-hw-{}-{}", std::process::id(), files.len())); + let _ = std::fs::remove_dir_all(&root); + for (path, contents) in files { + let path = root.join(path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, contents).unwrap(); + } + root + } + + #[test] + fn finds_a_snapdragon_x_laptop() { + let root = tree(&[ + ("dev/fastrpc-cdsp", b""), + ( + "sys/class/drm/card0/device/uevent", + b"DRIVER=msm\nOF_NAME=gpu\n", + ), + ( + "sys/firmware/devicetree/base/compatible", + b"lenovo,thinkpad-t14s\0qcom,x1e78100\0qcom,x1e80100\0", + ), + ]); + let devices = devices_in(&root); + let units: Vec<_> = devices.accelerators.iter().map(|a| a.unit).collect(); + assert_eq!(units, [ComputeUnit::Npu, ComputeUnit::Gpu]); + assert_eq!(devices.cpu_name.as_deref(), Some("Qualcomm X1E78100")); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn finds_qualcomm_kernels_and_nothing_elsewhere() { + let root = tree(&[ + ("sys/class/remoteproc/remoteproc1/name", b"cdsp\n"), + ("sys/class/kgsl/kgsl-3d0/gpu_model", b"Adreno643v1\n"), + ]); + let devices = devices_in(&root); + assert_eq!(devices.accelerators.len(), 2); + assert_eq!(devices.accelerators[1].name, "Qualcomm Adreno643v1"); + assert_eq!(devices.cpu_name, None); + let _ = std::fs::remove_dir_all(&root); + + let empty = tree(&[("sys/class/drm/card0/device/uevent", b"DRIVER=virtio_gpu\n")]); + assert_eq!(devices_in(&empty).accelerators, Vec::new()); + let _ = std::fs::remove_dir_all(empty); + } +} diff --git a/crates/calcine-hw/src/power.rs b/crates/calcine-hw/src/power.rs new file mode 100644 index 0000000..6f61fe2 --- /dev/null +++ b/crates/calcine-hw/src/power.rs @@ -0,0 +1,176 @@ +//! Power and energy from the device's energy metering, when it has some. +//! +//! The channels are named by the platform: on Snapdragon X, `SYS` is the +//! whole system, `CPU_CLUSTER_0`… the CPU clusters and `GPU` the Adreno GPU. +//! There is no channel for the NPU: its work shows in `SYS`. + +use calcine_core::hardware::{EnergyReading, PowerDraw}; + +#[cfg(windows)] +use crate::windows::energy::EnergyMeters; + +/// Energy a channel has used since its meter started. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Channel { + pub name: String, + pub joules: f64, + pub seconds: f64, +} + +/// Reads the meters and remembers the last reading, so each draw is the +/// average power since the previous one. +#[derive(Debug, Default)] +pub(crate) struct PowerMeter { + #[cfg(windows)] + meters: Option, + previous: Option>, +} + +impl PowerMeter { + pub(crate) fn open() -> Self { + Self { + #[cfg(windows)] + meters: EnergyMeters::open() + .inspect_err(|err| tracing::info!(%err, "power won't be shown")) + .ok(), + previous: None, + } + } + + #[cfg(windows)] + fn read(&self) -> Option> { + let channels = self.meters.as_ref()?.read(); + channels + .inspect_err(|err| tracing::debug!(%err, "couldn't read the energy meters")) + .ok() + .map(|channels| { + channels + .into_iter() + .map(|channel| Channel { + name: channel.name, + joules: channel.joules, + seconds: channel.seconds, + }) + .collect() + }) + } + + #[cfg(not(windows))] + #[allow(clippy::unused_self)] + fn read(&self) -> Option> { + None + } + + /// Power since the previous call. `None` without metering, and on the + /// first call. + pub(crate) fn draw(&mut self) -> Option { + let current = self.read()?; + let previous = self.previous.replace(current.clone())?; + draw_between(&previous, ¤t) + } + + /// Energy the whole system has used so far. + pub(crate) fn system_energy(&self) -> Option { + system(&self.read()?) + } +} + +fn is_system(name: &str) -> bool { + name.eq_ignore_ascii_case("SYS") +} + +fn is_cpu(name: &str) -> bool { + name.to_ascii_uppercase().starts_with("CPU") +} + +fn is_gpu(name: &str) -> bool { + name.eq_ignore_ascii_case("GPU") +} + +fn system(channels: &[Channel]) -> Option { + channels + .iter() + .find(|channel| is_system(&channel.name)) + .map(|channel| EnergyReading { + joules: channel.joules, + seconds: channel.seconds, + }) +} + +/// Average power of each kind of channel between two readings. +#[allow(clippy::cast_possible_truncation)] +fn draw_between(before: &[Channel], after: &[Channel]) -> Option { + let watts = |pick: fn(&str) -> bool| -> Option { + let mut total = None; + for channel in after.iter().filter(|channel| pick(&channel.name)) { + let earlier = before.iter().find(|other| other.name == channel.name)?; + let seconds = channel.seconds - earlier.seconds; + if seconds <= 0.0 { + return None; + } + let channel_watts = (channel.joules - earlier.joules) / seconds; + *total.get_or_insert(0.0) += channel_watts; + } + total.map(|watts: f64| watts.max(0.0) as f32) + }; + Some(PowerDraw { + system_watts: watts(is_system)?, + cpu_watts: watts(is_cpu), + gpu_watts: watts(is_gpu), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn channel(name: &str, joules: f64, seconds: f64) -> Channel { + Channel { + name: name.into(), + joules, + seconds, + } + } + + #[test] + fn averages_each_kind_of_channel() { + let before = [ + channel("SYS", 100.0, 10.0), + channel("CPU_CLUSTER_0", 10.0, 10.0), + channel("CPU_CLUSTER_1", 20.0, 10.0), + channel("GPU", 1.0, 10.0), + channel("PSU_USB", 5.0, 10.0), + ]; + let after = [ + channel("SYS", 130.0, 12.0), + channel("CPU_CLUSTER_0", 16.0, 12.0), + channel("CPU_CLUSTER_1", 22.0, 12.0), + channel("GPU", 1.0, 12.0), + channel("PSU_USB", 50.0, 12.0), + ]; + assert_eq!( + draw_between(&before, &after), + Some(PowerDraw { + system_watts: 15.0, + cpu_watts: Some(4.0), + gpu_watts: Some(0.0), + }) + ); + assert_eq!( + system(&after), + Some(EnergyReading { + joules: 130.0, + seconds: 12.0 + }) + ); + } + + #[test] + fn needs_a_system_channel_and_time_passing() { + let gpu = [channel("GPU", 1.0, 10.0)]; + assert_eq!(draw_between(&gpu, &gpu), None); + let sys = [channel("SYS", 1.0, 10.0)]; + assert_eq!(draw_between(&sys, &sys), None); + assert_eq!(system(&gpu), None); + } +} diff --git a/crates/calcine-hw/src/snapshot.rs b/crates/calcine-hw/src/snapshot.rs index f61dbfa..229f3d3 100644 --- a/crates/calcine-hw/src/snapshot.rs +++ b/crates/calcine-hw/src/snapshot.rs @@ -32,7 +32,12 @@ fn platform_devices() -> PlatformDevices { }) } -#[cfg(not(windows))] +#[cfg(target_os = "linux")] +fn platform_devices() -> PlatformDevices { + crate::linux::devices() +} + +#[cfg(not(any(windows, target_os = "linux")))] fn platform_devices() -> PlatformDevices { PlatformDevices::default() } diff --git a/crates/calcine-hw/src/usage.rs b/crates/calcine-hw/src/usage.rs index 4eff72e..6af3f94 100644 --- a/crates/calcine-hw/src/usage.rs +++ b/crates/calcine-hw/src/usage.rs @@ -1,8 +1,9 @@ //! Live load: CPU and memory from `sysinfo`, NPU and GPU from the OS. -use calcine_core::hardware::{HardwareUsage, MemoryInfo}; +use calcine_core::hardware::{EnergyReading, HardwareUsage, MemoryInfo}; use sysinfo::{CpuRefreshKind, MemoryRefreshKind, System}; +use crate::power::PowerMeter; #[cfg(windows)] use crate::windows::gpu_engines::GpuEngines; @@ -12,6 +13,7 @@ pub(crate) struct UsageSampler { system: System, #[cfg(windows)] engines: Option, + power: PowerMeter, } impl UsageSampler { @@ -24,6 +26,7 @@ impl UsageSampler { engines: GpuEngines::open() .inspect_err(|err| tracing::warn!(%err, "NPU and GPU load won't be shown")) .ok(), + power: PowerMeter::open(), } } @@ -41,9 +44,14 @@ impl UsageSampler { total_bytes: self.system.total_memory(), available_bytes: self.system.available_memory(), }, + power: self.power.draw(), } } + pub(crate) fn energy(&self) -> Option { + self.power.system_energy() + } + #[cfg(windows)] fn engine_load(&self) -> (Option, Option) { match self.engines.as_ref().map(GpuEngines::read) { diff --git a/crates/calcine-hw/src/windows/energy.rs b/crates/calcine-hw/src/windows/energy.rs new file mode 100644 index 0000000..0b56d8a --- /dev/null +++ b/crates/calcine-hw/src/windows/energy.rs @@ -0,0 +1,82 @@ +//! Energy from the `Energy Meter` performance counters, Windows' view of the +//! SoC's energy metering (EMI). The meters themselves only open for +//! administrators; the counters are readable by everyone. +//! +//! On Snapdragon X there are channels for the CPU clusters, the GPU and the +//! whole system (`SYS`), but none for the NPU: what the NPU costs shows in +//! `SYS`. + +use std::collections::HashMap; + +use super::pdh::{Query, translated_path}; + +/// Joules in a picowatt-hour. +const JOULES_PER_PICOWATT_HOUR: f64 = 3.6e-9; + +/// Energy a channel has used since its meter started. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ChannelEnergy { + pub name: String, + pub joules: f64, + /// When it was measured, in seconds since the meter started. + pub seconds: f64, +} + +/// Live `Energy Meter(*)\Energy` and `\Time` queries. +#[derive(Debug)] +pub(crate) struct EnergyMeters { + energy: Query, + time: Query, +} + +impl EnergyMeters { + /// An error on devices without energy metering. + pub(crate) fn open() -> Result { + let energy = Query::open_translated(&translated_path("Energy Meter", "Energy")?)?; + let time = Query::open_translated(&translated_path("Energy Meter", "Time")?)?; + let meters = Self { energy, time }; + if meters.read()?.is_empty() { + return Err("this device has no energy metering".into()); + } + Ok(meters) + } + + /// Energy of every channel, `_Total` left out (it isn't a sum here). + pub(crate) fn read(&self) -> Result, String> { + let times: HashMap = self.time.read()?.into_iter().collect(); + Ok(self + .energy + .read()? + .into_iter() + .filter(|(name, _)| name != "_Total") + .map(|(name, picowatt_hours)| ChannelEnergy { + // Milliseconds since the meter started. + seconds: times.get(&name).copied().unwrap_or(0.0) / 1000.0, + joules: picowatt_hours * JOULES_PER_PICOWATT_HOUR, + name, + }) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[ignore = "needs a device with energy metering"] + fn reads_the_meters() { + let meters = EnergyMeters::open().unwrap(); + let first = meters.read().unwrap(); + std::thread::sleep(std::time::Duration::from_secs(3)); + let second = meters.read().unwrap(); + for (a, b) in first.iter().zip(&second) { + let watts = (b.joules - a.joules) / (b.seconds - a.seconds); + println!( + "{}: {watts:.2} W over {:.3} s", + a.name, + b.seconds - a.seconds + ); + } + } +} diff --git a/crates/calcine-hw/src/windows/gpu_engines.rs b/crates/calcine-hw/src/windows/gpu_engines.rs index 07ca32c..1c6ce95 100644 --- a/crates/calcine-hw/src/windows/gpu_engines.rs +++ b/crates/calcine-hw/src/windows/gpu_engines.rs @@ -8,16 +8,9 @@ //! An adapter whose engines are all `Compute` is the NPU; one with 3D and //! video engines is the GPU. -// The PDH C API needs `unsafe`; it is confined to the `Query` type below. -#![allow(unsafe_code)] - use std::collections::{HashMap, HashSet}; -use windows_sys::Win32::System::Performance::{ - PDH_CSTATUS_NEW_DATA, PDH_CSTATUS_VALID_DATA, PDH_FMT_COUNTERVALUE_ITEM_W, PDH_FMT_DOUBLE, - PDH_HCOUNTER, PDH_HQUERY, PDH_MORE_DATA, PdhAddEnglishCounterW, PdhCloseQuery, - PdhCollectQueryData, PdhGetFormattedCounterArrayW, PdhOpenQueryW, -}; +use super::pdh::Query; /// Load of the NPU and GPU, 0-100. #[derive(Debug, Clone, Copy, Default, PartialEq)] @@ -111,122 +104,6 @@ impl Instance { } } -/// A PDH query holding one wildcard counter. -#[derive(Debug)] -struct Query { - query: PDH_HQUERY, - counter: PDH_HCOUNTER, -} - -// SAFETY: PDH handles aren't tied to the thread that created them; Calcine -// only uses a query from one thread at a time (behind a mutex). -unsafe impl Send for Query {} - -impl Query { - fn open(path: &str) -> Result { - let path: Vec = path.encode_utf16().chain(Some(0)).collect(); - let mut query: PDH_HQUERY = std::ptr::null_mut(); - // SAFETY: a null data source means live counters; `query` is a valid - // out pointer. - check(unsafe { PdhOpenQueryW(std::ptr::null(), 0, &raw mut query) })?; - let mut counter: PDH_HCOUNTER = std::ptr::null_mut(); - // SAFETY: `query` is open and `path` is NUL-terminated UTF-16 that - // outlives the call. - let added = unsafe { PdhAddEnglishCounterW(query, path.as_ptr(), 0, &raw mut counter) }; - if let Err(err) = check(added) { - // SAFETY: `query` is open and closed only here. - unsafe { PdhCloseQuery(query) }; - return Err(err); - } - Ok(Self { query, counter }) - } - - /// Collect a sample and return each instance's value. - fn read(&self) -> Result, String> { - // SAFETY: `self.query` stays open for `self`'s lifetime. - check(unsafe { PdhCollectQueryData(self.query) })?; - - let mut size = 0_u32; - let mut count = 0_u32; - // SAFETY: a null buffer with size 0 asks for the required size. - let status = unsafe { - PdhGetFormattedCounterArrayW( - self.counter, - PDH_FMT_DOUBLE, - &raw mut size, - &raw mut count, - std::ptr::null_mut(), - ) - }; - if status != PDH_MORE_DATA { - // No instances (nothing uses the GPU or NPU): an empty sample. - return Ok(Vec::new()); - } - - // The buffer holds the items followed by the strings they point to; - // u64 storage keeps it aligned for the items. - let mut buffer = vec![0_u64; (size as usize).div_ceil(8)]; - let items = buffer.as_mut_ptr().cast::(); - // SAFETY: `buffer` is at least `size` bytes and suitably aligned. - check(unsafe { - PdhGetFormattedCounterArrayW( - self.counter, - PDH_FMT_DOUBLE, - &raw mut size, - &raw mut count, - items, - ) - })?; - - let mut samples = Vec::with_capacity(count as usize); - for index in 0..count as usize { - // SAFETY: PDH wrote `count` items at the start of `buffer`. - let item = unsafe { *items.add(index) }; - let status = item.FmtValue.CStatus; - if status != PDH_CSTATUS_VALID_DATA && status != PDH_CSTATUS_NEW_DATA { - continue; - } - // SAFETY: `szName` points to a NUL-terminated string inside `buffer`. - let name = unsafe { wide_to_string(item.szName) }; - // SAFETY: PDH_FMT_DOUBLE fills the `doubleValue` member. - let value = unsafe { item.FmtValue.Anonymous.doubleValue }; - samples.push((name, value)); - } - Ok(samples) - } -} - -impl Drop for Query { - fn drop(&mut self) { - // SAFETY: the query is open and dropped once. - unsafe { PdhCloseQuery(self.query) }; - } -} - -/// # Safety -/// -/// `ptr` must be null or point to a NUL-terminated UTF-16 string. -unsafe fn wide_to_string(ptr: *const u16) -> String { - if ptr.is_null() { - return String::new(); - } - let mut len = 0; - // SAFETY: guaranteed NUL-terminated by the caller. - while unsafe { *ptr.add(len) } != 0 { - len += 1; - } - // SAFETY: `len` u16s before the NUL are readable. - String::from_utf16_lossy(unsafe { std::slice::from_raw_parts(ptr, len) }) -} - -fn check(status: u32) -> Result<(), String> { - if status == 0 { - Ok(()) - } else { - Err(format!("performance counter error 0x{status:08X}")) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/calcine-hw/src/windows/mod.rs b/crates/calcine-hw/src/windows/mod.rs index 9a113f6..92307c7 100644 --- a/crates/calcine-hw/src/windows/mod.rs +++ b/crates/calcine-hw/src/windows/mod.rs @@ -4,6 +4,10 @@ //! database (WMI) //! - `gpu_engines`: NPU and GPU load, from the `GPU Engine` performance //! counters (PDH), like Task Manager +//! - `energy`: power drawn by the system, CPU clusters and GPU, from the +//! `Energy Meter` counters (the SoC's energy metering) pub(crate) mod devices; +pub(crate) mod energy; pub(crate) mod gpu_engines; +mod pdh; diff --git a/crates/calcine-hw/src/windows/pdh.rs b/crates/calcine-hw/src/windows/pdh.rs new file mode 100644 index 0000000..6bafef7 --- /dev/null +++ b/crates/calcine-hw/src/windows/pdh.rs @@ -0,0 +1,262 @@ +//! A small safe wrapper over the PDH performance counter API. + +// The PDH C API needs `unsafe`; it is confined to this module. +#![allow(unsafe_code)] + +use windows_sys::Win32::System::Performance::{ + PDH_CSTATUS_NEW_DATA, PDH_CSTATUS_VALID_DATA, PDH_FMT_COUNTERVALUE_ITEM_W, PDH_FMT_DOUBLE, + PDH_HCOUNTER, PDH_HQUERY, PDH_MORE_DATA, PERF_REG_COUNTER_ENGLISH_NAMES, + PERF_REG_COUNTER_NAME_STRINGS, PERF_REG_COUNTERSET_ENGLISH_NAME, + PERF_REG_COUNTERSET_NAME_STRING, PdhAddCounterW, PdhAddEnglishCounterW, PdhCloseQuery, + PdhCollectQueryData, PdhGetFormattedCounterArrayW, PdhOpenQueryW, PerfEnumerateCounterSet, + PerfQueryCounterSetRegistrationInfo, PerfRegInfoType, +}; +use windows_sys::core::GUID; + +/// A PDH query holding one wildcard counter. +#[derive(Debug)] +pub(crate) struct Query { + query: PDH_HQUERY, + counter: PDH_HCOUNTER, +} + +// SAFETY: PDH handles aren't tied to the thread that created them; Calcine +// only uses a query from one thread at a time (behind a mutex). +unsafe impl Send for Query {} + +impl Query { + /// `path` uses English names, whatever the Windows language. + pub(crate) fn open(path: &str) -> Result { + Self::add(path, true) + } + + /// `path` uses the names Windows shows, see [`translated_path`]. + pub(crate) fn open_translated(path: &str) -> Result { + Self::add(path, false) + } + + fn add(path: &str, english: bool) -> Result { + let path: Vec = path.encode_utf16().chain(Some(0)).collect(); + let mut query: PDH_HQUERY = std::ptr::null_mut(); + // SAFETY: a null data source means live counters; `query` is a valid + // out pointer. + check(unsafe { PdhOpenQueryW(std::ptr::null(), 0, &raw mut query) })?; + let mut counter: PDH_HCOUNTER = std::ptr::null_mut(); + // SAFETY: `query` is open and `path` is NUL-terminated UTF-16 that + // outlives the call. + let added = unsafe { + if english { + PdhAddEnglishCounterW(query, path.as_ptr(), 0, &raw mut counter) + } else { + PdhAddCounterW(query, path.as_ptr(), 0, &raw mut counter) + } + }; + if let Err(err) = check(added) { + // SAFETY: `query` is open and closed only here. + unsafe { PdhCloseQuery(query) }; + return Err(err); + } + Ok(Self { query, counter }) + } + + /// Collect a sample and return each instance's value. + pub(crate) fn read(&self) -> Result, String> { + // SAFETY: `self.query` stays open for `self`'s lifetime. + check(unsafe { PdhCollectQueryData(self.query) })?; + + let mut size = 0_u32; + let mut count = 0_u32; + // SAFETY: a null buffer with size 0 asks for the required size. + let status = unsafe { + PdhGetFormattedCounterArrayW( + self.counter, + PDH_FMT_DOUBLE, + &raw mut size, + &raw mut count, + std::ptr::null_mut(), + ) + }; + if status != PDH_MORE_DATA { + // No instances (nothing uses the GPU or NPU): an empty sample. + return Ok(Vec::new()); + } + + // The buffer holds the items followed by the strings they point to; + // u64 storage keeps it aligned for the items. + let mut buffer = vec![0_u64; (size as usize).div_ceil(8)]; + let items = buffer.as_mut_ptr().cast::(); + // SAFETY: `buffer` is at least `size` bytes and suitably aligned. + check(unsafe { + PdhGetFormattedCounterArrayW( + self.counter, + PDH_FMT_DOUBLE, + &raw mut size, + &raw mut count, + items, + ) + })?; + + let mut samples = Vec::with_capacity(count as usize); + for index in 0..count as usize { + // SAFETY: PDH wrote `count` items at the start of `buffer`. + let item = unsafe { *items.add(index) }; + let status = item.FmtValue.CStatus; + if status != PDH_CSTATUS_VALID_DATA && status != PDH_CSTATUS_NEW_DATA { + continue; + } + // SAFETY: `szName` points to a NUL-terminated string inside `buffer`. + let name = unsafe { wide_to_string(item.szName) }; + // SAFETY: PDH_FMT_DOUBLE fills the `doubleValue` member. + let value = unsafe { item.FmtValue.Anonymous.doubleValue }; + samples.push((name, value)); + } + Ok(samples) + } +} + +impl Drop for Query { + fn drop(&mut self) { + // SAFETY: the query is open and dropped once. + unsafe { PdhCloseQuery(self.query) }; + } +} + +/// # Safety +/// +/// `ptr` must be null or point to a NUL-terminated UTF-16 string. +unsafe fn wide_to_string(ptr: *const u16) -> String { + if ptr.is_null() { + return String::new(); + } + let mut len = 0; + // SAFETY: guaranteed NUL-terminated by the caller. + while unsafe { *ptr.add(len) } != 0 { + len += 1; + } + // SAFETY: `len` u16s before the NUL are readable. + String::from_utf16_lossy(unsafe { std::slice::from_raw_parts(ptr, len) }) +} + +fn check(status: u32) -> Result<(), String> { + if status == 0 { + Ok(()) + } else { + Err(format!("performance counter error 0x{status:08X}")) + } +} + +/// The path of a counter in a counter set, both given by their English names, +/// written with the names Windows shows (`\Compteur Énergie(*)\Énergie` +/// on a French Windows). PDH can't add counter sets declared by a manifest, +/// like `Energy Meter`, by their English names on a non-English Windows. +pub(crate) fn translated_path(english_set: &str, english_counter: &str) -> Result { + for set in counter_sets()? { + if registration_string(&set, PERF_REG_COUNTERSET_ENGLISH_NAME).as_deref() + != Some(english_set) + { + continue; + } + let name = registration_string(&set, PERF_REG_COUNTERSET_NAME_STRING) + .ok_or_else(|| format!("{english_set} has no name"))?; + let id = counter_strings(&set, PERF_REG_COUNTER_ENGLISH_NAMES) + .into_iter() + .find(|(_, counter)| counter == english_counter) + .map(|(id, _)| id) + .ok_or_else(|| format!("{english_set} has no {english_counter} counter"))?; + let counter = counter_strings(&set, PERF_REG_COUNTER_NAME_STRINGS) + .into_iter() + .find(|(other, _)| *other == id) + .map(|(_, counter)| counter) + .ok_or_else(|| format!("{english_set} has no name for {english_counter}"))?; + return Ok(format!(r"\{name}(*)\{counter}")); + } + Err(format!("there is no {english_set} counter set")) +} + +/// Every counter set registered with PerfLib (manifest-based ones). +fn counter_sets() -> Result, String> { + let mut count = 0_u32; + // SAFETY: a null buffer with size 0 asks for the count. + unsafe { PerfEnumerateCounterSet(std::ptr::null(), std::ptr::null_mut(), 0, &raw mut count) }; + let mut sets = vec![GUID::from_u128(0); count as usize]; + // SAFETY: `sets` holds `count` GUIDs. + check(unsafe { + PerfEnumerateCounterSet(std::ptr::null(), sets.as_mut_ptr(), count, &raw mut count) + })?; + sets.truncate(count as usize); + Ok(sets) +} + +/// A counter set's registration data, in the current UI language. +fn registration(set: &GUID, request: PerfRegInfoType) -> Option> { + let mut size = 0_u32; + // SAFETY: a null buffer with size 0 asks for the size. + unsafe { + PerfQueryCounterSetRegistrationInfo( + std::ptr::null(), + set, + request, + 0, + std::ptr::null_mut(), + 0, + &raw mut size, + ) + }; + if size == 0 { + return None; + } + let mut buffer = vec![0_u8; size as usize]; + // SAFETY: `buffer` holds `size` bytes. + let status = unsafe { + PerfQueryCounterSetRegistrationInfo( + std::ptr::null(), + set, + request, + 0, + buffer.as_mut_ptr(), + size, + &raw mut size, + ) + }; + (status == 0).then_some(buffer) +} + +fn registration_string(set: &GUID, request: PerfRegInfoType) -> Option { + registration(set, request).map(|bytes| utf16_at(&bytes, 0)) +} + +/// `PERF_STRING_BUFFER_HEADER`, then one `(counter id, offset)` pair per +/// counter, then the strings the offsets point to. +fn counter_strings(set: &GUID, request: PerfRegInfoType) -> Vec<(u32, String)> { + let Some(bytes) = registration(set, request) else { + return Vec::new(); + }; + let u32_at = |offset: usize| { + bytes + .get(offset..offset + 4) + .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])) + }; + let count = u32_at(4).unwrap_or(0) as usize; + (0..count) + .filter_map(|index| { + let entry = 8 + index * 8; + let id = u32_at(entry)?; + let offset = u32_at(entry + 4)? as usize; + Some((id, utf16_at(&bytes, offset))) + }) + .collect() +} + +/// The NUL-terminated UTF-16 string at `offset` in `bytes`. +fn utf16_at(bytes: &[u8], offset: usize) -> String { + let units: Vec = bytes + .get(offset..) + .unwrap_or_default() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes(*pair)) + .take_while(|&c| c != 0) + .collect(); + String::from_utf16_lossy(&units) +} diff --git a/crates/calcine-mock/src/bench.rs b/crates/calcine-mock/src/bench.rs new file mode 100644 index 0000000..d6eaddd --- /dev/null +++ b/crates/calcine-mock/src/bench.rs @@ -0,0 +1,154 @@ +//! Simulated benchmarks: a fake tool download, then plausible numbers per +//! compute unit (the NPU fastest to the first token, the GPU close behind on +//! decode), with a little noise and a sample history. + +use std::sync::{MutexGuard, PoisonError}; + +use async_trait::async_trait; +use calcine_core::bench::{ + BenchMeasure, BenchRequest, BenchResult, BenchStat, BenchTool, Benchmarker, merge_history, +}; +use calcine_core::jobs::{JobCtx, JobPhase, JobProgress}; +use calcine_core::models::{ComputeUnit, Runtime}; +use calcine_core::{Error, Result}; + +use crate::MockBackend; + +const TOOL_BYTES: u64 = 84_772_012; +const STEPS: u64 = 12; + +impl MockBackend { + fn bench_tool(&self) -> MutexGuard<'_, Option> { + self.bench_tool + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + fn bench_history(&self) -> MutexGuard<'_, Vec> { + self.bench_history + .lock() + .unwrap_or_else(PoisonError::into_inner) + } +} + +#[async_trait] +impl Benchmarker for MockBackend { + async fn tool(&self) -> Result { + let installed = self.bench_tool().clone(); + let wanted = self.cli_version(); + Ok(BenchTool { + download_bytes: (installed.as_ref() != Some(&wanted)).then_some(TOOL_BYTES), + installed, + wanted: Some(wanted), + }) + } + + async fn install_tool(&self, ctx: JobCtx) -> Result<()> { + for step in 1..=STEPS { + ctx.report(JobProgress { + done_bytes: TOOL_BYTES * step / STEPS, + total_bytes: Some(TOOL_BYTES), + bytes_per_second: Some(TOOL_BYTES / 3), + phase: Some(JobPhase::Downloading), + step: None, + }); + tokio::select! { + () = tokio::time::sleep(self.pull_duration / 24) => {} + () = ctx.cancelled() => return Err(Error::Cancelled), + } + } + ctx.report(JobProgress { + phase: Some(JobPhase::Installing), + ..JobProgress::default() + }); + tokio::time::sleep(self.pull_duration / 12).await; + *self.bench_tool() = Some(self.cli_version()); + Ok(()) + } + + async fn measure( + &self, + request: &BenchRequest, + unit: ComputeUnit, + ctx: &JobCtx, + ) -> Result { + if self.bench_tool().is_none() { + return Err(Error::InvalidInput( + "download the benchmark tool first".into(), + )); + } + tokio::select! { + () = tokio::time::sleep(self.pull_duration / 3) => {} + () = ctx.cancelled() => return Err(Error::Cancelled), + } + if request.runtime == Runtime::Qairt && unit != ComputeUnit::Npu { + return Err(Error::Command { + command: "geniex-bench".into(), + message: + "geniex_llm_create: Model loading failed (AI Hub models only run on the NPU)" + .into(), + }); + } + Ok(sample( + request, + unit, + self.started.elapsed().subsec_millis(), + )) + } + + fn history(&self) -> Vec { + self.bench_history().clone() + } + + fn record(&self, results: Vec) -> Result<()> { + let mut history = self.bench_history(); + let merged = merge_history(std::mem::take(&mut *history), results); + *history = merged; + Ok(()) + } + + fn forget(&self, ids: &[String]) -> Result<()> { + self.bench_history() + .retain(|result| !ids.contains(&result.id)); + Ok(()) + } +} + +/// Numbers shaped like a Snapdragon X Elite, scaled by model size. +fn sample(request: &BenchRequest, unit: ComputeUnit, seed: u32) -> BenchMeasure { + let billions = if request.model.contains("4B") { + 4.0 + } else { + 0.6 + }; + let (ttft_per_k, prefill, decode) = match unit { + ComputeUnit::Npu => (80.0, 3300.0, 53.0), + ComputeUnit::Hybrid => (120.0, 2100.0, 48.0), + ComputeUnit::Gpu => (160.0, 2400.0, 45.0), + ComputeUnit::Cpu => (900.0, 380.0, 24.0), + }; + let size: f64 = billions / 0.6; + let noise = 1.0 + f64::from(seed % 7) / 100.0; + // Speculative decoding helps generation a little in the demo. + let decode = if request.spec_type.is_some() { + decode * 1.25 + } else { + decode + }; + let stat = |median: f64| BenchStat { + median, + min: median * 0.96, + max: median * 1.05, + stdev: median * 0.02, + }; + let prompt = f64::from(request.prompt_tokens); + BenchMeasure { + ttft_ms: stat(ttft_per_k * prompt / 1000.0 * size.sqrt() * noise), + prefill_tps: stat(prefill / size.sqrt() / noise), + decode_tps: stat(decode / size / noise), + generated_tokens: f64::from(request.generated_tokens), + prompt_tokens: prompt, + geniex_version: "v0.8.0".into(), + energy: None, + } +} diff --git a/crates/calcine-mock/src/context.rs b/crates/calcine-mock/src/context.rs new file mode 100644 index 0000000..59aff27 --- /dev/null +++ b/crates/calcine-mock/src/context.rs @@ -0,0 +1,46 @@ +//! Context windows and token counts, estimated: AI Hub models have 4096 +//! tokens, llama.cpp ones use the server option. + +use async_trait::async_trait; +use calcine_core::Result; +use calcine_core::context::{ContextMeter, TokenCount}; +use calcine_core::models::Runtime; +use serde_json::Value; + +use crate::MockBackend; + +#[async_trait] +impl ContextMeter for MockBackend { + async fn compiled_window(&self, model: &str) -> Result> { + let name = model.split(':').next().unwrap_or(model); + Ok(self + .models() + .iter() + .any(|cached| cached.name == name && cached.runtime == Runtime::Qairt) + .then_some(4096)) + } + + async fn count(&self, _model: &str, request: &Value) -> Result { + let per_message = request + .get("messages") + .and_then(Value::as_array) + .map(|messages| { + messages + .iter() + .map(|message| { + let text = message + .get("content") + .map(Value::to_string) + .unwrap_or_default(); + u32::try_from(text.len().div_ceil(3)).unwrap_or(u32::MAX) + 5 + }) + .collect() + }) + .unwrap_or_default(); + Ok(TokenCount { + per_message, + overhead: 20, + exact: false, + }) + } +} diff --git a/crates/calcine-mock/src/data.rs b/crates/calcine-mock/src/data.rs index c0d061e..44cc82f 100644 --- a/crates/calcine-mock/src/data.rs +++ b/crates/calcine-mock/src/data.rs @@ -3,7 +3,7 @@ use std::path::Path; use calcine_core::hardware::{ - Accelerator, DiskSpace, HardwareInfo, HardwareUsage, MemoryInfo, Processor, + Accelerator, DiskSpace, HardwareInfo, HardwareUsage, MemoryInfo, PowerDraw, Processor, }; use calcine_core::models::{ Chipset, ComputeUnit, HubCatalog, HubModel, LocalModel, ModelHub, ModelType, RemoteModel, @@ -117,6 +117,9 @@ pub(crate) fn hardware(models_dir: Option<&Path>) -> HardwareInfo { } } +/// What the mock device draws doing nothing. +pub(crate) const IDLE_WATTS: f64 = 8.0; + /// Gentle waves so the gauges move: the NPU works in bursts, like during /// generation. #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] @@ -136,6 +139,11 @@ pub(crate) fn usage(seconds: f32) -> HardwareUsage { total_bytes: 32 * GIB, available_bytes: (18 * GIB).saturating_sub(busy_mib << 20), }, + power: Some(PowerDraw { + system_watts: IDLE_WATTS as f32 + npu / 100.0 * 7.0, + cpu_watts: Some(wave(7.0, 1.2, 3.5)), + gpu_watts: Some(wave(11.0, 0.0, 0.4)), + }), } } diff --git a/crates/calcine-mock/src/hardware.rs b/crates/calcine-mock/src/hardware.rs index 60b4d4b..044b1b1 100644 --- a/crates/calcine-mock/src/hardware.rs +++ b/crates/calcine-mock/src/hardware.rs @@ -2,7 +2,7 @@ use std::path::Path; use async_trait::async_trait; use calcine_core::Result; -use calcine_core::hardware::{HardwareInfo, HardwareProbe, HardwareUsage}; +use calcine_core::hardware::{EnergyReading, HardwareInfo, HardwareProbe, HardwareUsage}; use crate::{MockBackend, data}; @@ -15,4 +15,13 @@ impl HardwareProbe for MockBackend { async fn usage(&self) -> Result { Ok(data::usage(self.started.elapsed().as_secs_f32())) } + + /// Idle draw since start, plus what the mock server's generations cost. + async fn energy(&self) -> Result> { + let seconds = self.started.elapsed().as_secs_f64(); + Ok(Some(EnergyReading { + joules: data::IDLE_WATTS * seconds + crate::server::generation_joules(), + seconds, + })) + } } diff --git a/crates/calcine-mock/src/installer.rs b/crates/calcine-mock/src/installer.rs index a2611bf..d30dc1e 100644 --- a/crates/calcine-mock/src/installer.rs +++ b/crates/calcine-mock/src/installer.rs @@ -58,6 +58,7 @@ impl RuntimeInstaller for MockBackend { total_bytes: Some(INSTALLER_BYTES), bytes_per_second: Some(INSTALLER_BYTES / 4), phase: Some(JobPhase::Downloading), + step: None, }); tokio::select! { () = tokio::time::sleep(tick) => {} diff --git a/crates/calcine-mock/src/lib.rs b/crates/calcine-mock/src/lib.rs index 037f049..beaec14 100644 --- a/crates/calcine-mock/src/lib.rs +++ b/crates/calcine-mock/src/lib.rs @@ -8,6 +8,8 @@ //! - `models`, `directory`, `runtime`, `hardware`, `server`: the service //! trait implementations +mod bench; +mod context; mod data; mod directory; mod hardware; @@ -21,6 +23,7 @@ pub use server::MockServer; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::{Duration, Instant}; +use calcine_core::bench::BenchResult; use calcine_core::jobs::JobManager; use calcine_core::models::LocalModel; use calcine_core::{BackendKind, Services}; @@ -36,6 +39,9 @@ pub struct MockBackend { version: Mutex, /// Drives the simulated hardware load. started: Instant, + /// "Installed" benchmark tool version. + bench_tool: Mutex>, + bench_history: Mutex>, } impl Default for MockBackend { @@ -46,6 +52,8 @@ impl Default for MockBackend { chipset: Mutex::new(None), version: Mutex::new(data::runtime_info().cli_version), started: Instant::now(), + bench_tool: Mutex::new(None), + bench_history: Mutex::new(Vec::new()), } } } @@ -71,7 +79,9 @@ impl MockBackend { directory: backend.clone(), runtime: backend.clone(), hardware: backend.clone(), - installer: backend, + installer: backend.clone(), + bench: backend.clone(), + context: backend, server: Arc::new(MockServer::default()), jobs: JobManager::new(), } @@ -280,3 +290,106 @@ mod server_tests { assert_eq!(*server.state().borrow(), ServerState::Stopped); } } + +#[cfg(test)] +mod bench_tests { + use std::time::Duration; + + use calcine_core::Services; + use calcine_core::jobs::JobState; + + use super::MockBackend; + + async fn wait(services: &Services, id: calcine_core::jobs::JobId) -> JobState { + let mut events = services.jobs.subscribe(); + while services + .jobs + .get(id) + .is_some_and(|job| job.state == JobState::Running) + { + let _ = tokio::time::timeout(Duration::from_secs(1), events.recv()).await; + } + services.jobs.get(id).unwrap().state + } + + #[tokio::test] + async fn profiles_energy_through_the_server() { + use calcine_core::bench::{BenchSource, EnergyRequest}; + use calcine_core::models::{ComputeUnit, Runtime}; + + let services = MockBackend::services(); + let request = EnergyRequest { + model: "qualcomm/Qwen3-4B".into(), + runtime: Runtime::Qairt, + units: vec![ComputeUnit::Npu], + power_modes: vec!["power_saver".into()], + generated_tokens: 128, + repetitions: 1, + }; + let job = services.start_energy_profile(request.clone()).unwrap(); + assert!( + services.start_energy_profile(request).is_err(), + "one at a time" + ); + assert_eq!(wait(&services, job).await, JobState::Succeeded); + + let history = services.bench.history(); + let result = &history[0]; + assert_eq!(result.source, BenchSource::EnergyProfile); + assert_eq!(result.power_mode, "power_saver"); + let measure = result.measure.as_ref().unwrap(); + assert_eq!(measure.decode_tps.median, 22.0); + let energy = measure.energy.unwrap(); + // The mock idles at 8 W and adds 2 W while generating in power_saver. + assert!((energy.idle_watts - 8.0).abs() < 0.1, "{energy:?}"); + assert!(energy.active_watts > energy.idle_watts, "{energy:?}"); + assert!(energy.joules_per_token > 0.0, "{energy:?}"); + } + + #[tokio::test] + async fn benchmarks_each_unit_and_keeps_failures() { + use calcine_core::bench::BenchRequest; + use calcine_core::models::{ComputeUnit, Runtime}; + + let services = MockBackend::default() + .with_pull_duration(Duration::from_millis(30)) + .into_services(); + assert!(!services.bench.tool().await.unwrap().is_ready()); + let install = services.start_bench_install("v0.8.0".into()); + assert_eq!(wait(&services, install).await, JobState::Succeeded); + assert!(services.bench.tool().await.unwrap().is_ready()); + + let request = BenchRequest { + model: "qualcomm/Qwen3-4B:W4A16".into(), + runtime: Runtime::Qairt, + units: vec![ComputeUnit::Npu, ComputeUnit::Gpu], + prompt_tokens: 512, + generated_tokens: 128, + repetitions: 3, + power_mode: "burst".into(), + spec_type: None, + draft_model: None, + }; + let job = services.start_benchmark(request.clone()).unwrap(); + assert!(services.start_benchmark(request).is_err(), "one at a time"); + assert_eq!(wait(&services, job).await, JobState::Succeeded); + + let history = services.bench.history(); + assert_eq!(history.len(), 2); + let npu = history.iter().find(|r| r.unit == ComputeUnit::Npu).unwrap(); + assert!( + npu.measure + .as_ref() + .is_some_and(|m| m.decode_tps.median > 0.0) + ); + let gpu = history.iter().find(|r| r.unit == ComputeUnit::Gpu).unwrap(); + assert!(gpu.measure.is_none() && gpu.error.is_some()); + assert_eq!(npu.session_id, gpu.session_id); + + services + .bench + .forget(std::slice::from_ref(&npu.id)) + .unwrap(); + assert_eq!(services.bench.history().len(), 1); + } +} diff --git a/crates/calcine-mock/src/models.rs b/crates/calcine-mock/src/models.rs index a9051ee..0cfacd8 100644 --- a/crates/calcine-mock/src/models.rs +++ b/crates/calcine-mock/src/models.rs @@ -36,6 +36,7 @@ impl ModelStore for MockBackend { total_bytes: Some(PULL_BYTES), bytes_per_second: Some(speed), phase: None, + step: None, }); tokio::select! { () = tokio::time::sleep(tick) => {} diff --git a/crates/calcine-mock/src/server.rs b/crates/calcine-mock/src/server.rs index c9e2e7a..0be1ba4 100644 --- a/crates/calcine-mock/src/server.rs +++ b/crates/calcine-mock/src/server.rs @@ -2,28 +2,34 @@ //! //! Streams a canned reply word by word (with a short reasoning preamble unless //! `enable_think` is false), so the gateway and the Chat page work anywhere. +//! Given tools, it calls the first one. use std::convert::Infallible; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, PoisonError}; use std::time::Duration; use async_trait::async_trait; use axum::Json; use axum::body::{Body, Bytes}; -use axum::http::header; +use axum::http::{StatusCode, header}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use calcine_core::Result; -use calcine_core::runtime::{InferenceServer, ServerState}; +use calcine_core::runtime::{InferenceServer, ServerOptions, ServerState}; use serde_json::{Value, json}; use tokio::sync::{mpsc, oneshot, watch}; use tokio_stream::wrappers::ReceiverStream; const TOKEN_DELAY: Duration = Duration::from_millis(25); +/// Like AI Hub models: GenieX refuses longer prompts. +const WINDOW: usize = 4096; #[derive(Debug)] pub struct MockServer { running: tokio::sync::Mutex)>>, state: watch::Sender, + options: Mutex, } impl Default for MockServer { @@ -31,6 +37,7 @@ impl Default for MockServer { Self { running: tokio::sync::Mutex::new(None), state: watch::Sender::new(ServerState::Stopped), + options: Mutex::new(ServerOptions::default()), } } } @@ -79,10 +86,58 @@ impl InferenceServer for MockServer { fn logs(&self) -> Vec { vec!["Mock inference server: replies are canned, no model runs.".into()] } + + fn options(&self) -> ServerOptions { + *self.options.lock().unwrap_or_else(PoisonError::into_inner) + } + + async fn set_options(&self, options: ServerOptions) -> Result<()> { + options + .validate() + .map_err(calcine_core::Error::InvalidInput)?; + *self.options.lock().unwrap_or_else(PoisonError::into_inner) = options; + Ok(()) + } + + /// A canned completion whose speed and energy follow the power mode, + /// returned within 0.4 s. + async fn complete(&self, request: &Value) -> Result { + let tokens = request["max_tokens"] + .as_u64() + .and_then(|tokens| u32::try_from(tokens).ok()) + .unwrap_or(16) + .min(512); + let speed = mode_speed(request["power_mode"].as_str().unwrap_or("burst")); + let seconds = (f64::from(tokens) / speed).min(0.4); + tokio::time::sleep(Duration::from_secs_f64(seconds)).await; + add_generation_joules( + mode_watts(request["power_mode"].as_str().unwrap_or("burst")) * seconds, + ); + Ok(json!({ + "object": "chat.completion", + "choices": [{ "index": 0, "finish_reason": "length", "message": { "role": "assistant", "content": "Once upon a time…" } }], + "usage": { "prompt_tokens": 48, "completion_tokens": tokens, "total_tokens": 48 + tokens }, + "timings": { "prompt_ms": 60.0, "prompt_per_second": 800.0, "predicted_per_second": speed }, + })) + } } async fn chat_completions(Json(request): Json) -> Response { let model = request["model"].as_str().unwrap_or("mock").to_owned(); + if prompt_size(&request) > WINDOW { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": { + "code": "context_length_exceeded", + "message": "prompt is longer than the model's context window", + "type": "invalid_request_error", + } })), + ) + .into_response(); + } + if let Some(call) = tool_call(&request) { + return call_tool(&model, &call, request["stream"].as_bool() == Some(true)); + } let prompt = request["messages"] .as_array() .and_then(|messages| messages.iter().rev().find(|m| m["role"] == "user")) @@ -95,10 +150,26 @@ async fn chat_completions(Json(request): Json) -> Response { } else { "" }; - let reply = format!( - "This is **Calcine's mock backend**: no model ran. You said: \"{prompt}\". Run Calcine \ - with GenieX on a Snapdragon device to talk to a real model." - ); + // Asked for JSON (Calcine's structured output): answer in JSON. A forced + // tool call is the example the instruction shows, filled in. + let system = request["messages"][0]["content"].as_str().unwrap_or(""); + let wants_json = system.contains("single valid JSON value"); + let example = system + .split_once("with a real value:\n") + .and_then(|(_, rest)| rest.lines().next()) + .and_then(|line| serde_json::from_str::(line).ok()); + let reply = + if let Some(mut call) = example.filter(|_| system.contains("Reply with only the call")) { + fill_in(&mut call); + call.to_string() + } else if wants_json { + json!({ "reply": "Mock answer", "said": prompt }).to_string() + } else { + format!( + "This is **Calcine's mock backend**: no model ran. You said: \"{prompt}\". Run \ + Calcine with GenieX on a Snapdragon device to talk to a real model." + ) + }; if request["stream"].as_bool() != Some(true) { return Json(json!({ @@ -145,6 +216,126 @@ async fn chat_completions(Json(request): Json) -> Response { .into_response() } +/// Tokens the prompt takes, counted as the mock's context meter does. +fn prompt_size(request: &Value) -> usize { + let messages = request["messages"] + .as_array() + .map(Vec::as_slice) + .unwrap_or_default(); + 20 + messages + .iter() + .map(|message| message["content"].to_string().len().div_ceil(3) + 5) + .sum::() +} + +/// With tools and a user message last, the mock calls the first tool, with +/// placeholder arguments. +fn tool_call(request: &Value) -> Option { + let tool = request["tools"].as_array()?.first()?; + let last = request["messages"].as_array()?.last()?; + if last["role"] != "user" { + return None; + } + let name = tool.pointer("/function/name")?.as_str()?; + let arguments: serde_json::Map = tool + .pointer("/function/parameters/properties") + .and_then(Value::as_object) + .map(|properties| { + properties + .iter() + .map(|(key, property)| { + let value = match property["type"].as_str() { + Some("integer" | "number") => json!(1), + Some("boolean") => json!(true), + Some("array") => json!([]), + Some("object") => json!({}), + _ => json!("mock"), + }; + (key.clone(), value) + }) + .collect() + }) + .unwrap_or_default(); + Some(json!({ + "id": "call_mock", + "type": "function", + "function": { "name": name, "arguments": Value::Object(arguments).to_string() }, + })) +} + +/// Replace the example's `` with "mock". +fn fill_in(value: &mut Value) { + match value { + Value::String(text) if text.starts_with('<') => *text = "mock".into(), + Value::Array(items) => items.iter_mut().for_each(fill_in), + Value::Object(fields) => fields.values_mut().for_each(fill_in), + _ => {} + } +} + +fn call_tool(model: &str, call: &Value, stream: bool) -> Response { + if !stream { + return Json(json!({ + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "finish_reason": "tool_calls", + "message": { "role": "assistant", "content": "", "tool_calls": [call] }, + }], + "usage": usage("call"), + })) + .into_response(); + } + let mut call = call.clone(); + call["index"] = json!(0); + let chunks = [ + json!({ "object": "chat.completion.chunk", "choices": [{ "index": 0, "delta": { "tool_calls": [call] }, "finish_reason": null }] }), + json!({ "object": "chat.completion.chunk", "choices": [{ "index": 0, "delta": {}, "finish_reason": "tool_calls" }] }), + ]; + let mut body: Vec = chunks + .iter() + .flat_map(|chunk| sse(chunk).to_vec()) + .collect(); + body.extend_from_slice(b"data:[DONE]\n\n"); + ([(header::CONTENT_TYPE, "text/event-stream")], body).into_response() +} + +/// Energy the mock's generations used, in millijoules, for the mock meter. +static GENERATION_MILLIJOULES: AtomicU64 = AtomicU64::new(0); + +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn add_generation_joules(joules: f64) { + GENERATION_MILLIJOULES.fetch_add((joules * 1000.0) as u64, Ordering::Relaxed); +} + +#[allow(clippy::cast_precision_loss)] +pub(crate) fn generation_joules() -> f64 { + GENERATION_MILLIJOULES.load(Ordering::Relaxed) as f64 / 1000.0 +} + +/// What generating adds to the idle draw, in watts, per power mode. +fn mode_watts(mode: &str) -> f64 { + match mode { + "burst" | "sustained_high_performance" => 7.0, + "high_performance" => 6.0, + "balanced" | "low_balanced" => 4.0, + "high_power_saver" => 3.0, + _ => 2.0, + } +} + +/// Plausible decode speeds per power mode, tokens per second. +pub(crate) fn mode_speed(mode: &str) -> f64 { + match mode { + "burst" | "sustained_high_performance" => 52.0, + "high_performance" => 48.0, + "balanced" | "low_balanced" => 38.0, + "high_power_saver" => 30.0, + _ => 22.0, + } +} + fn sse(value: &Value) -> Bytes { Bytes::from(format!("data:{value}\n\n")) } diff --git a/docs/RELEASING.md b/docs/RELEASING.md index ccc2a2c..aa2ac78 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -37,10 +37,12 @@ manager. 4. `release.yml` sees a version on `main` without a release and a changelog section for it, then: - - builds the NSIS installer on `windows-11-arm` with GenieX bundled - (`runtime/geniex.json`), + - builds the NSIS installer on `windows-11-arm` (Calcine only: GenieX is + downloaded on first launch, see below), and the Linux `.deb` on + `ubuntu-24.04-arm` (`build-linux.yml`, which both release workflows call), - signs it with Authenticode through SignPath (once configured, below), - - signs the update and writes `latest.json` for the Stable channel, + - signs the updates and writes `latest.json` for the Stable channel + (Windows and Linux), - attaches `SHA256SUMS`, a CycloneDX SBOM and a build provenance attestation (`gh attestation verify -R eraflo/Calcine`), - publishes `vX.Y.Z` as the latest release, and offers it on the Beta @@ -52,17 +54,26 @@ Pushes to `main` that don't change the version publish nothing. Pushes to `dev` only run the checks (no installer). To publish a beta, run **Actions › Prerelease › Run workflow** on `dev`. Each beta is a GitHub -pre-release with the installer and `SHA256SUMS`; the fixed pre-release +pre-release with the installer, the `.deb` and `SHA256SUMS`; the fixed pre-release `beta-channel` holds `latest-beta.json`, which the Beta channel reads. -The Windows ARM64 build runs in CI only for `main` and pull requests to it. +The Windows and Linux ARM64 builds run in CI only for `main` and pull +requests to it. The Rust checks run on x86_64 Linux for every push. -## GenieX bundled in the installer +## GenieX installed on first launch + +Installers contain Calcine only (on Linux, GenieX's official archive is +unpacked to `~/.local/share/geniex-cli`, checked against its own pin). GenieX includes Qualcomm's proprietary +runtimes (QAIRT), which can't be signed or redistributed under SignPath +Foundation's terms, so the welcome screen downloads the official installer +from Qualcomm and checks it against the SHA-256 pinned in +`runtime/geniex.json`, built into Calcine. `geniex-bump.yml` checks Qualcomm's release index daily. For a new stable GenieX it verifies the installer (size and SHA-256 from the manifest), installs it on a Windows ARM64 runner, checks `geniex version`, and opens a -PR to `dev` updating `runtime/geniex.json`. PRs opened by workflows don't +PR to `dev` updating `runtime/geniex.json`. CI checks that the pinned +installer is still published (`scripts/check-geniex.ts`). PRs opened by workflows don't start CI: close and reopen the PR to run the checks. Users can also update GenieX from the app (Hardware › GenieX runtime), on the diff --git a/docs/architecture.md b/docs/architecture.md index 390472d..9082bbb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -29,11 +29,12 @@ Security of the gateway is described in [security-model.md](security-model.md). | Crate | Role | Depends on Tauri? | |---|---|---| | `crates/calcine-core` | Domain types, model references (pasted names and links), service traits (`ModelStore`, `ModelCatalog`, `RuntimeManager`, `HardwareProbe`), the `JobManager` for long-running work, errors, `Services` | No | -| `crates/calcine-geniex` | GenieX CLI adapter: discovery, command runner, `pull` with live progress, `geniex serve` supervisor, output parsers tested on real captures (`tests/fixtures/`), installing and updating GenieX (`update/`: release index, verified download, signature, silent installer, cache for rolling back) | No | -| `crates/calcine-gateway` | Local HTTP API (axum): Host/Origin/API-key checks, body sanitizing, OpenAI-compatible proxy with streaming and a one-at-a-time queue, request log, `/calcine/v1` management API | No | +| `crates/calcine-geniex` | GenieX CLI adapter: discovery, command runner (`platform.rs`: what differs on Linux, `LD_LIBRARY_PATH`, parent-death signal, archive install), `pull` with live progress, `geniex serve` supervisor, output parsers tested on real captures (`tests/fixtures/`), installing and updating GenieX (`update/`: release index, verified download, signature, silent installer, cache for rolling back), context windows and token counts with each model's `tokenizer.json` (`context.rs`) | No | +| `crates/calcine-gateway` | Local HTTP API (axum): Host/Origin/API-key checks, optional local network port (`network/`: HTTPS with rustls and a self-signed certificate, allowed address ranges, key-guessing limiter), body sanitizing, OpenAI-compatible proxy with streaming and a one-at-a-time queue, request log, `/calcine/v1` management API | No | | `crates/calcine-hub` | Hub lookups over HTTPS: Hugging Face search, precisions with sizes (named like GenieX), AI Hub chipsets (`ModelDirectory`) | No | -| `crates/calcine-hw` | Hardware probe: CPU, memory and disk (sysinfo), NPU/GPU and drivers (Windows WMI), live NPU/GPU load (performance counters) | No | +| `crates/calcine-hw` | Hardware probe: CPU, memory and disk (sysinfo), NPU/GPU and drivers (Windows WMI; on Linux the FastRPC device, the `msm`/KGSL GPU and the device tree's SoC), live NPU/GPU load (performance counters), power and energy from the SoC's energy metering (`Energy Meter` counters, found by their English names through PerfLib) | No | | `crates/calcine-mock` | In-memory backend with simulated downloads, for UI work and tests | No | +| `src-tauri/src/cli.rs`, `src-tauri/src/bin/calcine-cli.rs` | `calcine-cli`: a console program (the app is a windowed one) built from the same crate: `serve` runs the same services and gateway as the window (`setup::build`), `keys` manages API keys | Tauri runtime only | | `src-tauri` | Desktop shell: builds `Services`, exposes commands, exports TypeScript bindings | Yes | Backend selection: `CALCINE_BACKEND=mock` uses the mock, anything else uses GenieX. @@ -63,7 +64,7 @@ calcine-gateway/src/ calcine-hub/src/ hf.rs (Hugging Face payloads) · quant.rs (precision names) · aihub.rs (chipsets) -calcine-hw/src/ snapshot.rs · usage.rs (live load) · system.rs (sysinfo) +calcine-hw/src/ snapshot.rs · usage.rs (live load) · power.rs (power, energy) · system.rs (sysinfo) windows/ devices.rs (WMI) · gpu_engines.rs (PDH, the only `unsafe` code) calcine-mock/src/ data.rs (sample data) · models.rs · directory.rs · runtime.rs · hardware.rs · server.rs @@ -71,7 +72,7 @@ src-tauri/ ├─ src/ipc/ mod.rs (bindings builder) · commands/ (one file per domain) · events.rs · error.rs ├─ src/setup/ services.rs (pick the backend) · gateway.rs · start and shutdown ├─ src/desktop/ tray.rs · window behaviour · notifications.rs · locale.rs · process.rs (Job Object) -├─ windows/ installer-hooks.nsh (installs GenieX) · test-manifest.xml +├─ windows/ license-and-privacy.txt (installer license page) · test-manifest.xml └─ tests/ tests that link Tauri (bindings export) ``` diff --git a/docs/cli-coverage.md b/docs/cli-coverage.md index 5f25326..877fd0f 100644 --- a/docs/cli-coverage.md +++ b/docs/cli-coverage.md @@ -9,17 +9,18 @@ command or flag becomes available in the app. Reference: GenieX v0.8.0. | `version` | ✅ `parse::version` | ✅ `RuntimeManager::info` | ✅ Hardware, sidebar | | | `config get chipset` | — (trimmed stdout) | ✅ `RuntimeManager::chipset` | ✅ Hardware, Welcome | Prints the reference device when detected, the saved id when set | | `config list` | ✅ `parse::config_list` | — | — | | -| `config set chipset ` / `""` | — | ✅ `RuntimeManager::set_chipset` | ✅ Hardware › Chipset | Always passes a value (an empty one resets to detection), so the interactive picker never opens. Chipset names come from AI Hub's `platform.json` (`calcine-hub`), like the picker | +| `config set chipset ` / `""` | — | ✅ `RuntimeManager::set_chipset` | ✅ Hardware › Chipset, `calcine-cli geniex chipset` | Always passes a value (an empty one resets to detection), so the interactive picker never opens. Chipset names come from AI Hub's `platform.json` (`calcine-hub`), like the picker | | `model list [--all]` | ✅ `parse::hub_table` | ✅ `ModelCatalog::aihub` | ✅ Discover › Qualcomm AI Hub, Welcome | `--all` lists chipset slugs (`x-elite`), not names | | `pull [:prec] [--model-hub] [--model-type]` | ✅ `parse::progress` (real capture) | ✅ `ModelStore::pull`, cancellable job | ✅ Discover (precision picker with sizes, memory and disk checks, type override), Welcome, task drawer | Precisions and sizes come from the Hugging Face API (`calcine-hub`), named like GenieX does. Without a TTY GenieX picks the recommended precision | | `pull --model-hub localfs --local-path ` | ✅ (same progress) | ✅ `PullRequest::import`, `Import` job | ✅ Library › Import, drag and drop | GGUF folders, extracted AI Hub bundles, AI Hub `.zip` | | `remove --yes ` | — | ✅ `Services::remove_models` | ✅ Library (model or one precision) | Stops `geniex serve` first (Windows can't delete a loaded model) | | `clean --yes` | — | ✅ `Services::clean_models` | ✅ Settings › Storage › Danger zone | Typed confirmation; stops `geniex serve` first | | `model set-type ` | — | ✅ `ModelStore::set_type` | ✅ Library (Text / Vision on each model) | Always passes the type, so the interactive picker never opens | -| `serve --host 127.0.0.1: --origins --keepalive` | — | ✅ `InferenceServer` (`GeniexServer`, supervised) | ✅ Server page (start/stop, output), on demand from the gateway | Never exposed directly; see security-model.md | -| `infer` / `run` | — | via `/v1/chat/completions` | ✅ Chat (streaming, reasoning, temperature, max tokens, think, compute, power mode, images and microphone for VLMs), Hardware › Self-test | The REPL itself isn't wrapped; the HTTP API takes the same options | +| `serve --host 127.0.0.1: --origins --keepalive --nctx [--compute cpu]` | — | ✅ `InferenceServer` (`GeniexServer`, supervised), `ServerOptions` saved in `server.json` | ✅ Server page (start/stop, output, Model loading: unload delay and llama.cpp context window), on demand from the gateway | Never exposed directly; see security-model.md. Changing the options restarts it. `nctx` sent per request is ignored by GenieX 0.8, hence a server option. AI Hub models have their window compiled in (`genie_config.json`, `dialog.context.size`): GenieX refuses longer prompts (`context_length_exceeded`), `--sliding-window` only exists for `infer`/`run` and `sliding_window` in API requests does nothing. GenieX reuses its cached conversation when the next request continues it, but counts it against the window when it doesn't (a regenerated reply, another client): the gateway empties the cache with a one-token request and retries. Prompts are counted with the model's `tokenizer.json` plus the chat template's share (`calcine-geniex` › `context`, matches GenieX's counts within 2 tokens); `"truncation": "auto"` forgets the oldest exchanges (always on for Ollama clients, as with Ollama). On Linux without the compute DSP (no `/dev/fastrpc-cdsp`), `--compute cpu`: GenieX otherwise loads llama.cpp models on the Hexagon and fails | +| `infer` / `run` | — | via `/v1/chat/completions` | ✅ Chat (streaming, reasoning, presets, temperature, top-p/k, min-p, penalties, seed, stop sequences, max tokens, think, compute, offloaded layers, vision encoder, power mode, saved system prompts, images and microphone for VLMs), Hardware › Self-test | The REPL itself isn't wrapped; the HTTP API takes the same options. Measured on GenieX 0.8: `seed` and `ngl` only apply to llama.cpp models, `vit_compute` takes `CPU` or `HTP0` (not the GPU), and `stop` is ignored over the API, so the Chat cuts the reply itself. Speculative decoding (`spec_type` as one comma-separated string, `spec_draft_model` by name, `spec_n_max/min`, `spec_p_min`; acceptance from `timings.draft_n`/`draft_n_accepted`) in the Chat and the Benchmark, llama.cpp only. `response_format` (JSON, JSON Schema) is ignored by GenieX 0.8's API on both runtimes, so the gateway enforces it (`structured/`: instruction with a filled-in example, check, up to 2 retries, 422 if still invalid), for the OpenAI and Ollama APIs and the Chat; works on the NPU. GBNF grammars only work in `geniex infer` (llama.cpp) and aren't offered. Tool calling works natively (`tools`, parallel calls, streamed, tool results), on the NPU too, but `tool_choice` is ignored: the gateway removes the tools for `none`, and for `required` or a named function asks for the call as JSON (only that function offered), checks the name and the arguments against the parameters, retries, and answers `tool_calls` (`structured/tools.rs`) | | `update` | — | ✅ `RuntimeInstaller` (`calcine-geniex::update`), `Services::start_runtime_install` | ✅ Hardware › GenieX runtime › Updates (stable / pre-releases, roll back, repair), sidebar badge | Not wrapped: Calcine reads the same `index.json` and manifests, downloads the installer with progress, checks its SHA-256 and Authenticode signature, stops `geniex serve`, runs the installer silently, checks `geniex version`, restarts the server. Installers are cached to roll back | | `completion` | — | — | — | M5 | +| `geniex-bench` (separate tool) | `calcine-geniex::bench::report` (JSON report schema 6, `ERROR:` lines) | ✅ `Benchmarker` (`GeniexBench`), `Services::start_benchmark`, history in `benchmarks.json` | ✅ Benchmark page (download, run per unit, results, fastest models, history, CSV export) | Downloaded from GenieX's GitHub release at the installed GenieX version; GitHub's `digest` and the published `.sha256` must agree. Unpacked with Windows' `tar`. One process per unit (in matrix mode a failure stops the rest). `geniex serve` is paused meanwhile | Global flags Calcine always passes: `--skip-update`, plus `--data-dir` when configured, with `NO_COLOR=1` in the environment. @@ -28,7 +29,9 @@ configured, with `NO_COLOR=1` in the environment. | What | Where | Notes | |---|---|---| +| Ollama API (`/api/version`, `tags`, `show`, `ps`, `chat`, `generate`) | `calcine-gateway` › `ollama.rs`, `routes/ollama.rs` | Translated to `/v1/chat/completions` (images, options, `think`, `format`, tools), answered as JSON lines or one object. On the main port with a key, and on 11434 without one when turned on (Settings › Local API). Model management (`pull`, `delete`, ...) answers 501: it stays in the app | | Hugging Face search (GGUF) | `calcine-hub` › `ModelDirectory::search` | `GET /api/models?filter=gguf`, honours `HF_ENDPOINT` | | Precisions with download sizes | `calcine-hub` › `ModelDirectory::details` | `GET /api/models/{repo}?blobs=true`; shards summed, vision projector added | | AI Hub chipsets | `calcine-hub` › `ModelDirectory::chipsets` | `releases/latest/platform.json`, filtered to the host OS like GenieX | | NPU / GPU / CPU load | `calcine-hw` › `HardwareProbe::usage` | `GPU Engine` performance counters (PDH): compute-only adapter = NPU | +| Power and energy | `calcine-hw` › `power.rs`, `windows/energy.rs`; `calcine-core` › `bench/energy.rs` | `Energy Meter` counters (EMI): on Snapdragon X, channels `SYS`, `CPU_CLUSTER_*`, `GPU`, `PSU_USB`, none for the NPU. The EMI devices only open for administrators; the counters are readable by all but translated, so PDH paths are built from PerfLib's English names. Energy profiles run through `geniex serve`: load in the power mode, measure idle, read the meter around each generation. Measured on GenieX 0.8 on mains power: power modes (per request or `serve --power-mode`) changed neither speed nor energy | diff --git a/docs/security-model.md b/docs/security-model.md index 89aff31..36df04a 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -21,12 +21,13 @@ outside request goes through the gateway. | Check | Defends against | |---|---| -| Listens on `127.0.0.1` only | Other machines on the network | +| Listens on `127.0.0.1` only (the optional local network port is separate, see below) | Other machines on the network | | `Host` must be `127.0.0.1:`, `localhost:` or `[::1]:` | DNS rebinding: a web page resolving its own domain to 127.0.0.1 to read the API | | A browser `Origin`, when present, must be allowed (Calcine's webview, plus the origins listed in Settings › Local API) | Web pages calling the API from your browser (CSRF), even with simple requests that skip CORS preflight | | `Authorization: Bearer calcine_…` must be a known key (on by default) | Other local programs using your models without your consent | | Key scopes: `inference` (`/v1/*`) and `manage` (`/calcine/v1/*`) | An app meant to chat downloading or deleting models | | Request bodies: local paths and URLs refused in `image_url`, `grammar_path`, `spec_draft_model` unless the key allows local files (`input_audio` is always decoded as base64 by GenieX, so it needs no check) | GenieX reading arbitrary files (`C:/Users/…`) or fetching internal URLs (SSRF) on an app's behalf | +| Model names used to count tokens: only `owner/name[:precision]` made of letters, digits, `-`, `_` and `.`, never `..`, read inside GenieX's model cache | A client making Calcine read `tokenizer.json` or `genie_config.json` outside the cache | | 64 MiB body limit | Memory exhaustion | Errors use the OpenAI shape (`{"error": {"message", "type"}}`) so client @@ -43,6 +44,46 @@ libraries show the reason. - **Require an API key** can be turned off for clients that can't send one. In that mode, keyless callers can only run models: no management, no local files. +## Ollama apps + +Ollama's API (`/api/chat`, `/api/generate`, `/api/tags`, ...) is translated to +GenieX's OpenAI API. On the main port it follows the rules above. Settings › +Local API › **Ollama apps** also opens Ollama's port, `127.0.0.1:11434`, for +apps that can't send a key: + +- The same Host and Origin checks apply: other machines and web pages are + refused. +- Callers without a key there can only run models (no management, no local + files), like keyless callers when keys aren't required. They show up as + "Ollama app" in the request log. +- Which port a request came in on is decided by the socket it arrived on, + never by its headers, so a request to the main port can't pass for an + Ollama one. +- It's off by default. If Ollama itself is running, the port is taken and + Calcine says so. + +## Local network + +Settings › **Local network** lets other devices (a laptop, a phone, a home +server) use the models. It's off by default, and it's a separate port, so +turning it on changes nothing for apps on this PC: + +| Rule | Defends against | +|---|---| +| HTTPS only, on its own port (`0.0.0.0:18443` by default); plain HTTP isn't answered there | Keys and prompts read on the network | +| A self-signed certificate made on this PC with `rcgen` (5 years, names: the PC's name, `name.local`, its address, `localhost`), kept in Calcine's data folder; its SHA-256 fingerprint is shown to pin it, `certificate.pem` to import it; **New certificate** replaces it | Someone between the device and this PC pretending to be it | +| Only addresses in the allowed ranges may connect: private networks by default (`10/8`, `172.16/12`, `192.168/16`, `100.64/10` for Tailscale, link-local, IPv6 ULA), or the ranges you list | Exposure beyond the networks you meant, e.g. a public Wi-Fi with a routed address | +| A key is always needed (whatever *Require an API key* says), and it must be allowed for other devices (**From other devices** when creating it) | A key made for a local app being usable from anywhere | +| Calcine's own per-launch token is refused there | The UI's full-rights token leaving this PC | +| Keys never reach local files there, even with *Read local files* | A device making GenieX read files on this PC | +| 10 invalid keys within a minute from an address block it for 5 minutes | Guessing keys | +| The `Host` check is replaced by the address checks above: the certificate and the key already pin who talks to whom | — | +| Turning it off closes the port and the connections still open | A device keeping a connection alive after you turned it off | + +Windows asks the first time whether Calcine may use the network: allow +private networks only. On Linux, open the port in your firewall (ufw, +firewalld) for your local network only. Request bodies are checked as on the main port. + ## Privacy The request log records who called which model, the status, duration, token @@ -51,9 +92,10 @@ Chat conversations are stored in the app's local storage on this PC only. ## Process lifetime -Calcine runs inside a Windows Job Object that kills its child processes +On Windows, Calcine runs inside a Job Object that kills its child processes (`geniex serve`, downloads) when Calcine exits, including after a crash, so no -unauthenticated server or loaded model is left behind. +unauthenticated server or loaded model is left behind. On Linux, each GenieX +process asks the kernel for `SIGKILL` when Calcine exits (`PR_SET_PDEATHSIG`). ## Updates @@ -71,9 +113,8 @@ unauthenticated server or loaded model is left behind. ## Not in scope (yet) -- Exposing the API on the local network (planned: opt-in, TLS and keys required). - Rate limiting per key. -- Programs running as your Windows user can read Calcine's data folder or talk +- Programs running as your user can read Calcine's data folder or talk to `geniex serve` directly if they find its port; they already have your privileges, so this is outside what a local API can defend against. diff --git a/package.json b/package.json index 861db4c..5b42c90 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "calcine", "private": true, - "version": "0.1.0", + "version": "1.0.0", "description": "Desktop app and local API gateway for Qualcomm GenieX", "license": "MIT", "type": "module", @@ -11,15 +11,14 @@ "app:mock": "bun scripts/dev-mock.ts", "dev": "vite", "build": "tsc -b && vite build", - "build:release": "bun run fetch:geniex && tauri build --config src-tauri/tauri.release.conf.json", + "build:release": "tauri build --config src-tauri/tauri.release.conf.json", "preview": "vite preview", "tauri": "tauri", "lint": "biome check .", "lint:fix": "biome check --write .", "typecheck": "tsc -b", "test": "vitest run", - "test:watch": "vitest", - "fetch:geniex": "bun scripts/fetch-geniex.ts" + "test:watch": "vitest" }, "dependencies": { "@fontsource-variable/geist": "^5.3.0", diff --git a/runtime/geniex.json b/runtime/geniex.json index 7330bd7..bbbf6ab 100644 --- a/runtime/geniex.json +++ b/runtime/geniex.json @@ -1,5 +1,5 @@ { - "$comment": "GenieX version bundled in Calcine installers. Bumped by CI (geniex-bump workflow); values come from the official manifest at https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-geniex/manifest-.json", + "$comment": "GenieX version Calcine was tested with and installs on first launch, per platform (downloaded from Qualcomm, checked against this SHA-256). Bumped by CI (geniex-bump workflow); values come from the official manifest at https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-geniex/manifest-.json", "version": "v0.8.0", "assets": { "windows-arm64": { @@ -7,6 +7,12 @@ "url": "https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-geniex/geniex-cli-setup-windows-arm64-v0.8.0.exe", "size": 65196910, "sha256": "fcc4d932f371a9b3ad3d08a1b34912fe21a7fdf7c54d47c869250b50d81ec734" + }, + "linux-arm64": { + "name": "geniex-cli-linux-arm64-v0.8.0.tar.gz", + "url": "https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-geniex/geniex-cli-linux-arm64-v0.8.0.tar.gz", + "size": 105309808, + "sha256": "a8ff7f90850b9f10629bc3c729d0fb93594e8c0b01d31ac5cf5ef9ea7b3e1c37" } } } diff --git a/scripts/check-geniex.ts b/scripts/check-geniex.ts new file mode 100644 index 0000000..b1dbd79 --- /dev/null +++ b/scripts/check-geniex.ts @@ -0,0 +1,33 @@ +/** + * Check that the GenieX packages pinned in `runtime/geniex.json` (Windows + * installer, Linux archive) are still published with the pinned size. + * Calcine downloads its platform's on first launch and verifies it against + * the pinned SHA-256, which ships inside the app. + * + * bun scripts/check-geniex.ts + */ +import { join } from "node:path"; + +type Asset = { name: string; url: string; size: number; sha256: string }; +type Pin = { version: string; assets: Record }; + +const PLATFORMS = ["windows-arm64", "linux-arm64"]; + +const pin: Pin = await Bun.file(join(import.meta.dir, "..", "runtime", "geniex.json")).json(); +for (const platform of PLATFORMS) { + const asset = pin.assets[platform]; + if (!asset) fail(`runtime/geniex.json has no "${platform}" asset`); + if (!asset.name.includes(pin.version)) fail(`${asset.name} isn't GenieX ${pin.version}`); + if (!/^[0-9a-f]{64}$/.test(asset.sha256)) fail(`"${asset.sha256}" isn't a SHA-256`); + + const response = await fetch(asset.url, { method: "HEAD" }); + if (!response.ok) fail(`HEAD ${asset.url} → ${response.status}`); + const length = Number(response.headers.get("content-length")); + if (length !== asset.size) fail(`size mismatch: remote ${length}, pinned ${asset.size}`); + console.log(`GenieX ${pin.version}: ${asset.name} is available (${asset.size} bytes)`); +} + +function fail(message: string): never { + console.error(`check-geniex: ${message}`); + process.exit(1); +} diff --git a/scripts/fetch-geniex.ts b/scripts/fetch-geniex.ts deleted file mode 100644 index f51fa4c..0000000 --- a/scripts/fetch-geniex.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Download the pinned GenieX installer into `src-tauri/resources/geniex/` so - * release builds can bundle it (see `src-tauri/tauri.release.conf.json`). - * - * bun scripts/fetch-geniex.ts # download (skipped if already verified) - * bun scripts/fetch-geniex.ts --check # only verify the remote file exists and matches the pinned size - * - * The file is verified against the pinned SHA-256 before it is kept. - */ -import { mkdir, rename, rm } from "node:fs/promises"; -import { join } from "node:path"; - -type Asset = { name: string; url: string; size: number; sha256: string }; -type Pin = { version: string; assets: Record }; - -const ROOT = join(import.meta.dir, ".."); -const PLATFORM = "windows-arm64"; -const OUTPUT_DIR = join(ROOT, "src-tauri", "resources", "geniex"); -const OUTPUT_FILE = join(OUTPUT_DIR, "geniex-cli-setup.exe"); - -const pin: Pin = await Bun.file(join(ROOT, "runtime", "geniex.json")).json(); -const asset = pin.assets[PLATFORM]; -if (!asset) fail(`runtime/geniex.json has no "${PLATFORM}" asset`); - -if (process.argv.includes("--check")) { - const response = await fetch(asset.url, { method: "HEAD" }); - if (!response.ok) fail(`HEAD ${asset.url} → ${response.status}`); - const length = Number(response.headers.get("content-length")); - if (length !== asset.size) fail(`size mismatch: remote ${length}, pinned ${asset.size}`); - console.log(`GenieX ${pin.version}: ${asset.name} is available (${asset.size} bytes)`); - process.exit(0); -} - -if ((await Bun.file(OUTPUT_FILE).exists()) && (await sha256(OUTPUT_FILE)) === asset.sha256) { - console.log(`GenieX ${pin.version} already fetched and verified`); - process.exit(0); -} - -console.log(`Downloading GenieX ${pin.version} (${(asset.size / 1024 ** 2).toFixed(1)} MiB)…`); -await mkdir(OUTPUT_DIR, { recursive: true }); -const partial = `${OUTPUT_FILE}.partial`; -const response = await fetch(asset.url); -if (!response.ok) fail(`GET ${asset.url} → ${response.status}`); -await Bun.write(partial, response); - -const actual = await sha256(partial); -if (actual !== asset.sha256) { - await rm(partial, { force: true }); - fail(`checksum mismatch: expected ${asset.sha256}, got ${actual}`); -} -await rename(partial, OUTPUT_FILE); -console.log(`GenieX ${pin.version} verified → ${OUTPUT_FILE}`); - -async function sha256(path: string): Promise { - const hasher = new Bun.CryptoHasher("sha256"); - for await (const chunk of Bun.file(path).stream()) hasher.update(chunk); - return hasher.digest("hex"); -} - -function fail(message: string): never { - console.error(`fetch-geniex: ${message}`); - process.exit(1); -} diff --git a/scripts/geniex-bump.ts b/scripts/geniex-bump.ts index 2df6f9c..5dccdf3 100644 --- a/scripts/geniex-bump.ts +++ b/scripts/geniex-bump.ts @@ -1,6 +1,6 @@ /** - * Pin the latest stable GenieX in `runtime/geniex.json` (the version bundled - * in Calcine's installer), from Qualcomm's release index. + * Pin the latest stable GenieX in `runtime/geniex.json` (the version Calcine + * installs on first launch), from Qualcomm's release index. * * bun scripts/geniex-bump.ts # update the pin if a newer stable exists * @@ -30,27 +30,29 @@ if (!latest || Bun.semver.order(latest.slice(1), pin.version.slice(1)) <= 0) pro const manifest = (await (await fetch(`${ENDPOINT}/manifest-${latest}.json`)).json()) as { assets: Asset[]; }; -const asset = manifest.assets.find( - (candidate) => - candidate.kind === "cli-installer" && - candidate.platform === "windows" && - candidate.arch === "arm64", -); -if (!asset) throw new Error(`${latest} has no Windows ARM64 installer`); - -const bytes = new Uint8Array(await (await fetch(asset.url)).arrayBuffer()); -const digest = new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); -if (bytes.length !== asset.size || digest !== asset.sha256.toLowerCase()) { - throw new Error(`${asset.name} doesn't match its manifest (size or SHA-256)`); +/** The package of `kind` for `platform`, downloaded and checked against the manifest. */ +async function verified(kind: string, platform: string) { + const asset = manifest.assets.find( + (candidate) => + candidate.kind === kind && candidate.platform === platform && candidate.arch === "arm64", + ); + if (!asset) throw new Error(`${latest} has no ${platform} ARM64 ${kind}`); + const bytes = new Uint8Array(await (await fetch(asset.url)).arrayBuffer()); + const digest = new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); + if (bytes.length !== asset.size || digest !== asset.sha256.toLowerCase()) { + throw new Error(`${asset.name} doesn't match its manifest (size or SHA-256)`); + } + const pinned = { name: asset.name, url: asset.url, size: asset.size, sha256: digest }; + return { bytes, pinned }; } -await Bun.write(join(import.meta.dir, "..", "geniex-cli-setup.exe"), bytes); + +const windows = await verified("cli-installer", "windows"); +const linux = await verified("cli-archive", "linux"); +// The workflow installs it on a Windows runner before opening the PR. +await Bun.write(join(import.meta.dir, "..", "geniex-cli-setup.exe"), windows.bytes); pin.version = latest; -pin.assets["windows-arm64"] = { - name: asset.name, - url: asset.url, - size: asset.size, - sha256: digest, -}; +pin.assets["windows-arm64"] = windows.pinned; +pin.assets["linux-arm64"] = linux.pinned; await Bun.write(PIN, `${JSON.stringify(pin, null, 2)}\n`); console.log(latest); diff --git a/scripts/installer-notice.test.ts b/scripts/installer-notice.test.ts new file mode 100644 index 0000000..7dd935d --- /dev/null +++ b/scripts/installer-notice.test.ts @@ -0,0 +1,26 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const root = join(__dirname, ".."); +const read = (path: string) => readFileSync(join(root, path), "utf8").replaceAll("\r\n", "\n"); + +// The installer shows this file on its license page: SignPath requires the +// privacy policy to be shown during installation. +describe("installer license page", () => { + const notice = read("src-tauri/windows/license-and-privacy.txt"); + + it("ends with Calcine's license, unchanged", () => { + expect(notice.endsWith(read("LICENSE"))).toBe(true); + }); + + it("summarizes the privacy policy and links to it", () => { + expect(notice).toContain("PRIVACY"); + expect(notice).toContain("https://github.com/eraflo/Calcine/blob/main/PRIVACY.md"); + }); + + it("is what the installer uses", () => { + const config = JSON.parse(read("src-tauri/tauri.conf.json")); + expect(config.bundle.licenseFile).toBe("windows/license-and-privacy.txt"); + }); +}); diff --git a/scripts/updater-manifest.test.ts b/scripts/updater-manifest.test.ts index f179d04..7caa782 100644 --- a/scripts/updater-manifest.test.ts +++ b/scripts/updater-manifest.test.ts @@ -1,12 +1,14 @@ import { describe, expect, it } from "vitest"; -import { manifest } from "./updater-manifest"; +import { LINUX, manifest, WINDOWS } from "./updater-manifest"; describe("updater manifest", () => { - it("points both Windows ARM64 keys at the release asset", () => { + it("points each platform's keys at its release asset", () => { const result = manifest( "1.0.0", - "Calcine_1.0.0_arm64-setup.exe", - "c2lnbmF0dXJl\n", + [ + { file: "Calcine_1.0.0_arm64-setup.exe", signature: "c2lnbmF0dXJl\n", platforms: WINDOWS }, + { file: "Calcine_1.0.0_arm64.deb", signature: "ZGVi", platforms: LINUX }, + ], "Notes", new Date("2026-10-09T10:00:00Z"), ); @@ -18,5 +20,19 @@ describe("updater manifest", () => { expect(entry?.url).toBe( "https://github.com/eraflo/Calcine/releases/download/v1.0.0/Calcine_1.0.0_arm64-setup.exe", ); + expect(result.platforms["linux-aarch64-deb"]).toEqual({ + signature: "ZGVi", + url: "https://github.com/eraflo/Calcine/releases/download/v1.0.0/Calcine_1.0.0_arm64.deb", + }); + expect(result.platforms["linux-aarch64"]).toEqual(result.platforms["linux-aarch64-deb"]); + }); + + it("leaves Linux out when it wasn't built", () => { + const result = manifest( + "1.0.0", + [{ file: "Calcine_1.0.0_arm64-setup.exe", signature: "x", platforms: WINDOWS }], + "", + ); + expect(Object.keys(result.platforms)).toEqual(WINDOWS); }); }); diff --git a/scripts/updater-manifest.ts b/scripts/updater-manifest.ts index 46d93c8..4f54e99 100644 --- a/scripts/updater-manifest.ts +++ b/scripts/updater-manifest.ts @@ -1,11 +1,12 @@ /** * Write the manifest the Tauri updater reads (`latest.json`), from the - * signed installer of a release build. + * signed packages of a release build. * - * bun scripts/updater-manifest.ts [notes-file] + * bun scripts/updater-manifest.ts [notes-file] [linux-dir] * - * The installer and its `.sig` come from `target/release/bundle/nsis/`; the - * download URL points at the GitHub release `v`. + * The Windows installer and its `.sig` come from `target/release/bundle/nsis/`; + * the Linux `.deb` and its `.sig`, when built, from `linux-dir`. Download URLs + * point at the GitHub release `v`. */ import { readdir } from "node:fs/promises"; import { join } from "node:path"; @@ -19,39 +20,48 @@ export type UpdaterManifest = { platforms: Record; }; +/** A signed package, and the updater keys that should install it. */ +export type Package = { file: string; signature: string; platforms: string[] }; + +// The updater looks up `--` first, then `-`. +export const WINDOWS = ["windows-aarch64-nsis", "windows-aarch64"]; +export const LINUX = ["linux-aarch64-deb", "linux-aarch64"]; + export function manifest( version: string, - installer: string, - signature: string, + packages: Package[], notes: string, now: Date = new Date(), ): UpdaterManifest { - const url = `https://github.com/${REPOSITORY}/releases/download/v${version}/${encodeURIComponent(installer)}`; - const entry = { signature: signature.trim(), url }; - return { - version, - notes, - pub_date: now.toISOString(), - // The updater looks up `--` first, then `-`. - platforms: { "windows-aarch64-nsis": entry, "windows-aarch64": entry }, - }; + const platforms: UpdaterManifest["platforms"] = {}; + for (const { file, signature, platforms: keys } of packages) { + const url = `https://github.com/${REPOSITORY}/releases/download/v${version}/${encodeURIComponent(file)}`; + for (const key of keys) platforms[key] = { signature: signature.trim(), url }; + } + return { version, notes, pub_date: now.toISOString(), platforms }; +} + +async function signed(dir: string, matches: (file: string) => boolean, platforms: string[]) { + const file = (await readdir(dir)).find(matches); + if (!file) throw new Error(`no package in ${dir}`); + const signature = await Bun.file(join(dir, `${file}.sig`)).text(); + return { file, signature, platforms }; } if (import.meta.main) { - const [version, out, notesFile] = process.argv.slice(2); + const [version, out, notesFile, linuxDir] = process.argv.slice(2); if (!version || !out) { - console.error("usage: bun scripts/updater-manifest.ts [notes-file]"); + console.error( + "usage: bun scripts/updater-manifest.ts [notes-file] [linux-dir]", + ); process.exit(1); } - const bundle = join(import.meta.dir, "..", "target", "release", "bundle", "nsis"); - const files = await readdir(bundle); - const installer = files.find((file) => file.endsWith("-setup.exe") && file.includes(version)); - if (!installer) throw new Error(`no ${version} installer in ${bundle}`); - const signature = await Bun.file(join(bundle, `${installer}.sig`)).text(); + const nsis = join(import.meta.dir, "..", "target", "release", "bundle", "nsis"); + const packages = [ + await signed(nsis, (file) => file.endsWith("-setup.exe") && file.includes(version), WINDOWS), + ]; + if (linuxDir) packages.push(await signed(linuxDir, (file) => file.endsWith(".deb"), LINUX)); const notes = notesFile ? await Bun.file(notesFile).text() : ""; - await Bun.write( - out, - `${JSON.stringify(manifest(version, installer, signature, notes), null, 2)}\n`, - ); - console.log(`${out}: Calcine ${version} → ${installer}`); + await Bun.write(out, `${JSON.stringify(manifest(version, packages, notes), null, 2)}\n`); + console.log(`${out}: Calcine ${version} → ${packages.map((p) => p.file).join(", ")}`); } diff --git a/site/index.html b/site/index.html index d022e96..9a59bcd 100644 --- a/site/index.html +++ b/site/index.html @@ -33,6 +33,9 @@ + + + @@ -73,7 +76,7 @@
-

Free & open source · Windows 11 on Snapdragon

+

Free & open source · Windows 11 on Snapdragon · Linux ARM64 preview

Your Snapdragon NPU,
as a local AI server. @@ -93,7 +96,8 @@

View on GitHub

-

Windows 11 ARM64 · GenieX included

+

Windows 11 ARM64

+
@@ -163,7 +167,37 @@

OpenAI-compatible API

Always up to date

-

GenieX ships in the installer and updates, or rolls back, from the app. Calcine updates itself with signed packages.

+

Calcine installs GenieX on first launch and updates, or rolls back, from the app. Calcine updates itself with signed packages.

+
+
+ +

Structured output and tools

+

JSON Schema replies and tool calls on every model, NPU included: Calcine checks each reply and has the model fix it.

+
+
+ +

Long conversations

+

Counted with each model's tokenizer. The oldest messages are summarized and forgotten instead of failing at the context limit.

+
+
+ +

Benchmark and energy

+

Qualcomm's geniex-bench, plus live power and energy per token on Snapdragon X, per compute unit.

+
+
+ +

Your other devices

+

Optional, over HTTPS with a certificate made on your PC, for keys you allow on the network and private networks only.

+
+
+ +

Works with Ollama apps

+

Ollama's API is translated too, and can answer on port 11434 for apps that only speak Ollama.

+
+
+ +

Without the window

+

calcine-cli serve runs the same API from a terminal, with the app's keys, settings and models.

@@ -294,12 +328,12 @@

A local API that doesn't trust the browser

    -
  1. 1

    Loopback only

    Listens on 127.0.0.1. Other machines on your network can't reach it.

    2. +
  2. 1

    Loopback first

    Listens on 127.0.0.1. Other devices get in only if you turn on the network port: HTTPS, keys allowed on the network, private addresses.

  3. 2

    Host header

    Blocks DNS rebinding, where a web page points its domain at 127.0.0.1.

  4. 3

    Browser origin

    Web pages can't call the API from your browser unless you allow their origin.

  5. 4

    Per-app keys

    Shown once, stored as SHA-256 fingerprints, revocable instantly, scoped to inference or management.

  6. 5

    Request bodies

    Local file paths and internal URLs in requests are refused, so apps can't make GenieX read your files.

    7. -
  7. 6

    Clean exit

    A Windows Job Object stops GenieX with Calcine, even after a crash. Nothing is left listening.

    8. +
  8. 6

    Clean exit

    GenieX stops with Calcine, even after a crash (a Job Object on Windows, a parent-death signal on Linux). Nothing is left listening.

Signed updates, verified GenieX installers, and an SBOM with build provenance on every stable release. @@ -312,7 +346,7 @@

A local API that doesn't trust the browser

Get started in three steps

    -
  1. Install Calcine. The installer also sets up GenieX, the Qualcomm runtime.
    2. +
  2. Install Calcine. On first launch it downloads GenieX, Qualcomm's runtime, and checks it.
  3. Download a model. The welcome screen suggests a few that fit your PC.
  4. Chat, or connect an app. Create an API key and use any OpenAI client.
@@ -322,10 +356,13 @@

Get started in three steps

Download for Windows -

Windows 11 ARM64 · GenieX included

+

Windows 11 ARM64

+

- Calcine is in beta and not code-signed yet: Windows SmartScreen asks for confirmation - before installing (More info › Run anyway). + Calcine isn't code-signed yet: Windows SmartScreen asks for confirmation + before installing (More info › Run anyway). Calcine has applied to the + SignPath Foundation for free code signing: see the + code signing policy.

@@ -334,15 +371,15 @@

Get started in three steps

Questions

Which PCs does Calcine run on? -

Windows 11 on Arm with a Snapdragon processor that GenieX supports, such as the Snapdragon X Elite and X Plus. Linux on ARM64 is planned.

+

Windows 11 on Arm with a Snapdragon processor that GenieX supports, such as the Snapdragon X Elite and X Plus. Linux on ARM64 (Ubuntu 24.04 or later) is a preview: GGUF models run on the CPU, and on the NPU where the kernel exposes the compute DSP.

Does anything leave my PC? -

No. Models run locally, the API only listens on this PC, and conversations are stored on this PC only. Calcine connects to the internet to download models and check for updates, and that's it: no telemetry.

+

No. Models run locally, the API only listens on this PC unless you turn on the network port, and conversations are stored on this PC only. Calcine connects to the internet to download models and check for updates, and that's it: no telemetry.

Do I need to install GenieX first? -

No. GenieX is bundled in the installer. You can update it, switch to pre-releases, or roll back from Hardware › GenieX runtime.

+

No. On first launch, Calcine downloads Qualcomm's official GenieX installer (an archive on Linux), checks it against the checksum built into Calcine, and installs it. Without a screen, calcine-cli geniex install does the same. You can update it, switch to pre-releases, or roll back from Hardware › GenieX runtime.

Which models can I run? @@ -352,10 +389,34 @@

Questions

I already use geniex serve. Will my apps still work?

Yes. Calcine listens on GenieX's default port, 18181, with the same OpenAI-compatible routes. Add an API key to your client, or turn off "Require an API key" for clients that can't send one.

+
+ Can I run the API without the window? +

Yes. calcine-cli serve serves the same API from a terminal, with the app's settings, keys and models. The installer puts calcine-cli on your PATH; the Linux package puts it in /usr/bin.

+
Is Calcine made by Qualcomm? -

No. Calcine is an independent open-source project under the MIT license. It redistributes GenieX under its BSD 3-Clause license.

+

No. Calcine is an independent open-source project under the MIT license. It doesn't redistribute GenieX: it downloads Qualcomm's official installer on your PC.

+
+ What does Calcine send over the internet? +

No prompts, conversations or telemetry. Calcine checks for updates at startup (you can turn that off), and downloads GenieX, models and tools when you ask. The privacy policy lists every connection.

+
+ + +
+

Code signing policy

+

+ Calcine has applied to the SignPath Foundation's free code signing program for + open-source projects. Once accepted: Free code signing provided by + SignPath.io, certificate by + SignPath Foundation. +

+
    +
  • Committers and reviewers: eraflo
    • +
  • Approvers: eraflo
    • +
  • Installers are built by GitHub Actions from the public repository and signed only after a manual approval. They contain Calcine only.
    • +
  • Privacy policy: PRIVACY.md
    • +
@@ -371,6 +432,8 @@

Questions

Releases Changelog Contribute + Code signing policy + Privacy diff --git a/site/main.js b/site/main.js index ca5befe..7600b66 100644 --- a/site/main.js +++ b/site/main.js @@ -1,5 +1,5 @@ // Calcine site: code tabs, copy button, reveal on scroll, and the download -// buttons pointing at the newest installer. +// buttons pointing at the newest installer (and Linux package). document.documentElement.classList.add("js"); @@ -84,7 +84,14 @@ if ("IntersectionObserver" in window) { label.textContent = `Download Calcine ${version}`; } for (const meta of document.querySelectorAll("[data-download-meta]")) { - meta.textContent = `${release.prerelease ? "Beta · " : ""}Windows 11 ARM64 · ${size} · GenieX included`; + meta.textContent = `${release.prerelease ? "Beta · " : ""}Windows 11 ARM64 · ${size}`; + } + // The Linux package of the same release, when it has one. + const deb = release.assets.find((candidate) => candidate.name.endsWith(".deb")); + if (deb) { + for (const link of document.querySelectorAll("[data-download-linux-link]")) + link.href = deb.browser_download_url; + for (const line of document.querySelectorAll("[data-download-linux]")) line.hidden = false; } } catch { // Keep the releases page link. diff --git a/site/style.css b/site/style.css index 50b832b..78f780b 100644 --- a/site/style.css +++ b/site/style.css @@ -812,6 +812,13 @@ details[open] summary::after { transform: rotate(45deg); } +.sub a, +.faq a, +.checks a, +.fine a { + color: var(--primary); +} + .faq p { margin: 0 4px 22px; color: var(--muted); diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index cdfb708..6425a0d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -8,6 +8,8 @@ license.workspace = true repository.workspace = true authors.workspace = true publish = false +# `tauri dev` runs the app; `calcine-cli` is the other binary. +default-run = "calcine" [lib] # The `_lib` suffix keeps the lib name distinct from the bin name on Windows @@ -37,12 +39,17 @@ tauri-plugin-opener.workspace = true tauri-plugin-single-instance.workspace = true tauri-plugin-updater.workspace = true tauri-specta.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["signal"] } tracing.workspace = true tracing-subscriber.workspace = true [target.'cfg(windows)'.dependencies] win32job.workspace = true +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_System_Registry", + "Win32_UI_WindowsAndMessaging", +] } [lints] workspace = true diff --git a/src-tauri/src/bin/calcine-cli.rs b/src-tauri/src/bin/calcine-cli.rs new file mode 100644 index 0000000..3eebd41 --- /dev/null +++ b/src-tauri/src/bin/calcine-cli.rs @@ -0,0 +1,7 @@ +//! `calcine-cli`: Calcine's API without the window. A console program, so +//! terminals wait for it and show its output (the app itself is a windowed +//! program). + +fn main() { + std::process::exit(calcine_lib::cli_main()); +} diff --git a/src-tauri/src/cli.rs b/src-tauri/src/cli.rs new file mode 100644 index 0000000..f02e075 --- /dev/null +++ b/src-tauri/src/cli.rs @@ -0,0 +1,541 @@ +//! `calcine-cli`: the API without the window, for servers, scripts and +//! terminals. Same services, keys, settings and data folder as the app. +//! +//! ```text +//! calcine-cli serve [--port N] [--network] [--network-port N] +//! calcine-cli keys list +//! calcine-cli keys create [--manage] [--network] [--local-files] +//! calcine-cli keys revoke +//! calcine-cli geniex install +//! calcine-cli geniex chipset [|auto] +//! calcine-cli path add|remove|status +//! ``` + +use std::collections::HashSet; +use std::fmt::Write as _; +use std::io::Write as _; + +use calcine_core::jobs::{JobPhase, JobState}; +use calcine_core::runtime::InstallSource; +use calcine_gateway::{KeyScope, NewApiKey, RequestEntry}; + +use crate::setup::{self, DataDirs}; + +const USAGE: &str = "\ +Calcine: your local models through an OpenAI-compatible API, without the window. + +Usage: + calcine-cli serve [options] Serve the API until Ctrl+C + --port Port on this PC (default: the app's, 18181) + --network Also answer other devices over HTTPS + --network-port Port for other devices (default 18443) + calcine-cli keys list List the API keys + calcine-cli keys create [--manage] [--network] [--local-files] + Create a key and print it, once + calcine-cli keys revoke Revoke a key + calcine-cli geniex install Install the GenieX version Calcine was tested with + calcine-cli geniex chipset [|auto] + Show or set the chipset models are downloaded for + calcine-cli path add|remove|status + Put calcine-cli on your PATH (the installer does) + calcine-cli --version + +Settings, keys and models are the app's own. --port and --network only +apply to this run."; + +/// Run a command; the process exit code. +pub fn main(args: &[String]) -> i32 { + let words: Vec<&str> = args.iter().map(String::as_str).collect(); + let result = match words.as_slice() { + [] | ["help" | "-h" | "--help", ..] => { + println!("{USAGE}"); + Ok(()) + } + ["-V" | "--version" | "version"] => { + println!("calcine-cli {}", env!("CARGO_PKG_VERSION")); + Ok(()) + } + ["serve", options @ ..] => ServeOptions::parse(options).and_then(|options| serve(&options)), + ["keys", rest @ ..] => keys(rest), + ["geniex", "install"] => install_geniex(), + ["geniex", "chipset"] => chipset(None), + ["geniex", "chipset", id] => chipset(Some(id)), + ["path", action] => path(action), + [other, ..] => Err(format!("unknown command {other}. Run calcine-cli --help.")), + }; + match result { + Ok(()) => 0, + Err(message) => { + eprintln!("calcine-cli: {message}"); + 1 + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct ServeOptions { + port: Option, + network: bool, + network_port: Option, +} + +impl ServeOptions { + fn parse(options: &[&str]) -> Result { + let mut parsed = Self::default(); + let mut words = options.iter(); + while let Some(word) = words.next() { + let mut port = |name: &str| { + words + .next() + .and_then(|value| value.parse::().ok()) + .filter(|port| *port > 0) + .ok_or_else(|| format!("{name} needs a port between 1 and 65535")) + }; + match *word { + "--port" => parsed.port = Some(port("--port")?), + "--network" => parsed.network = true, + "--network-port" => parsed.network_port = Some(port("--network-port")?), + other => return Err(format!("unknown option {other}. Run calcine-cli --help.")), + } + } + Ok(parsed) + } +} + +fn data_dirs() -> Result { + DataDirs::from_env().ok_or_else(|| "can't find the user's data folder".to_owned()) +} + +fn serve(options: &ServeOptions) -> Result<(), String> { + let dirs = data_dirs()?; + let ServeOptions { + port, + network, + network_port, + } = *options; + tauri::async_runtime::block_on(async move { + let core = setup::build(&dirs, None).await; + let gateway = core.gateway; + gateway.override_settings(|settings| { + if let Some(port) = port { + settings.port = port; + } + if network { + settings.network_enabled = true; + } + if let Some(port) = network_port { + settings.network_port = port; + } + }); + if let Err(err) = core.services.runtime.info().await + && core.services.backend == calcine_core::BackendKind::Geniex + { + return Err(format!( + "GenieX isn't ready ({err}). Install it with: calcine-cli geniex install" + )); + } + if let Err(message) = gateway.start().await { + return Err(if message.contains("already used") { + format!("{message}\nIf the Calcine window is open, it already serves the API.") + } else { + message + }); + } + announce(&gateway.status()); + let mut requests = gateway.subscribe_requests(); + let mut printed = HashSet::new(); + // Made once, so a signal between two requests isn't missed. + let stop = stop_requested(); + tokio::pin!(stop); + loop { + tokio::select! { + () = &mut stop => break, + entry = requests.recv() => { + if let Ok(entry) = entry + && entry.duration_ms.is_some() + && printed.insert(entry.id) + { + // Only recent requests can still be updated. + printed.retain(|id: &u32| id + 1000 > entry.id); + println!("{}", request_line(&entry)); + } + } + } + } + println!("Stopping…"); + gateway.stop().await; + let _ = core.services.server.stop().await; + Ok(()) + }) +} + +/// Install the GenieX version Calcine was tested with, as the welcome screen +/// does: downloaded from Qualcomm, checked against the SHA-256 built into +/// Calcine. For machines without a screen. +fn install_geniex() -> Result<(), String> { + let dirs = data_dirs()?; + tauri::async_runtime::block_on(async move { + let services = setup::build(&dirs, None).await.services; + let release = setup::pinned_release(); + println!("Installing GenieX {} from Qualcomm…", release.version); + // Subscribed first, so no update is missed. + let mut updates = services.jobs.subscribe(); + let id = services.start_runtime_install(InstallSource::Release { release }); + let mut shown = (None, 0); + loop { + let job = match updates.recv().await { + Ok(job) if job.id == id => job, + Ok(_) => continue, + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + match services.jobs.get(id) { + Some(job) => job, + None => continue, + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err("the install stopped".into()); + } + }; + match job.state { + JobState::Running => {} + JobState::Succeeded => { + let version = services.runtime.info().await.map_or_else( + |_| "GenieX".to_owned(), + |info| format!("GenieX {}", info.cli_version), + ); + println!("{version} is installed. Start the API with: calcine-cli serve"); + return Ok(()); + } + JobState::Cancelled => return Err("the install was cancelled".into()), + JobState::Failed { message } => return Err(message), + } + let Some(progress) = job.progress else { + continue; + }; + // A line per phase, and per quarter of the download. + let quarter = progress + .total_bytes + .filter(|total| *total > 0) + .map_or(0, |total| progress.done_bytes * 4 / total); + if (progress.phase, quarter) == shown { + continue; + } + shown = (progress.phase, quarter); + match progress.phase { + Some(JobPhase::Downloading) => println!(" downloading {}%", quarter * 25), + Some(JobPhase::Verifying) => println!(" checking the SHA-256"), + Some(JobPhase::Installing) => println!(" installing"), + _ => {} + } + } + }) +} + +/// Show the chipset GenieX downloads models for, with the ones AI Hub knows, +/// or set it (`auto`: let GenieX detect it). Linux machines GenieX can't +/// identify need it before downloading. +fn chipset(id: Option<&str>) -> Result<(), String> { + let dirs = data_dirs()?; + tauri::async_runtime::block_on(async move { + let services = setup::build(&dirs, None).await.services; + let known = services.directory.chipsets().await.unwrap_or_default(); + match id { + Some("auto") => { + services + .runtime + .set_chipset(None) + .await + .map_err(|err| err.to_string())?; + println!("GenieX detects the chipset again."); + } + Some(id) => { + let found = known.iter().find(|chipset| { + chipset.id.eq_ignore_ascii_case(id) + || chipset + .aliases + .iter() + .any(|alias| alias.eq_ignore_ascii_case(id)) + }); + // AI Hub lists Linux models for Dragonwing boards only: a + // Snapdragon X laptop on Linux still names its chipset (for + // GenieX), and gets llama.cpp (GGUF) models. + if found.is_none() { + println!( + "AI Hub has no NPU models for {id} on this system: \ + only llama.cpp (GGUF) models will download." + ); + } + let id = found.map_or(id, |chipset| chipset.id.as_str()); + services + .runtime + .set_chipset(Some(id)) + .await + .map_err(|err| err.to_string())?; + println!("Models will be downloaded for {id}."); + } + None => { + let current = services + .runtime + .chipset() + .await + .map_err(|err| err.to_string())?; + println!( + "Chipset: {}", + current + .as_deref() + .unwrap_or("unknown (GenieX couldn't detect it)") + ); + if !known.is_empty() { + println!("\nSet it with: calcine-cli geniex chipset "); + } + for chipset in &known { + let name = chipset.marketing_name.as_deref().unwrap_or(&chipset.device); + println!(" {:<40} {name}", chipset.id); + } + } + } + Ok(()) + }) +} + +/// Ctrl+C, and on Windows also Ctrl+Break and closing the console window. +async fn stop_requested() { + #[cfg(windows)] + { + use tokio::signal::windows; + let (Ok(mut ctrl_break), Ok(mut close)) = (windows::ctrl_break(), windows::ctrl_close()) + else { + let _ = tokio::signal::ctrl_c().await; + return; + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = ctrl_break.recv() => {} + _ = close.recv() => {} + } + } + #[cfg(not(windows))] + { + let _ = tokio::signal::ctrl_c().await; + } +} + +fn announce(status: &calcine_gateway::GatewayStatus) { + println!("Calcine API on {}", status.base_url); + if let Some(network) = &status.network { + println!("Other devices: {}", network.base_url); + println!(" certificate SHA-256 {}", network.fingerprint); + println!(" certificate file {}", network.certificate_path); + } else if let Some(error) = &status.network_error { + println!("Other devices: not listening ({error})"); + } + if status.require_api_key { + println!("Apps need an API key: calcine-cli keys create "); + } + println!("Models load on the first request. Press Ctrl+C to stop."); + let _ = std::io::stdout().flush(); +} + +/// `200 POST /v1/chat/completions qualcomm/Qwen3-4B 1.2 s 45 tok/s (App)`. +fn request_line(entry: &RequestEntry) -> String { + let mut line = format!( + "{} {} {}", + entry + .status + .map_or_else(|| "---".to_owned(), |status| status.to_string()), + entry.method, + entry.path + ); + if let Some(model) = &entry.model { + line.push(' '); + line.push_str(model); + } + if let Some(ms) = entry.duration_ms { + #[allow(clippy::cast_precision_loss)] + let seconds = ms as f64 / 1000.0; + let _ = write!(line, " {seconds:.1} s"); + } + if let Some(speed) = entry.tokens_per_second { + let _ = write!(line, " {speed:.0} tok/s"); + } + let _ = write!(line, " ({})", entry.client); + if let Some(error) = &entry.error { + let _ = write!(line, ": {error}"); + } + line +} + +/// Put the folder holding `calcine-cli` on the user's `PATH`, or take it off. +#[cfg(windows)] +fn path(action: &str) -> Result<(), String> { + const NAME: &str = "Path"; + let exe = std::env::current_exe().map_err(|err| err.to_string())?; + let dir = exe + .parent() + .ok_or("can't tell where calcine-cli is")? + .display() + .to_string(); + match action { + "add" => { + if crate::user_path::add(NAME, &dir)? { + println!("Added {dir} to your PATH. New terminals will find calcine-cli."); + } else { + println!("{dir} is already on your PATH."); + } + } + "remove" => { + if crate::user_path::remove(NAME, &dir)? { + println!("Removed {dir} from your PATH."); + } else { + println!("{dir} wasn't on your PATH."); + } + } + "status" => { + if crate::user_path::is_added(NAME, &dir)? { + println!("{dir} is on your PATH."); + } else { + println!("{dir} isn't on your PATH. Add it with: calcine-cli path add"); + } + } + other => return Err(format!("unknown action {other}: add, remove or status")), + } + Ok(()) +} + +/// The Linux package installs calcine-cli in `/usr/bin`: nothing to do. +#[cfg(not(windows))] +fn path(action: &str) -> Result<(), String> { + match action { + "add" | "remove" | "status" => { + println!("The Calcine package installs calcine-cli in /usr/bin, already on your PATH."); + Ok(()) + } + other => Err(format!("unknown action {other}: add, remove or status")), + } +} + +fn keys(words: &[&str]) -> Result<(), String> { + let store = calcine_gateway::KeyStore::load(data_dirs()?.data.join("api-keys.json")) + .map_err(|err| err.to_string())?; + match words { + ["list"] | [] => { + let keys = store.list(); + if keys.is_empty() { + println!("No API keys. Create one with: calcine-cli keys create "); + } + for key in keys { + let mut rights: Vec<&str> = key + .scopes + .iter() + .map(|scope| match scope { + KeyScope::Inference => "run models", + KeyScope::Manage => "manage models", + }) + .collect(); + if key.network { + rights.push("other devices"); + } + if key.allow_local_files { + rights.push("local files"); + } + println!( + "{} {} {} {}", + key.id, + key.preview, + key.name, + rights.join(", ") + ); + } + Ok(()) + } + ["create", name, flags @ ..] => { + let mut scopes = vec![KeyScope::Inference]; + let mut network = false; + let mut allow_local_files = false; + for flag in flags { + match *flag { + "--manage" => scopes.push(KeyScope::Manage), + "--network" => network = true, + "--local-files" => allow_local_files = true, + other => return Err(format!("unknown option {other}")), + } + } + let created = store + .create(NewApiKey { + name: (*name).to_owned(), + scopes, + allow_local_files, + network, + }) + .map_err(|err| err.to_string())?; + println!("{}", created.token); + eprintln!( + "Key \"{}\" created ({}). It's shown only now: keep it somewhere safe.", + created.key.name, created.key.id + ); + Ok(()) + } + ["revoke", id] => { + if store.revoke(id).map_err(|err| err.to_string())? { + println!("Revoked {id}."); + Ok(()) + } else { + Err(format!( + "there's no key {id}. List them with: calcine-cli keys list" + )) + } + } + _ => Err("usage: calcine-cli keys list | create [...] | revoke ".into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_serve_options() { + assert_eq!(ServeOptions::parse(&[]), Ok(ServeOptions::default())); + assert_eq!( + ServeOptions::parse(&["--port", "18200", "--network", "--network-port", "9443"]), + Ok(ServeOptions { + port: Some(18200), + network: true, + network_port: Some(9443), + }) + ); + assert!(ServeOptions::parse(&["--port"]).is_err()); + assert!(ServeOptions::parse(&["--port", "0"]).is_err()); + assert!(ServeOptions::parse(&["--verbose"]).is_err()); + } + + #[test] + fn prints_one_line_per_request() { + let entry = RequestEntry { + id: 1, + started_at_ms: 0, + client: "Phone".into(), + method: "POST".into(), + path: "/v1/chat/completions".into(), + model: Some("qualcomm/Qwen3-4B".into()), + stream: true, + status: Some(200), + duration_ms: Some(1300), + first_token_ms: Some(150), + prompt_tokens: Some(20), + completion_tokens: Some(40), + tokens_per_second: Some(21.4), + error: None, + }; + assert_eq!( + request_line(&entry), + "200 POST /v1/chat/completions qualcomm/Qwen3-4B 1.3 s 21 tok/s (Phone)" + ); + } + + #[test] + fn unknown_commands_fail() { + assert_eq!(main(&["frobnicate".into()]), 1); + assert_eq!(main(&["--version".into()]), 0); + } +} diff --git a/src-tauri/src/desktop/locale.rs b/src-tauri/src/desktop/locale.rs index 581bd72..9227727 100644 --- a/src-tauri/src/desktop/locale.rs +++ b/src-tauri/src/desktop/locale.rs @@ -46,10 +46,13 @@ pub struct Strings { pub download: JobMessages, pub import: JobMessages, pub geniex_update: JobMessages, + pub benchmark: JobMessages, /// After a model name: "… is ready to use." pub model_ready: &'static str, /// After a version: "… is installed." pub version_installed: &'static str, + /// After a model name: "…: results are on the Benchmark page." + pub results_ready: &'static str, } impl Language { @@ -70,8 +73,13 @@ impl Language { done: "GenieX updated", failed: "GenieX update failed", }, + benchmark: JobMessages { + done: "Benchmark finished", + failed: "Benchmark failed", + }, model_ready: "is ready to use.", version_installed: "is installed.", + results_ready: ": results are on the Benchmark page.", }, Self::Fr => Strings { open: "Ouvrir Calcine", @@ -88,8 +96,13 @@ impl Language { done: "GenieX mis à jour", failed: "Échec de la mise à jour de GenieX", }, + benchmark: JobMessages { + done: "Benchmark terminé", + failed: "Échec du benchmark", + }, model_ready: "est prêt à l'emploi.", version_installed: "est installé.", + results_ready: " : les résultats sont sur la page Benchmark.", }, } } diff --git a/src-tauri/src/desktop/notifications.rs b/src-tauri/src/desktop/notifications.rs index ba2d0f0..70c0dc8 100644 --- a/src-tauri/src/desktop/notifications.rs +++ b/src-tauri/src/desktop/notifications.rs @@ -44,8 +44,22 @@ fn message(job: &Job, strings: &Strings) -> Option<(String, String)> { JobKind::InstallRuntime { version } => { (version, &strings.geniex_update, strings.version_installed) } - // Calcine restarts into the new version: nothing to announce. - JobKind::UpdateApp { .. } => return None, + JobKind::Benchmark { model } | JobKind::EnergyProfile { model } => { + return match &job.state { + JobState::Succeeded => Some(( + strings.benchmark.done.to_owned(), + format!("{model}{}", strings.results_ready), + )), + JobState::Failed { message } => Some(( + strings.benchmark.failed.to_owned(), + format!("{model}: {message}"), + )), + JobState::Running | JobState::Cancelled => None, + }; + } + // Calcine restarts into the new version: nothing to announce. The + // benchmark tool downloads while its page is open. + JobKind::UpdateApp { .. } | JobKind::InstallBench { .. } => return None, }; match &job.state { JobState::Succeeded => Some((messages.done.to_owned(), format!("{subject} {success}"))), diff --git a/src-tauri/src/ipc/commands/app.rs b/src-tauri/src/ipc/commands/app.rs index 274026f..a10b6ee 100644 --- a/src-tauri/src/ipc/commands/app.rs +++ b/src-tauri/src/ipc/commands/app.rs @@ -1,9 +1,10 @@ use calcine_core::{BackendKind, Services}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use specta::Type; use tauri::{AppHandle, State}; use tauri_plugin_autostart::ManagerExt; +use tauri_plugin_dialog::DialogExt; use tauri_plugin_opener::OpenerExt; use crate::desktop::locale::{Language, Locale}; @@ -75,3 +76,90 @@ pub fn set_autostart(app: AppHandle, enabled: bool) -> ApiResult<()> { } .map_err(ApiError::io) } + +/// Exports larger than this are refused (a conversation is a few hundred KB). +const MAX_EXPORT_BYTES: usize = 16 << 20; + +/// What an export contains, for the save dialog's file type. +#[derive(Debug, Clone, Copy, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +pub enum ExportFormat { + Markdown, + Csv, +} + +impl ExportFormat { + fn filter(self) -> (&'static str, &'static str) { + match self { + Self::Markdown => ("Markdown", "md"), + Self::Csv => ("CSV", "csv"), + } + } +} + +/// Save an export where the user picks in the system dialog. Returns the +/// saved path, or `None` when the user cancels. +#[tauri::command] +#[specta::specta] +pub async fn save_export( + app: AppHandle, + file_name: String, + contents: String, + format: ExportFormat, +) -> ApiResult> { + if contents.len() > MAX_EXPORT_BYTES { + return Err(ApiError::invalid_input("this export is too large".into())); + } + let (label, extension) = format.filter(); + let (sender, receiver) = tokio::sync::oneshot::channel(); + app.dialog() + .file() + .set_file_name(export_file_name(&file_name, extension)) + .add_filter(label, &[extension]) + .save_file(move |path| { + let _ = sender.send(path); + }); + let Some(path) = receiver.await.ok().flatten() else { + return Ok(None); + }; + let path = path.into_path().map_err(ApiError::io)?; + tokio::fs::write(&path, contents) + .await + .map_err(ApiError::io)?; + Ok(Some(path.display().to_string())) +} + +/// A safe Windows file name ending in `.extension`. +fn export_file_name(name: &str, extension: &str) -> String { + let cleaned: String = name + .chars() + .map(|c| { + if c.is_control() || r#"<>:"/\|?*"#.contains(c) { + ' ' + } else { + c + } + }) + .collect(); + let words = cleaned.split_whitespace().collect::>().join(" "); + let stem: String = words.trim_end_matches('.').chars().take(80).collect(); + let stem = stem.trim_end(); + if stem.is_empty() { + format!("Calcine.{extension}") + } else { + format!("{stem}.{extension}") + } +} + +#[cfg(test)] +mod tests { + use super::export_file_name; + + #[test] + fn makes_safe_file_names() { + assert_eq!(export_file_name("What's 2/3?", "md"), "What's 2 3.md"); + assert_eq!(export_file_name(" \u{7} ", "csv"), "Calcine.csv"); + assert_eq!(export_file_name("notes...", "md"), "notes.md"); + assert_eq!(export_file_name(&"a".repeat(200), "md").len(), 83); + } +} diff --git a/src-tauri/src/ipc/commands/bench.rs b/src-tauri/src/ipc/commands/bench.rs new file mode 100644 index 0000000..fdf80a0 --- /dev/null +++ b/src-tauri/src/ipc/commands/bench.rs @@ -0,0 +1,58 @@ +use calcine_core::Services; +use calcine_core::bench::{BenchRequest, BenchResult, BenchTool, EnergyRequest}; +use calcine_core::jobs::JobId; +use tauri::State; + +use crate::ipc::error::{ApiError, ApiResult}; + +/// Whether `geniex-bench` is downloaded, and for which GenieX. +#[tauri::command] +#[specta::specta] +pub async fn bench_tool(services: State<'_, Services>) -> ApiResult { + Ok(services.bench.tool().await?) +} + +/// Download `geniex-bench` for the installed GenieX, as a job. +#[tauri::command] +#[specta::specta] +pub async fn install_bench_tool(services: State<'_, Services>) -> ApiResult { + let tool = services.bench.tool().await?; + let version = tool + .wanted + .ok_or_else(|| ApiError::invalid_input("install GenieX first".into()))?; + Ok(services.start_bench_install(version)) +} + +/// Benchmark a model on each requested compute unit, as a job. +#[tauri::command] +#[specta::specta] +pub async fn start_benchmark( + services: State<'_, Services>, + request: BenchRequest, +) -> ApiResult { + Ok(services.start_benchmark(request)?) +} + +/// Measure a model's speed and energy in each requested power mode, as a +/// job. Needs a device with energy metering. +#[tauri::command] +#[specta::specta] +pub async fn start_energy_profile( + services: State<'_, Services>, + request: EnergyRequest, +) -> ApiResult { + Ok(services.start_energy_profile(request)?) +} + +/// Past results, newest first. +#[tauri::command] +#[specta::specta] +pub fn bench_history(services: State<'_, Services>) -> Vec { + services.bench.history() +} + +#[tauri::command] +#[specta::specta] +pub fn forget_bench_results(services: State<'_, Services>, ids: Vec) -> ApiResult<()> { + Ok(services.bench.forget(&ids)?) +} diff --git a/src-tauri/src/ipc/commands/mod.rs b/src-tauri/src/ipc/commands/mod.rs index fe6fcf0..413b8cc 100644 --- a/src-tauri/src/ipc/commands/mod.rs +++ b/src-tauri/src/ipc/commands/mod.rs @@ -5,6 +5,7 @@ #![allow(clippy::needless_pass_by_value)] pub mod app; +pub mod bench; pub mod jobs; pub mod models; pub mod runtime; diff --git a/src-tauri/src/ipc/commands/runtime.rs b/src-tauri/src/ipc/commands/runtime.rs index 1d1be15..55a87c8 100644 --- a/src-tauri/src/ipc/commands/runtime.rs +++ b/src-tauri/src/ipc/commands/runtime.rs @@ -3,7 +3,7 @@ use calcine_core::hardware::{HardwareInfo, HardwareUsage}; use calcine_core::jobs::JobId; use calcine_core::models::Chipset; use calcine_core::runtime::{ - CachedInstaller, InstallSource, ReleaseChannel, RuntimeInfo, RuntimeUpdateCheck, + CachedInstaller, InstallSource, ReleaseChannel, RuntimeInfo, RuntimeRelease, RuntimeUpdateCheck, }; use tauri::State; @@ -53,6 +53,13 @@ pub async fn hardware_usage(services: State<'_, Services>) -> ApiResult RuntimeRelease { + crate::setup::pinned_release() +} + /// The newest GenieX on `channel`, compared with the installed one. #[tauri::command] #[specta::specta] diff --git a/src-tauri/src/ipc/commands/server.rs b/src-tauri/src/ipc/commands/server.rs index eee53fc..b00a3b7 100644 --- a/src-tauri/src/ipc/commands/server.rs +++ b/src-tauri/src/ipc/commands/server.rs @@ -1,15 +1,19 @@ use std::sync::Arc; use calcine_core::Services; +use calcine_core::context::ContextUsage; +use calcine_core::runtime::ServerOptions; use calcine_gateway::{ ApiKeyInfo, CreatedApiKey, Gateway, GatewaySettings, GatewayStatus, KeyError, NewApiKey, RequestEntry, }; use serde::Serialize; use specta::Type; -use tauri::State; +use tauri::{AppHandle, State}; +use tauri_plugin_opener::OpenerExt; use crate::ipc::error::{ApiError, ApiResult}; +use crate::setup::ServerOptionsFile; /// How Calcine's own UI talks to the gateway. #[derive(Debug, Serialize, Type)] @@ -56,6 +60,40 @@ pub fn server_logs(services: State<'_, Services>) -> Vec { services.server.logs() } +/// How much of the model's context window a chat request takes. `request` +/// is the request's JSON, as sent to `/v1/chat/completions`. +#[tauri::command] +#[specta::specta] +pub async fn context_usage( + services: State<'_, Services>, + model: String, + request: String, +) -> ApiResult { + let request: serde_json::Value = serde_json::from_str(&request) + .map_err(|err| ApiError::invalid_input(format!("request isn't JSON: {err}")))?; + Ok(services.context_usage(&model, &request).await?) +} + +/// What `geniex serve` starts with: model unload delay, context window. +#[tauri::command] +#[specta::specta] +pub fn server_options(services: State<'_, Services>) -> ServerOptions { + services.server.options() +} + +/// Save new server options. A running GenieX restarts to apply them. +#[tauri::command] +#[specta::specta] +pub async fn set_server_options( + services: State<'_, Services>, + file: State<'_, ServerOptionsFile>, + options: ServerOptions, +) -> ApiResult<()> { + options.validate().map_err(ApiError::invalid_input)?; + file.save(options).map_err(ApiError::io)?; + Ok(services.server.set_options(options).await?) +} + #[tauri::command] #[specta::specta] pub async fn set_require_api_key(gateway: State<'_, Arc>, require: bool) -> ApiResult<()> { @@ -94,6 +132,17 @@ pub fn revoke_api_key(gateway: State<'_, Arc>, id: String) -> ApiResult gateway.keys().revoke(&id).map_err(key_error) } +/// Let a key be used from other devices on the local network, or not. +#[tauri::command] +#[specta::specta] +pub fn set_api_key_network( + gateway: State<'_, Arc>, + id: String, + network: bool, +) -> ApiResult> { + gateway.keys().set_network(&id, network).map_err(key_error) +} + fn key_error(err: KeyError) -> ApiError { match err { KeyError::MissingName | KeyError::NoScope => ApiError::invalid_input(err.to_string()), @@ -129,3 +178,47 @@ pub fn set_allowed_origins( .set_allowed_origins(origins) .map_err(ApiError::invalid_input) } + +/// Also answer Ollama apps on port 11434, without a key (inference only). +#[tauri::command] +#[specta::specta] +pub async fn set_ollama_port(gateway: State<'_, Arc>, enabled: bool) -> ApiResult<()> { + gateway.set_ollama_port(enabled).await.map_err(ApiError::io) +} + +/// Answer other devices on the local network over HTTPS, or stop. `allowed` +/// lists addresses or ranges; empty means private networks. +#[tauri::command] +#[specta::specta] +pub async fn set_network( + gateway: State<'_, Arc>, + enabled: bool, + port: u16, + allowed: Vec, +) -> ApiResult<()> { + gateway + .set_network(enabled, port, allowed) + .await + .map_err(ApiError::invalid_input) +} + +/// Show the network port's certificate (`certificate.pem`) in Explorer, to +/// copy it to other devices. +#[tauri::command] +#[specta::specta] +pub fn show_network_certificate(app: AppHandle, gateway: State<'_, Arc>) -> ApiResult<()> { + let path = gateway + .status() + .network + .map(|network| network.certificate_path) + .ok_or_else(|| ApiError::invalid_input("the local network port is off".into()))?; + app.opener().reveal_item_in_dir(path).map_err(ApiError::io) +} + +/// Make a new certificate for the network port. Devices that trusted the +/// old one must trust the new one. +#[tauri::command] +#[specta::specta] +pub async fn renew_network_certificate(gateway: State<'_, Arc>) -> ApiResult<()> { + gateway.renew_certificate().await.map_err(ApiError::io) +} diff --git a/src-tauri/src/ipc/commands/updates.rs b/src-tauri/src/ipc/commands/updates.rs index a59bb01..bcf46a7 100644 --- a/src-tauri/src/ipc/commands/updates.rs +++ b/src-tauri/src/ipc/commands/updates.rs @@ -117,6 +117,7 @@ pub async fn install_app_update( total_bytes: total, bytes_per_second: None, phase: Some(JobPhase::Downloading), + step: None, }); }, || {}, diff --git a/src-tauri/src/ipc/mod.rs b/src-tauri/src/ipc/mod.rs index 195507b..c5aa072 100644 --- a/src-tauri/src/ipc/mod.rs +++ b/src-tauri/src/ipc/mod.rs @@ -23,6 +23,7 @@ pub fn builder() -> Builder { commands::app::app_info, commands::app::set_language, commands::app::open_url, + commands::app::save_export, commands::app::autostart_enabled, commands::app::set_autostart, commands::updates::check_app_update, @@ -47,8 +48,15 @@ pub fn builder() -> Builder { commands::runtime::hardware_usage, commands::runtime::check_runtime_update, commands::runtime::install_runtime, + commands::runtime::recommended_runtime, commands::runtime::cached_runtimes, commands::jobs::list_jobs, + commands::bench::bench_tool, + commands::bench::install_bench_tool, + commands::bench::start_benchmark, + commands::bench::start_energy_profile, + commands::bench::bench_history, + commands::bench::forget_bench_results, commands::jobs::cancel_job, commands::jobs::dismiss_job, commands::server::gateway_status, @@ -56,10 +64,18 @@ pub fn builder() -> Builder { commands::server::start_server, commands::server::stop_server, commands::server::server_logs, + commands::server::context_usage, + commands::server::server_options, + commands::server::set_server_options, commands::server::set_require_api_key, commands::server::gateway_settings, commands::server::set_gateway_port, commands::server::set_allowed_origins, + commands::server::set_ollama_port, + commands::server::set_network, + commands::server::renew_network_certificate, + commands::server::show_network_certificate, + commands::server::set_api_key_network, commands::server::list_requests, commands::server::list_api_keys, commands::server::create_api_key, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index cab74a1..509b674 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -7,10 +7,14 @@ //! - `ipc`: commands and events the webview uses (typed bindings) //! - `setup`: building the app state at startup, tearing it down on exit //! - `desktop`: tray, window behaviour, child process lifetime +//! - [`cli`]: `calcine-cli`, the API without the window +pub mod cli; mod desktop; mod ipc; mod setup; +#[cfg(windows)] +mod user_path; use tauri::RunEvent; use tracing_subscriber::EnvFilter; @@ -22,7 +26,7 @@ use tracing_subscriber::EnvFilter; /// If Tauri fails to start (for example, no WebView2 runtime). #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { - init_tracing(); + init_tracing("info"); desktop::process::kill_children_on_exit(); let specta = ipc::builder(); @@ -69,13 +73,26 @@ pub fn run() { }); } +/// Entry point of `calcine-cli`: run the command in the arguments and +/// return the exit code. Logs only warnings, to stderr. +pub fn cli_main() -> i32 { + init_tracing("warn"); + desktop::process::kill_children_on_exit(); + let args: Vec = std::env::args().skip(1).collect(); + cli::main(&args) +} + /// Regenerate `src/lib/bindings.ts` (used by `tests/bindings.rs`). #[doc(hidden)] pub fn export_bindings() -> Result<(), specta_typescript::Error> { ipc::export(&ipc::builder()) } -fn init_tracing() { - let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init(); +fn init_tracing(default_level: &str) { + let filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_level)); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .with_writer(std::io::stderr) + .try_init(); } diff --git a/src-tauri/src/setup/gateway.rs b/src-tauri/src/setup/gateway.rs index 57a080c..0377b3a 100644 --- a/src-tauri/src/setup/gateway.rs +++ b/src-tauri/src/setup/gateway.rs @@ -25,6 +25,7 @@ pub fn create(services: Services, data_dir: &Path) -> Gateway { keys: Arc::new(keys), settings_path: Some(data_dir.join("gateway.json")), builtin_origins: origins, + network_dir: Some(data_dir.join("network")), }, ) } diff --git a/src-tauri/src/setup/mod.rs b/src-tauri/src/setup/mod.rs index a4b56f1..324cb17 100644 --- a/src-tauri/src/setup/mod.rs +++ b/src-tauri/src/setup/mod.rs @@ -1,8 +1,10 @@ //! Build the app state at startup and tear it down on exit. mod gateway; +mod server_options; mod services; +use std::path::PathBuf; use std::sync::Arc; use calcine_core::Services; @@ -10,26 +12,95 @@ use calcine_gateway::Gateway; use tauri::path::BaseDirectory; use tauri::{App, AppHandle, Manager}; +pub use self::server_options::ServerOptionsFile; +pub use self::services::pinned_release; use self::services::{AppPaths, services_from_env}; -/// Build the services, start the gateway, and forward state changes to the -/// webview. Returns the services, also managed as Tauri state. -pub fn start(app: &App) -> tauri::Result { +/// The app's identifier, which names its data folders (as in +/// `tauri.conf.json`). +pub const IDENTIFIER: &str = "com.eraflo.calcine"; + +/// Where Calcine keeps its files. +#[derive(Debug, Clone)] +pub struct DataDirs { + /// Settings, keys, history (roaming on Windows). + pub data: PathBuf, + /// Caches: GenieX installers, the benchmark tool. + pub local: PathBuf, +} + +impl DataDirs { + /// The folders Tauri gives the window, found without Tauri (for + /// `calcine-cli`): `%APPDATA%` and `%LOCALAPPDATA%` on Windows, + /// `$XDG_DATA_HOME` (or `~/.local/share`) elsewhere. + pub fn from_env() -> Option { + let var = |name: &str| std::env::var_os(name).map(PathBuf::from); + let (data, local) = if cfg!(windows) { + (var("APPDATA")?, var("LOCALAPPDATA")?) + } else { + let share = var("XDG_DATA_HOME") + .or_else(|| var("HOME").map(|home| home.join(".local").join("share")))?; + (share.clone(), share) + }; + Some(Self { + data: data.join(IDENTIFIER), + local: local.join(IDENTIFIER), + }) + } +} + +/// What both the window and `calcine-cli serve` run on. +pub struct Core { + pub services: Services, + pub gateway: Arc, + pub options_file: ServerOptionsFile, +} + +/// Build the services and the gateway (not started yet) the same way for +/// the window and for `calcine-cli`. +pub async fn build(dirs: &DataDirs, bundled_geniex: Option) -> Core { let paths = AppPaths { - runtime_cache: app.path().app_local_data_dir()?.join("runtime-cache"), - // Only release builds bundle it (`tauri.release.conf.json`). - bundled_geniex: app - .path() - .resolve("geniex/geniex-cli-setup.exe", BaseDirectory::Resource) - .ok() - .filter(|path| path.is_file()), + runtime_cache: dirs.local.join("runtime-cache"), + bench_tools: dirs.local.join("geniex-bench"), + bench_history: dirs.data.join("benchmarks.json"), + bundled_geniex, }; let services = services_from_env(&paths); tracing::info!(backend = ?services.backend, "starting Calcine"); - app.manage(services.clone()); - let data_dir = app.path().app_data_dir()?; - let gateway = Arc::new(gateway::create(services.clone(), &data_dir)); + // The server isn't running yet: this only sets what it starts with. + let options_file = ServerOptionsFile::new(&dirs.data); + if let Err(err) = services.server.set_options(options_file.load()).await { + tracing::warn!(%err, "ignoring the saved server options"); + } + let gateway = Arc::new(gateway::create(services.clone(), &dirs.data)); + Core { + services, + gateway, + options_file, + } +} + +/// Build the services, start the gateway, and forward state changes to the +/// webview. Returns the services, also managed as Tauri state. +pub fn start(app: &App) -> tauri::Result { + let dirs = DataDirs { + data: app.path().app_data_dir()?, + local: app.path().app_local_data_dir()?, + }; + // Only release builds bundle it (`tauri.release.conf.json`). + let bundled = app + .path() + .resolve("geniex/geniex-cli-setup.exe", BaseDirectory::Resource) + .ok() + .filter(|path| path.is_file()); + let Core { + services, + gateway, + options_file, + } = tauri::async_runtime::block_on(build(&dirs, bundled)); + app.manage(services.clone()); + app.manage(options_file); app.manage(gateway.clone()); crate::ipc::events::forward(app.handle(), &services.jobs, &gateway); tauri::async_runtime::spawn(async move { @@ -57,3 +128,27 @@ pub fn shutdown(handle: &AppHandle) { } }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_identifier_matches_the_tauri_config() { + let config: serde_json::Value = + serde_json::from_str(include_str!("../../tauri.conf.json")).unwrap(); + assert_eq!(config["identifier"], IDENTIFIER); + } + + #[test] + #[cfg(windows)] + fn finds_the_same_folders_as_tauri() { + let dirs = DataDirs::from_env().unwrap(); + assert!(dirs.data.ends_with(IDENTIFIER)); + assert!(dirs.data.starts_with(std::env::var_os("APPDATA").unwrap())); + assert!( + dirs.local + .starts_with(std::env::var_os("LOCALAPPDATA").unwrap()) + ); + } +} diff --git a/src-tauri/src/setup/server_options.rs b/src-tauri/src/setup/server_options.rs new file mode 100644 index 0000000..0ccdaa2 --- /dev/null +++ b/src-tauri/src/setup/server_options.rs @@ -0,0 +1,58 @@ +//! `geniex serve` options chosen on the Server page, kept across launches. + +use std::path::{Path, PathBuf}; + +use calcine_core::runtime::ServerOptions; + +/// Where the options are saved (`server.json` in the app's data folder). +#[derive(Debug, Clone)] +pub struct ServerOptionsFile(PathBuf); + +impl ServerOptionsFile { + pub fn new(data_dir: &Path) -> Self { + Self(data_dir.join("server.json")) + } + + /// The saved options, or GenieX's defaults when missing or invalid. + pub fn load(&self) -> ServerOptions { + std::fs::read(&self.0) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .filter(|options| options.validate().is_ok()) + .unwrap_or_default() + } + + pub fn save(&self, options: ServerOptions) -> std::io::Result<()> { + if let Some(dir) = self.0.parent() { + std::fs::create_dir_all(dir)?; + } + let json = serde_json::to_vec_pretty(&options).map_err(std::io::Error::other)?; + std::fs::write(&self.0, json) + } +} + +#[cfg(test)] +mod tests { + use calcine_core::runtime::ServerOptions; + + use super::ServerOptionsFile; + + #[test] + fn round_trips_and_falls_back_to_defaults() { + let dir = + std::env::temp_dir().join(format!("calcine-server-options-{}", std::process::id())); + let file = ServerOptionsFile::new(&dir); + assert_eq!(file.load(), ServerOptions::default()); + + let options = ServerOptions { + keepalive_secs: 900, + context_size: 8192, + }; + file.save(options).unwrap(); + assert_eq!(file.load(), options); + + std::fs::write(dir.join("server.json"), br#"{"keepaliveSecs": 1}"#).unwrap(); + assert_eq!(file.load(), ServerOptions::default()); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/src-tauri/src/setup/services.rs b/src-tauri/src/setup/services.rs index 597f74d..1be2fc3 100644 --- a/src-tauri/src/setup/services.rs +++ b/src-tauri/src/setup/services.rs @@ -4,11 +4,12 @@ use std::path::PathBuf; use std::sync::Arc; use calcine_core::jobs::JobManager; +use calcine_core::runtime::{InstallerAsset, RuntimeRelease, ServerOptions}; use calcine_core::{BackendKind, Services}; use calcine_geniex::update::RELEASE_ENDPOINT; use calcine_geniex::{ - BundledInstaller, Geniex, GeniexConfig, GeniexInstaller, GeniexServer, InstallerConfig, - ServeOptions, + BenchConfig, BundledInstaller, Geniex, GeniexBench, GeniexConfig, GeniexContext, + GeniexInstaller, GeniexServer, InstallerConfig, }; use calcine_hub::{HubConfig, HubDirectory}; use calcine_hw::SystemProbe; @@ -18,7 +19,8 @@ use serde::Deserialize; /// Environment variable selecting the backend: `mock` or `geniex` (default). const BACKEND_ENV: &str = "CALCINE_BACKEND"; -/// The GenieX version bundled in release installers (`runtime/geniex.json`). +/// The GenieX version Calcine was tested with and installs on first launch +/// (`runtime/geniex.json`), with its official download and checksum. const PINNED_GENIEX: &str = include_str!("../../../runtime/geniex.json"); /// Where Calcine keeps files the services need. @@ -28,6 +30,10 @@ pub struct AppPaths { pub runtime_cache: PathBuf, /// The GenieX installer shipped in Calcine's installer, if any. pub bundled_geniex: Option, + /// `geniex-bench` versions, downloaded on demand. + pub bench_tools: PathBuf, + /// Benchmark history. + pub bench_history: PathBuf, } pub fn services_from_env(paths: &AppPaths) -> Services { @@ -46,7 +52,7 @@ pub fn services_from_env(paths: &AppPaths) -> Services { fn geniex_services(paths: &AppPaths) -> Services { let geniex = Geniex::new(GeniexConfig::default()); - let server = Arc::new(GeniexServer::new(geniex.clone(), ServeOptions::default())); + let server = Arc::new(GeniexServer::new(geniex.clone(), ServerOptions::default())); let installer = GeniexInstaller::new( geniex.clone(), InstallerConfig { @@ -54,11 +60,20 @@ fn geniex_services(paths: &AppPaths) -> Services { cache_dir: paths.runtime_cache.clone(), bundled: paths.bundled_geniex.clone().map(|path| BundledInstaller { path, - version: pinned_geniex_version(), + version: pinned_release().version, }), }, ) .expect("the system TLS stack should load"); + let bench = GeniexBench::new( + geniex.clone(), + BenchConfig { + tools_dir: paths.bench_tools.clone(), + history_path: paths.bench_history.clone(), + }, + ) + .expect("the system TLS stack should load"); + let context = Arc::new(GeniexContext::new(geniex.clone())); let geniex = Arc::new(geniex); Services { backend: BackendKind::Geniex, @@ -71,24 +86,66 @@ fn geniex_services(paths: &AppPaths) -> Services { server, hardware: Arc::new(SystemProbe::new()), installer: Arc::new(installer), + bench: Arc::new(bench), + context, jobs: JobManager::new(), } } -fn pinned_geniex_version() -> String { +/// The pinned GenieX release. Its SHA-256 ships inside Calcine, so the +/// download is checked against a value from the signed app itself. +pub fn pinned_release() -> RuntimeRelease { #[derive(Deserialize)] struct Pin { version: String, + assets: Assets, + } + #[derive(Deserialize)] + struct Assets { + #[serde(rename = "windows-arm64")] + windows_arm64: Asset, + #[serde(rename = "linux-arm64")] + linux_arm64: Asset, + } + #[derive(Deserialize)] + struct Asset { + name: String, + url: String, + size: u64, + sha256: String, + } + let pin: Pin = serde_json::from_str(PINNED_GENIEX).expect("runtime/geniex.json is valid"); + // The Windows installer, or the Linux archive. + let asset = if cfg!(windows) { + pin.assets.windows_arm64 + } else { + pin.assets.linux_arm64 + }; + RuntimeRelease { + notes_url: Some(format!( + "https://github.com/qualcomm/GenieX/releases/tag/{}", + pin.version + )), + version: pin.version, + prerelease: false, + released_at: None, + installer: InstallerAsset { + name: asset.name, + url: asset.url, + size: asset.size, + sha256: asset.sha256.to_ascii_lowercase(), + }, } - serde_json::from_str::(PINNED_GENIEX) - .map(|pin| pin.version) - .expect("runtime/geniex.json has a version") } #[cfg(test)] mod tests { #[test] - fn reads_the_pinned_version() { - assert!(super::pinned_geniex_version().starts_with('v')); + fn reads_the_pinned_release() { + let release = super::pinned_release(); + assert!(release.version.starts_with('v')); + assert!(release.installer.url.starts_with("https://")); + assert_eq!(release.installer.sha256.len(), 64); + assert!(release.installer.name.contains(&release.version)); } } diff --git a/src-tauri/src/user_path.rs b/src-tauri/src/user_path.rs new file mode 100644 index 0000000..7cbb659 --- /dev/null +++ b/src-tauri/src/user_path.rs @@ -0,0 +1,220 @@ +//! Putting `calcine-cli` on the user's `PATH` (Windows), so terminals find +//! it. Done here rather than in the NSIS installer: NSIS strings stop at +//! 1024 characters, and a longer `PATH` would be cut short. +//! +//! Only the user's own `PATH` (`HKCU\Environment`) is touched, keeping its +//! type (`REG_EXPAND_SZ`, so `%VARIABLES%` keep working). + +/// `path` with `dir` appended, or `None` when it's already there. +pub fn with_dir(path: &str, dir: &str) -> Option { + if contains(path, dir) { + return None; + } + let path = path.trim_end_matches(';'); + Some(if path.is_empty() { + dir.to_owned() + } else { + format!("{path};{dir}") + }) +} + +/// `path` without `dir`, or `None` when it isn't there. +pub fn without_dir(path: &str, dir: &str) -> Option { + if !contains(path, dir) { + return None; + } + Some( + path.split(';') + .filter(|entry| !entry.is_empty() && !same(entry, dir)) + .collect::>() + .join(";"), + ) +} + +pub fn contains(path: &str, dir: &str) -> bool { + path.split(';').any(|entry| same(entry, dir)) +} + +fn same(entry: &str, dir: &str) -> bool { + let normalize = |text: &str| text.trim().trim_end_matches(['\\', '/']).to_lowercase(); + !entry.trim().is_empty() && normalize(entry) == normalize(dir) +} + +#[cfg(windows)] +pub use self::registry::{add, is_added, remove}; + +#[cfg(windows)] +mod registry { + // The registry and window messages need `unsafe`; it stays here. + #![allow(unsafe_code)] + + use windows_sys::Win32::Foundation::{ERROR_FILE_NOT_FOUND, ERROR_SUCCESS}; + use windows_sys::Win32::System::Registry::{ + HKEY_CURRENT_USER, REG_EXPAND_SZ, REG_VALUE_TYPE, RRF_NOEXPAND, RRF_RT_REG_EXPAND_SZ, + RRF_RT_REG_SZ, RegGetValueW, RegSetKeyValueW, + }; + use windows_sys::Win32::UI::WindowsAndMessaging::{ + HWND_BROADCAST, SMTO_ABORTIFHUNG, SendMessageTimeoutW, WM_SETTINGCHANGE, + }; + + const ENVIRONMENT: &str = "Environment"; + + fn wide(text: &str) -> Vec { + text.encode_utf16().chain(Some(0)).collect() + } + + /// The value and its type; `None` when it doesn't exist yet. + fn read(name: &str) -> Result, String> { + let (key, name) = (wide(ENVIRONMENT), wide(name)); + let flags = RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ | RRF_NOEXPAND; + let mut kind: REG_VALUE_TYPE = 0; + let mut size = 0_u32; + // SAFETY: valid NUL-terminated strings; a null buffer asks for the size. + let status = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + key.as_ptr(), + name.as_ptr(), + flags, + &raw mut kind, + std::ptr::null_mut(), + &raw mut size, + ) + }; + if status == ERROR_FILE_NOT_FOUND { + return Ok(None); + } + if status != ERROR_SUCCESS { + return Err(format!("can't read the PATH (error {status})")); + } + let mut buffer = vec![0_u16; (size as usize).div_ceil(2)]; + // SAFETY: `buffer` holds `size` bytes. + let status = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + key.as_ptr(), + name.as_ptr(), + flags, + &raw mut kind, + buffer.as_mut_ptr().cast(), + &raw mut size, + ) + }; + if status != ERROR_SUCCESS { + return Err(format!("can't read the PATH (error {status})")); + } + let end = buffer.iter().position(|&c| c == 0).unwrap_or(buffer.len()); + Ok(Some((String::from_utf16_lossy(&buffer[..end]), kind))) + } + + fn write(name: &str, value: &str, kind: REG_VALUE_TYPE) -> Result<(), String> { + let (key, name, data) = (wide(ENVIRONMENT), wide(name), wide(value)); + let bytes = u32::try_from(data.len() * 2).map_err(|_| "the PATH is too long")?; + // SAFETY: valid NUL-terminated strings; `data` is `bytes` long. + let status = unsafe { + RegSetKeyValueW( + HKEY_CURRENT_USER, + key.as_ptr(), + name.as_ptr(), + kind, + data.as_ptr().cast(), + bytes, + ) + }; + if status != ERROR_SUCCESS { + return Err(format!("can't change the PATH (error {status})")); + } + // Tell Explorer, so new terminals see it without signing out. + let environment = wide(ENVIRONMENT); + // SAFETY: broadcasting a string pointer that outlives the call. + unsafe { + SendMessageTimeoutW( + HWND_BROADCAST, + WM_SETTINGCHANGE, + 0, + environment.as_ptr() as isize, + SMTO_ABORTIFHUNG, + 5000, + std::ptr::null_mut(), + ); + } + Ok(()) + } + + /// Add `dir` to the user variable `name` (`Path`). `false` when it was + /// already there. + pub fn add(name: &str, dir: &str) -> Result { + let (current, kind) = read(name)?.unwrap_or_default(); + let kind = if kind == 0 { REG_EXPAND_SZ } else { kind }; + match super::with_dir(¤t, dir) { + Some(updated) => write(name, &updated, kind).map(|()| true), + None => Ok(false), + } + } + + /// Remove `dir` from the user variable `name`. `false` when it wasn't there. + pub fn remove(name: &str, dir: &str) -> Result { + let Some((current, kind)) = read(name)? else { + return Ok(false); + }; + match super::without_dir(¤t, dir) { + Some(updated) => write(name, &updated, kind).map(|()| true), + None => Ok(false), + } + } + + pub fn is_added(name: &str, dir: &str) -> Result { + Ok(read(name)?.is_some_and(|(current, _)| super::contains(¤t, dir))) + } + + #[cfg(test)] + mod tests { + use super::*; + + /// Uses its own variable: the real `Path` is never touched. + #[test] + fn adds_and_removes_a_folder() { + let name = format!("CalcineTestPath{}", std::process::id()); + write(&name, r"C:\Tools;%USERPROFILE%\bin", REG_EXPAND_SZ).unwrap(); + assert!(add(&name, r"C:\Apps\Calcine").unwrap()); + assert!(!add(&name, r"c:\apps\calcine\").unwrap(), "already there"); + assert!(is_added(&name, r"C:\Apps\Calcine").unwrap()); + let (value, kind) = read(&name).unwrap().unwrap(); + assert_eq!(value, r"C:\Tools;%USERPROFILE%\bin;C:\Apps\Calcine"); + assert_eq!(kind, REG_EXPAND_SZ); + assert!(remove(&name, r"C:\Apps\Calcine").unwrap()); + assert_eq!( + read(&name).unwrap().unwrap().0, + r"C:\Tools;%USERPROFILE%\bin" + ); + assert!(!remove(&name, r"C:\Apps\Calcine").unwrap()); + // Clean up: an empty value is harmless, but leave nothing behind. + delete(&name); + } + + fn delete(name: &str) { + use windows_sys::Win32::System::Registry::RegDeleteKeyValueW; + let (key, name) = (wide(ENVIRONMENT), wide(name)); + // SAFETY: valid NUL-terminated strings. + unsafe { RegDeleteKeyValueW(HKEY_CURRENT_USER, key.as_ptr(), name.as_ptr()) }; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn appends_once_and_removes_every_copy() { + assert_eq!(with_dir("", r"C:\A").as_deref(), Some(r"C:\A")); + assert_eq!(with_dir(r"C:\X;", r"C:\A").as_deref(), Some(r"C:\X;C:\A")); + assert_eq!(with_dir(r"C:\X;c:\a\", r"C:\A"), None); + assert_eq!( + without_dir(r"C:\X;C:\A;;C:\Y;c:\a", r"C:\A").as_deref(), + Some(r"C:\X;C:\Y") + ); + assert_eq!(without_dir(r"C:\X", r"C:\A"), None); + assert!(!contains(";;", "")); + } +} diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index e8ac8a2..4781f4e 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -42,7 +42,7 @@ "publisher": "The Calcine Authors", "copyright": "Copyright (c) 2026 The Calcine Authors", "license": "MIT", - "licenseFile": "../LICENSE", + "licenseFile": "windows/license-and-privacy.txt", "homepage": "https://github.com/eraflo/calcine", "category": "DeveloperTool", "shortDescription": "Run LLMs on the Snapdragon NPU", @@ -56,7 +56,13 @@ ], "windows": { "nsis": { - "installMode": "currentUser" + "installMode": "currentUser", + "installerHooks": "windows/installer-hooks.nsh" + } + }, + "linux": { + "deb": { + "depends": ["ocl-icd-libopencl1"] } } }, diff --git a/src-tauri/tauri.release.conf.json b/src-tauri/tauri.release.conf.json index 49b5961..d50663d 100644 --- a/src-tauri/tauri.release.conf.json +++ b/src-tauri/tauri.release.conf.json @@ -1,14 +1,6 @@ { "$schema": "https://schema.tauri.app/config/2", "bundle": { - "resources": { - "resources/geniex/geniex-cli-setup.exe": "geniex/geniex-cli-setup.exe" - }, - "windows": { - "nsis": { - "installerHooks": "./windows/installer-hooks.nsh" - } - }, "createUpdaterArtifacts": true } } diff --git a/src-tauri/windows/installer-hooks.nsh b/src-tauri/windows/installer-hooks.nsh index 112d982..8e3fd13 100644 --- a/src-tauri/windows/installer-hooks.nsh +++ b/src-tauri/windows/installer-hooks.nsh @@ -1,18 +1,10 @@ -; Calcine NSIS installer hooks (merged in by tauri.release.conf.json). -; -; The official GenieX installer is bundled as a resource and run silently when -; GenieX isn't installed yet. It is per-user (no admin), installs to -; %LOCALAPPDATA%\GenieX CLI and adds itself to the user PATH. Upgrades of an -; existing GenieX are handled by the app, which first stops the server: the -; GenieX installer kills any running geniex.exe. +; Lets terminals find calcine-cli. calcine-cli changes the user's PATH itself: +; NSIS strings stop at 1024 characters and would cut a longer PATH short. !macro NSIS_HOOK_POSTINSTALL - IfFileExists "$LOCALAPPDATA\GenieX CLI\geniex.exe" calcine_geniex_present - DetailPrint "Installing GenieX CLI..." - ExecWait '"$INSTDIR\geniex\geniex-cli-setup.exe" /VERYSILENT /SUPPRESSMSGBOXES /NORESTART' $0 - DetailPrint "GenieX CLI installer exited with code $0" - Goto calcine_geniex_done - calcine_geniex_present: - DetailPrint "GenieX CLI already installed, leaving it as is" - calcine_geniex_done: + nsExec::Exec '"$INSTDIR\calcine-cli.exe" path add' +!macroend + +!macro NSIS_HOOK_PREUNINSTALL + nsExec::Exec '"$INSTDIR\calcine-cli.exe" path remove' !macroend diff --git a/src-tauri/windows/license-and-privacy.txt b/src-tauri/windows/license-and-privacy.txt new file mode 100644 index 0000000..9b710d8 --- /dev/null +++ b/src-tauri/windows/license-and-privacy.txt @@ -0,0 +1,45 @@ +CALCINE + +Calcine is free and open-source software under the MIT License (below). + +PRIVACY + +Calcine runs language models on this PC. Your prompts, replies, images, +recordings and conversations never leave it. Calcine has no account, no +telemetry, no analytics and no crash reporting. + +Calcine connects to the internet only to: +- check for updates at startup (GitHub for Calcine, Qualcomm for GenieX), + unless you turn off Settings > Calcine updates > Check for updates + automatically; +- download GenieX, Qualcomm's runtime, on first launch and when you update it; +- search and download the models you choose (Hugging Face, Qualcomm AI Hub, + ModelScope, Docker Hub); +- download Qualcomm's benchmark tool when you ask for it. + +The full privacy policy lists every connection: +https://github.com/eraflo/Calcine/blob/main/PRIVACY.md + +LICENSE + +MIT License + +Copyright (c) 2026 The Calcine Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/app/messages.ts b/src/app/messages.ts index a25915c..6c7cdd2 100644 --- a/src/app/messages.ts +++ b/src/app/messages.ts @@ -8,6 +8,7 @@ export const messages = defineMessages({ nav_discover: "Discover", nav_server: "Server", nav_hardware: "Hardware", + nav_benchmark: "Benchmark", nav_settings: "Settings", mainNavigation: "Main", mock: "Mock", @@ -34,6 +35,7 @@ export const messages = defineMessages({ stopGeniex: "Stop GenieX", createKey: "Create an API key", refreshLibrary: "Refresh model library", + importModel: "Import a model from this PC", copyBaseUrl: "Copy API base URL", theme: "Theme", darkTheme: "Dark theme", @@ -49,6 +51,7 @@ export const messages = defineMessages({ nav_discover: "Découvrir", nav_server: "Serveur", nav_hardware: "Matériel", + nav_benchmark: "Benchmark", nav_settings: "Réglages", mainNavigation: "Principale", mock: "Démo", @@ -74,6 +77,7 @@ export const messages = defineMessages({ stopGeniex: "Arrêter GenieX", createKey: "Créer une clé API", refreshLibrary: "Actualiser la bibliothèque", + importModel: "Importer un modèle depuis ce PC", copyBaseUrl: "Copier l'URL de base de l'API", theme: "Thème", darkTheme: "Thème sombre", diff --git a/src/app/navigation.ts b/src/app/navigation.ts index 7f489df..089938b 100644 --- a/src/app/navigation.ts +++ b/src/app/navigation.ts @@ -1,5 +1,5 @@ import type { LucideIcon } from "lucide-react"; -import { Compass, Cpu, Library, MessagesSquare, Server, Settings } from "lucide-react"; +import { Compass, Cpu, Gauge, Library, MessagesSquare, Server, Settings } from "lucide-react"; import type { messages } from "./messages"; type NavItem = { to: string; label: keyof (typeof messages)["en"]; icon: LucideIcon }; @@ -14,6 +14,7 @@ export const mainNavigation = [ { to: "/discover", label: "nav_discover", icon: Compass }, { to: "/server", label: "nav_server", icon: Server }, { to: "/hardware", label: "nav_hardware", icon: Cpu }, + { to: "/benchmark", label: "nav_benchmark", icon: Gauge }, ] as const satisfies readonly NavItem[]; export const settingsNavigation = { diff --git a/src/app/router.tsx b/src/app/router.tsx index 0fed337..98ceca2 100644 --- a/src/app/router.tsx +++ b/src/app/router.tsx @@ -6,6 +6,7 @@ import { redirect, } from "@tanstack/react-router"; import type { ReactNode } from "react"; +import { BenchmarkPage } from "@/features/benchmark/benchmark-page"; import { ChatPage } from "@/features/chat/chat-page"; import { DiscoverPage } from "@/features/discover/discover-page"; import { HardwarePage } from "@/features/hardware/hardware-page"; @@ -37,6 +38,7 @@ const routeTree = rootRoute.addChildren([ page("/discover", DiscoverPage), page("/server", ServerPage), page("/hardware", HardwarePage), + page("/benchmark", BenchmarkPage), page("/settings", SettingsPage), page("/welcome", WelcomePage), ]); diff --git a/src/app/shell/command-palette.tsx b/src/app/shell/command-palette.tsx index cb9dac4..5e8bb7a 100644 --- a/src/app/shell/command-palette.tsx +++ b/src/app/shell/command-palette.tsx @@ -2,6 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useNavigate } from "@tanstack/react-router"; import { Copy, + FolderInput, KeyRound, Languages, MessageSquarePlus, @@ -92,6 +93,15 @@ export function CommandPalette() { {t("createKey")} + { + useUi.getState().setImporting({ path: null }); + void navigate({ to: "/library" }); + })} + > + + {t("importModel")} + queryClient.invalidateQueries({ queryKey: modelsQuery.queryKey }), diff --git a/src/app/shell/sidebar.tsx b/src/app/shell/sidebar.tsx index eac612e..c990cb6 100644 --- a/src/app/shell/sidebar.tsx +++ b/src/app/shell/sidebar.tsx @@ -73,7 +73,11 @@ function RuntimeStatus() { const t = useT(messages); const { data, error, isPending } = useQuery(runtimeQuery); const channel = useUpdates((state) => state.geniexChannel); - const { data: update } = useQuery({ ...runtimeUpdateQuery(channel), enabled: Boolean(data) }); + const autoCheck = useUpdates((state) => state.autoCheck); + const { data: update } = useQuery({ + ...runtimeUpdateQuery(channel), + enabled: Boolean(data) && autoCheck, + }); const kind = error instanceof CalcineError ? error.kind : undefined; const [tone, label] = isPending diff --git a/src/components/calcine/feedback/error-state.tsx b/src/components/calcine/feedback/error-state.tsx index 458e819..7a35b90 100644 --- a/src/components/calcine/feedback/error-state.tsx +++ b/src/components/calcine/feedback/error-state.tsx @@ -1,3 +1,4 @@ +import { Link } from "@tanstack/react-router"; import { MonitorSmartphone, PackageX, TriangleAlert } from "lucide-react"; import { Button } from "@/components/ui/button"; import { useT } from "@/i18n"; @@ -24,7 +25,15 @@ export function ErrorState({ error, onRetry }: { error: unknown; onRetry?: () => if (kind === "runtime_not_found") { return ( - + + {t("setUpGeniex")} + + } + > {t.rich("geniexMissingHint", { strong: (text) => {text}, })} diff --git a/src/components/calcine/messages.ts b/src/components/calcine/messages.ts index a6b863d..540e196 100644 --- a/src/components/calcine/messages.ts +++ b/src/components/calcine/messages.ts @@ -19,7 +19,8 @@ export const messages = defineMessages({ "This page needs Calcine's backend, which isn't available in a regular browser. Run bun run app or bun run app:mock.", geniexMissing: "GenieX isn't installed", geniexMissingHint: - "Calcine's installer sets up GenieX for you. In development, install it from the GenieX releases page, then refresh.", + "Calcine runs models with GenieX, Qualcomm's runtime. Install it from the welcome screen.", + setUpGeniex: "Set up GenieX", somethingWrong: "Something went wrong", }, fr: { @@ -37,7 +38,8 @@ export const messages = defineMessages({ "Cette page a besoin du backend de Calcine, indisponible dans un navigateur classique. Lancez bun run app ou bun run app:mock.", geniexMissing: "GenieX n'est pas installé", geniexMissingHint: - "L'installeur de Calcine installe GenieX pour vous. En développement, installez-le depuis la page GenieX releases, puis actualisez.", + "Calcine exécute les modèles avec GenieX, le moteur de Qualcomm. Installez-le depuis l'écran d'accueil.", + setUpGeniex: "Installer GenieX", somethingWrong: "Une erreur est survenue", }, }); diff --git a/src/features/benchmark/api.ts b/src/features/benchmark/api.ts new file mode 100644 index 0000000..95e38e5 --- /dev/null +++ b/src/features/benchmark/api.ts @@ -0,0 +1,50 @@ +import { queryOptions, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { jobsQuery } from "@/features/tasks/api"; +import { type BenchRequest, call, commands, type EnergyRequest, type Job, unwrap } from "@/lib/api"; +import { useUi } from "@/stores/ui"; + +export const benchToolQuery = queryOptions({ + queryKey: ["bench-tool"], + queryFn: () => unwrap(commands.benchTool), +}); + +export const benchHistoryQuery = queryOptions({ + queryKey: ["bench-history"], + queryFn: () => call(commands.benchHistory), +}); + +/** The running benchmark or tool download, if any. */ +export function useRunningJob( + type: "benchmark" | "install_bench" | "energy_profile", +): Job | undefined { + const { data: jobs = [] } = useQuery(jobsQuery); + return jobs.find((job) => job.kind.type === type && job.state.state === "running"); +} + +export function useInstallBenchTool() { + const openTasks = useUi((state) => state.setTasksOpen); + return useMutation({ + mutationFn: () => unwrap(commands.installBenchTool), + onSuccess: () => openTasks(true), + }); +} + +export function useStartBenchmark() { + return useMutation({ + mutationFn: (request: BenchRequest) => unwrap(() => commands.startBenchmark(request)), + }); +} + +export function useStartEnergyProfile() { + return useMutation({ + mutationFn: (request: EnergyRequest) => unwrap(() => commands.startEnergyProfile(request)), + }); +} + +export function useForgetResults() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (ids: string[]) => unwrap(() => commands.forgetBenchResults(ids)), + onSettled: () => queryClient.invalidateQueries({ queryKey: benchHistoryQuery.queryKey }), + }); +} diff --git a/src/features/benchmark/benchmark-page.tsx b/src/features/benchmark/benchmark-page.tsx new file mode 100644 index 0000000..1f5deb8 --- /dev/null +++ b/src/features/benchmark/benchmark-page.tsx @@ -0,0 +1,47 @@ +import { useQuery } from "@tanstack/react-query"; +import { BarChart3 } from "lucide-react"; +import { useState } from "react"; +import { EmptyState } from "@/components/calcine/feedback/empty-state"; +import { Page } from "@/components/calcine/layout/page"; +import { useT } from "@/i18n"; +import { benchHistoryQuery } from "./api"; +import { EnergyCard } from "./components/energy-card"; +import { HistoryCard, LeaderboardCard, SessionCard, UsageCard } from "./components/results-cards"; +import { RunCard } from "./components/run-card"; +import { ToolCard } from "./components/tool-card"; +import { sessions as groupSessions } from "./lib/results"; +import { messages } from "./messages"; + +export function BenchmarkPage() { + const t = useT(messages); + const { data: history = [] } = useQuery(benchHistoryQuery); + // Energy profiles have their own card. + const sessions = groupSessions(history).filter((session) => session.source !== "energy_profile"); + const [selected, setSelected] = useState(null); + const shown = sessions.find((session) => session.id === selected) ?? sessions[0]; + + return ( + + + + {shown ? ( + + ) : ( + + {t("noResults")} + + )} + + + + {sessions.length > 0 && ( + + )} + + ); +} diff --git a/src/features/benchmark/components/energy-card.tsx b/src/features/benchmark/components/energy-card.tsx new file mode 100644 index 0000000..fca2add --- /dev/null +++ b/src/features/benchmark/components/energy-card.tsx @@ -0,0 +1,398 @@ +import { useQuery } from "@tanstack/react-query"; +import { Leaf, Play, Square, Zap } from "lucide-react"; +import { useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Field, Select } from "@/components/ui/field"; +import { Progress } from "@/components/ui/progress"; +import type { PowerMode } from "@/features/chat/lib/request"; +import { messages as chatMessages } from "@/features/chat/messages"; +import { usageQuery } from "@/features/hardware/api"; +import { modelsQuery } from "@/features/library/api"; +import { useCancelJob } from "@/features/tasks/api"; +import { useT } from "@/i18n"; +import { common } from "@/i18n/common"; +import type { BenchResult, ComputeUnit } from "@/lib/api"; +import { formatNumber, formatRelative } from "@/lib/format"; +import { cn } from "@/lib/utils"; +import { benchHistoryQuery, useRunningJob, useStartEnergyProfile } from "../api"; +import { + benchTargets, + type EnergyRow, + energyRows, + mostEfficient, + POWER_MODE_ORDER, + sessions, +} from "../lib/results"; +import { messages } from "../messages"; + +const GENERATED = [128, 256, 512] as const; +const REPETITIONS = [1, 2, 3] as const; +/** Power modes made no difference on the devices measured: one is enough. */ +const DEFAULT_MODES = ["burst"]; +const ENERGY_UNITS: ComputeUnit[] = ["npu", "gpu", "cpu"]; + +/** Compare a model's speed and energy in each power mode. */ +export function EnergyCard() { + const t = useT(messages); + const { data: usage } = useQuery(usageQuery); + const { data: history = [] } = useQuery(benchHistoryQuery); + const metered = Boolean(usage?.power); + const latest = sessions(history).find((session) => session.source === "energy_profile"); + + return ( + + + + + {t("energyTitle")} + + {t("energyHint")} + + + {usage && !metered ? ( +

{t("energyUnavailable")}

+ ) : ( + + )} + {latest && } + + + ); +} + +function EnergyForm() { + const t = useT(messages); + const tc = useT(common); + const tchat = useT(chatMessages); + const { data: models = [] } = useQuery(modelsQuery); + const targets = benchTargets(models); + const [targetId, setTargetId] = useState(""); + const target = targets.find((candidate) => candidate.id === targetId) ?? targets[0]; + const llamaCpp = target?.model.runtime === "llama_cpp"; + const [units, setUnits] = useState(ENERGY_UNITS); + const [modes, setModes] = useState(DEFAULT_MODES); + const [generatedTokens, setGeneratedTokens] = useState(256); + const [repetitions, setRepetitions] = useState(2); + const start = useStartEnergyProfile(); + const cancel = useCancelJob(); + const running = useRunningJob("energy_profile"); + const busy = Boolean(running); + + if (targets.length === 0) { + return

{t("noModels")}

; + } + const toggleMode = (mode: string) => + setModes((current) => + current.includes(mode) ? current.filter((other) => other !== mode) : [...current, mode], + ); + const toggleUnit = (unit: ComputeUnit) => + setUnits((current) => + current.includes(unit) ? current.filter((other) => other !== unit) : [...current, unit], + ); + const chosen = POWER_MODE_ORDER.filter((mode) => modes.includes(mode)); + // AI Hub models only run on the NPU. + const available: ComputeUnit[] = llamaCpp ? ENERGY_UNITS : ["npu"]; + const chosenUnits = available.filter((unit) => units.includes(unit)); + + return ( +
+
+ + + + + + + + + +
+
+ + ({ + value: unit, + label: tc(unit), + disabled: !available.includes(unit), + }))} + selected={chosenUnits} + onToggle={toggleUnit} + disabled={busy} + /> + + + ({ + value: mode, + label: tchat(`power_${mode}`), + }))} + selected={chosen} + onToggle={toggleMode} + disabled={busy} + /> + +
+ + {running ? ( +
+
+ +

+ {running.progress?.step + ? t("running", { + current: String(running.progress.step.current), + total: String(running.progress.step.total), + }) + : t("startingRun")} +

+
+ +
+ ) : ( +
+

+ {start.isError ? start.error.message : t("energyQuiet")} +

+ +
+ )} +
+ ); +} + +/** Toggle buttons for a multiple choice. */ +function Chips({ + options, + selected, + onToggle, + disabled, +}: { + options: { value: T; label: string; disabled?: boolean }[]; + selected: readonly T[]; + onToggle: (value: T) => void; + disabled: boolean; +}) { + return ( +
+ {options.map((option) => { + const active = selected.includes(option.value); + return ( + + ); + })} +
+ ); +} + +function EnergyResults({ + results, + startedAtMs, +}: { + results: readonly BenchResult[]; + startedAtMs: number; +}) { + const t = useT(messages); + const tc = useT(common); + const tchat = useT(chatMessages); + const rows = energyRows(results); + if (rows.length === 0) return null; + const efficient = mostEfficient(rows); + const fastest = rows.reduce((best, row) => (row.decodeTps > best.decodeTps ? row : best)); + const top = Math.max(...rows.map((row) => row.tokensPerJoule)); + const idle = rows[0]?.result.measure?.energy?.idleWatts; + // The unit only shows when several were measured, likewise the mode. + const severalUnits = new Set(rows.map((row) => row.unit)).size > 1; + const severalModes = new Set(rows.map((row) => row.mode)).size > 1; + const mode = (row: EnergyRow) => + [ + severalUnits || !severalModes ? tc(row.unit) : null, + severalModes ? tchat(`power_${row.mode as PowerMode}`) : null, + ] + .filter(Boolean) + .join(" · "); + + return ( +
+
+ {results[0]?.model} + + {formatRelative(startedAtMs)} + {idle != null && ` · ${t("energyIdle", { value: watts(idle) })}`} + +
+
+ + + + + + + + + + + + {rows.map((row) => ( + + + + + + + + ))} + +
+ {severalModes + ? severalUnits + ? t("energyUnitMode") + : t("energyMode") + : t("energyUnit")} + {t("energySpeed")}{t("energyPower")}{t("energyPerToken")}{t("energyEfficiency")}
+ + {mode(row)} + {row === efficient && ( + + + {t("mostEfficient")} + + )} + + + {t("tps", { value: formatNumber(row.decodeTps, 1) })} + + {watts(row.activeWatts)} + + {t("energyExtra", { value: watts(row.extraWatts) })} + + + {t("millijoules", { value: formatNumber(row.joulesPerToken * 1000, 0) })} + + {t("energySystem", { + value: t("millijoules", { + value: formatNumber(row.systemJoulesPerToken * 1000, 0), + }), + })} + + + + + 0 ? (row.tokensPerJoule / top) * 100 : 0}%` }} + /> + + + {formatNumber(row.tokensPerJoule, 1)} + + +
+
+ {efficient && rows.length > 1 && ( +

+ {efficient === fastest + ? t("energySame", { mode: mode(efficient) }) + : t("energySummary", { + efficient: mode(efficient), + fastest: mode(fastest), + ratio: formatNumber(efficient.tokensPerJoule / fastest.tokensPerJoule, 1), + slower: formatNumber((1 - efficient.decodeTps / fastest.decodeTps) * 100, 0), + })} +

+ )} +
+ ); +} + +function watts(value: number): string { + return `${formatNumber(value, 1)} W`; +} diff --git a/src/features/benchmark/components/metric-bars.tsx b/src/features/benchmark/components/metric-bars.tsx new file mode 100644 index 0000000..480f422 --- /dev/null +++ b/src/features/benchmark/components/metric-bars.tsx @@ -0,0 +1,82 @@ +import { Trophy } from "lucide-react"; +import { useT } from "@/i18n"; +import { common } from "@/i18n/common"; +import type { BenchResult, ComputeUnit } from "@/lib/api"; +import { formatNumber } from "@/lib/format"; +import { cn } from "@/lib/utils"; +import { best, type Metric, median } from "../lib/results"; +import { messages } from "../messages"; + +/** Compute-unit colors, as everywhere else in Calcine. */ +export const UNIT_COLOR: Record = { + npu: { bar: "bg-npu", text: "text-npu" }, + gpu: { bar: "bg-gpu", text: "text-gpu" }, + cpu: { bar: "bg-cpu", text: "text-cpu" }, + hybrid: { bar: "bg-info", text: "text-info" }, +}; + +/** One metric for each unit of a session, as bars scaled to the largest. */ +export function MetricBars({ + results, + metric, +}: { + results: readonly BenchResult[]; + metric: Metric; +}) { + const t = useT(messages); + const tc = useT(common); + const winner = results.length > 1 ? best(results, metric) : null; + const largest = Math.max(1, ...results.map((result) => median(result, metric) ?? 0)); + const title = { decodeTps: t("decode"), prefillTps: t("prefill"), ttftMs: t("ttft") }[metric]; + const hint = { decodeTps: t("decodeHint"), prefillTps: t("prefillHint"), ttftMs: t("ttftHint") }[ + metric + ]; + const format = (value: number) => + metric === "ttftMs" + ? t("ms", { value: formatNumber(value, value < 100 ? 1 : 0) }) + : t("tps", { value: formatNumber(value, value < 100 ? 1 : 0) }); + + return ( +
+
+ {title} + {hint} +
+ {results.map((result) => { + const value = median(result, metric); + const spread = result.measure?.[metric].stdev ?? null; + const color = UNIT_COLOR[result.unit]; + return ( +
+ {tc(result.unit)} +
+ {value !== null && ( +
+ )} +
+ + {value === null ? ( + {t("failed")} + ) : ( + <> + {winner?.id === result.id && ( + + )} + {format(value)} + {spread !== null && spread > 0 && ( + + {t("spread", { value: formatNumber(spread, spread < 10 ? 1 : 0) })} + + )} + + )} + +
+ ); + })} +
+ ); +} diff --git a/src/features/benchmark/components/results-cards.tsx b/src/features/benchmark/components/results-cards.tsx new file mode 100644 index 0000000..b05fdb1 --- /dev/null +++ b/src/features/benchmark/components/results-cards.tsx @@ -0,0 +1,259 @@ +import { useMutation, useQuery } from "@tanstack/react-query"; +import { Check, FileDown, Trash2 } from "lucide-react"; +import { useEffect, useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { messages as chatMessages } from "@/features/chat/messages"; +import { requestsQuery } from "@/features/server/api"; +import { useT } from "@/i18n"; +import { common } from "@/i18n/common"; +import type { BenchResult } from "@/lib/api"; +import { commands, unwrap } from "@/lib/api"; +import { formatCount, formatNumber, formatRelative } from "@/lib/format"; +import { cn } from "@/lib/utils"; +import { useForgetResults } from "../api"; +import { leaderboard, median, type Session, toCsv, usageByModel } from "../lib/results"; +import { messages } from "../messages"; +import { MetricBars, UNIT_COLOR } from "./metric-bars"; + +/** One session: generation, prompt reading and first token, per unit. */ +export function SessionCard({ session, latest }: { session: Session; latest: boolean }) { + const t = useT(messages); + const tchat = useT(chatMessages); + const failures = session.results.filter((result) => result.error); + return ( + + + + {latest ? t("latest") : t("resultsTitle")} + + {session.model} + + + · {formatRelative(session.startedAtMs)} + + + + {t("setup", { + prompt: String(session.promptTokens), + generated: String(session.generatedTokens), + runs: String(session.repetitions), + power: tchat(`power_${session.powerMode}` as "power_burst"), + })} + {session.specType && + ` · ${t("withSpec", { + method: [tchat(`spec_${session.specType}` as "spec_off"), session.draftModel] + .filter(Boolean) + .join(" + "), + })}`} + + + + + + + {failures.length > 0 && ( +
    + {failures.map((result) => ( +
  • + + {result.unit.toUpperCase()} + {" "} + {result.error} +
    • + ))} +
+ )} + + + ); +} + +/** Each model's best generation speed, across the history. */ +export function LeaderboardCard({ history }: { history: readonly BenchResult[] }) { + const t = useT(messages); + const tc = useT(common); + const board = leaderboard(history); + if (board.length < 2) return null; + const top = median(board[0] as BenchResult, "decodeTps") ?? 1; + return ( + + + {t("leaderboardTitle")} + {t("leaderboardHint")} + + + {board.map((result) => { + const value = median(result, "decodeTps") ?? 0; + return ( +
+ + {result.model} + {result.specType && ( + {result.specType} + )} + +
+
+
+ + {formatNumber(value, value < 100 ? 1 : 0)}{" "} + {tc(result.unit)} + +
+ ); + })} + + + ); +} + +/** Past sessions: show one, delete it, or export everything as CSV. */ +export function HistoryCard({ + sessions, + history, + selected, + onSelect, +}: { + sessions: readonly Session[]; + history: readonly BenchResult[]; + selected: string | null; + onSelect: (id: string) => void; +}) { + const t = useT(messages); + const tc = useT(common); + const forget = useForgetResults(); + const [saved, setSaved] = useState(false); + const exportCsv = useMutation({ + mutationFn: () => + unwrap(() => commands.saveExport("Calcine benchmarks", toCsv(history), "csv")), + onSuccess: (path) => setSaved(path !== null), + }); + useEffect(() => { + if (!saved) return; + const timer = setTimeout(() => setSaved(false), 2000); + return () => clearTimeout(timer); + }, [saved]); + + return ( + + + {t("historyTitle")} + + + +
    + {sessions.map((session) => ( +
  • + + +
    • + ))} +
+ {exportCsv.isError && ( +

{exportCsv.error.message}

+ )} + + + ); +} + +/** Speed with real prompts, from the request log since Calcine started. */ +export function UsageCard() { + const t = useT(messages); + const { data: entries = [] } = useQuery(requestsQuery); + const usage = usageByModel(entries); + return ( + + + {t("usageTitle")} + {t("usageHint")} + + + {usage.length === 0 ? ( +

{t("usageEmpty")}

+ ) : ( +
    + {usage.map((row) => ( +
  • + {row.model} + + {t.plural("requests", row.requests, { count: formatCount(row.requests) })} + + {row.firstTokenMs !== null && ( + + {t("firstTokenShort", { + value: t("ms", { value: formatNumber(row.firstTokenMs, 0) }), + })} + + )} + {row.tokensPerSecond !== null && ( + + {t("speedShort", { value: formatNumber(row.tokensPerSecond, 1) })} + + )} +
    • + ))} +
+ )} + + + ); +} diff --git a/src/features/benchmark/components/run-card.tsx b/src/features/benchmark/components/run-card.tsx new file mode 100644 index 0000000..9e6919a --- /dev/null +++ b/src/features/benchmark/components/run-card.tsx @@ -0,0 +1,330 @@ +import { useQuery } from "@tanstack/react-query"; +import { Gauge, Play, Square } from "lucide-react"; +import { useEffect, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Field, Select } from "@/components/ui/field"; +import { Progress } from "@/components/ui/progress"; +import { + needsDraftModel, + POWER_MODES, + type PowerMode, + SPEC_TYPES, + type SpecType, +} from "@/features/chat/lib/request"; +import { messages as chatMessages } from "@/features/chat/messages"; +import { modelsQuery } from "@/features/library/api"; +import { useCancelJob } from "@/features/tasks/api"; +import { useT } from "@/i18n"; +import { common } from "@/i18n/common"; +import type { ComputeUnit } from "@/lib/api"; +import { cn } from "@/lib/utils"; +import { benchToolQuery, useRunningJob, useStartBenchmark } from "../api"; +import { benchTargets, benchUnits } from "../lib/results"; +import { messages } from "../messages"; +import { UNIT_COLOR } from "./metric-bars"; + +const PROMPTS = [128, 512, 1024, 2048] as const; +const GENERATED = [64, 128, 256, 512] as const; +const REPETITIONS = [3, 5, 10] as const; + +/** Pick a model, units and sizes, then run geniex-bench on each unit. */ +export function RunCard() { + const t = useT(messages); + const tc = useT(common); + const tchat = useT(chatMessages); + const { data: models = [] } = useQuery(modelsQuery); + const { data: tool } = useQuery(benchToolQuery); + const targets = benchTargets(models); + const [targetId, setTargetId] = useState(""); + const target = targets.find((candidate) => candidate.id === targetId) ?? targets[0]; + const available = target ? benchUnits(target.model.runtime) : []; + const [units, setUnits] = useState(["npu"]); + const [promptTokens, setPromptTokens] = useState(512); + const [generatedTokens, setGeneratedTokens] = useState(128); + const [repetitions, setRepetitions] = useState(5); + const [powerMode, setPowerMode] = useState("burst"); + const [specType, setSpecType] = useState(null); + const [draftModel, setDraftModel] = useState(null); + const start = useStartBenchmark(); + const cancel = useCancelJob(); + const running = useRunningJob("benchmark"); + const ready = tool?.installed !== null && tool?.installed === tool?.wanted; + + // Changing model keeps the units it supports, or falls back to all of them. + // Keyed on the runtime: `target` is a new object on every render. + const runtime = target?.model.runtime; + useEffect(() => { + if (!runtime) return; + const supported = benchUnits(runtime); + setUnits((current) => { + const kept = current.filter((unit) => supported.includes(unit)); + if (kept.length === current.length) return current; + return kept.length ? kept : supported; + }); + }, [runtime]); + + const toggle = (unit: ComputeUnit) => + setUnits((current) => + current.includes(unit) ? current.filter((u) => u !== unit) : [...current, unit], + ); + const chosen = available.filter((unit) => units.includes(unit)); + // Speculative decoding is a llama.cpp feature; drafts share the vocabulary. + const speculative = target?.model.runtime === "llama_cpp" ? specType : null; + const drafts = targets.filter( + (candidate) => + candidate.model.runtime === "llama_cpp" && candidate.model.name !== target?.model.name, + ); + + return ( + + + + + {t("runTitle")} + + {t("runHint")} + + + {targets.length === 0 ? ( +

{t("noModels")}

+ ) : ( + <> +
+ + + + +
+ {(["npu", "hybrid", "gpu", "cpu"] as const).map((unit) => { + const supported = available.includes(unit); + const active = supported && units.includes(unit); + return ( + + ); + })} +
+ +
+ +
+ t("tokens", { count: String(count) })} + onChange={setPromptTokens} + disabled={Boolean(running)} + /> + t("tokens", { count: String(count) })} + onChange={setGeneratedTokens} + disabled={Boolean(running)} + /> + t("runs", { count: String(count) })} + onChange={setRepetitions} + disabled={Boolean(running)} + /> + + + +
+ + {target?.model.runtime === "llama_cpp" && ( +
+ + + + {needsDraftModel(specType) && ( + + + + )} +
+ )} + + {running ? ( +
+
+ +

+ {running.progress?.step + ? t("running", { + current: String(running.progress.step.current), + total: String(running.progress.step.total), + }) + : t("startingRun")} +

+
+ +
+ ) : ( +
+ {start.isError && ( +

{start.error.message}

+ )} + +
+ )} + + )} + + + ); +} + +function Choice({ + id, + label, + value, + options, + format, + onChange, + disabled, +}: { + id: string; + label: string; + value: T; + options: readonly T[]; + format: (value: T) => string; + onChange: (value: T) => void; + disabled: boolean; +}) { + return ( + + + + ); +} diff --git a/src/features/benchmark/components/tool-card.tsx b/src/features/benchmark/components/tool-card.tsx new file mode 100644 index 0000000..4d90226 --- /dev/null +++ b/src/features/benchmark/components/tool-card.tsx @@ -0,0 +1,76 @@ +import { useQuery } from "@tanstack/react-query"; +import { Download, PackageCheck } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Progress } from "@/components/ui/progress"; +import { progressLabel } from "@/features/tasks/lib/format"; +import { messages as taskMessages } from "@/features/tasks/messages"; +import { useT } from "@/i18n"; +import { formatBytes } from "@/lib/format"; +import { benchToolQuery, useInstallBenchTool, useRunningJob } from "../api"; +import { messages } from "../messages"; + +/** Download `geniex-bench`, matching the installed GenieX. Hidden once ready. */ +export function ToolCard() { + const t = useT(messages); + const tt = useT(taskMessages); + const { data: tool, error } = useQuery(benchToolQuery); + const install = useInstallBenchTool(); + const downloading = useRunningJob("install_bench"); + + if (!tool && !error) return null; + const ready = tool && tool.installed !== null && tool.installed === tool.wanted; + if (ready) return null; + + const status = !tool?.wanted + ? t("noGeniex") + : tool.installed + ? t("toolOutdated", { installed: tool.installed, version: tool.wanted }) + : t("toolMissing", { version: tool.wanted }); + + return ( + + + + + {t("toolTitle")} + + {t("toolHint")} + + + {downloading ? ( +
+ +

+ {progressLabel(downloading, tt)} +

+
+ ) : ( +
+

{error ? error.message : status}

+ {tool?.wanted && ( + + )} +
+ )} + {install.isError &&

{install.error.message}

} + + + ); +} diff --git a/src/features/benchmark/lib/results.test.ts b/src/features/benchmark/lib/results.test.ts new file mode 100644 index 0000000..017ba94 --- /dev/null +++ b/src/features/benchmark/lib/results.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it } from "vitest"; +import type { BenchResult, RequestEntry } from "@/lib/api"; +import { + benchTargets, + best, + energyRows, + leaderboard, + mostEfficient, + sessions, + toCsv, + usageByModel, +} from "./results"; + +const stat = (median: number) => ({ median, min: median, max: median, stdev: 1 }); + +const result = ( + overrides: Partial & { decode?: number; ttft?: number }, +): BenchResult => { + const { decode, ttft, ...rest } = overrides; + return { + id: "r", + sessionId: "s1", + startedAtMs: 1000, + model: "qualcomm/Qwen3-4B:W4A16", + runtime: "qairt", + unit: "npu", + promptTokens: 512, + generatedTokens: 128, + repetitions: 5, + powerMode: "burst", + measure: + decode === undefined + ? null + : { + ttftMs: stat(ttft ?? 100), + prefillTps: stat(2000), + decodeTps: stat(decode), + generatedTokens: 128, + promptTokens: 512, + geniexVersion: "v0.8.0", + energy: null, + }, + error: decode === undefined ? "failed" : null, + ...rest, + }; +}; + +describe("sessions", () => { + it("groups by session, newest first, NPU before CPU", () => { + const grouped = sessions([ + result({ id: "a", unit: "cpu", decode: 20 }), + result({ id: "b", unit: "npu", decode: 50 }), + result({ id: "c", sessionId: "s2", startedAtMs: 5000, decode: 10 }), + ]); + expect(grouped.map((session) => session.id)).toEqual(["s2", "s1"]); + expect(grouped[1]?.results.map((r) => r.unit)).toEqual(["npu", "cpu"]); + }); +}); + +describe("best", () => { + it("prefers the fastest decoding and the shortest wait", () => { + const results = [ + result({ id: "npu", decode: 50, ttft: 80 }), + result({ id: "gpu", unit: "gpu", decode: 60, ttft: 120 }), + result({ id: "cpu", unit: "cpu" }), + ]; + expect(best(results, "decodeTps")?.id).toBe("gpu"); + expect(best(results, "ttftMs")?.id).toBe("npu"); + expect(best([result({})], "decodeTps")).toBeNull(); + }); +}); + +describe("leaderboard", () => { + it("keeps each model's fastest unit, from its latest run", () => { + const board = leaderboard([ + result({ id: "old", decode: 90, startedAtMs: 1 }), + result({ id: "new", decode: 40, startedAtMs: 2 }), + result({ id: "gpu", unit: "gpu", decode: 45, startedAtMs: 2 }), + result({ id: "small", model: "qualcomm/Qwen3-0.6B:W4A16", decode: 80 }), + ]); + expect(board.map((r) => r.id)).toEqual(["small", "gpu"]); + }); +}); + +describe("toCsv", () => { + it("writes one row per unit and quotes errors", () => { + const csv = toCsv([result({ decode: 50 }), result({ error: 'said "no", twice' })]); + const lines = csv.trim().split("\n"); + expect(lines[0]).toMatch(/^date,model,runtime,unit/); + expect(lines[1]).toContain(",50,1,"); + expect(lines[2]).toContain('"said ""no"", twice"'); + }); +}); + +describe("benchTargets", () => { + it("lists every precision", () => { + const targets = benchTargets([ + { + name: "a/b", + sizeBytes: 1, + runtime: "llama_cpp", + modelType: "llm", + precisions: ["Q4_0", "Q8_0"], + }, + { name: "c/d", sizeBytes: 1, runtime: "qairt", modelType: "llm", precisions: [] }, + ]); + expect(targets.map((target) => target.id)).toEqual(["a/b:Q4_0", "a/b:Q8_0", "c/d"]); + }); +}); + +describe("usageByModel", () => { + const entry = (overrides: Partial): RequestEntry => ({ + id: 1, + startedAtMs: 0, + client: "Calcine", + method: "POST", + path: "/v1/chat/completions", + model: "m", + stream: true, + status: 200, + durationMs: 1000, + firstTokenMs: 100, + promptTokens: 10, + completionTokens: 50, + tokensPerSecond: 20, + error: null, + ...overrides, + }); + + it("summarizes successful requests per model", () => { + const usage = usageByModel([ + entry({ firstTokenMs: 100, tokensPerSecond: 20 }), + entry({ firstTokenMs: 300, tokensPerSecond: 30 }), + entry({ status: 500 }), + entry({ model: "other" }), + ]); + expect(usage[0]).toEqual({ + model: "m", + requests: 2, + completionTokens: 100, + firstTokenMs: 200, + tokensPerSecond: 25, + }); + expect(usage).toHaveLength(2); + }); +}); + +describe("energy profiles", () => { + const profile = (mode: string, decode: number, activeWatts: number, joulesPerToken: number) => { + const base = result({ id: mode, powerMode: mode, decode, source: "energy_profile" }); + return { + ...base, + measure: base.measure && { + ...base.measure, + energy: { idleWatts: 15, activeWatts, joulesPerToken, systemJoulesPerToken: 0.4 }, + }, + }; + }; + + it("lists modes fastest first and finds the most efficient", () => { + const rows = energyRows([ + profile("power_saver", 30, 18, 0.1), + profile("burst", 52, 22, 0.135), + result({ id: "failed", powerMode: "balanced", source: "energy_profile" }), + ]); + expect(rows.map((row) => row.mode)).toEqual(["burst", "power_saver"]); + expect(rows[0]?.extraWatts).toBe(7); + expect(rows[1]?.tokensPerJoule).toBeCloseTo(10); + expect(mostEfficient(rows)?.mode).toBe("power_saver"); + expect(mostEfficient([])).toBeNull(); + }); + + it("keeps energy profiles out of the leaderboard", () => { + expect(leaderboard([profile("burst", 52, 22, 0.1)])).toEqual([]); + }); +}); diff --git a/src/features/benchmark/lib/results.ts b/src/features/benchmark/lib/results.ts new file mode 100644 index 0000000..62bc059 --- /dev/null +++ b/src/features/benchmark/lib/results.ts @@ -0,0 +1,295 @@ +import type { + BenchResult, + BenchSource, + ComputeUnit, + LocalModel, + RequestEntry, + Runtime, +} from "@/lib/api"; + +/** Units in the order they're shown: NPU first, CPU last. */ +export const UNIT_ORDER: readonly ComputeUnit[] = ["npu", "hybrid", "gpu", "cpu"]; + +/** Units `geniex-bench` can run a model on. AI Hub bundles only run on the NPU. */ +export function benchUnits(runtime: Runtime): ComputeUnit[] { + return runtime === "llama_cpp" ? ["npu", "hybrid", "gpu", "cpu"] : ["npu"]; +} + +/** One entry per downloaded precision, named like GenieX ids. */ +export function benchTargets(models: readonly LocalModel[]) { + return models.flatMap((model) => + (model.precisions.length ? model.precisions : [null]).map((precision) => ({ + id: precision ? `${model.name}:${precision}` : model.name, + model, + })), + ); +} + +/** Results measured together. */ +export type Session = { + id: string; + startedAtMs: number; + model: string; + runtime: Runtime; + promptTokens: number; + generatedTokens: number; + repetitions: number; + powerMode: string; + specType: string | null; + draftModel: string | null; + source: BenchSource; + results: BenchResult[]; +}; + +/** Group the history by session, newest first, units in display order. */ +export function sessions(history: readonly BenchResult[]): Session[] { + const groups = new Map(); + for (const result of history) { + const session = groups.get(result.sessionId); + if (session) { + session.results.push(result); + session.startedAtMs = Math.min(session.startedAtMs, result.startedAtMs); + } else { + groups.set(result.sessionId, { + id: result.sessionId, + startedAtMs: result.startedAtMs, + model: result.model, + runtime: result.runtime, + promptTokens: result.promptTokens, + generatedTokens: result.generatedTokens, + repetitions: result.repetitions, + powerMode: result.powerMode, + specType: result.specType ?? null, + draftModel: result.draftModel ?? null, + source: result.source ?? "geniex_bench", + results: [result], + }); + } + } + return [...groups.values()] + .map((session) => ({ + ...session, + results: session.results.sort( + (a, b) => UNIT_ORDER.indexOf(a.unit) - UNIT_ORDER.indexOf(b.unit), + ), + })) + .sort((a, b) => b.startedAtMs - a.startedAtMs); +} + +export type Metric = "decodeTps" | "prefillTps" | "ttftMs"; + +/** Median of a metric, `null` if the run failed. */ +export function median(result: BenchResult, metric: Metric): number | null { + return result.measure?.[metric].median ?? null; +} + +/** Lower time to first token is better; higher speeds are. */ +export const LOWER_IS_BETTER: Record = { + decodeTps: false, + prefillTps: false, + ttftMs: true, +}; + +/** The best result of a session for a metric. */ +export function best(results: readonly BenchResult[], metric: Metric): BenchResult | null { + let winner: BenchResult | null = null; + for (const result of results) { + const value = median(result, metric); + if (value === null) continue; + const current = winner ? median(winner, metric) : null; + if (current === null || (LOWER_IS_BETTER[metric] ? value < current : value > current)) { + winner = result; + } + } + return winner; +} + +/** Each model's fastest decoding, from its latest successful measurement per unit. */ +export function leaderboard(history: readonly BenchResult[]) { + const latest = new Map(); + for (const result of history) { + // Energy profiles use another prompt: their speeds don't compare. + if (!result.measure || result.source === "energy_profile") continue; + const key = `${result.model}|${result.specType ?? ""}|${result.unit}`; + const seen = latest.get(key); + if (!seen || result.startedAtMs > seen.startedAtMs) latest.set(key, result); + } + const perModel = new Map(); + for (const result of latest.values()) { + // With and without speculative decoding are listed apart. + const key = `${result.model}|${result.specType ?? ""}`; + const current = perModel.get(key); + if (!current || (median(result, "decodeTps") ?? 0) > (median(current, "decodeTps") ?? 0)) { + perModel.set(key, result); + } + } + return [...perModel.values()].sort( + (a, b) => (median(b, "decodeTps") ?? 0) - (median(a, "decodeTps") ?? 0), + ); +} + +/** The whole history as CSV, one row per unit. */ +export function toCsv(history: readonly BenchResult[]): string { + const header = [ + "date", + "model", + "runtime", + "unit", + "prompt_tokens", + "generated_tokens", + "repetitions", + "power_mode", + "spec_type", + "draft_model", + "ttft_ms_median", + "ttft_ms_stdev", + "prefill_tps_median", + "prefill_tps_stdev", + "decode_tps_median", + "decode_tps_stdev", + "source", + "idle_watts", + "active_watts", + "joules_per_token", + "geniex_version", + "error", + ]; + const cell = (value: string | number | null | undefined) => { + const text = value === null || value === undefined ? "" : String(value); + return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; + }; + const rows = history.map((result) => { + const measure = result.measure; + return [ + new Date(result.startedAtMs).toISOString(), + result.model, + result.runtime, + result.unit, + result.promptTokens, + result.generatedTokens, + result.repetitions, + result.powerMode, + result.specType, + result.draftModel, + measure?.ttftMs.median, + measure?.ttftMs.stdev, + measure?.prefillTps.median, + measure?.prefillTps.stdev, + measure?.decodeTps.median, + measure?.decodeTps.stdev, + result.source ?? "geniex_bench", + measure?.energy?.idleWatts, + measure?.energy?.activeWatts, + measure?.energy?.joulesPerToken, + measure?.geniexVersion, + result.error, + ] + .map(cell) + .join(","); + }); + return `${[header.join(","), ...rows].join("\n")}\n`; +} + +/** Everyday speed per model, from the request log (Chat and other apps). */ +export type Usage = { + model: string; + requests: number; + completionTokens: number; + firstTokenMs: number | null; + tokensPerSecond: number | null; +}; + +export function usageByModel(entries: readonly RequestEntry[]): Usage[] { + const byModel = new Map(); + for (const entry of entries) { + if (entry.status !== 200 || !entry.model || entry.error) continue; + const list = byModel.get(entry.model) ?? []; + list.push(entry); + byModel.set(entry.model, list); + } + return [...byModel.entries()] + .map(([model, list]) => ({ + model, + requests: list.length, + completionTokens: list.reduce((sum, entry) => sum + (entry.completionTokens ?? 0), 0), + firstTokenMs: middle(list.map((entry) => entry.firstTokenMs)), + tokensPerSecond: middle(list.map((entry) => entry.tokensPerSecond)), + })) + .sort((a, b) => b.requests - a.requests); +} + +function middle(values: readonly (number | null)[]): number | null { + const known = values.filter((value): value is number => value !== null).sort((a, b) => a - b); + if (known.length === 0) return null; + const half = Math.floor(known.length / 2); + return known.length % 2 + ? (known[half] ?? null) + : ((known[half - 1] ?? 0) + (known[half] ?? 0)) / 2; +} + +/** GenieX's power modes, fastest first (as `POWER_MODES` in the backend). */ +export const POWER_MODE_ORDER = [ + "burst", + "sustained_high_performance", + "high_performance", + "balanced", + "low_balanced", + "high_power_saver", + "power_saver", + "low_power_saver", +] as const; + +/** One unit and power mode of an energy profile. */ +export type EnergyRow = { + result: BenchResult; + unit: ComputeUnit; + mode: string; + decodeTps: number; + activeWatts: number; + /** What generating adds to the idle draw. */ + extraWatts: number; + joulesPerToken: number; + /** The whole system's energy per token, screen and all. */ + systemJoulesPerToken: number; + tokensPerJoule: number; +}; + +/** An energy profile's measurements, NPU first, then fastest mode first. + * Failed ones are left out. */ +export function energyRows(results: readonly BenchResult[]): EnergyRow[] { + return results + .flatMap((result) => { + const energy = result.measure?.energy; + if (!result.measure || !energy) return []; + // Numbers come as `null` when they aren't finite. + const activeWatts = energy.activeWatts ?? 0; + const joulesPerToken = energy.joulesPerToken ?? 0; + return [ + { + result, + unit: result.unit, + mode: result.powerMode, + decodeTps: result.measure.decodeTps.median ?? 0, + activeWatts, + extraWatts: Math.max(0, activeWatts - (energy.idleWatts ?? 0)), + joulesPerToken, + systemJoulesPerToken: energy.systemJoulesPerToken ?? 0, + tokensPerJoule: joulesPerToken > 0 ? 1 / joulesPerToken : 0, + }, + ]; + }) + .sort( + (a, b) => + UNIT_ORDER.indexOf(a.unit) - UNIT_ORDER.indexOf(b.unit) || + POWER_MODE_ORDER.indexOf(a.mode as (typeof POWER_MODE_ORDER)[number]) - + POWER_MODE_ORDER.indexOf(b.mode as (typeof POWER_MODE_ORDER)[number]), + ); +} + +/** The unit and mode that generate the most tokens per joule. */ +export function mostEfficient(rows: readonly EnergyRow[]): EnergyRow | null { + return rows.reduce( + (winner, row) => (!winner || row.tokensPerJoule > winner.tokensPerJoule ? row : winner), + null, + ); +} diff --git a/src/features/benchmark/messages.ts b/src/features/benchmark/messages.ts new file mode 100644 index 0000000..bc51a33 --- /dev/null +++ b/src/features/benchmark/messages.ts @@ -0,0 +1,191 @@ +import { defineMessages } from "@/i18n"; + +export const messages = defineMessages({ + en: { + title: "Benchmark", + description: + "How fast your models run on each compute unit, measured with Qualcomm's geniex-bench.", + // Tool + toolTitle: "Benchmark tool", + toolHint: + "Calcine measures with geniex-bench, Qualcomm's own tool, downloaded from GenieX's GitHub release and checked against its published SHA-256.", + toolMissing: "geniex-bench {version} isn't downloaded yet.", + toolOutdated: "geniex-bench {installed} is installed; GenieX {version} needs its own version.", + toolReady: "geniex-bench {version} is ready.", + downloadTool: "Download ({size})", + downloadToolNoSize: "Download", + noGeniex: "Install GenieX first: the tool must match its version.", + // Run + runTitle: "New benchmark", + runHint: + "Each unit runs one warmup, then the measured repetitions, on a fixed random prompt. GenieX is paused meanwhile so nothing else uses the hardware.", + model: "Model", + noModels: "Download a model first.", + units: "Compute units", + unitsQairt: "AI Hub models are compiled for the NPU only.", + hybrid: "Hybrid", + promptTokens: "Prompt", + generatedTokens: "Generated", + repetitions: "Repetitions", + powerMode: "Power mode", + specHint: "Run once with and once without to measure the gain.", + tokens: "{count} tokens", + runs: "{count} runs", + start: "Run benchmark", + running: "Measuring {current} of {total}…", + startingRun: "Starting…", + cancel: "Cancel", + // Energy + energyTitle: "Energy", + energyHint: + "Speed and energy on each compute unit and in each power mode, measured by the chip while the model generates. The model stays loaded, and the idle draw measured just before is subtracted.", + energyUnavailable: + "This device doesn't report its power use, so energy can't be measured here.", + energyQuiet: + "Leave the PC alone meanwhile: other apps, and requests to Calcine's API, would skew the numbers.", + energyModes: "Power modes", + energyUnit: "Compute unit", + energyStart: "Measure", + energyMode: "Mode", + energyUnitMode: "Unit · mode", + energySpeed: "Speed", + energyPower: "Power", + energyPerToken: "Energy per token", + energySystem: "{value} whole system", + energyEfficiency: "Tokens per joule", + energyExtra: "+{value} above idle", + energyIdle: "Idle with the model loaded: {value}", + mostEfficient: "Most efficient", + energySummary: + "{efficient} makes {ratio}× more tokens per joule than {fastest}, for {slower}% less speed.", + energySame: "{mode} is both the fastest and the most efficient here.", + watts: "{value} W", + millijoules: "{value} mJ", + // Results + resultsTitle: "Results", + latest: "Latest results", + noResults: "Run a benchmark to compare compute units.", + decode: "Generation", + decodeHint: "Tokens per second once the reply has started.", + prefill: "Prompt reading", + prefillHint: "Tokens per second while reading the prompt.", + ttft: "First token", + ttftHint: "Wait before the reply starts. Lower is better.", + tps: "{value} tok/s", + ms: "{value} ms", + fastest: "Fastest", + failed: "Failed", + spread: "± {value}", + setup: "{prompt} prompt tokens · {generated} generated · {runs} runs · {power}", + withSpec: "speculative: {method}", + // Leaderboard + leaderboardTitle: "Fastest models", + leaderboardHint: "Each model's best generation speed, on its fastest unit.", + // History + historyTitle: "History", + exportCsv: "Export CSV", + exported: "Saved", + deleteSession: "Delete these results", + show: "Show", + // Usage + usageTitle: "Real-world speed", + usageHint: + "From the requests since Calcine started: the Chat and your other apps, with their real prompts.", + usageEmpty: "Chat with a model, or call the API, to see its everyday speed here.", + requests_one: "{count} request", + requests_other: "{count} requests", + firstTokenShort: "first token {value}", + speedShort: "{value} tok/s", + }, + fr: { + title: "Benchmark", + description: + "La vitesse de vos modèles sur chaque unité de calcul, mesurée avec geniex-bench, l'outil de Qualcomm.", + toolTitle: "Outil de benchmark", + toolHint: + "Calcine mesure avec geniex-bench, l'outil de Qualcomm, téléchargé depuis la release GitHub de GenieX et vérifié avec son SHA-256 publié.", + toolMissing: "geniex-bench {version} n'est pas encore téléchargé.", + toolOutdated: + "geniex-bench {installed} est installé ; GenieX {version} a besoin de sa propre version.", + toolReady: "geniex-bench {version} est prêt.", + downloadTool: "Télécharger ({size})", + downloadToolNoSize: "Télécharger", + noGeniex: "Installez d'abord GenieX : l'outil doit correspondre à sa version.", + runTitle: "Nouveau benchmark", + runHint: + "Chaque unité fait un tour de chauffe, puis les répétitions mesurées, sur un prompt aléatoire fixe. GenieX est mis en pause pendant ce temps pour que rien d'autre n'utilise le matériel.", + model: "Modèle", + noModels: "Téléchargez d'abord un modèle.", + units: "Unités de calcul", + unitsQairt: "Les modèles AI Hub sont compilés pour le NPU uniquement.", + hybrid: "Hybride", + promptTokens: "Prompt", + generatedTokens: "Générés", + repetitions: "Répétitions", + powerMode: "Mode d'alimentation", + specHint: "Lancez avec et sans pour mesurer le gain.", + tokens: "{count} tokens", + runs: "{count} passes", + start: "Lancer le benchmark", + running: "Mesure {current} sur {total}…", + startingRun: "Démarrage…", + cancel: "Annuler", + energyTitle: "Énergie", + energyHint: + "Vitesse et énergie sur chaque unité de calcul et dans chaque mode d'alimentation, mesurées par la puce pendant que le modèle génère. Le modèle reste chargé, et la consommation au repos mesurée juste avant est déduite.", + energyUnavailable: + "Cet appareil n'indique pas sa consommation : l'énergie ne peut pas être mesurée ici.", + energyQuiet: + "Laissez le PC tranquille pendant la mesure : d'autres applications, ou des requêtes à l'API de Calcine, fausseraient les chiffres.", + energyModes: "Modes d'alimentation", + energyUnit: "Unité de calcul", + energyStart: "Mesurer", + energyMode: "Mode", + energyUnitMode: "Unité · mode", + energySpeed: "Vitesse", + energyPower: "Puissance", + energyPerToken: "Énergie par token", + energySystem: "{value} pour tout le système", + energyEfficiency: "Tokens par joule", + energyExtra: "+{value} au-dessus du repos", + energyIdle: "Au repos avec le modèle chargé : {value}", + mostEfficient: "Le plus efficace", + energySummary: + "{efficient} produit {ratio}× plus de tokens par joule que {fastest}, pour {slower} % de vitesse en moins.", + energySame: "{mode} est ici à la fois le plus rapide et le plus efficace.", + watts: "{value} W", + millijoules: "{value} mJ", + resultsTitle: "Résultats", + latest: "Derniers résultats", + noResults: "Lancez un benchmark pour comparer les unités de calcul.", + decode: "Génération", + decodeHint: "Tokens par seconde une fois la réponse commencée.", + prefill: "Lecture du prompt", + prefillHint: "Tokens par seconde pendant la lecture du prompt.", + ttft: "Premier token", + ttftHint: "Attente avant le début de la réponse. Plus bas, c'est mieux.", + tps: "{value} tok/s", + ms: "{value} ms", + fastest: "Le plus rapide", + failed: "Échec", + spread: "± {value}", + setup: "{prompt} tokens de prompt · {generated} générés · {runs} passes · {power}", + withSpec: "spéculatif : {method}", + leaderboardTitle: "Modèles les plus rapides", + leaderboardHint: + "La meilleure vitesse de génération de chaque modèle, sur son unité la plus rapide.", + historyTitle: "Historique", + exportCsv: "Exporter en CSV", + exported: "Enregistré", + deleteSession: "Supprimer ces résultats", + show: "Afficher", + usageTitle: "Vitesse au quotidien", + usageHint: + "D'après les requêtes depuis le lancement de Calcine : le chat et vos autres applis, avec leurs vrais prompts.", + usageEmpty: "Discutez avec un modèle, ou appelez l'API, pour voir ici sa vitesse au quotidien.", + requests_one: "{count} requête", + requests_other: "{count} requêtes", + firstTokenShort: "premier token {value}", + speedShort: "{value} tok/s", + }, +}); diff --git a/src/features/chat/chat-page.tsx b/src/features/chat/chat-page.tsx index 7945731..a8175f6 100644 --- a/src/features/chat/chat-page.tsx +++ b/src/features/chat/chat-page.tsx @@ -5,10 +5,14 @@ import { ErrorState } from "@/components/calcine/feedback/error-state"; import { Button } from "@/components/ui/button"; import { Tooltip } from "@/components/ui/tooltip"; import { modelsQuery } from "@/features/library/api"; +import { useMediaQuery } from "@/hooks/use-media-query"; import { useT } from "@/i18n"; import { Composer } from "./components/composer"; +import { ContextMeter } from "./components/context-meter"; import { ConversationList } from "./components/conversation-list"; +import { ExportButton } from "./components/export-button"; import { ModelPicker } from "./components/model-picker"; +import { PreloadButton } from "./components/preload-button"; import { SettingsPanel } from "./components/settings-panel"; import { Thread } from "./components/thread"; import { supportsMedia } from "./lib/request"; @@ -16,6 +20,9 @@ import { messages } from "./messages"; import { useChat } from "./store"; import { type PendingAttachment, useSend } from "./use-send"; +/** Wide enough for the conversations, the chat and the settings side by side. */ +const WIDE_WINDOW = "(min-width: 1200px)"; + export function ChatPage() { const t = useT(messages); const models = useQuery(modelsQuery); @@ -25,7 +32,11 @@ export function ChatPage() { const create = useChat((state) => state.create); const setModel = useChat((state) => state.setModel); const { send, stop, streaming, ready } = useSend(); - const [showSettings, setShowSettings] = useState(true); + // The settings panel squeezes the chat on small windows: hidden there + // unless opened. Opening or closing it holds for that size of window. + const wide = useMediaQuery(WIDE_WINDOW); + const [settingsChoice, setSettingsChoice] = useState<{ wide: boolean; shown: boolean }>(); + const showSettings = settingsChoice?.wide === wide ? settingsChoice.shown : wide; const installed = models.data ?? []; const active = conversations.find((conversation) => conversation.id === activeId); @@ -53,7 +64,8 @@ export function ChatPage() {
modelId && create(modelId)} />
-
+ {/* A container: the header adapts to the chat column, not the window. */} +
- {missingModel && {t("missingModel")}} + {missingModel && ( + {t("missingModel")} + )} +
+ {/* Left out when the column is too narrow for them (settings panel open on a small window). */} +
+ + + +
+ + ); +} diff --git a/src/features/chat/components/export-button.tsx b/src/features/chat/components/export-button.tsx new file mode 100644 index 0000000..9e6a251 --- /dev/null +++ b/src/features/chat/components/export-button.tsx @@ -0,0 +1,60 @@ +import { useMutation } from "@tanstack/react-query"; +import { Check, FileDown } from "lucide-react"; +import { useEffect, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Tooltip } from "@/components/ui/tooltip"; +import { useT } from "@/i18n"; +import { commands, unwrap } from "@/lib/api"; +import { toMarkdown } from "../lib/export"; +import { messages } from "../messages"; +import type { Conversation } from "../store"; + +/** Save the conversation as a Markdown file. */ +export function ExportButton({ conversation }: { conversation: Conversation | undefined }) { + const t = useT(messages); + const [saved, setSaved] = useState(false); + const title = conversation?.title || t("newChat"); + const save = useMutation({ + mutationFn: (current: Conversation) => { + const markdown = toMarkdown( + current, + { + title, + you: t("exportYou"), + model: t("model"), + reasoning: t("reasoning"), + exported: t("exportedOn"), + image: t("exportImage"), + recording: t("recordingName"), + stopped: t("stopped"), + }, + new Date(), + ); + return unwrap(() => commands.saveExport(title, markdown, "markdown")); + }, + onSuccess: (path) => setSaved(path !== null), + }); + + useEffect(() => { + if (!saved) return; + const timer = setTimeout(() => setSaved(false), 2000); + return () => clearTimeout(timer); + }, [saved]); + + const empty = !conversation?.messages.some((message) => message.content); + const label = save.isError ? t("exportFailed", { message: save.error.message }) : t("export"); + return ( + + + + ); +} diff --git a/src/features/chat/components/message.tsx b/src/features/chat/components/message.tsx index 5d5a088..d36f257 100644 --- a/src/features/chat/components/message.tsx +++ b/src/features/chat/components/message.tsx @@ -10,6 +10,7 @@ import { useT } from "@/i18n"; import { formatNumber } from "@/lib/format"; import { cn } from "@/lib/utils"; import type { Attachment } from "../lib/attachments"; +import { prettyJson } from "../lib/request"; import type { StreamStats } from "../lib/sse"; import { messages } from "../messages"; import { type ChatMessage, useMediaPayloads } from "../store"; @@ -20,7 +21,7 @@ export function Message({ live, }: { message: ChatMessage; - live?: { content: string; reasoning: string }; + live?: { content: string; reasoning: string; status?: "summarizing" | null }; }) { const t = useT(messages); const content = live?.content ?? message.content; @@ -49,7 +50,15 @@ export function Message({ return (
{reasoning && } - {content ? : live && !reasoning ? : null} + {content ? ( + + ) : live?.status === "summarizing" ? ( +

+ {t("summarizing")} +

+ ) : live && !reasoning ? ( + + ) : null} {message.error && (

@@ -60,6 +69,11 @@ export function Message({ {t("loadFailedHint")} )} + {message.error.includes("context window") && ( + + {t("contextErrorHint")} + + )}

)} @@ -127,6 +141,12 @@ function Reasoning({ text, active }: { text: string; active: boolean }) { ); } +/** JSON replies (structured output) show as a formatted code block. */ +function asMarkdown(content: string): string { + const json = prettyJson(content); + return json === null ? content : `\`\`\`json\n${json}\n\`\`\``; +} + const MarkdownBody = memo(function MarkdownBody({ text }: { text: string }) { return (
@@ -143,6 +163,12 @@ function Stats({ stats }: { stats: StreamStats }) { t("tokensPerSecond", { value: formatNumber(stats.tokensPerSecond, 0) }), stats.firstTokenMs !== undefined && t("firstToken", { value: formatNumber(stats.firstTokenMs / 1000, 1) }), + stats.draftTokens && + t("draftsAccepted", { + accepted: String(stats.draftAccepted ?? 0), + drafted: String(stats.draftTokens), + percent: formatNumber(((stats.draftAccepted ?? 0) / stats.draftTokens) * 100, 0), + }), ].filter(Boolean); return {parts.join(" · ")}; } diff --git a/src/features/chat/components/model-picker.tsx b/src/features/chat/components/model-picker.tsx index 218743b..8cb9dc6 100644 --- a/src/features/chat/components/model-picker.tsx +++ b/src/features/chat/components/model-picker.tsx @@ -28,12 +28,12 @@ export function ModelPicker({ } const selected = models.find((model) => model.name === value); return ( -
+
- {selected && } + {selected && ( + + + + )}
); } diff --git a/src/features/chat/components/preload-button.tsx b/src/features/chat/components/preload-button.tsx new file mode 100644 index 0000000..6033f33 --- /dev/null +++ b/src/features/chat/components/preload-button.tsx @@ -0,0 +1,84 @@ +import { useMutation, useQuery } from "@tanstack/react-query"; +import { Check, Loader2, Zap } from "lucide-react"; +import { useEffect, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Tooltip } from "@/components/ui/tooltip"; +import { connectionQuery } from "@/features/server/api"; +import { useT } from "@/i18n"; +import type { LocalModel } from "@/lib/api"; +import { buildChatRequest } from "../lib/request"; +import { streamChat } from "../lib/sse"; +import { messages } from "../messages"; +import { useChat } from "../store"; + +/** + * Load the model now, with the chat's settings (compute unit, offloaded + * layers), so the first reply doesn't wait for it. GenieX keeps it loaded + * for the unload delay set on the Server page. + */ +export function PreloadButton({ + model, + modelId, + disabled, +}: { + model: LocalModel | undefined; + modelId: string | undefined; + disabled: boolean; +}) { + const t = useT(messages); + const { data: connection } = useQuery(connectionQuery); + const [loaded, setLoaded] = useState(false); + const preload = useMutation({ + mutationFn: async ({ id, url, token }: { id: string; url: string; token: string }) => { + const request = buildChatRequest(model, id, useChat.getState().settings, [ + { role: "user", content: "Hi" }, + ]); + await streamChat({ + url: `${url}/chat/completions`, + token, + body: { ...request, max_tokens: 1, enable_think: false }, + signal: AbortSignal.timeout(5 * 60_000), + onDelta: () => {}, + }); + }, + onSuccess: () => setLoaded(true), + }); + + useEffect(() => { + if (!loaded) return; + const timer = setTimeout(() => setLoaded(false), 2500); + return () => clearTimeout(timer); + }, [loaded]); + + const label = preload.isPending + ? t("preloading") + : loaded + ? t("preloaded") + : preload.isError + ? t("preloadFailed", { message: preload.error.message }) + : t("preload"); + return ( + + + + ); +} diff --git a/src/features/chat/components/settings-panel.tsx b/src/features/chat/components/settings-panel.tsx index fb6309b..5a2d4eb 100644 --- a/src/features/chat/components/settings-panel.tsx +++ b/src/features/chat/components/settings-panel.tsx @@ -1,26 +1,43 @@ import * as Collapsible from "@radix-ui/react-collapsible"; import { useQuery } from "@tanstack/react-query"; -import { ChevronRight } from "lucide-react"; -import { type ReactNode, useState } from "react"; +import { ChevronRight, Dices, X } from "lucide-react"; +import { type KeyboardEvent, type ReactNode, useState } from "react"; import { CopyButton } from "@/components/calcine/actions/copy-button"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; import { Field, Input, Select, Textarea } from "@/components/ui/field"; +import { SegmentedControl } from "@/components/ui/segmented-control"; import { Switch } from "@/components/ui/switch"; +import { modelsQuery } from "@/features/library/api"; import { gatewayQuery } from "@/features/server/api"; import { useT } from "@/i18n"; import { common } from "@/i18n/common"; import type { LocalModel } from "@/lib/api"; import { formatNumber } from "@/lib/format"; import { cn } from "@/lib/utils"; +import { clamp, escapeStop, parseOptional, unescapeStop } from "../lib/inputs"; import { + activePreset, buildChatRequest, type ChatSettings, + DEFAULT_SETTINGS, + needsDraftModel, + type OutputFormat, POWER_MODES, type PowerMode, + PRESETS, + type Preset, + parseSchema, + SPEC_TYPES, + type SpecType, supportsComputeChoice, + supportsMedia, toCurl, + type VisionCompute, } from "../lib/request"; import { messages } from "../messages"; import { useChat } from "../store"; +import { SystemPromptField } from "./system-prompts"; const COMPUTE_UNITS = [ { value: "npu", tone: "text-npu" }, @@ -28,6 +45,30 @@ const COMPUTE_UNITS = [ { value: "cpu", tone: "text-cpu" }, ] as const; +const ADVANCED_SAMPLING = [ + "topP", + "topK", + "minP", + "repetitionPenalty", + "presencePenalty", + "frequencyPenalty", + "seed", + "stop", +] as const satisfies readonly (keyof ChatSettings)[]; + +const MODEL_LOADING = [ + "gpuLayers", + "visionCompute", +] as const satisfies readonly (keyof ChatSettings)[]; + +const SPECULATIVE = [ + "specType", + "draftModel", + "draftMax", + "draftMin", + "draftPMin", +] as const satisfies readonly (keyof ChatSettings)[]; + /** Generation options, adapted to what the selected model supports. */ export function SettingsPanel({ model, @@ -40,19 +81,12 @@ export function SettingsPanel({ const tc = useT(common); const settings = useChat((state) => state.settings); const setSettings = useChat((state) => state.setSettings); - const canChooseCompute = supportsComputeChoice(model); + const llamaCpp = supportsComputeChoice(model); + const preset = activePreset(settings); return (