diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96aba4d88..735490f1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,10 @@ permissions: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - EMULEBB_BUILD_REF: 3b9ddf42466b07f852f798f6967ca782484056a4 - EMULEBB_ANDROID_BUILD_REF: 3b9ddf42466b07f852f798f6967ca782484056a4 - EMULEBB_BUILD_TESTS_REF: 7c680ed68e478c2f22c56c65f21f12c3790d3826 - EMULEBB_TOOLING_REF: 26abd1eab97b28977f2d55adba17e6ba28821498 + EMULEBB_BUILD_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_ANDROID_BUILD_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_BUILD_TESTS_REF: 281ccbcdf9f4c9960c5efda4705df966418b6bb2 + EMULEBB_TOOLING_REF: f88a8ce608bc75a619aea6909f54a9cdecdddb6e # Pin the native NAT sources linked by the sys crates. MINIUPNP_REF: 4e7a10948004ca2085ac8103e14fc77edb898b4c PCPNATPMP_REF: a6e787f837e507336a8774522c33b54264ef4a4b @@ -602,7 +602,7 @@ jobs: with: persist-credentials: false repository: emulebb/emulebb-build-tests - ref: 7c680ed68e478c2f22c56c65f21f12c3790d3826 + ref: 281ccbcdf9f4c9960c5efda4705df966418b6bb2 path: repos/emulebb-build-tests - name: Checkout pinned Rust OpenAPI contract @@ -610,7 +610,7 @@ jobs: with: persist-credentials: false repository: emulebb/emulebb-tooling - ref: 26abd1eab97b28977f2d55adba17e6ba28821498 + ref: f88a8ce608bc75a619aea6909f54a9cdecdddb6e path: repos/emulebb-tooling - name: Download tested Linux daemon diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 83893901b..34b53c057 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,18 +18,56 @@ on: required: false type: boolean default: true + diagnostic_run_id: + description: "Optional original Nightly run for read-only APK replay" + required: false + type: string + default: "" + diagnostic_source_sha: + description: "Original producer's exact Rust commit (40 lowercase hex characters)" + required: false + type: string + default: "" + diagnostic_manifest_sha256: + description: "Independently verified original APK manifest SHA-256" + required: false + type: string + default: "" + diagnostic_image: + description: "Replay one original APK on an isolated guest; never eligible for publication" + required: false + type: choice + options: [none, apk31, apk32, apk33, apk34, apk35, apk36, apk37] + default: none concurrency: - group: rust-nightly + group: >- + ${{ (inputs.diagnostic_run_id || inputs.diagnostic_source_sha || + inputs.diagnostic_manifest_sha256 || + (inputs.diagnostic_image && inputs.diagnostic_image != 'none')) && + format('rust-nightly-android-diagnostic-{0}-{1}', github.ref, inputs.diagnostic_image) || 'rust-nightly' }} cancel-in-progress: false permissions: actions: read contents: read +env: + EMULEBB_APK_DIAGNOSTIC_BUILD_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_APK_DIAGNOSTIC_HARNESS_REF: 281ccbcdf9f4c9960c5efda4705df966418b6bb2 + EMULEBB_APK_DIAGNOSTIC_TOOLING_REF: f88a8ce608bc75a619aea6909f54a9cdecdddb6e + EMULEBB_APK_DIAGNOSTIC_MINIUPNP_REF: 4e7a10948004ca2085ac8103e14fc77edb898b4c + EMULEBB_APK_DIAGNOSTIC_PCPNATPMP_REF: a6e787f837e507336a8774522c33b54264ef4a4b + EMULEBB_APK_DIAGNOSTIC_SIGNER_SHA256: 9edc742d7230a714a0149c20eb2afb235b8ad5237166090f10e7ac69e2bb359f + jobs: prepare: name: Derive immutable nightly metadata + # Any diagnostic input closes the normal release path, even when incomplete. + if: >- + ${{ !inputs.diagnostic_run_id && !inputs.diagnostic_source_sha && + !inputs.diagnostic_manifest_sha256 && + (!inputs.diagnostic_image || inputs.diagnostic_image == 'none') }} runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: @@ -38,7 +76,7 @@ jobs: previous_ref: ${{ steps.metadata.outputs.previous_ref }} should_build: ${{ steps.metadata.outputs.should_build }} steps: - - name: Checkout main with release tags + - name: Checkout verified source with release tags uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} @@ -55,7 +93,7 @@ jobs: run: python tools/nightly_release.py metadata --force "${{ github.event_name == 'workflow_dispatch' && inputs.force || false }}" --publish "${{ github.event_name == 'schedule' || inputs.publish }}" --scheduled "${{ github.event_name == 'schedule' }}" --github-output "$GITHUB_OUTPUT" verify-source: - name: Require green main source checks + name: Require green exact-source checks needs: prepare if: ${{ needs.prepare.outputs.should_build == 'true' }} runs-on: ubuntu-24.04 @@ -93,3 +131,139 @@ jobs: channel: nightly publish: ${{ github.event_name == 'schedule' || inputs.publish }} secrets: inherit + + apk-diagnostic: + name: Read-only original APK diagnostic replay + if: >- + ${{ github.event_name == 'workflow_dispatch' && + (inputs.diagnostic_run_id || inputs.diagnostic_source_sha || + inputs.diagnostic_manifest_sha256 || + (inputs.diagnostic_image && inputs.diagnostic_image != 'none')) }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + actions: read + contents: read + env: + EMULEBB_WORKSPACE_ROOT: ${{ github.workspace }} + EMULEBB_RUST_REPO: ${{ github.workspace }}/.ci/original-rust + EMULEBB_TESTS_REPO: ${{ github.workspace }}/.ci/emulebb-build-tests + EMULEBB_TOOLING_REPO: ${{ github.workspace }}/.ci/emulebb-tooling + MINIUPNP_ROOT: ${{ github.workspace }}/.ci/emulebb-miniupnp + PCPNATPMP_ROOT: ${{ github.workspace }}/.ci/emulebb-libpcpnatpmp + steps: + - name: Validate the isolated read-only diagnostic request + shell: python + env: + DIAGNOSTIC_PUBLISH: ${{ inputs.publish }} + PRODUCER_RUN_ID: ${{ inputs.diagnostic_run_id }} + PRODUCER_SOURCE_SHA: ${{ inputs.diagnostic_source_sha }} + APK_MANIFEST_SHA256: ${{ inputs.diagnostic_manifest_sha256 }} + DIAGNOSTIC_IMAGE: ${{ inputs.diagnostic_image }} + run: | + import os + import re + checks = { + "PRODUCER_RUN_ID": r"[1-9][0-9]*", + "PRODUCER_SOURCE_SHA": r"[0-9a-f]{40}", + "APK_MANIFEST_SHA256": r"[0-9a-f]{64}", + "DIAGNOSTIC_IMAGE": r"apk3[1-7]", + } + if os.environ.get("DIAGNOSTIC_PUBLISH") != "false" or any( + re.fullmatch(pattern, os.environ.get(name, "")) is None + for name, pattern in checks.items() + ): + raise SystemExit("Diagnostic replay requires publish=false and complete immutable inputs") + - name: Checkout the independently pinned diagnostic executor + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-build + ref: ${{ env.EMULEBB_APK_DIAGNOSTIC_BUILD_REF }} + path: .ci/emulebb-build + persist-credentials: false + - name: Checkout the independently pinned diagnostic Harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-build-tests + ref: ${{ env.EMULEBB_APK_DIAGNOSTIC_HARNESS_REF }} + path: .ci/emulebb-build-tests + persist-credentials: false + - name: Checkout original producer documentation authority + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-tooling + ref: ${{ env.EMULEBB_APK_DIAGNOSTIC_TOOLING_REF }} + path: .ci/emulebb-tooling + persist-credentials: false + - name: Checkout original producer MiniUPnP authority + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-miniupnp + ref: ${{ env.EMULEBB_APK_DIAGNOSTIC_MINIUPNP_REF }} + path: .ci/emulebb-miniupnp + persist-credentials: false + - name: Checkout original producer PCP/NAT-PMP authority + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-libpcpnatpmp + ref: ${{ env.EMULEBB_APK_DIAGNOSTIC_PCPNATPMP_REF }} + path: .ci/emulebb-libpcpnatpmp + persist-credentials: false + - name: Set up pinned Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.15.2" + - name: Install pinned diagnostic owner dependencies + run: python -m pip install click==8.4.1 pydantic==2.14.3 PyYAML==6.0.3 + - name: Set up pinned Docker + uses: docker/setup-docker-action@3f1ab1ab0e29231562e4db4ca600c1a49653bac6 # v5.7.0 + with: + version: v29.6.1 + daemon-config: '{"features":{"containerd-snapshotter":true}}' + - name: Set up pinned Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 + with: + version: v0.35.0 + driver: docker + use: false + - name: Grant this runner access to KVM + run: | + test -c /dev/kvm + sudo setfacl -m "u:$(id -un):rw" /dev/kvm + # WHY: finish host setup/cache actions before admitting the user-selected + # tree. Only the pinned executor may then read it for original-byte replay. + - name: Checkout the exact original producer source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.diagnostic_source_sha }} + path: .ci/original-rust + persist-credentials: false + - name: Replay admitted original APK bytes without publication eligibility + id: replay + working-directory: .ci/emulebb-build + env: + EMULEBB_WORKSPACE_OUTPUT_ROOT: ${{ runner.temp }}/emulebb-rust-out + GH_TOKEN: ${{ github.token }} + PRODUCER_RUN_ID: ${{ inputs.diagnostic_run_id }} + PRODUCER_SOURCE_SHA: ${{ inputs.diagnostic_source_sha }} + APK_MANIFEST_SHA256: ${{ inputs.diagnostic_manifest_sha256 }} + DIAGNOSTIC_IMAGE: ${{ inputs.diagnostic_image }} + run: | + mkdir -p "$EMULEBB_WORKSPACE_OUTPUT_ROOT" + python -B -m emule_workspace diagnose-rust-android-apk-ci \ + --producer-run-id "$PRODUCER_RUN_ID" --producer-source-sha "$PRODUCER_SOURCE_SHA" \ + --apk-manifest-sha256 "$APK_MANIFEST_SHA256" --image "$DIAGNOSTIC_IMAGE" \ + --diagnostic-build-ref "$EMULEBB_APK_DIAGNOSTIC_BUILD_REF" \ + --diagnostic-harness-ref "$EMULEBB_APK_DIAGNOSTIC_HARNESS_REF" \ + --expected-cert-sha256 "$EMULEBB_APK_DIAGNOSTIC_SIGNER_SHA256" \ + --github-output "$GITHUB_OUTPUT" + # Diagnostic receipts remain ineligible for publication, even after a + # successful replay. Private guest reports, logs and AVDs stay excluded. + - name: Retain only sanitized diagnostic replay evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: emulebb-rust-android-apk-diagnostic-${{ inputs.diagnostic_image }}-${{ github.sha }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/emulebb-rust-out/reports/android-emulator-boot/*/validation.json + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/optional-platforms.yml b/.github/workflows/optional-platforms.yml index bb3c59501..c0f4e5acd 100644 --- a/.github/workflows/optional-platforms.yml +++ b/.github/workflows/optional-platforms.yml @@ -17,10 +17,10 @@ permissions: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - EMULEBB_BUILD_REF: 3b9ddf42466b07f852f798f6967ca782484056a4 - EMULEBB_ANDROID_BUILD_REF: 3b9ddf42466b07f852f798f6967ca782484056a4 - EMULEBB_BUILD_TESTS_REF: 7c680ed68e478c2f22c56c65f21f12c3790d3826 - EMULEBB_TOOLING_REF: 26abd1eab97b28977f2d55adba17e6ba28821498 + EMULEBB_BUILD_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_ANDROID_BUILD_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_BUILD_TESTS_REF: 281ccbcdf9f4c9960c5efda4705df966418b6bb2 + EMULEBB_TOOLING_REF: f88a8ce608bc75a619aea6909f54a9cdecdddb6e # Pin the native NAT sources linked by the sys crates. MINIUPNP_REF: 4e7a10948004ca2085ac8103e14fc77edb898b4c PCPNATPMP_REF: a6e787f837e507336a8774522c33b54264ef4a4b diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3f7c93c9a..4ea90b000 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,10 +51,14 @@ permissions: env: RELEASE_PLATFORM_PROFILE: ${{ inputs.platform_profile || 'full' }} RELEASE_CHANNEL: ${{ inputs.channel || (startsWith(github.ref, 'refs/tags/rust-v') && 'beta') || 'candidate' }} - EMULEBB_BUILD_REF: 3b9ddf42466b07f852f798f6967ca782484056a4 - EMULEBB_LINUX_PREPARED_IMAGE: ghcr.io/emulebb/emulebb-build-ci/ubuntu-rust@sha256:98da7751087bc9a47a597571f99e4bebdccadb5ea80e9757d0f15452a35308bb - EMULEBB_BUILD_TESTS_REF: 7c680ed68e478c2f22c56c65f21f12c3790d3826 - EMULEBB_TOOLING_REF: 26abd1eab97b28977f2d55adba17e6ba28821498 + EMULEBB_BUILD_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_LINUX_PREPARED_IMAGE: ghcr.io/emulebb/emulebb-build-ci/ubuntu-rust@sha256:c94812f3ec606ada847fed87aea7192ba64c1ae3fe11c5cb117f75b6319da62b + EMULEBB_BUILD_TESTS_REF: 281ccbcdf9f4c9960c5efda4705df966418b6bb2 + EMULEBB_TOOLING_REF: f88a8ce608bc75a619aea6909f54a9cdecdddb6e + EMULEBB_RELEASE_VALIDATOR_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_APK_BUILD_REF: 43515795e85beceeb73b1bbbd77e5ab167ef13b6 + EMULEBB_APK_BUILD_TESTS_REF: 281ccbcdf9f4c9960c5efda4705df966418b6bb2 + EMULEBB_ANDROID_SIGNER_SHA256: 9edc742d7230a714a0149c20eb2afb235b8ad5237166090f10e7ac69e2bb359f MINIUPNP_REF: 4e7a10948004ca2085ac8103e14fc77edb898b4c PCPNATPMP_REF: a6e787f837e507336a8774522c33b54264ef4a4b @@ -68,6 +72,7 @@ jobs: native_matrix: ${{ steps.profile.outputs.native_matrix }} release_version: ${{ steps.identity.outputs.version }} source_sha: ${{ steps.source.outputs.sha }} + apk_version_code: ${{ steps.android-code.outputs.apk_version_code }} steps: - name: Checkout trusted release-control helper uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -110,6 +115,9 @@ jobs: env: REQUESTED_VERSION: ${{ inputs.release_version }} run: python tools/release_publish.py release-identity --requested-version "$REQUESTED_VERSION" --channel "$RELEASE_CHANNEL" --ref-name "$GITHUB_REF_NAME" --ref-type "$GITHUB_REF_TYPE" --github-output "$GITHUB_OUTPUT" --source-root .ci/candidate + - name: Derive one UTC Android version code for every producer and proof + id: android-code + run: python tools/release_publish.py android-version-code --github-output "$GITHUB_OUTPUT" - name: Generate and preflight immutable-source release documents env: RELEASE_VERSION: ${{ steps.identity.outputs.version }} @@ -625,6 +633,222 @@ jobs: if-no-files-found: warn retention-days: 30 + android-apk-package: + name: Signed universal Android application + if: ${{ inputs.channel == 'nightly' }} + needs: [prepare, verify-source] + runs-on: ubuntu-24.04 + timeout-minutes: 60 + outputs: + manifest_sha256: ${{ steps.apk.outputs.manifest_sha256 }} + apk_sha256: ${{ steps.apk.outputs.apk_sha256 }} + env: + RELEASE_VERSION: ${{ needs.prepare.outputs.release_version }} + EMULEBB_ANDROID_VERSION_CODE: ${{ needs.prepare.outputs.apk_version_code }} + EMULEBB_WORKSPACE_ROOT: ${{ github.workspace }} + EMULEBB_RUST_REPO: ${{ github.workspace }} + EMULEBB_TOOLING_REPO: ${{ github.workspace }}/.ci/emulebb-tooling + MINIUPNP_ROOT: ${{ github.workspace }}/.ci/emulebb-miniupnp + PCPNATPMP_ROOT: ${{ github.workspace }}/.ci/emulebb-libpcpnatpmp + steps: + - &apk-checkout-source + name: Checkout workflow-bound application source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + - &apk-checkout-build + name: Checkout pinned APK producer + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-build + ref: ${{ env.EMULEBB_APK_BUILD_REF }} + path: .ci/emulebb-build + persist-credentials: false + - &apk-checkout-tooling + name: Checkout pinned APK contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-tooling + ref: ${{ env.EMULEBB_TOOLING_REF }} + path: .ci/emulebb-tooling + persist-credentials: false + - &apk-checkout-miniupnp + name: Checkout pinned APK MiniUPnP + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-miniupnp + ref: ${{ env.MINIUPNP_REF }} + path: .ci/emulebb-miniupnp + persist-credentials: false + - &apk-checkout-pcp + name: Checkout pinned APK PCP/NAT-PMP + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-libpcpnatpmp + ref: ${{ env.PCPNATPMP_REF }} + path: .ci/emulebb-libpcpnatpmp + persist-credentials: false + - &apk-python + name: Set up pinned APK Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.15.2" + - &apk-python-dependencies + name: Install pinned APK owner dependencies + run: python -m pip install click==8.4.1 pydantic==2.14.3 + - name: Set up pinned APK Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "26.13.0" + package-manager-cache: false + - name: Install pinned APK npm + run: npm install --global npm@11.20.0 + - &apk-docker + name: Set up pinned APK Docker + uses: docker/setup-docker-action@3f1ab1ab0e29231562e4db4ca600c1a49653bac6 # v5.7.0 + with: + version: v29.6.1 + daemon-config: '{"features":{"containerd-snapshotter":true}}' + - &apk-buildx + name: Set up pinned APK Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 + with: + version: v0.35.0 + driver: docker + use: false + - name: Build and verify one production-signed APK + id: apk + working-directory: .ci/emulebb-build + env: + EMULEBB_ANDROID_KEYSTORE_BASE64: ${{ secrets.EMULEBB_ANDROID_KEYSTORE_BASE64 }} + EMULEBB_WORKSPACE_OUTPUT_ROOT: ${{ runner.temp }}/emulebb-rust-out + CARGO_TARGET_DIR: ${{ runner.temp }}/emulebb-rust-out/builds/rust/target + EMULEBB_ANDROID_KEY_ALIAS: ${{ secrets.EMULEBB_ANDROID_KEY_ALIAS }} + EMULEBB_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.EMULEBB_ANDROID_KEYSTORE_PASSWORD }} + EMULEBB_ANDROID_KEY_PASSWORD: ${{ secrets.EMULEBB_ANDROID_KEY_PASSWORD }} + EXPECTED_SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + run: | + set -euo pipefail + umask 077 + signing_dir="$(mktemp -d "$RUNNER_TEMP/emulebb-android-signing.XXXXXX")" + trap 'rm -f -- "$signing_dir/release.keystore"; rmdir -- "$signing_dir"' EXIT + printf '%s' "$EMULEBB_ANDROID_KEYSTORE_BASE64" | base64 --decode > "$signing_dir/release.keystore" + export EMULEBB_ANDROID_KEYSTORE="$signing_dir/release.keystore" + unset EMULEBB_ANDROID_KEYSTORE_BASE64 + mkdir -p "$CARGO_TARGET_DIR" + python -B -m emule_workspace package-rust-android-apk-ci \ + --release-version "$RELEASE_VERSION" --channel nightly --expected-source-sha "$EXPECTED_SOURCE_SHA" \ + --expected-cert-sha256 "$EMULEBB_ANDROID_SIGNER_SHA256" --version-code "$EMULEBB_ANDROID_VERSION_CODE" \ + --github-output "$GITHUB_OUTPUT" + - name: Retain exact signed APK payload + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: emulebb-rust-android-apk-payload + path: | + ${{ steps.apk.outputs.apk_path }} + ${{ steps.apk.outputs.manifest_path }} + ${{ steps.apk.outputs.sbom_path }} + if-no-files-found: error + retention-days: 30 + # WHY: compilation can fail before payload publication; retain only the + # producer's nonsecret logs and summary, excluding caches and signing scratch. + - name: Retain Android application build failure diagnostics + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: emulebb-rust-android-apk-failure-${{ github.sha }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/emulebb-rust-out/reports/android-apk/**/*.log + ${{ runner.temp }}/emulebb-rust-out/reports/android-apk/**/build.json + if-no-files-found: warn + retention-days: 30 + + android-apk-qualification: + name: Android application runtime (${{ matrix.image }}) + if: ${{ inputs.channel == 'nightly' }} + needs: [prepare, android-apk-package, android-package] + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + image: [apk31, apk32, apk33, apk34, apk35, apk36, apk37] + env: + RELEASE_VERSION: ${{ needs.prepare.outputs.release_version }} + EMULEBB_ANDROID_VERSION_CODE: ${{ needs.prepare.outputs.apk_version_code }} + EMULEBB_WORKSPACE_ROOT: ${{ github.workspace }} + EMULEBB_RUST_REPO: ${{ github.workspace }} + EMULEBB_TOOLING_REPO: ${{ github.workspace }}/.ci/emulebb-tooling + MINIUPNP_ROOT: ${{ github.workspace }}/.ci/emulebb-miniupnp + PCPNATPMP_ROOT: ${{ github.workspace }}/.ci/emulebb-libpcpnatpmp + EMULEBB_TESTS_REPO: ${{ github.workspace }}/.ci/emulebb-build-tests + steps: + - *apk-checkout-source + - *apk-checkout-build + - *apk-checkout-tooling + - *apk-checkout-miniupnp + - *apk-checkout-pcp + - name: Checkout pinned APK runtime Harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-build-tests + ref: ${{ env.EMULEBB_APK_BUILD_TESTS_REF }} + path: .ci/emulebb-build-tests + persist-credentials: false + - *apk-python + - *apk-python-dependencies + - *apk-docker + - *apk-buildx + - name: Grant this runner access to KVM + run: | + test -c /dev/kvm + sudo setfacl -m "u:$(id -un):rw" /dev/kvm + - name: Download the single signed APK payload + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: emulebb-rust-android-apk-payload + path: ${{ runner.temp }}/emulebb-rust-out/apk-input + - name: Download the same-source native x86_64 peer archive + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: emulebb-rust-android-developer-payload-x86_64 + path: ${{ runner.temp }}/emulebb-rust-out/peer-input + - name: Qualify identical APK bytes on the selected API + id: qualify + working-directory: .ci/emulebb-build + env: + APK_MANIFEST_SHA256: ${{ needs.android-apk-package.outputs.manifest_sha256 }} + EMULEBB_WORKSPACE_OUTPUT_ROOT: ${{ runner.temp }}/emulebb-rust-out + CARGO_TARGET_DIR: ${{ runner.temp }}/emulebb-rust-out/builds/rust/target + EXPECTED_SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + run: >- + python -B -m emule_workspace qualify-rust-android-apk-ci --image ${{ matrix.image }} + --apk-payload "$EMULEBB_WORKSPACE_OUTPUT_ROOT/apk-input" + --apk-manifest-name "emulebb-rust-v$RELEASE_VERSION-android-universal.apk.manifest.json" + --apk-manifest-sha256 "$APK_MANIFEST_SHA256" --apk-source-sha "$EXPECTED_SOURCE_SHA" + --expected-cert-sha256 "$EMULEBB_ANDROID_SIGNER_SHA256" --apk-version-code "$EMULEBB_ANDROID_VERSION_CODE" + --release-version "$RELEASE_VERSION" --peer-artifacts "$EMULEBB_WORKSPACE_OUTPUT_ROOT/peer-input" + --peer-build-ref "$EMULEBB_BUILD_REF" --apk-build-ref "$EMULEBB_APK_BUILD_REF" + --apk-harness-ref "$EMULEBB_APK_BUILD_TESTS_REF" --soak-seconds 60 --github-output "$GITHUB_OUTPUT" + - name: Retain sanitized API-specific proof + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: emulebb-rust-android-apk-proof-${{ matrix.image }} + path: ${{ steps.qualify.outputs.qualification_path }} + if-no-files-found: error + retention-days: 30 + # WHY: public failure evidence can exist before qualification outputs; + # retain only the outer sanitized report, never private guest data or logs. + - name: Retain sanitized Android application runtime failure + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: emulebb-rust-android-apk-runtime-failure-${{ matrix.image }}-${{ github.sha }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/emulebb-rust-out/reports/android-emulator-boot/*/validation.json + if-no-files-found: warn + retention-days: 30 + image-candidate: name: Build multiarch ${{ matrix.variant }} image candidate needs: [prepare, native-package, musl-package] @@ -821,21 +1045,32 @@ jobs: retention-days: 30 assemble-candidate: - name: Assemble verified native and container candidate + name: Assemble verified application, native and container candidate if: >- ${{ !cancelled() && needs.prepare.result == 'success' && needs.native-package-smoke.result == 'success' && needs.musl-package.result == 'success' && needs.image-native-validation.result == 'success' && (needs.android-package.result == 'success' || - (inputs.channel != 'nightly' && needs.android-package.result == 'skipped')) }} - needs: [prepare, native-package-smoke, musl-package, image-native-validation, android-package] + (inputs.channel != 'nightly' && needs.android-package.result == 'skipped')) && + (needs.android-apk-package.result == 'success' || + (inputs.channel != 'nightly' && needs.android-apk-package.result == 'skipped')) && + (needs.android-apk-qualification.result == 'success' || + (inputs.channel != 'nightly' && needs.android-apk-qualification.result == 'skipped')) }} + needs: [prepare, native-package-smoke, musl-package, image-native-validation, android-package, android-apk-package, android-apk-qualification] runs-on: ubuntu-24.04 timeout-minutes: 15 outputs: candidate_sha256: ${{ steps.candidate.outputs.candidate_sha256 }} env: + EMULEBB_WORKSPACE_ROOT: ${{ github.workspace }} + EMULEBB_RUST_REPO: ${{ github.workspace }}/emulebb-rust + EMULEBB_TOOLING_REPO: ${{ github.workspace }}/emulebb-tooling + EMULEBB_TESTS_REPO: ${{ github.workspace }}/emulebb-apk-tests + MINIUPNP_ROOT: ${{ github.workspace }}/.ci/emulebb-miniupnp + PCPNATPMP_ROOT: ${{ github.workspace }}/.ci/emulebb-libpcpnatpmp RELEASE_VERSION: ${{ needs.prepare.outputs.release_version }} + EMULEBB_ANDROID_VERSION_CODE: ${{ needs.prepare.outputs.apk_version_code }} EMULEBB_SOURCE_REVISION: ${{ needs.prepare.outputs.source_sha }} steps: - name: Checkout trusted release-control source @@ -848,7 +1083,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: emulebb/emulebb-build - ref: ${{ env.EMULEBB_BUILD_REF }} + ref: ${{ env.EMULEBB_RELEASE_VALIDATOR_REF }} path: emulebb-build persist-credentials: false - name: Checkout pinned image evidence reader @@ -865,6 +1100,16 @@ jobs: ref: ${{ env.EMULEBB_TOOLING_REF }} path: emulebb-tooling persist-credentials: false + - name: Checkout independently pinned APK evidence reader + if: ${{ inputs.channel == 'nightly' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-build-tests + ref: ${{ env.EMULEBB_APK_BUILD_TESTS_REF }} + path: emulebb-apk-tests + persist-credentials: false + - *apk-checkout-miniupnp + - *apk-checkout-pcp - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: @@ -890,6 +1135,19 @@ jobs: pattern: emulebb-rust-android-developer-payload-* merge-multiple: true path: ${{ runner.temp }}/emulebb-rust-out/release/rust-v${{ env.RELEASE_VERSION }} + - name: Download the single signed Android application + if: ${{ inputs.channel == 'nightly' }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: emulebb-rust-android-apk-payload + path: ${{ runner.temp }}/emulebb-rust-out/release/rust-v${{ env.RELEASE_VERSION }} + - name: Download seven separate Android runtime proofs + if: ${{ inputs.channel == 'nightly' }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: emulebb-rust-android-apk-proof-* + merge-multiple: false + path: ${{ runner.temp }}/emulebb-rust-out/apk-evidence - name: Download every selected native package lifecycle proof uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -936,6 +1194,23 @@ jobs: with: name: emulebb-rust-release-documents path: ${{ runner.temp }}/emulebb-rust-out/release/rust-v${{ env.RELEASE_VERSION }} + - name: Bind seven API proofs to the same signed APK and trusted authorities + if: ${{ inputs.channel == 'nightly' }} + working-directory: emulebb-build + env: + APK_MANIFEST_SHA256: ${{ needs.android-apk-package.outputs.manifest_sha256 }} + EMULEBB_WORKSPACE_OUTPUT_ROOT: ${{ runner.temp }}/emulebb-rust-out + CARGO_TARGET_DIR: ${{ runner.temp }}/emulebb-rust-out/builds/rust/target + RELEASE_DIR: ${{ runner.temp }}/emulebb-rust-out/release/rust-v${{ env.RELEASE_VERSION }} + run: >- + python -B -m emule_workspace assemble-rust-android-apk-evidence-ci + --apk-payload "$RELEASE_DIR" + --apk-manifest-name "emulebb-rust-v$RELEASE_VERSION-android-universal.apk.manifest.json" + --apk-manifest-sha256 "$APK_MANIFEST_SHA256" --apk-source-sha "$EMULEBB_SOURCE_REVISION" + --expected-cert-sha256 "$EMULEBB_ANDROID_SIGNER_SHA256" --apk-version-code "$EMULEBB_ANDROID_VERSION_CODE" + --release-version "$RELEASE_VERSION" --apk-build-ref "$EMULEBB_APK_BUILD_REF" + --apk-harness-ref "$EMULEBB_APK_BUILD_TESTS_REF" + --evidence-dir "$EMULEBB_WORKSPACE_OUTPUT_ROOT/apk-evidence" --release-dir "$RELEASE_DIR" - name: Verify all payloads and checksum the final public evidence set working-directory: emulebb-build env: @@ -954,7 +1229,7 @@ jobs: --release-dir "$RELEASE_DIR" --output "$RELEASE_DIR/emulebb-rust-v$RELEASE_VERSION-candidate-manifest.json" --archives-dir "$EMULEBB_WORKSPACE_OUTPUT_ROOT/artifacts" - --build-repo emulebb-build --harness-repo emulebb-build-tests + --build-repo emulebb-build --apk-harness-repo emulebb-apk-tests --harness-repo emulebb-build-tests --github-output "$GITHUB_OUTPUT" - name: Retain the exact public release candidate uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -985,6 +1260,7 @@ jobs: alpine_digest: ${{ steps.published.outputs.alpine_digest }} env: RELEASE_VERSION: ${{ needs.prepare.outputs.release_version }} + EMULEBB_ANDROID_VERSION_CODE: ${{ needs.prepare.outputs.apk_version_code }} EMULEBB_SOURCE_REVISION: ${{ needs.prepare.outputs.source_sha }} steps: - name: Checkout trusted publication helpers @@ -997,7 +1273,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: emulebb/emulebb-build - ref: ${{ env.EMULEBB_BUILD_REF }} + ref: ${{ env.EMULEBB_RELEASE_VALIDATOR_REF }} path: emulebb-build persist-credentials: false - name: Checkout pinned OCI graph reader @@ -1007,6 +1283,14 @@ jobs: ref: ${{ env.EMULEBB_BUILD_TESTS_REF }} path: emulebb-build-tests persist-credentials: false + - name: Checkout independently pinned APK evidence reader + if: ${{ inputs.channel == 'nightly' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-build-tests + ref: ${{ env.EMULEBB_APK_BUILD_TESTS_REF }} + path: emulebb-apk-tests + persist-credentials: false - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: @@ -1044,7 +1328,7 @@ jobs: run: >- python -B emulebb-rust/tools/release_publish.py publish-images --release-dir "$RELEASE_DIR" --archives-dir "$EMULEBB_WORKSPACE_OUTPUT_ROOT/artifacts" - --build-repo emulebb-build --harness-repo emulebb-build-tests + --build-repo emulebb-build --apk-harness-repo emulebb-apk-tests --harness-repo emulebb-build-tests --assembled-candidate-sha256 "$ASSEMBLED_CANDIDATE_SHA" --github-output "$GITHUB_OUTPUT" - name: Attest the published Ubuntu digest uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 @@ -1075,6 +1359,7 @@ jobs: id-token: write env: RELEASE_VERSION: ${{ needs.prepare.outputs.release_version }} + EMULEBB_ANDROID_VERSION_CODE: ${{ needs.prepare.outputs.apk_version_code }} EMULEBB_SOURCE_REVISION: ${{ needs.prepare.outputs.source_sha }} VERIFIED_CANDIDATE_SHA: ${{ needs.publish-image.outputs.candidate_sha256 }} steps: @@ -1088,9 +1373,17 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: emulebb/emulebb-build - ref: ${{ env.EMULEBB_BUILD_REF }} + ref: ${{ env.EMULEBB_RELEASE_VALIDATOR_REF }} path: emulebb-build persist-credentials: false + - name: Checkout independently pinned APK evidence reader + if: ${{ inputs.channel == 'nightly' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: emulebb/emulebb-build-tests + ref: ${{ env.EMULEBB_APK_BUILD_TESTS_REF }} + path: emulebb-apk-tests + persist-credentials: false - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: @@ -1109,7 +1402,7 @@ jobs: run: | set -euo pipefail python -B emulebb-rust/tools/release_publish.py release-assets \ - --release-dir "$RELEASE_DIR" --build-repo emulebb-build \ + --release-dir "$RELEASE_DIR" --build-repo emulebb-build --apk-harness-repo emulebb-apk-tests \ --verified-candidate-sha256 "$VERIFIED_CANDIDATE_SHA" --phase before-upload - name: Attest the exact native release candidate uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 @@ -1135,7 +1428,7 @@ jobs: run: | set -euo pipefail python -B emulebb-rust/tools/release_publish.py release-assets \ - --release-dir "$RELEASE_DIR" --build-repo emulebb-build \ + --release-dir "$RELEASE_DIR" --build-repo emulebb-build --apk-harness-repo emulebb-apk-tests \ --verified-candidate-sha256 "$VERIFIED_CANDIDATE_SHA" --phase after-upload --release-id "$RELEASE_ID" gh api --method PATCH "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" -F draft=false diff --git a/Cargo.lock b/Cargo.lock index 514ebbf2b..9ab5fca93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -569,6 +569,19 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba92c8f4591bf29f89716b45e5f5f56577adbb81426e5620c6e994da7c88bb7a" +[[package]] +name = "emulebb-android" +version = "0.1.0-beta.2" +dependencies = [ + "anyhow", + "emulebb-daemon", + "jni", + "serde", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "emulebb-core" version = "0.1.0-beta.2" @@ -614,9 +627,11 @@ version = "0.1.0-beta.2" dependencies = [ "anyhow", "axum", + "chrono", "clap", "embed-manifest", "emulebb-core", + "emulebb-diagnostics", "emulebb-ed2k", "emulebb-index", "emulebb-kad-dht", @@ -826,10 +841,12 @@ dependencies = [ "axum", "chrono", "emulebb-core", + "emulebb-diagnostics", "emulebb-ed2k", "emulebb-index", "futures-util", "http-body-util", + "libc", "percent-encoding", "rusqlite", "serde", @@ -838,6 +855,7 @@ dependencies = [ "tokio", "tower", "tower-http 0.7.1", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1a20c52b7..fa459a498 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "crates/emulebb-android", "crates/emulebb-core", "crates/emulebb-daemon", "crates/emulebb-diagnostics", @@ -57,6 +58,7 @@ if-addrs = "0.15" ipnet = "2" ipconfig = "0.3" libc = "0.2" +jni = "0.22.4" md4 = "0.11" md5 = "0.8" notify = "8" diff --git a/README.md b/README.md index b8a5c7133..637c9ce6b 100644 --- a/README.md +++ b/README.md @@ -1,147 +1,156 @@

- eMuleBB pixel donkey + eMuleBB Rust: eD2K and Kad with browser control on Windows, Linux and macOS

-# eMuleBB Rust +

eMuleBB Rust

-An IPv4 eD2K/Kad client for Windows, Linux and macOS. Search, download and share -from one headless daemon, with a browser console for everyday control and a REST -API for automation. - -**Active beta.** Dated nightly prereleases are the current public testing channel. - -| ↓ Use eMuleBB Rust | ⌘ Develop with eMuleBB Rust | -| --- | --- | -| [Download a nightly](https://github.com/emulebb/emulebb-rust/releases) | [Build and contribute](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/reference/GUIDE-CONTRIBUTING/) | -| [Get started](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/reference/GUIDE-GETTING-STARTED/) | [Connect to the API](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/api/GUIDE-API-CONSUMERS/) | +

+ eD2K and Kad. Browser control. REST automation.
+ Search, download and share with one headless IPv4 daemon. +

-## Contents +

+ Use eMuleBB Rust  ·  + Develop with eMuleBB Rust +

+

+ Nightly downloads · + Platforms · + REST API · + Roadmap · + Help +

+

+ Rust CI + CodeQL workflow + GPL-2.0-only +

-- [What you can do](#what-you-can-do) -- [Downloads and platforms](#downloads-and-platforms) -- [Get started](#get-started) -- [Use the API](#use-the-api) -- [Develop and contribute](#develop-and-contribute) -- [Help and security](#help-and-security) -- [License](#license) +**Active beta.** Dated nightly prereleases are the current public testing +channel. Prebuilt packages need no Rust toolchain. ## What you can do -- Search eD2K/Kad and manage downloads from the browser console. -- Share monitored folder trees and choose network and sharing settings. -- Automate client operations through an authenticated, documented REST API. -- Inspect transfer and log state, and identify packages by source revision and - published checksums. + + + + + + + + + +
+

Search and transfer

+

Search eD2K/Kad, add downloads and follow transfer progress from the packaged browser console.

+
+

Share chosen folders

+

Monitor recursive folder trees, review hashing and manage sharing settings.

+
+

Automate through REST

+

Connect integrations to an authenticated API with a documented OpenAPI contract.

+
+

Inspect your client

+

Review network status, diagnostics, logs and source-bound package checksums.

+
## Downloads and platforms -Choose the matching runnable package from the -[nightly releases](https://github.com/emulebb/emulebb-rust/releases). -Verify it against that release's `SHA256SUMS` before launch. No Rust toolchain is -needed to use a prebuilt package. +Choose a runnable asset from the [nightly releases](https://github.com/emulebb/emulebb-rust/releases), +then compare its exact filename against that release's `SHA256SUMS`. +The [platform run guide](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md) +owns installation commands, profile paths, requirements and qualification scope. -| Platform | Architecture | Distribution | +| Platform | Choose | Install and run | | --- | --- | --- | -| Windows | x64 / ARM64 | ZIP | -| Linux | x64 / ARM64 | DEB, AppImage, static-musl tar/zstd | -| macOS | Intel / Apple Silicon | DMG; unsigned and not notarized | -| Docker / OCI | Linux amd64 / arm64 | GHCR Ubuntu and Alpine images | -| Android developers | arm64-v8a / x86_64 | Experimental native CLI bundles; **not an APK** | - -Container images are published at -[`ghcr.io/emulebb/emulebb-rust`](https://github.com/emulebb/emulebb-rust/pkgs/container/emulebb-rust). -Use `nightly` or `nightly-ubuntu` for Ubuntu, and `nightly-alpine` for Alpine. -Versioned tags are available; pin a verified digest for a repeatable deployment. -There is no `latest` tag. - -Native macOS package smoke runs on Intel and Apple Silicon. Android bundles -remain experimental native CLI packages. An API 37/x86_64/16 KiB emulator -[check](https://github.com/emulebb/emulebb-build/actions/runs/43369243640) -for the [tested source](https://github.com/emulebb/emulebb-rust/commit/6429b952eac6f19ee9bf084d3a84d5649dfba0c2) -verifies six TLS cases, fresh-profile authenticated REST startup and graceful -shutdown with a source-built release-mode daemon on the development channel. -Published Android archive execution, physical devices, ARM runtime, APK, WebUI, -completed-file delivery and VPN behavior remain unqualified. Linux musl bundles -are separate from Android. - -**Android HTTPS trust:** native CLI bundles verify certificates and hostnames -with packaged Mozilla CA roots, without a JVM. Android/Termux CA stores and -`SSL_CERT_FILE` do not extend this trust set. Updated roots arrive with new -client builds. - -See [downloads and platforms](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS/) -for requirements, installation, container mounts and the scope of package checks. +| Windows | x64 or ARM64 ZIP | [Extract and launch](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md#windows-x64-and-arm64) | +| Linux | x64 or ARM64 DEB, AppImage or static-musl tar/zstd | [Choose a Linux variant](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md#linux) | +| macOS | Intel or Apple Silicon DMG | [Copy and launch the app](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md#macos-intel-and-apple-silicon) | +| Docker / OCI | Linux amd64 or arm64; Ubuntu or Alpine | [Run with persistent volumes](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md#containers) | +| Android APK | Universal ARM64 + x86_64; Android 12+ | [Permissions, manual Start and WebUI](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md#android-application) | +| Android / Termux native | Experimental `android-arm64-v8a.tar.zst` or `android-x86_64.tar.zst` | [Extract under private HOME and launch](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md#termux-native-shell) | + +The macOS app is unsigned and not notarized. For containers, both +[`ghcr.io/emulebb/emulebb-rust`](https://github.com/emulebb/emulebb-rust/pkgs/container/emulebb-rust) +variants publish versioned tags and moving `nightly` / `nightly-ubuntu` / +`nightly-alpine` aliases; there is no `latest` tag. + +**Android remains experimental.** Use the signed APK only when the selected +published release includes it after qualification. Both native ABI archives +remain separate public nightly assets alongside that APK. Their API 28 binary +floor and compile/ELF evidence do not qualify Termux runtime, the full Android +API matrix, Fire OS or 24-hour continuous operation. The APK minimum is Android +12/API 31; its foreground service starts manually and has no boot-resume option. +See [Android evidence and remaining acceptance](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/active/items/RUST-FEAT-056.md). ## Get started -1. Install or extract the package for your system. Keep the complete layout, - including the adjacent `webui` directory when using an archive. -2. Run `emulebb-rust.exe` on Windows, `./emulebb-rust` from a Linux archive, or - the copied macOS app. Direct daemon launches use the directory containing - the executable for TOML and database files, regardless of the shell working - directory. Use `--profile ` to select another profile; - missing profiles are created. `--profile .` selects the shell working - directory, so it matches the default only when that is the executable folder. - Desktop app launchers and Docker explicitly select the documented writable - profile paths; the daemon never silently falls back to a home directory. -3. Open **** and enter - **`change_me_123change_me_123change_me_123`**. Review network and sharing - settings before starting transfers. - -The shipped API key is a shared starter value. Change `rest.api_key` in your -private bootstrap file before exposing the API outside this computer. The -default P2P ports are TCP **20019** and UDP **20020**; mutable client settings -remain in the profile database. - -The default REST address is local control, not an offline-P2P guarantee: startup -may fetch discovery inputs and start configured networking. - -**Profiles during beta:** an incompatible database requires an entirely fresh -profile. Keep the stopped old profile, payloads and matching binary intact for -explicit rollback; automatic unfinished-transfer continuation is not promised. - -The [getting-started guide](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/reference/GUIDE-GETTING-STARTED/) -covers profile locations, discovery, search, downloads, sharing and troubleshooting. +1. Follow your [platform's run instructions](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-DOWNLOADS-PLATFORMS.md). + Keep the complete extracted archive layout, including `webui`. +2. Open **** on the daemon host. The default listener + is **`0.0.0.0:20017`**. Native packages use the ready key + **`change_me_123change_me_123change_me_123`**; the APK supplies a generated + key through **Copy API key** and **Open WebUI**. +3. Review network, incoming and shared-folder settings, then search or add a + download. Default P2P ports are TCP **20019** and UDP **20020**. + +Direct daemon launches keep the profile beside the executable, independently +of the working directory. Select another writable profile with `--profile`. +Desktop launchers, containers and the APK explicitly choose their documented +profile paths. The private `emulebb-rust-settings.toml` owns REST bootstrap +settings; mutable client settings live in the profile database. + +Startup may fetch discovery inputs and start configured networking. During beta, +an incompatible database requires an entirely fresh profile; preserve the +stopped old profile, payloads and matching binary for rollback. The +[getting-started guide](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-GETTING-STARTED.md) +covers everyday operation, shutdown and troubleshooting. ## Use the API -All `/api/v1` routes use the **`X-API-Key`** header. The daemon serves HTTP; -keep control on loopback or use a trusted TLS reverse proxy with restricted -access. Keep the key out of URLs, committed scripts and public logs. +All `/api/v1` routes use the **`X-API-Key`** header. Start with read-only +`GET /api/v1/capabilities` and `GET /api/v1/status`; keep credentials in headers. -Start with authenticated, read-only requests such as -`GET /api/v1/capabilities` and `GET /api/v1/status`. -The [API guide](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/api/GUIDE-API-CONSUMERS/) -includes private-key prompts, request examples and response handling. The -[OpenAPI contract](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/api/REST-API-OPENAPI.yaml) -defines the routes and schemas. The beta API is still evolving; check the -contract for the version you use. +- [API consumer guide](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/api/GUIDE-API-CONSUMERS/) +- [OpenAPI contract](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/api/REST-API-OPENAPI.yaml) -## Develop and contribute +The beta API is still evolving; use the contract for your selected build. -The [contributor guide](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/reference/GUIDE-CONTRIBUTING/) -walks through prerequisites, managed workspace setup, source orientation and -supported checks. See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution details -and the [roadmap](https://github.com/orgs/emulebb/projects/3) for current work. +
+Android native HTTPS trust -Contributions can improve the client, browser console, integrations, tests or -documentation. Coordinate an actionable issue before implementation. +Native CLI bundles verify certificates and hostnames with packaged Mozilla CA +roots, without a JVM. Android/Termux CA stores and `SSL_CERT_FILE` do not extend +this trust set. Updated roots arrive with new client builds. + +
+ +## Develop and contribute -Eligible root `README.md` edits use a trusted documentation-only CI path. Source, -packaging and workflow changes receive full qualification; required checks remain -mandatory. For parallel work in the managed workspace, the parent coordinator -assigns non-overlapping branches and batches compatible authored changes for -final integration testing. Follow the contributor guide for supported checks. + + + + + +
+

Work on the client

+

Improve the daemon, browser console, tests or documentation.

+Build and validation guide · Contributing +
+

Find the next change

+

Review current work and coordinate an actionable issue before implementation.

+Roadmap · Issues +
## Help and security -- **Troubleshooting:** [startup, listeners and logs](https://emulebb.github.io/emulebb-tooling/products/emulebb-rust/reference/GUIDE-GETTING-STARTED/#common-problems). -- **Bug reports:** [open an issue](https://github.com/emulebb/emulebb-rust/issues) - with the package version and sanitized reproduction steps. -- **Vulnerabilities:** follow the [private security reporting policy](SECURITY.md). +- [Troubleshoot startup, listeners and logs](https://github.com/emulebb/emulebb-tooling/blob/f88a8ce608bc75a619aea6909f54a9cdecdddb6e/docs/products/emulebb-rust/reference/GUIDE-GETTING-STARTED.md#common-problems). +- [Report a bug](https://github.com/emulebb/emulebb-rust/issues) with the package + version and sanitized reproduction steps. +- Follow [private security reporting](SECURITY.md) for vulnerabilities. -Diagnostic bundles are private operator evidence and can contain API keys, -paths, identifiers and logs. Never attach them unreviewed to a public report. +Diagnostic bundles are private operator evidence and can contain credentials, +paths, identifiers and logs. Review them before sharing publicly. ## License diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts new file mode 100644 index 000000000..d3f5be333 --- /dev/null +++ b/android/app/build.gradle.kts @@ -0,0 +1,179 @@ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import java.security.MessageDigest +import java.util.HexFormat +import javax.xml.XMLConstants +import javax.xml.parsers.DocumentBuilderFactory +import org.gradle.api.artifacts.component.ModuleComponentIdentifier +import org.w3c.dom.Element + +plugins { id("com.android.application") } + +dependencies { implementation("androidx.core:core:1.19.1") } + +abstract class StageAssets : Sync() { + @get:OutputDirectory + abstract val outputDirectory: DirectoryProperty +} + +/** Export only selected application runtime artifacts, never the compiler/plugin graph. */ +abstract class EmulebbRuntimeInventory : DefaultTask() { + @get:Input abstract val variant: Property + @get:Input abstract val componentDescriptions: ListProperty + @get:InputFiles @get:PathSensitive(PathSensitivity.NONE) + abstract val sourceFiles: ConfigurableFileCollection + @get:OutputDirectory abstract val destination: DirectoryProperty + + @TaskAction fun export() { + val root = destination.get().asFile + require(!root.exists() || root.listFiles()?.isEmpty() == true) { "Runtime inventory output must be new or empty" } + check(root.mkdirs() || root.isDirectory) + fun exportFile(source: File, folder: File): Map { + require(source.isFile && source.name.matches(Regex("[A-Za-z0-9_.-]+"))) + val target = File(folder, source.name) + check(target.parentFile.mkdirs() || target.parentFile.isDirectory) + source.copyTo(target, overwrite = false) + val digest = MessageDigest.getInstance("SHA-256") + source.inputStream().use { input -> + val buffer = ByteArray(65536) + while (true) { + val size = input.read(buffer) + if (size < 0) break + digest.update(buffer, 0, size) + } + } + return mapOf("path" to target.relativeTo(root).invariantSeparatorsPath, + "sha256" to HexFormat.of().formatHex(digest.digest()), "sizeBytes" to target.length()) + } + val components = componentDescriptions.get().map { description -> + val component = JsonSlurper().parseText(description) as Map<*, *> + val group = component["group"] as String + val name = component["name"] as String + val version = component["version"] as String + require(listOf(group, name, version).all { it.matches(Regex("[A-Za-z0-9_.-]+")) && it != "." && it != ".." }) + val folder = File(root, "artifacts/${group.replace('.', '/')}/$name/$version") + val pom = exportFile(File(component["pom"] as String), folder) + val parents = (component["pomParents"] as List<*>).mapIndexed { index, parent -> + exportFile(File(parent as String), File(folder, "parents/$index")).filterKeys { it != "sizeBytes" } + } + val artifacts = (component["artifacts"] as List<*>).map { exportFile(File(it as String), folder) } + mapOf("group" to group, "name" to name, "version" to version, + "pom" to pom.filterKeys { it != "sizeBytes" }, "pomParents" to parents, "artifacts" to artifacts) + } + val report = mapOf("schema" to "emulebb.android.runtime-classpath/1", "variant" to variant.get(), + "configuration" to "${variant.get()}RuntimeClasspath", "components" to components) + File(root, "report.json").writeText(JsonOutput.prettyPrint(JsonOutput.toJson(report)) + "\n") + } +} + +android { + namespace = "org.emulebb.android" + compileSdk { version = release(37) { minorApiLevel = 2 } } + buildToolsVersion = "37.0.0" + ndkVersion = "30.0.16248370" + defaultConfig { + applicationId = "org.emulebb.android" + minSdk = 31 + targetSdk = 37 + versionCode = providers.gradleProperty("emulebbVersionCode").get().toInt() + versionName = providers.gradleProperty("emulebbVersionName").get() + ndk { abiFilters.addAll(listOf("arm64-v8a", "x86_64")) } + } + buildFeatures { buildConfig = true } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + sourceSets.getByName("main") { + jniLibs.directories.add(providers.gradleProperty("emulebbNativeDir").get()) + } + packaging { + jniLibs { + useLegacyPackaging = false + // Exact source-bound native bytes are audited by the producer and APK harness. + keepDebugSymbols.add("**/*.so") + } + } +} + +val stageAssets = tasks.register("stageAssets") { + outputDirectory.set(layout.buildDirectory.dir("generated/assets")) + into(outputDirectory) + from(providers.gradleProperty("emulebbWebuiDir")) { into("webui") } + from(providers.gradleProperty("emulebbNoticesDir")) { into("licenses") } +} +androidComponents.onVariants { variant -> + variant.sources.assets?.addGeneratedSourceDirectory(stageAssets, StageAssets::outputDirectory) +} + +val inventoryVariant = providers.gradleProperty("emulebbRuntimeVariant").orElse("release") +val inventoryComponents = providers.provider { + val variant = inventoryVariant.get() + require(variant in listOf("debug", "release")) + fun resolvePom(group: String, name: String, version: String): File = + configurations.detachedConfiguration(dependencies.create("$group:$name:$version@pom")) + .apply { isTransitive = false }.singleFile + fun child(element: Element, name: String): Element? { + val found = (0 until element.childNodes.length).map { element.childNodes.item(it) } + .filterIsInstance().filter { (it.localName ?: it.tagName) == name } + require(found.size <= 1) { "Maven POM has duplicate $name elements" } + return found.singleOrNull() + } + fun parentOf(pom: File): Triple? { + val factory = DocumentBuilderFactory.newInstance().apply { + isNamespaceAware = true + isXIncludeAware = false + isExpandEntityReferences = false + setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true) + setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "") + setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "") + } + val project = factory.newDocumentBuilder().parse(pom).documentElement + require((project.localName ?: project.tagName) == "project") { "Runtime POM must contain a Maven project" } + val parent = child(project, "parent") ?: return null + fun coordinate(name: String): String { + val value = child(parent, name)?.textContent?.trim() ?: error("Maven parent lacks $name") + require(value.matches(Regex("[A-Za-z0-9_.-]+")) && value != "." && value != "..") { + "Maven parent requires concrete coordinates" + } + return value + } + return Triple(coordinate("groupId"), coordinate("artifactId"), coordinate("version")) + } + val artifacts = configurations.getByName("${variant}RuntimeClasspath").incoming.artifacts.resolvedArtifacts.get() + artifacts.groupBy { + it.id.componentIdentifier as? ModuleComponentIdentifier + ?: error("Runtime inventory requires resolved external Maven components") + }.entries.sortedBy { "${it.key.group}:${it.key.module}:${it.key.version}" }.map { (id, files) -> + val pom = resolvePom(id.group, id.module, id.version) + val parents = mutableListOf() + val seen = mutableSetOf(Triple(id.group, id.module, id.version)) + var parent = parentOf(pom) + while (parent != null) { + require(parents.size < 8 && seen.add(parent)) { "Maven parent chain exceeds its bound or contains a cycle" } + val selected = parent + val original = resolvePom(selected.first, selected.second, selected.third) + parents.add(original) + parent = parentOf(original) + } + mapOf("group" to id.group, "name" to id.module, "version" to id.version, + "pom" to pom.absolutePath, "pomParents" to parents.map { it.absolutePath }, + "artifacts" to files.map { it.file.absolutePath }.sorted()) + } +} +tasks.register("emulebbRuntimeInventory") { + variant.set(inventoryVariant) + componentDescriptions.set(inventoryComponents.map { components -> components.map(JsonOutput::toJson) }) + sourceFiles.from(inventoryComponents.map { components -> components.flatMap { + listOf(File(it["pom"] as String)) + (it["artifacts"] as List<*>).map { artifact -> File(artifact as String) } + + (it["pomParents"] as List<*>).map { parent -> File(parent as String) } + } }) + destination.set(layout.dir(providers.gradleProperty("emulebbRuntimeInventoryDir").map { output -> + val folder = file(output) + require(folder.isAbsolute && !folder.canonicalFile.toPath().startsWith(rootProject.projectDir.canonicalFile.toPath())) { + "Runtime inventory must use an absolute external output directory" + } + folder + })) +} diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..4c97bbbda --- /dev/null +++ b/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/app/src/main/java/org/emulebb/android/DaemonService.kt b/android/app/src/main/java/org/emulebb/android/DaemonService.kt new file mode 100644 index 000000000..37f86d505 --- /dev/null +++ b/android/app/src/main/java/org/emulebb/android/DaemonService.kt @@ -0,0 +1,356 @@ +package org.emulebb.android + +import android.Manifest +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Context +import android.content.Intent +import android.content.pm.ServiceInfo +import android.content.pm.PackageManager +import android.net.ConnectivityManager +import android.net.Network +import android.net.LinkProperties +import android.os.Bundle +import android.os.Build +import android.os.Environment +import android.os.Handler +import android.os.IBinder +import android.os.Looper +import android.os.Message +import android.os.Messenger +import android.os.PowerManager +import android.os.SystemClock +import org.json.JSONObject +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicBoolean + +/** Owns service intent and the native runtime; no client settings are duplicated here. */ +class DaemonService : Service() { + companion object { + const val START = "org.emulebb.android.START" + const val STOP = "org.emulebb.android.STOP" + const val STATUS = 1 + const val PREFERENCES = "service-lifecycle" + const val DESIRED = "desired-running" + const val LOCAL_NETWORK = Manifest.permission.ACCESS_LOCAL_NETWORK + private const val CHANNEL = "daemon" + private const val NOTICE = 1 + } + + private fun hasLocalNetworkPermission() = Build.VERSION.SDK_INT < 37 || + checkSelfPermission(LOCAL_NETWORK) == PackageManager.PERMISSION_GRANTED + + private val handler = Handler(Looper.getMainLooper()) + private val commands = Executors.newSingleThreadExecutor() + private val desired = AtomicBoolean(false) + private val networkNotificationPending = AtomicBoolean(false) + private val prefs by lazy { getSharedPreferences(PREFERENCES, Context.MODE_PRIVATE) } + private var phase = "Stopped" + private var error: String? = null + private var address: String? = null + private var starting = false + private var wakeLock: PowerManager.WakeLock? = null + private var network: Network? = null + private var restartForNetwork = false + private var stopFailures = 0 + private var notifiedPhase: String? = null + private var waitingForNetwork = false + private var destroyed = false + private val connectivity by lazy { requireNotNull(getSystemService(ConnectivityManager::class.java)) } + + private val networkCallback = object : ConnectivityManager.NetworkCallback() { + override fun onAvailable(available: Network) { + handler.post { + if (destroyed) return@post + val changed = network != null && network != available + network = available + notifyNetworkChanged() + if (changed && desired.get() && phase == "Running") { + restartForNetwork = true + phase = "Stopping" + requestNativeStop() + } + } + } + + override fun onLinkPropertiesChanged(network: Network, properties: LinkProperties) { + handler.post { notifyNetworkChanged() } + } + + override fun onLost(network: Network) { + handler.post { notifyNetworkChanged() } + } + } + + private fun notifyNetworkChanged() { + if (destroyed || !desired.get()) return + if (!networkNotificationPending.compareAndSet(false, true)) return + commands.execute { + // Coalesce callbacks while setup occupies the command executor. + // Reset before JNI so an event racing this call retains a final wake. + networkNotificationPending.set(false) + try { if (desired.get()) NativeBridge.call("networkChanged") } + catch (failure: Exception) { + handler.post { if (!destroyed && desired.get()) fail(failure.message ?: "Cannot notify native network change") } + } + } + } + + private val messenger = Messenger(object : Handler(Looper.getMainLooper()) { + override fun handleMessage(message: Message) { + // The service is non-exported; Binder calls remain within this application's UID. + if (message.what == STATUS) { + val response = Message.obtain(null, STATUS).apply { + data = Bundle().apply { + putString("phase", phase) + putString("error", error) + putString("address", address) + putBoolean("waitingForNetwork", waitingForNetwork) + } + } + try { message.replyTo?.send(response) } catch (_: android.os.RemoteException) { } + } + } + }) + + override fun onCreate() { + super.onCreate() + ServiceRecovery.mayResume(this) + error = prefs.getString("last-error", null) + if (error != null) phase = "Failed" + requireNotNull(getSystemService(NotificationManager::class.java)).createNotificationChannel( + NotificationChannel(CHANNEL, "eMuleBB service", NotificationManager.IMPORTANCE_LOW)) + network = connectivity.activeNetwork + connectivity.registerDefaultNetworkCallback(networkCallback) + } + + override fun onBind(intent: Intent): IBinder = messenger.binder + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + if (intent?.action == STOP) { + stopExplicitly() + return START_NOT_STICKY + } + if (intent?.action != START && !ServiceRecovery.mayResume(this)) { + stopSelf() + return START_NOT_STICKY + } + if (intent?.action == START) { + if (!prefs.edit().putInt("restarts", 0).putLong("restart-window", SystemClock.elapsedRealtime()) + .putLong(ServiceRecovery.LAST_EXPLICIT_START, System.currentTimeMillis()).commit()) { + fail("Cannot persist explicit service start") + return START_NOT_STICKY + } + } else if (!admitRecovery()) { + fail("Repeated service termination; open the app and press Start to retry") + return START_NOT_STICKY + } + if (!prefs.edit().putBoolean(DESIRED, true).remove("last-error").commit()) { + fail("Cannot persist service state") + return START_NOT_STICKY + } + desired.set(true) + error = null + stopFailures = 0 + try { + val foregroundType = if (Build.VERSION.SDK_INT >= 34) { + ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE + } else { + ServiceInfo.FOREGROUND_SERVICE_TYPE_NONE + } + startForeground(NOTICE, notification("Starting"), foregroundType) + notifiedPhase = "Starting" + } catch (failure: Exception) { + // Promotion happens before native startup. A platform/AppOp denial + // must be visible rather than becoming an automatic crash loop. + error = "Android refused the foreground service: ${failure.javaClass.simpleName}" + setDesired(false) + phase = "Failed" + prefs.edit().putString("last-error", error).commit() + releaseWakeLock() + stopSelf() + return START_NOT_STICKY + } + if (!Environment.isExternalStorageManager()) { + fail("All-files access is required. Open the app to grant it.") + return START_NOT_STICKY + } + if (!hasLocalNetworkPermission()) { + fail("Local-network permission is required. Open the app to grant it.") + return START_NOT_STICKY + } + if (wakeLock == null) { + wakeLock = requireNotNull(getSystemService(PowerManager::class.java)).newWakeLock( + PowerManager.PARTIAL_WAKE_LOCK, "$packageName:continuous-service").apply { + setReferenceCounted(false) + acquire() + } + } + if (!starting && phase != "Running" && phase != "Stopping") { + phase = "Starting" + val delay = if (intent?.action == START) 0L else (1L shl prefs.getInt("restarts", 1)) * 1_000 + handler.postDelayed({ if (desired.get() && !starting) startNative() }, delay) + } + return START_STICKY + } + + private fun admitRecovery(): Boolean { + val now = SystemClock.elapsedRealtime() + val window = prefs.getLong("restart-window", now) + val fresh = now < window || now - window > 600_000 + val attempts = if (fresh) 0 else prefs.getInt("restarts", 0) + return attempts < 3 && prefs.edit().putInt("restarts", attempts + 1) + .putLong("restart-window", if (fresh) now else window).commit() + } + + private fun startNative() { + starting = true + phase = "Starting" + commands.execute { + try { + ShellFiles.prepareStart(this) + // Commands are serialized: Stop queued during preparation runs after + // any Start already entering JNI, and cancellation before it skips Start. + if (desired.get()) ShellFiles.start(this) + handler.post { starting = false; handler.removeCallbacks(poll); handler.post(poll) } + } catch (failure: Exception) { + handler.post { starting = false; fail(failure.message ?: "Service startup failed") } + } + } + } + + private val poll = object : Runnable { + override fun run() { + try { + if (!Environment.isExternalStorageManager() && desired.get()) { + restartForNetwork = false + error = "All-files access was revoked; stopping the daemon" + setDesired(false) + requestNativeStop() + } + if (!hasLocalNetworkPermission() && desired.get()) { + restartForNetwork = false + error = "Local-network permission was revoked; stopping the daemon" + setDesired(false) + requestNativeStop() + } + val status = NativeBridge.call("status") as JSONObject + phase = status.getString("phase") + waitingForNetwork = status.getBoolean("waitingForNetwork") + address = status.optString("address").takeUnless { it.isEmpty() || it == "null" } + if (phase == "Failed") { + error = status.optString("error").takeUnless { it.isBlank() || it == "null" } + ?: error ?: "Native runtime failed" + finishStopped() + return + } + if (!desired.get() && (phase == "Running" || phase == "Starting")) { + phase = "Stopping" + if (stopFailures in 1..2) requestNativeStop() + if (stopFailures >= 3) error = "Native Stop failed. Use Android app settings to force-stop eMuleBB." + } + if (phase == "Stopped") { + if (desired.get() && restartForNetwork) { + restartForNetwork = false + startNative() + return + } + // A graceful REST shutdown is an explicit stop, too. Process + // death never reaches this path, retaining restart intent. + finishStopped() + return + } + val noticeState = if (waitingForNetwork) "Waiting for network" else phase + if (notifiedPhase != noticeState) { + requireNotNull(getSystemService(NotificationManager::class.java)).notify(NOTICE, notification(noticeState)) + notifiedPhase = noticeState + } + handler.postDelayed(this, 1_000) + } catch (failure: Exception) { + error = failure.message ?: "Cannot inspect native runtime" + setDesired(false) + restartForNetwork = false + phase = "Stopping" + if (!NativeBridge.isAvailable()) finishStopped() + else { + // Unknown native state is not Stopped. Retain the foreground + // owner until JNI reports drained; never start a second core. + if (stopFailures < 3) requestNativeStop() + else error = "Native shutdown could not be confirmed. Use Android app settings to force-stop eMuleBB." + handler.postDelayed(this, 1_000) + } + } + } + } + + private fun setDesired(enabled: Boolean) { + desired.set(enabled) + if (!prefs.edit().putBoolean(DESIRED, enabled).commit()) error = "Cannot persist service state" + } + + private fun stopExplicitly() { + setDesired(false) + restartForNetwork = false + phase = "Stopping" + requestNativeStop() + } + + private fun fail(message: String) { + error = message + setDesired(false) + restartForNetwork = false + phase = "Stopping" + requestNativeStop() + } + + private fun requestNativeStop() { + commands.execute { + try { NativeBridge.call("stop") } + catch (failure: Exception) { + handler.post { stopFailures++; error = failure.message ?: "Native Stop failed" } + } + handler.post { handler.removeCallbacks(poll); handler.post(poll) } + } + } + + private fun finishStopped() { + setDesired(false) + phase = if (error == null) "Stopped" else "Failed" + waitingForNetwork = false + if (error != null) prefs.edit().putString("last-error", error).commit() + releaseWakeLock() + stopForeground(STOP_FOREGROUND_REMOVE) + stopSelf() + } + + private fun releaseWakeLock() { + wakeLock?.let { if (it.isHeld) it.release() } + wakeLock = null + } + + private fun notification(state: String): Notification { + val open = PendingIntent.getActivity(this, 0, Intent(this, MainActivity::class.java), PendingIntent.FLAG_IMMUTABLE) + val stop = PendingIntent.getService(this, 1, Intent(this, DaemonService::class.java).setAction(STOP), PendingIntent.FLAG_IMMUTABLE) + return Notification.Builder(this, CHANNEL) + .setSmallIcon(R.drawable.ic_notification) + .setContentTitle("eMuleBB: $state") + .setContentText("Continuous eD2K/Kad peer service") + .setContentIntent(open).setOngoing(true) + .addAction(Notification.Action.Builder(null, "Stop", stop).build()).build() + } + + override fun onDestroy() { + destroyed = true + handler.removeCallbacksAndMessages(null) + connectivity.unregisterNetworkCallback(networkCallback) + // Deliberately retain desired-running on OS teardown. Explicit Stop and + // graceful REST shutdown clear it before entering this path. + commands.execute { try { NativeBridge.call("stop") } catch (_: Exception) { } } + commands.shutdown() + releaseWakeLock() + super.onDestroy() + } +} diff --git a/android/app/src/main/java/org/emulebb/android/DiagnosticsSharing.kt b/android/app/src/main/java/org/emulebb/android/DiagnosticsSharing.kt new file mode 100644 index 000000000..66da14e97 --- /dev/null +++ b/android/app/src/main/java/org/emulebb/android/DiagnosticsSharing.kt @@ -0,0 +1,76 @@ +package org.emulebb.android + +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Build +import androidx.core.content.FileProvider +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.nio.file.Files +import java.nio.file.StandardOpenOption +import java.util.UUID +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** Shares only the daemon's closed diagnostic projection and non-identifying app metadata. */ +internal object DiagnosticsSharing { + private const val MAX_JSON_BYTES = 256 * 1024 + private const val RETAINED_REPORTS = 4 + private val reportName = Regex("emulebb-diagnostics-[0-9a-f-]+\\.zip") + + @Synchronized + fun prepare(context: Context, phaseAtRequest: String): Uri { + val diagnostics = NativeBridge.call("shareDiagnostics", "profile_dir" to ShellFiles.profile(context).absolutePath) as JSONObject + check(diagnostics.getString("schema") == "emulebb.support-diagnostics/1") { "Unexpected diagnostic projection" } + val phase = phaseAtRequest.takeIf { it in setOf("Stopped", "Starting", "Running", "Stopping", "Failed") } ?: "Unknown" + val metadata = JSONObject().apply { + put("schema", "emulebb.android.support-share/1") + put("versionName", BuildConfig.VERSION_NAME) + put("versionCode", BuildConfig.VERSION_CODE) + put("servicePhaseAtRequest", phase) + put("android", JSONObject().apply { + put("apiLevel", Build.VERSION.SDK_INT) + put("release", Build.VERSION.RELEASE) + put("manufacturer", Build.MANUFACTURER) + put("model", Build.MODEL) + put("supported64BitAbis", JSONArray(Build.SUPPORTED_64_BIT_ABIS.toList())) + }) + } + val diagnosticBytes = diagnostics.toString().toByteArray(Charsets.UTF_8) + val metadataBytes = metadata.toString().toByteArray(Charsets.UTF_8) + check(diagnosticBytes.size <= MAX_JSON_BYTES && metadataBytes.size <= 8192) { "Diagnostic report exceeds its sharing budget" } + val cache = context.cacheDir.canonicalFile + val root = File(cache, "diagnostics") + check(!Files.isSymbolicLink(root.toPath())) { "Diagnostic cache must be owned" } + check(root.mkdirs() || root.isDirectory) { "Cannot create diagnostic cache" } + check(root.canonicalFile.parentFile == cache) { "Diagnostic cache escaped app storage" } + // Cache is private to this app. Bound its own immutable reports and + // revoke earlier read grants before removing older share files. + val reports = root.listFiles().orEmpty().filter { + reportName.matches(it.name) && it.isFile && !Files.isSymbolicLink(it.toPath()) + }.sortedByDescending { it.lastModified() } + reports.drop(RETAINED_REPORTS - 1).forEach { old -> + context.revokeUriPermission(uri(context, old), Intent.FLAG_GRANT_READ_URI_PERMISSION) + check(old.delete()) { "Cannot expire an old diagnostic report" } + } + val report = File(root, "emulebb-diagnostics-${UUID.randomUUID()}.zip") + try { + ZipOutputStream(Files.newOutputStream(report.toPath(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)).use { zip -> + for ((name, bytes) in listOf("metadata.json" to metadataBytes, "diagnostics.json" to diagnosticBytes)) { + zip.putNextEntry(ZipEntry(name)) + zip.write(bytes) + zip.closeEntry() + } + } + return uri(context, report) + } catch (failure: Exception) { + report.delete() + throw failure + } + } + + private fun uri(context: Context, report: File): Uri = + FileProvider.getUriForFile(context, "${context.packageName}.diagnostics", report) +} diff --git a/android/app/src/main/java/org/emulebb/android/MainActivity.kt b/android/app/src/main/java/org/emulebb/android/MainActivity.kt new file mode 100644 index 000000000..977d01a46 --- /dev/null +++ b/android/app/src/main/java/org/emulebb/android/MainActivity.kt @@ -0,0 +1,386 @@ +package org.emulebb.android + +import android.Manifest +import android.app.Activity +import android.app.ActivityManager +import android.app.NotificationManager +import android.content.ClipData +import android.content.ClipDescription +import android.content.ClipboardManager +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import android.content.ServiceConnection +import android.content.pm.PackageManager +import android.net.Uri +import android.os.Bundle +import android.os.Build +import android.os.Environment +import android.os.Handler +import android.os.IBinder +import android.os.Looper +import android.os.Message +import android.os.Messenger +import android.os.PersistableBundle +import android.os.PowerManager +import android.provider.Settings +import android.text.InputType +import android.view.View +import android.view.WindowManager +import android.view.WindowInsets +import android.widget.Button +import android.widget.EditText +import android.widget.LinearLayout +import android.widget.ScrollView +import android.widget.TextView +import android.widget.Toast +import java.util.concurrent.Executors + +/** Bootstrap and service control only. Client operations belong to the browser WebUI. */ +class MainActivity : Activity() { + private val handler = Handler(Looper.getMainLooper()) + private val setup = Executors.newSingleThreadExecutor() + private var remote: Messenger? = null + private var bound = false + private var resumed = false + private var busy = true + private var phase = "Stopped" + private var address: String? = null + private var pendingLink: String? = null + private var pendingLinkError = "" + private var sharingDiagnostics = false + private lateinit var status: TextView + private lateinit var error: TextView + private lateinit var networkWait: TextView + private lateinit var permissions: TextView + private lateinit var pending: TextView + private lateinit var linkError: TextView + private lateinit var discardLink: Button + private lateinit var bind: EditText + private lateinit var key: EditText + private lateinit var start: Button + private lateinit var stop: Button + private lateinit var save: Button + private lateinit var regenerate: Button + private lateinit var open: Button + private lateinit var shareDiagnostics: Button + + private val replies = Messenger(object : Handler(Looper.getMainLooper()) { + override fun handleMessage(message: Message) { + if (message.what != DaemonService.STATUS) return + phase = message.data.getString("phase") ?: "Stopped" + address = message.data.getString("address") + status.show(phase) + networkWait.show(if (message.data.getBoolean("waitingForNetwork")) "Waiting for a usable default network" else "") + error.show(message.data.getString("error") ?: "") + updateControls() + } + }) + + private val connection = object : ServiceConnection { + override fun onServiceConnected(name: ComponentName, service: IBinder) { + remote = Messenger(service) + queryStatus() + } + override fun onServiceDisconnected(name: ComponentName) { + remote = null + phase = "Starting" + status.text = "Service connection interrupted" + updateControls() + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + // The setup screen deliberately shows credentials; keep them out of + // screenshots, recent-task previews, autofill and saved instance state. + window.addFlags(WindowManager.LayoutParams.FLAG_SECURE) + val panel = LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + val padding = (16 * resources.displayMetrics.density).toInt() + setPadding(padding, padding, padding, padding) + } + fun label(text: String, id: Int = View.NO_ID) = TextView(this).apply { + this.text = text; this.id = id; panel.addView(this) + } + fun edit(label: String, id: Int) = EditText(this).apply { + this.id = id + hint = label + contentDescription = label + inputType = InputType.TYPE_CLASS_TEXT or InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS + isSingleLine = true + isSaveEnabled = false + importantForAutofill = View.IMPORTANT_FOR_AUTOFILL_NO_EXCLUDE_DESCENDANTS + panel.addView(this) + } + fun button(text: String, id: Int = View.NO_ID, action: () -> Unit) = Button(this).apply { + this.text = text; this.id = id; setOnClickListener { action() }; panel.addView(this) + } + label("eMuleBB ${BuildConfig.VERSION_NAME}", R.id.app_version) + status = label("Stopped", R.id.service_status) + error = label("") + networkWait = label("", R.id.network_wait_status) + permissions = label("") + pending = label("", R.id.pending_link) + linkError = label("", R.id.pending_link_error) + discardLink = button("Discard pending link") { + pendingLink = null + pendingLinkError = "" + updatePendingLink() + } + button("Grant all-files access", R.id.grant_all_files_access) { + launchSettings(Intent(Settings.ACTION_MANAGE_APP_ALL_FILES_ACCESS_PERMISSION, Uri.parse("package:$packageName"))) + } + button("Allow notifications", R.id.allow_notifications) { + if (Build.VERSION.SDK_INT >= 33) { + requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), 1) + } else { + launchSettings(Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS).putExtra(Settings.EXTRA_APP_PACKAGE, packageName)) + } + } + button("Allow local network", R.id.allow_local_network) { + if (Build.VERSION.SDK_INT >= 37) requestPermissions(arrayOf(DaemonService.LOCAL_NETWORK), 2) + }.isEnabled = Build.VERSION.SDK_INT >= 37 + button("Allow continuous background operation", R.id.allow_continuous_background) { + launchSettings(Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS, Uri.parse("package:$packageName"))) + } + button("App battery settings", R.id.app_battery_settings) { + launchSettings(Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.parse("package:$packageName"))) + } + label("REST bind address") + bind = edit("REST bind address", R.id.bind_addr).apply { setText("0.0.0.0:20017") } + label("API key") + key = edit("API key", R.id.api_key) + button("Copy API key", R.id.copy_api_key) { + val clip = ClipData.newPlainText("eMuleBB API key", key.text.toString()) + clip.description.extras = PersistableBundle().apply { putBoolean(ClipDescription.EXTRA_IS_SENSITIVE, true) } + requireNotNull(getSystemService(ClipboardManager::class.java)).setPrimaryClip(clip) + Toast.makeText(this, "API key copied", Toast.LENGTH_SHORT).show() + } + regenerate = button("Regenerate API key") { key.setText(ShellFiles.newKey()) } + save = button("Save settings", R.id.save_settings) { saveSettings(false) } + label("Starting enables the eD2K/Kad peer service; new profiles connect automatically. Incoming files are shared automatically.") + label("New-profile folders:\nCompleted and shared: ${ShellFiles.defaultIncoming()}\nAdditional sharing: ${ShellFiles.defaultShared()}\nIncomplete files stay in app-managed storage. Manage current folders in WebUI Settings and Sharing; changes are preserved.", R.id.storage_paths) + start = button("Start", R.id.start_service) { saveSettings(true) } + stop = button("Stop", R.id.stop_service) { + phase = "Stopping" + status.text = phase + updateControls() + startService(Intent(this, DaemonService::class.java).setAction(DaemonService.STOP)) + } + open = button("Open WebUI", R.id.open_webui) { + address?.let { endpoint -> + try { + val uri = Uri.parse(ShellFiles.browserUrl(endpoint)).buildUpon().apply { + pendingLink?.let { link -> + path("/transfers") + encodedFragment("ed2k=" + Uri.encode(link)) + } + }.build() + startActivity(Intent(Intent.ACTION_VIEW, uri)) + pendingLink = null + updatePendingLink() + } + catch (_: android.content.ActivityNotFoundException) { error.text = "Install a browser to open the WebUI" } + } + } + label("Share version, service and Android details plus up to 200 recent log summaries. Keys, configuration, paths, peer/content data and free-form messages are excluded.") + shareDiagnostics = button("Share diagnostics", R.id.share_diagnostics) { shareDiagnosticReport() } + label("Client settings, sharing, transfers and diagnostics are available in the WebUI. Copy the API key into its sign-in screen.") + label("Continuous operation uses battery and network traffic. All-files access is required; notifications and battery exemption are recommended.") + val scroll = ScrollView(this).apply { addView(panel) } + window.setDecorFitsSystemWindows(false) + scroll.setOnApplyWindowInsetsListener { view, insets -> + val bars = insets.getInsets(WindowInsets.Type.systemBars() or WindowInsets.Type.displayCutout() or WindowInsets.Type.ime()) + view.setPadding(bars.left, bars.top, bars.right, bars.bottom) + insets + } + setContentView(scroll) + scroll.requestApplyInsets() + if (savedInstanceState != null) { + pendingLink = savedInstanceState.getString("pending-link") + pendingLinkError = savedInstanceState.getString("pending-link-error") ?: "" + updatePendingLink() + } else acceptLink(intent) + updateControls() + setup.execute { + try { + var settings = ShellFiles.readSettings(this) + if (settings == null) { + ShellFiles.saveSettings(this, "0.0.0.0:20017", ShellFiles.newKey(), ShellFiles.prepareWebui(this)) + settings = ShellFiles.readSettings(this) + } + val loaded = requireNotNull(settings) + handler.post { + if (!isDestroyed) { + bind.setText(loaded.getString("bind_addr")) + key.setText(loaded.getString("api_key")) + busy = false + updateControls() + } + } + } catch (failure: Exception) { + handler.post { + if (!isDestroyed) { + busy = false + error.text = failure.message ?: "Cannot load bootstrap settings" + updateControls() + } + } + } + } + } + + override fun onNewIntent(intent: Intent) { + super.onNewIntent(intent) + setIntent(intent) + acceptLink(intent) + } + + override fun onSaveInstanceState(outState: Bundle) { + outState.putString("pending-link", pendingLink) + outState.putString("pending-link-error", pendingLinkError) + super.onSaveInstanceState(outState) + } + + private fun acceptLink(intent: Intent?) { + if (intent?.action == Intent.ACTION_VIEW) { + pendingLink = null + pendingLinkError = "" + try { + val raw = requireNotNull(intent.dataString) + require(raw.length <= 2048) + pendingLink = NativeBridge.call("validateLink", "link" to raw) as String + } catch (_: Exception) { pendingLinkError = "Unsupported or malformed eD2K file link" } + } + updatePendingLink() + } + + private fun updatePendingLink() { + pending.show(pendingLink?.let { "Pending link: $it\nStart the service, then Open WebUI to review and add this file." } ?: "") + linkError.show(pendingLinkError) + discardLink.visibility = if (pendingLink != null || pendingLinkError.isNotEmpty()) View.VISIBLE else View.GONE + } + + private fun TextView.show(value: String) { + // Avoid recurring accessibility changes while the service state is unchanged. + if (text.toString() != value) text = value + } + + private fun launchSettings(intent: Intent) { + try { startActivity(intent) } + catch (_: android.content.ActivityNotFoundException) { error.text = "This device does not expose the requested permission settings" } + } + + private fun shareDiagnosticReport() { + if (sharingDiagnostics) return + sharingDiagnostics = true + shareDiagnostics.isEnabled = false + val phaseAtRequest = phase + setup.execute { + try { + val uri = DiagnosticsSharing.prepare(this, phaseAtRequest) + handler.post { + if (!isDestroyed) { + sharingDiagnostics = false + shareDiagnostics.isEnabled = true + if (resumed) { + val send = Intent(Intent.ACTION_SEND).apply { + type = "application/zip" + putExtra(Intent.EXTRA_STREAM, uri) + clipData = ClipData.newUri(contentResolver, "eMuleBB diagnostics", uri) + addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + try { startActivity(Intent.createChooser(send, "Share eMuleBB diagnostics")) } + catch (_: android.content.ActivityNotFoundException) { error.show("No application can receive this diagnostic report") } + } else error.show("Return to the app and share diagnostics again") + } + } + } catch (failure: Exception) { + handler.post { + if (!isDestroyed) { + sharingDiagnostics = false + shareDiagnostics.isEnabled = true + error.show("Cannot prepare diagnostics (${failure.javaClass.simpleName})") + } + } + } + } + } + + private fun saveSettings(andStart: Boolean) { + if (busy || phase !in listOf("Stopped", "Failed")) return + if (andStart && !Environment.isExternalStorageManager()) { + error.text = "Grant all-files access before starting" + return + } + if (andStart && Build.VERSION.SDK_INT >= 37 && checkSelfPermission(DaemonService.LOCAL_NETWORK) != PackageManager.PERMISSION_GRANTED) { + error.text = "Allow local-network access before starting" + return + } + val bindValue = bind.text.toString().trim() + val keyValue = key.text.toString() + busy = true + error.text = "" + updateControls() + setup.execute { + try { + ShellFiles.saveSettings(this, bindValue, keyValue, ShellFiles.prepareWebui(this)) + handler.post { + if (!isDestroyed) { + busy = false + if (andStart && resumed) { + try { + startForegroundService(Intent(this, DaemonService::class.java).setAction(DaemonService.START)) + phase = "Starting" + status.text = phase + } catch (failure: Exception) { error.text = failure.message ?: "Cannot start service" } + } else if (andStart) error.text = "Return to the app and press Start" + updateControls() + } + } + } catch (failure: Exception) { + handler.post { if (!isDestroyed) { busy = false; error.text = failure.message; updateControls() } } + } + } + } + + private fun updateControls() { + val stopped = phase == "Stopped" || phase == "Failed" + bind.isEnabled = !busy && stopped + key.isEnabled = !busy && stopped + regenerate.isEnabled = !busy && stopped + save.isEnabled = !busy && stopped + start.isEnabled = !busy && stopped + stop.isEnabled = phase == "Starting" || phase == "Running" + open.isEnabled = phase == "Running" && address != null + } + + private fun queryStatus() { + try { remote?.send(Message.obtain(null, DaemonService.STATUS).apply { replyTo = replies }) } + catch (_: android.os.RemoteException) { remote = null } + } + private val poll = object : Runnable { + override fun run() { + queryStatus() + val notifications = requireNotNull(getSystemService(NotificationManager::class.java)).areNotificationsEnabled() + val exemption = requireNotNull(getSystemService(PowerManager::class.java)).isIgnoringBatteryOptimizations(packageName) + val local = Build.VERSION.SDK_INT < 37 || checkSelfPermission(DaemonService.LOCAL_NETWORK) == PackageManager.PERMISSION_GRANTED + val restricted = requireNotNull(getSystemService(ActivityManager::class.java)).isBackgroundRestricted + permissions.show("Files: ${if (Environment.isExternalStorageManager()) "allowed" else "required"}; local network: ${if (local) "allowed" else "required"}; notifications: ${if (notifications) "allowed" else "disabled"}; battery exemption: ${if (exemption) "allowed" else "disabled"}; background restriction: ${if (restricted) "enabled" else "disabled"}") + handler.postDelayed(this, 1_000) + } + } + + override fun onStart() { + super.onStart() + bound = bindService(Intent(this, DaemonService::class.java), connection, Context.BIND_AUTO_CREATE) + } + override fun onResume() { super.onResume(); resumed = true; handler.post(poll) } + override fun onPause() { resumed = false; handler.removeCallbacks(poll); super.onPause() } + override fun onStop() { + if (bound) { unbindService(connection); bound = false; remote = null } + super.onStop() + } + override fun onDestroy() { handler.removeCallbacksAndMessages(null); setup.shutdown(); super.onDestroy() } +} diff --git a/android/app/src/main/java/org/emulebb/android/NativeBridge.kt b/android/app/src/main/java/org/emulebb/android/NativeBridge.kt new file mode 100644 index 000000000..5ccf85a30 --- /dev/null +++ b/android/app/src/main/java/org/emulebb/android/NativeBridge.kt @@ -0,0 +1,22 @@ +package org.emulebb.android + +import org.json.JSONObject + +/** Private, in-process JNI protocol. Never expose it through an Android component. */ +internal object NativeBridge { + private val loadFailure = try { + System.loadLibrary("emulebb_android") + null + } catch (_: LinkageError) { "Cannot load the packaged native daemon" } + private external fun request(payload: String): String + fun isAvailable(): Boolean = loadFailure == null + + fun call(op: String, vararg fields: Pair): Any? { + check(loadFailure == null) { requireNotNull(loadFailure) } + val input = JSONObject().put("op", op) + fields.forEach { input.put(it.first, it.second) } + val output = JSONObject(request(input.toString())) + check(output.getBoolean("ok")) { output.optString("error", "Native operation failed") } + return output.opt("data").takeUnless { it === JSONObject.NULL } + } +} diff --git a/android/app/src/main/java/org/emulebb/android/ServiceRecovery.kt b/android/app/src/main/java/org/emulebb/android/ServiceRecovery.kt new file mode 100644 index 000000000..b8159d70b --- /dev/null +++ b/android/app/src/main/java/org/emulebb/android/ServiceRecovery.kt @@ -0,0 +1,29 @@ +package org.emulebb.android + +import android.app.ActivityManager +import android.app.ApplicationExitInfo +import android.content.Context + +/** Android's Active apps Stop kills the process without any service callback. */ +internal object ServiceRecovery { + const val LAST_EXPLICIT_START = "last-explicit-start" + + fun mayResume(context: Context): Boolean { + val prefs = context.getSharedPreferences(DaemonService.PREFERENCES, Context.MODE_PRIVATE) + if (!prefs.getBoolean(DaemonService.DESIRED, false)) return false + val started = prefs.getLong(LAST_EXPLICIT_START, 0) + val exits = requireNotNull(context.getSystemService(ActivityManager::class.java)) + .getHistoricalProcessExitReasons(context.packageName, 0, 0) + val stoppedByUser = exits.any { + it.processName == "${context.packageName}:daemon" && + it.reason == ApplicationExitInfo.REASON_USER_REQUESTED && it.timestamp >= started + } + if (stoppedByUser) { + // Only daemon-process exits count: dismissing the setup activity's + // task must not cancel a user-enabled foreground peer service. + prefs.edit().putBoolean(DaemonService.DESIRED, false).commit() + return false + } + return true + } +} diff --git a/android/app/src/main/java/org/emulebb/android/ShellFiles.kt b/android/app/src/main/java/org/emulebb/android/ShellFiles.kt new file mode 100644 index 000000000..093e6bb89 --- /dev/null +++ b/android/app/src/main/java/org/emulebb/android/ShellFiles.kt @@ -0,0 +1,82 @@ +package org.emulebb.android + +import android.content.Context +import android.os.Environment +import org.json.JSONObject +import java.io.File +import java.io.RandomAccessFile +import java.security.SecureRandom + +internal object ShellFiles { + fun profile(context: Context) = File(context.filesDir, "profile") + fun publicRoot() = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "eMuleBB") + fun defaultIncoming() = File(publicRoot(), "Incoming") + fun defaultShared() = File(publicRoot(), "Shared") + + fun newKey(): String = ByteArray(32).also { SecureRandom().nextBytes(it) } + .joinToString("") { "%02x".format(it.toInt() and 255) } + + /** Assets are immutable per APK version. Cross-process locking prevents partial reads. */ + fun prepareWebui(context: Context): File { + val root = File(context.filesDir, "webui-${BuildConfig.VERSION_CODE}") + RandomAccessFile(File(context.filesDir, "webui.lock"), "rw").use { lockFile -> + lockFile.channel.lock().use { + val ready = File(root, ".ready") + if (!ready.isFile) { + check(root.mkdirs() || root.isDirectory) { "Cannot create WebUI directory" } + copyAssets(context, "webui", root) + check(File(root, "index.html").isFile) { "Packaged WebUI is incomplete" } + ready.outputStream().use { it.fd.sync() } + } + } + } + return root + } + + private fun copyAssets(context: Context, source: String, target: File) { + val children = context.assets.list(source) ?: error("Missing packaged WebUI") + if (children.isEmpty()) { + context.assets.open(source).use { input -> + target.outputStream().use { output -> input.copyTo(output); output.fd.sync() } + } + } else { + check(target.mkdirs() || target.isDirectory) { "Cannot create WebUI assets" } + children.forEach { copyAssets(context, "$source/$it", File(target, it)) } + } + } + + fun readSettings(context: Context): JSONObject? = NativeBridge.call( + "readSettings", "profile_dir" to profile(context).absolutePath + ) as JSONObject? + + fun saveSettings(context: Context, bind: String, key: String, webui: File) { + NativeBridge.call("writeSettings", "profile_dir" to profile(context).absolutePath, + "bind_addr" to bind, "api_key" to key, "web_root_dir" to webui.absolutePath) + NativeBridge.call("pruneAssets", "profile_dir" to profile(context).absolutePath, + "current_web_root" to webui.absolutePath) + } + + fun prepareStart(context: Context) { + val settings = readSettings(context) ?: error("Open eMuleBB and complete setup before starting") + val webui = prepareWebui(context) + if (settings.optString("web_root_dir") != webui.absolutePath) { + saveSettings(context, settings.getString("bind_addr"), settings.getString("api_key"), webui) + } + } + + fun start(context: Context) { + val external = context.getExternalFilesDir(null) ?: error("Payload storage is unavailable") + NativeBridge.call("start", "profile_dir" to profile(context).absolutePath, + "transfer_root" to File(external, "transfers").absolutePath, + "incoming_dir" to defaultIncoming().absolutePath, "shared_dir" to defaultShared().absolutePath) + } + + fun browserUrl(address: String): String { + val authority = when { + address.startsWith("0.0.0.0:") -> "127.0.0.1:" + address.substringAfterLast(':') + address.startsWith("[::]:") -> "[::1]:" + address.substringAfterLast(':') + else -> address + } + return "http://$authority/" + } +} diff --git a/android/app/src/main/res/drawable-nodpi/donkey_launcher.png b/android/app/src/main/res/drawable-nodpi/donkey_launcher.png new file mode 100644 index 000000000..42fdbd1c7 Binary files /dev/null and b/android/app/src/main/res/drawable-nodpi/donkey_launcher.png differ diff --git a/android/app/src/main/res/drawable/ic_launcher_foreground.xml b/android/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 000000000..71fa6c014 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,5 @@ + + + diff --git a/android/app/src/main/res/drawable/ic_launcher_monochrome.xml b/android/app/src/main/res/drawable/ic_launcher_monochrome.xml new file mode 100644 index 000000000..514177c97 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_launcher_monochrome.xml @@ -0,0 +1,4 @@ + + diff --git a/android/app/src/main/res/drawable/ic_notification.xml b/android/app/src/main/res/drawable/ic_notification.xml new file mode 100644 index 000000000..607bcb0ac --- /dev/null +++ b/android/app/src/main/res/drawable/ic_notification.xml @@ -0,0 +1,7 @@ + + + + + diff --git a/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 000000000..80a5340e4 --- /dev/null +++ b/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/android/app/src/main/res/mipmap-anydpi-v33/ic_launcher.xml b/android/app/src/main/res/mipmap-anydpi-v33/ic_launcher.xml new file mode 100644 index 000000000..df4ab4217 --- /dev/null +++ b/android/app/src/main/res/mipmap-anydpi-v33/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/android/app/src/main/res/values-night/themes.xml b/android/app/src/main/res/values-night/themes.xml new file mode 100644 index 000000000..b74094612 --- /dev/null +++ b/android/app/src/main/res/values-night/themes.xml @@ -0,0 +1,9 @@ + + + + diff --git a/android/app/src/main/res/values/ids.xml b/android/app/src/main/res/values/ids.xml new file mode 100644 index 000000000..de7106124 --- /dev/null +++ b/android/app/src/main/res/values/ids.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/app/src/main/res/values/themes.xml b/android/app/src/main/res/values/themes.xml new file mode 100644 index 000000000..f2f712311 --- /dev/null +++ b/android/app/src/main/res/values/themes.xml @@ -0,0 +1,9 @@ + + + + diff --git a/android/app/src/main/res/xml/data_extraction_rules.xml b/android/app/src/main/res/xml/data_extraction_rules.xml new file mode 100644 index 000000000..ae178b2ef --- /dev/null +++ b/android/app/src/main/res/xml/data_extraction_rules.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/app/src/main/res/xml/diagnostic_paths.xml b/android/app/src/main/res/xml/diagnostic_paths.xml new file mode 100644 index 000000000..ff5705e6b --- /dev/null +++ b/android/app/src/main/res/xml/diagnostic_paths.xml @@ -0,0 +1,4 @@ + + + + diff --git a/android/build.gradle.kts b/android/build.gradle.kts new file mode 100644 index 000000000..c5daa4108 --- /dev/null +++ b/android/build.gradle.kts @@ -0,0 +1,9 @@ +plugins { id("com.android.application") version "9.5.1" apply false } + +val externalBuildRoot = providers.gradleProperty("androidBuildRoot").get() +require(file(externalBuildRoot).isAbsolute) { "androidBuildRoot must be an absolute output directory" } +require(!file(externalBuildRoot).canonicalFile.toPath().startsWith(projectDir.canonicalFile.toPath())) { + "Android build output must be outside the source project" +} +layout.buildDirectory.set(file(externalBuildRoot)) +subprojects { layout.buildDirectory.set(rootProject.layout.buildDirectory.dir(name)) } diff --git a/android/gradle.properties b/android/gradle.properties new file mode 100644 index 000000000..aec357bc5 --- /dev/null +++ b/android/gradle.properties @@ -0,0 +1,3 @@ +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +org.gradle.daemon=false +android.nonTransitiveRClass=true diff --git a/android/gradle/verification-metadata.xml b/android/gradle/verification-metadata.xml new file mode 100644 index 000000000..8b98c0c32 --- /dev/null +++ b/android/gradle/verification-metadata.xml @@ -0,0 +1,1679 @@ + + + + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts new file mode 100644 index 000000000..1d6db57bb --- /dev/null +++ b/android/settings.gradle.kts @@ -0,0 +1,7 @@ +pluginManagement { repositories { google(); mavenCentral(); gradlePluginPortal() } } +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { google(); mavenCentral() } +} +rootProject.name = "emulebb-android" +include(":app") diff --git a/crates/emulebb-android/Cargo.toml b/crates/emulebb-android/Cargo.toml new file mode 100644 index 000000000..a3a23744f --- /dev/null +++ b/crates/emulebb-android/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "emulebb-android" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "Android application host for the eMuleBB daemon" + +[lib] +crate-type = ["cdylib", "rlib"] + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +emulebb-daemon = { path = "../emulebb-daemon" } +jni.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] } + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/emulebb-android/src/lib.rs b/crates/emulebb-android/src/lib.rs new file mode 100644 index 000000000..00d86d053 --- /dev/null +++ b/crates/emulebb-android/src/lib.rs @@ -0,0 +1,520 @@ +//! Narrow application-host boundary; the Android shell owns no client settings. + +use std::{ + path::PathBuf, + sync::{Arc, Mutex, OnceLock}, + thread::JoinHandle, +}; + +use anyhow::{Context, Result, ensure}; +use emulebb_daemon::{DaemonProfile, RestBootstrapSettings, ShutdownRequest, host_bootstrap}; +use jni::{ + Env, native_method, + objects::{JObject, JString}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +mod startup_network; +use startup_network::StartupNetwork; + +#[derive(Deserialize)] +#[serde(tag = "op", rename_all = "camelCase", deny_unknown_fields)] +enum Request { + ShareDiagnostics { + profile_dir: PathBuf, + }, + ValidateLink { + link: String, + }, + ReadSettings { + profile_dir: PathBuf, + }, + WriteSettings { + profile_dir: PathBuf, + bind_addr: String, + api_key: String, + web_root_dir: PathBuf, + }, + PruneAssets { + profile_dir: PathBuf, + current_web_root: PathBuf, + }, + Start { + profile_dir: PathBuf, + transfer_root: PathBuf, + incoming_dir: PathBuf, + shared_dir: PathBuf, + }, + Stop {}, + NetworkChanged {}, + Status {}, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +enum Phase { + #[default] + Stopped, + Starting, + Running, + Stopping, + Failed, +} + +#[derive(Default)] +struct Host { + phase: Phase, + error: Option, + address: Option, + shutdown: Option, + worker: Option>, + startup_network: Option>, + waiting_for_network: bool, +} + +fn host() -> &'static Mutex { + static HOST: OnceLock> = OnceLock::new(); + HOST.get_or_init(Mutex::default) +} + +struct ProcessLogging { + profile: PathBuf, + _guard: emulebb_daemon::logging::LoggingGuard, +} + +fn ensure_process_logging(profile: &std::path::Path) -> Result<()> { + static LOGGING: OnceLock>> = OnceLock::new(); + let mut logging = LOGGING + .get_or_init(Mutex::default) + .lock() + .map_err(|_| anyhow::anyhow!("process logging state unavailable"))?; + let profile = profile.canonicalize()?; + if let Some(existing) = logging.as_ref() { + ensure!( + existing.profile == profile, + "one Android service process owns one logging profile" + ); + } else { + *logging = Some(ProcessLogging { + _guard: emulebb_daemon::logging::init_process(&profile)?, + profile, + }); + } + Ok(()) +} + +fn handle(request: Request) -> Result { + match request { + Request::ShareDiagnostics { profile_dir } => { + emulebb_daemon::support_diagnostics::capture(&profile_dir) + } + Request::ValidateLink { link } => { + emulebb_daemon::validate_ed2k_file_link(&link)?; + Ok(Value::String(link)) + } + Request::PruneAssets { + profile_dir, + current_web_root, + } => { + host_bootstrap::while_stopped(&profile_dir, || { + prune_assets(&profile_dir, ¤t_web_root) + })?; + Ok(Value::Null) + } + Request::ReadSettings { profile_dir } => { + ensure!( + profile_dir.is_absolute(), + "profile directory must be absolute" + ); + Ok(json!(host_bootstrap::read(&profile_dir)?)) + } + Request::WriteSettings { + profile_dir, + bind_addr, + api_key, + web_root_dir, + } => { + ensure!( + profile_dir.is_absolute() && web_root_dir.is_absolute(), + "host directories must be absolute" + ); + let rest = RestBootstrapSettings { + bind_addr: Some(bind_addr.parse().context("invalid REST bind address")?), + api_key, + web_root_dir: Some(web_root_dir), + }; + host_bootstrap::write(&profile_dir, rest)?; + Ok(Value::Null) + } + Request::Start { + profile_dir, + transfer_root, + incoming_dir, + shared_dir, + } => { + ensure!( + profile_dir.is_absolute() + && transfer_root.is_absolute() + && incoming_dir.is_absolute() + && shared_dir.is_absolute(), + "host directories must be absolute" + ); + let mut state = host() + .lock() + .map_err(|_| anyhow::anyhow!("host state unavailable"))?; + if matches!( + state.phase, + Phase::Starting | Phase::Running | Phase::Stopping + ) { + return Ok(Value::Null); + } + if let Some(worker) = &state.worker { + ensure!(worker.is_finished(), "previous runtime is still draining"); + } + if let Some(worker) = state.worker.take() { + let _ = worker.join(); + } + let shutdown = ShutdownRequest::default(); + state.phase = Phase::Starting; + state.error = None; + state.address = None; + state.shutdown = Some(shutdown.clone()); + let startup_network = Arc::new(StartupNetwork::default()); + state.startup_network = Some(Arc::clone(&startup_network)); + state.waiting_for_network = false; + let worker = std::thread::Builder::new() + .name("emulebb-android".into()) + .spawn(move || { + let outcome = std::panic::catch_unwind(|| { + run_worker( + profile_dir, + transfer_root, + incoming_dir, + shared_dir, + shutdown, + startup_network, + ) + }); + if let Ok(mut state) = host().lock() { + match outcome { + Ok(Ok(())) => { + state.phase = Phase::Stopped; + state.error = None; + } + Ok(Err(error)) => { + state.phase = Phase::Failed; + state.error = Some(format!("{error:#}")); + } + Err(_) => { + state.phase = Phase::Failed; + state.error = Some("native runtime failed".into()); + } + } + state.shutdown = None; + state.startup_network = None; + state.waiting_for_network = false; + state.address = None; + } + }); + match worker { + Ok(worker) => state.worker = Some(worker), + Err(error) => { + state.phase = Phase::Failed; + state.shutdown = None; + state.startup_network = None; + return Err(error.into()); + } + } + Ok(Value::Null) + } + Request::Stop {} => { + let mut state = host() + .lock() + .map_err(|_| anyhow::anyhow!("host state unavailable"))?; + if let Some(shutdown) = &state.shutdown { + shutdown.request(); + state.phase = Phase::Stopping; + } + if let Some(network) = &state.startup_network { + network.cancel()?; + } + Ok(Value::Null) + } + Request::NetworkChanged {} => { + let state = host() + .lock() + .map_err(|_| anyhow::anyhow!("host state unavailable"))?; + if let Some(network) = &state.startup_network { + network.changed()?; + } + Ok(Value::Null) + } + Request::Status {} => { + let state = host() + .lock() + .map_err(|_| anyhow::anyhow!("host state unavailable"))?; + Ok( + json!({"phase":state.phase,"error":state.error,"address":state.address, + "waitingForNetwork":state.waiting_for_network}), + ) + } + } +} + +fn run_worker( + profile_dir: PathBuf, + transfer_root: PathBuf, + incoming_dir: PathBuf, + shared_dir: PathBuf, + shutdown: ShutdownRequest, + network: Arc, +) -> Result<()> { + network.run( + |ready| { + if shutdown.subscribe().is_requested() { + return Ok(()); + } + run_attempt( + &profile_dir, + &transfer_root, + &incoming_dir, + &shared_dir, + &shutdown, + ready, + ) + }, + |waiting| { + // The failed future, runtime and profile lease have drained before this + // wait. No host mutex is held while waiting or entering another attempt. + host() + .lock() + .map_err(|_| anyhow::anyhow!("host state unavailable"))? + .waiting_for_network = waiting; + Ok(()) + }, + ) +} + +fn run_attempt( + profile_dir: &std::path::Path, + transfer_root: &std::path::Path, + incoming_dir: &std::path::Path, + shared_dir: &std::path::Path, + shutdown: &ShutdownRequest, + readiness_reported: &mut bool, +) -> Result<()> { + let mut profile = + DaemonProfile::load_hosted(profile_dir.into(), incoming_dir.into(), shared_dir.into())?; + ensure_process_logging(profile.profile_io_dir())?; + profile.host_transfer_root = Some(transfer_root.into()); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + let (ready, readiness) = tokio::sync::oneshot::channel(); + let outcome = runtime.block_on(await_readiness( + emulebb_daemon::run_hosted(profile, shutdown.subscribe(), ready), + readiness, + readiness_reported, + |address| { + let mut state = host() + .lock() + .map_err(|_| anyhow::anyhow!("host state unavailable"))?; + state.address = Some(address.to_string()); + if state.phase == Phase::Starting { + state.phase = Phase::Running; + } + Ok(()) + }, + )); + // WHY: Tokio blocking work can outlive the daemon future. Drain the runtime + // before retrying startup or publishing Stopped/releasing ownership. + drop(runtime); + outcome +} + +async fn await_readiness( + run: impl std::future::Future>, + mut readiness: tokio::sync::oneshot::Receiver, + reported: &mut bool, + ready: impl FnOnce(std::net::SocketAddr) -> Result<()>, +) -> Result<()> { + tokio::pin!(run); + tokio::select! { + result = &mut run => { + // WHY: the future can send readiness and fail in the same poll. + // Such a failure is after readiness regardless of select order. + *reported |= readiness.try_recv().is_ok(); + result + }, + result = &mut readiness => { + if let Ok(address) = result { + *reported = true; + ready(address)?; + } + run.await + } + } +} + +fn prune_assets(profile_dir: &std::path::Path, current_web_root: &std::path::Path) -> Result<()> { + let parent = profile_dir + .parent() + .context("profile has no parent")? + .canonicalize()?; + let current = current_web_root.canonicalize()?; + ensure!( + current.parent() == Some(parent.as_path()), + "WebUI assets must belong to the application directory" + ); + ensure!( + current.join(".ready").is_file(), + "current WebUI assets are not ready" + ); + let mut versions = Vec::new(); + for entry in std::fs::read_dir(&parent)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let name = entry.file_name(); + let Some(version) = name + .to_str() + .and_then(|name| name.strip_prefix("webui-")) + .and_then(|value| value.parse::().ok()) + else { + continue; + }; + versions.push((version, entry.path())); + } + versions.sort_by_key(|item| std::cmp::Reverse(item.0)); + let mut retained_previous = false; + for (_, path) in versions { + if path == current { + continue; + } + if !retained_previous { + retained_previous = true; + continue; + } + // WHY: source assets are disposable, but metadata/payloads are not. + // Limit cleanup to real, directly owned version directories while the + // profile lease excludes every running daemon, including other processes. + ensure!( + path.canonicalize()?.parent() == Some(parent.as_path()), + "asset cleanup escaped application directory" + ); + std::fs::remove_dir_all(path)?; + } + Ok(()) +} + +const _: jni::NativeMethod = native_method! { + java_type = "org.emulebb.android.NativeBridge", + extern fn request(payload: JString) -> JString, +}; + +fn request<'local>( + env: &mut Env<'local>, + _this: JObject<'local>, + payload: JString<'local>, +) -> Result, jni::errors::Error> { + let input = payload.to_string(); + // The JNI macro catches panics and prevents unwinding into the JVM. + let response = match serde_json::from_str::(&input) + .map_err(|_| anyhow::anyhow!("invalid host request")) + .and_then(handle) + { + Ok(data) => json!({"ok":true,"data":data}), + Err(error) => json!({"ok":false,"error":format!("{error:#}")}), + }; + JString::from_str(env, response.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn failure_in_same_poll_as_readiness_is_after_startup() { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let (ready, readiness) = tokio::sync::oneshot::channel(); + let mut reported = false; + let outcome = runtime.block_on(await_readiness( + async { + ready.send("127.0.0.1:20017".parse().unwrap()).unwrap(); + Err(emulebb_daemon::NetworkStartupError::DefaultRouteUnavailable.into()) + }, + readiness, + &mut reported, + |_| panic!("terminal result must not publish Running"), + )); + assert!(reported); + assert!( + outcome + .unwrap_err() + .is::() + ); + } + + #[test] + fn asset_cleanup_retains_current_previous_and_unrelated_data() { + let root = tempfile::tempdir().unwrap(); + let profile = root.path().join("profile"); + for name in [ + "profile", + "webui-1", + "webui-2", + "webui-3", + "webui-custom", + "payloads", + ] { + std::fs::create_dir(root.path().join(name)).unwrap(); + } + std::fs::write(root.path().join("webui-3/.ready"), b"").unwrap(); + handle(Request::PruneAssets { + profile_dir: profile, + current_web_root: root.path().join("webui-3"), + }) + .unwrap(); + assert!(!root.path().join("webui-1").exists()); + for name in ["profile", "webui-2", "webui-3", "webui-custom", "payloads"] { + assert!(root.path().join(name).is_dir()); + } + } + + #[test] + fn private_host_protocol_rejects_unknown_fields_and_retired_commands() { + assert!(serde_json::from_str::(r#"{"op":"status","api_key":"hidden"}"#).is_err()); + assert!(serde_json::from_str::(r#"{"op":"migrate"}"#).is_err()); + assert!( + handle(Request::ReadSettings { + profile_dir: "relative".into() + }) + .is_err() + ); + } + + #[test] + fn external_link_validation_has_no_profile_or_runtime_side_effects() { + let link = "ed2k://|file|Sample%20%26%20Guide.bin|42|00112233445566778899aabbccddeeff|/"; + assert_eq!( + handle(Request::ValidateLink { link: link.into() }).unwrap(), + json!(link) + ); + for invalid in [ + "ed2k://|server|192.0.2.1|4661|/".to_string(), + "ed2k://|file|Sample.bin|42|invalid|/".to_string(), + "ed2k://|file|Sample.bin|invalid|00112233445566778899aabbccddeeff|/".to_string(), + "ed2k://|file|Sample%0Abin|42|00112233445566778899aabbccddeeff|/".to_string(), + format!( + "ed2k://|file|{}|42|00112233445566778899aabbccddeeff|/", + "A".repeat(2048) + ), + ] { + assert!(handle(Request::ValidateLink { link: invalid }).is_err()); + } + assert_eq!(handle(Request::Status {}).unwrap()["phase"], "Stopped"); + } +} diff --git a/crates/emulebb-android/src/startup_network.rs b/crates/emulebb-android/src/startup_network.rs new file mode 100644 index 000000000..bbc58ad5d --- /dev/null +++ b/crates/emulebb-android/src/startup_network.rs @@ -0,0 +1,245 @@ +//! Cancellable network generations for one Android-hosted daemon worker. + +use std::sync::{Condvar, Mutex}; +use std::time::Duration; + +use anyhow::{Result, anyhow}; +use emulebb_daemon::NetworkStartupError; + +#[derive(Default)] +struct State { + generation: u64, + cancelled: bool, +} + +#[derive(Default)] +pub(super) struct StartupNetwork { + state: Mutex, + changed: Condvar, +} + +#[derive(Debug, PartialEq, Eq)] +pub(super) enum Wake { + Changed, + Retry, + Cancelled, +} + +impl StartupNetwork { + pub(super) fn run( + &self, + mut attempt: impl FnMut(&mut bool) -> Result<()>, + mut waiting: impl FnMut(bool) -> Result<()>, + ) -> Result<()> { + let mut failures = 0; + loop { + if self.cancelled()? { + return Ok(()); + } + // Snapshot before the attempt so a callback racing the native + // probe cannot be lost between a failure and its wait. + let generation = self.generation()?; + let mut ready = false; + let outcome = attempt(&mut ready); + if self.cancelled()? { + return Ok(()); + } + let Err(error) = outcome else { + return Ok(()); + }; + if ready + || !matches!( + error.downcast_ref::(), + Some(NetworkStartupError::DefaultRouteUnavailable) + ) + { + return Err(error); + } + waiting(true)?; + let wake = self.wait(generation, retry_delay(failures))?; + waiting(false)?; + match wake { + Wake::Cancelled => return Ok(()), + Wake::Changed => failures = 0, + Wake::Retry => failures = failures.saturating_add(1), + } + } + } + + fn cancelled(&self) -> Result { + Ok(self.state.lock().map_err(|_| unavailable())?.cancelled) + } + + pub(super) fn generation(&self) -> Result { + Ok(self.state.lock().map_err(|_| unavailable())?.generation) + } + + pub(super) fn changed(&self) -> Result<()> { + let mut state = self.state.lock().map_err(|_| unavailable())?; + state.generation = state.generation.wrapping_add(1); + self.changed.notify_all(); + Ok(()) + } + + pub(super) fn cancel(&self) -> Result<()> { + self.state.lock().map_err(|_| unavailable())?.cancelled = true; + self.changed.notify_all(); + Ok(()) + } + + pub(super) fn wait(&self, generation: u64, delay: Duration) -> Result { + let state = self.state.lock().map_err(|_| unavailable())?; + let (state, _) = self + .changed + .wait_timeout_while(state, delay, |state| { + !state.cancelled && state.generation == generation + }) + .map_err(|_| unavailable())?; + Ok(if state.cancelled { + Wake::Cancelled + } else if state.generation != generation { + Wake::Changed + } else { + Wake::Retry + }) + } +} + +pub(super) fn retry_delay(failures: u32) -> Duration { + // WHY: LinkProperties callbacks can precede kernel route readiness. Retain + // liveness without requiring another callback, with bounded probe frequency. + Duration::from_secs((1u64 << failures.min(5)).min(30)) +} + +fn unavailable() -> anyhow::Error { + anyhow!("network startup signal unavailable") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + #[test] + fn network_change_during_attempt_cannot_be_lost_before_wait() { + let signal = StartupNetwork::default(); + let generation = signal.generation().unwrap(); + signal.changed().unwrap(); + assert_eq!( + signal.wait(generation, Duration::from_secs(60)).unwrap(), + Wake::Changed + ); + } + + #[test] + fn stop_cancels_existing_and_future_waits_despite_late_network_changes() { + let signal = Arc::new(StartupNetwork::default()); + let worker_signal = Arc::clone(&signal); + let worker = + std::thread::spawn(move || worker_signal.wait(0, Duration::from_secs(60)).unwrap()); + signal.cancel().unwrap(); + signal.changed().unwrap(); + assert_eq!(worker.join().unwrap(), Wake::Cancelled); + assert_eq!( + signal + .wait(signal.generation().unwrap(), Duration::from_secs(60)) + .unwrap(), + Wake::Cancelled + ); + } + + #[test] + fn absent_callback_still_allows_a_rate_limited_recheck() { + let signal = StartupNetwork::default(); + assert_eq!(signal.wait(0, Duration::ZERO).unwrap(), Wake::Retry); + assert_eq!(retry_delay(0), Duration::from_secs(1)); + assert_eq!(retry_delay(4), Duration::from_secs(16)); + assert_eq!(retry_delay(5), Duration::from_secs(30)); + assert_eq!(retry_delay(u32::MAX), Duration::from_secs(30)); + } + + #[test] + fn typed_unavailable_route_waits_then_ready_attempt_completes() { + let signal = StartupNetwork::default(); + let mut attempts = 0; + let mut waiting = Vec::new(); + signal + .run( + |_| { + attempts += 1; + if attempts == 1 { + signal.changed()?; + Err(NetworkStartupError::DefaultRouteUnavailable.into()) + } else { + Ok(()) + } + }, + |state| { + waiting.push(state); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(attempts, 2); + assert_eq!(waiting, [true, false]); + } + + #[test] + fn permanent_failure_does_not_wait_or_retry() { + let signal = StartupNetwork::default(); + let mut attempts = 0; + let error = signal + .run( + |_| { + attempts += 1; + Err(anyhow!("invalid binding configuration")) + }, + |_| panic!("permanent failure must not wait"), + ) + .unwrap_err(); + assert_eq!(error.to_string(), "invalid binding configuration"); + assert_eq!(attempts, 1); + } + + #[test] + fn cancellation_during_attempt_and_late_callback_never_retry() { + let signal = StartupNetwork::default(); + let mut attempts = 0; + signal + .run( + |_| { + attempts += 1; + signal.cancel()?; + signal.changed()?; + Err(NetworkStartupError::DefaultRouteUnavailable.into()) + }, + |_| panic!("cancelled startup must not wait"), + ) + .unwrap(); + signal + .run( + |_| panic!("late callback must not start another attempt"), + |_| Ok(()), + ) + .unwrap(); + assert_eq!(attempts, 1); + } + + #[test] + fn network_error_after_readiness_is_fatal() { + let signal = StartupNetwork::default(); + let mut attempts = 0; + let error = signal + .run( + |ready| { + attempts += 1; + *ready = true; + Err(NetworkStartupError::DefaultRouteUnavailable.into()) + }, + |_| panic!("a ready runtime must not enter startup waiting"), + ) + .unwrap_err(); + assert!(error.is::()); + assert_eq!(attempts, 1); + } +} diff --git a/crates/emulebb-core/src/ed2k_net_drivers.rs b/crates/emulebb-core/src/ed2k_net_drivers.rs index c16f9d81d..8b565196e 100644 --- a/crates/emulebb-core/src/ed2k_net_drivers.rs +++ b/crates/emulebb-core/src/ed2k_net_drivers.rs @@ -134,13 +134,13 @@ const ED2K_RELOGIN_MIN_INTERVAL: Duration = Duration::from_secs(300); /// a stale server address. Gentle: one STUN race per interval, more often only /// while the effective address is unknown. pub(crate) async fn run_ed2k_public_ip_probe( - bind_ip: Ipv4Addr, + binding: emulebb_ed2k::networking::EgressBinding, public_ip: ExternalReachability, public_ip_writer: RuntimeReachabilityWriter, shutdown: Arc, ) { while !shutdown.load(Ordering::Relaxed) { - if let Ok(ip) = stun_probe(bind_ip, DEFAULT_STUN_TIMEOUT).await + if let Ok(ip) = stun_probe(binding, DEFAULT_STUN_TIMEOUT).await && public_ip_writer.set_fallback_public_ip(ip) == ReachabilityUpdate::Changed { tracing::info!("ED2K fallback public IP refreshed via STUN: {ip}"); @@ -162,11 +162,14 @@ pub(crate) async fn run_ed2k_public_ip_probe( /// fall back to TCP. Informational only (no behavior change): STUN reports the probe /// socket's source-port mapping, not a listen port, so it is never used to advertise /// a port — the advertised port stays UPnP-mapped / Kad-observed. -pub(crate) async fn run_ed2k_nat_type_probe(bind_ip: Ipv4Addr, shutdown: Arc) { +pub(crate) async fn run_ed2k_nat_type_probe( + binding: emulebb_ed2k::networking::EgressBinding, + shutdown: Arc, +) { if shutdown.load(Ordering::Relaxed) { return; } - match stun_probe_mapping_behavior(bind_ip, DEFAULT_STUN_TIMEOUT).await { + match stun_probe_mapping_behavior(binding, DEFAULT_STUN_TIMEOUT).await { NatMappingBehavior::EndpointIndependent => tracing::info!( "NAT mapping behavior: endpoint-independent (cone) — eD2k reask/HighID reachability is solid" ), diff --git a/crates/emulebb-core/src/hostname_lookup.rs b/crates/emulebb-core/src/hostname_lookup.rs index a390e656e..424ee9972 100644 --- a/crates/emulebb-core/src/hostname_lookup.rs +++ b/crates/emulebb-core/src/hostname_lookup.rs @@ -200,16 +200,17 @@ async fn run_hostname_lookup_tick(core: &EmulebbCore, settings: &HostnameLookupS return; } let bind_ip = network.bind_ip; - let bind_if_index = - match emulebb_ed2k::networking::require_bind_if_index(bind_ip, "hostname reverse DNS") { - Ok(index) => Some(index), - Err(error) if network.p2p_bind_ip.is_some() || network.p2p_bind_interface.is_some() => { - core.hostname_lookup_cache - .set_blocked(queued, error.to_string()); - return; - } - Err(_) => None, - }; + let bind_if_index = match network + .egress_route + .interface_index(bind_ip, "hostname reverse DNS") + { + Ok(index) => index, + Err(error) => { + core.hostname_lookup_cache + .set_blocked(queued, error.to_string()); + return; + } + }; for ip in queued { let result = reverse_lookup(ip, bind_ip, bind_if_index, &dns_servers).await; core.hostname_lookup_cache diff --git a/crates/emulebb-core/src/kad_hello.rs b/crates/emulebb-core/src/kad_hello.rs index 6930ab081..7f6885d26 100644 --- a/crates/emulebb-core/src/kad_hello.rs +++ b/crates/emulebb-core/src/kad_hello.rs @@ -9,7 +9,7 @@ //! dispatch and the test module reach them by their bare names. use std::{ - net::{IpAddr, Ipv4Addr, SocketAddr}, + net::{IpAddr, SocketAddr}, sync::Arc, time::Duration, }; @@ -275,12 +275,12 @@ async fn send_kad_firewalled_response(dht: &DhtNode, from: SocketAddr) -> Result } async fn probe_kad_firewalled_tcp( - bind_ip: Ipv4Addr, + binding: emulebb_ed2k::networking::EgressBinding, peer_addr: SocketAddr, timeout: Duration, ) -> Result<()> { - let bind_if_index = - emulebb_ed2k::networking::require_bind_if_index(bind_ip, "Kad TCP firewall probe")?; + let bind_ip = binding.address; + let bind_if_index = binding.interface_index("Kad TCP firewall probe")?; let socket = match peer_addr { SocketAddr::V4(_) => TcpSocket::new_v4(), SocketAddr::V6(_) => { @@ -296,7 +296,7 @@ async fn probe_kad_firewalled_tcp( // interface used by the eD2K TCP paths before connect. emulebb_kad_dht::socket_opts::pin_egress_to_interface( socket2::SockRef::from(&socket), - Some(bind_if_index), + bind_if_index, ) .with_context(|| format!("failed to pin Kad TCP firewall probe egress for {bind_ip}"))?; tokio::time::timeout(timeout, socket.connect(peer_addr)) @@ -354,7 +354,7 @@ async fn kad_firewalled_request_allowed( pub(crate) fn spawn_kad_firewalled_response( dht: DhtNode, - bind_ip: Ipv4Addr, + binding: emulebb_ed2k::networking::EgressBinding, reachability: ExternalReachability, kad_firewall: Arc>, our_tcp_port: u16, @@ -384,7 +384,7 @@ pub(crate) fn spawn_kad_firewalled_response( let peer_addr = SocketAddr::new(from.ip(), tcp_port); let timeout = Duration::from_secs(KAD_FIREWALLED_TCP_PROBE_TIMEOUT_SECS); - match probe_kad_firewalled_tcp(bind_ip, peer_addr, timeout).await { + match probe_kad_firewalled_tcp(binding, peer_addr, timeout).await { Ok(()) => { if let Err(error) = dht.send_packet(from, &KadPacket::FirewalledAckRes).await { tracing::debug!("Kad FIREWALLED_ACK_RES failed for {from}: {error:#}"); @@ -485,7 +485,7 @@ pub(crate) fn spawn_modern_kad_firewalled_response( }; match send_kad_firewall_tcp_ack( - network.bind_ip, + emulebb_ed2k::networking::EgressBinding::new(network.bind_ip, network.egress_route), peer_addr, hello_identity, req.user_hash.0, @@ -508,6 +508,7 @@ pub(crate) fn spawn_modern_kad_firewalled_response( #[cfg(test)] mod tests { use super::*; + use std::net::Ipv4Addr; #[test] fn kad_hello_advertises_source_uport_even_when_firewalled() { @@ -537,7 +538,10 @@ mod tests { #[tokio::test] async fn kad_tcp_firewall_probe_requires_resolved_bind_interface_index() { let err = probe_kad_firewalled_tcp( - Ipv4Addr::new(203, 0, 113, 234), + emulebb_ed2k::networking::EgressBinding::new( + Ipv4Addr::new(203, 0, 113, 234), + emulebb_ed2k::networking::EgressRoute::StrictInterface, + ), SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 44)), 4662), Duration::from_millis(1), ) diff --git a/crates/emulebb-core/src/kad_udp_firewall_check.rs b/crates/emulebb-core/src/kad_udp_firewall_check.rs index f2533d5cb..000f23c11 100644 --- a/crates/emulebb-core/src/kad_udp_firewall_check.rs +++ b/crates/emulebb-core/src/kad_udp_firewall_check.rs @@ -214,7 +214,7 @@ async fn run_kad_udp_firewall_check_round( let helper_addr = SocketAddr::new(IpAddr::V4(helper.ip), helper.tcp_port); match request_udp_firewall_check( Some(dht.clone()), - network.bind_ip, + emulebb_ed2k::networking::EgressBinding::new(network.bind_ip, network.egress_route), helper_addr, hello_identity, Arc::clone(&network.secure_ident), diff --git a/crates/emulebb-core/src/lib.rs b/crates/emulebb-core/src/lib.rs index 0686cd0ab..a7c3e87d7 100644 --- a/crates/emulebb-core/src/lib.rs +++ b/crates/emulebb-core/src/lib.rs @@ -580,6 +580,22 @@ impl EmulebbCore { ed2k_network: Option, progressive_catalog: bool, ) -> Result { + if let Some(network) = ed2k_network.as_ref() + && network.egress_route == emulebb_ed2k::networking::EgressRoute::DirectSystemRoute + { + // WHY: public runtime constructors must not accept a direct route + // alongside an explicit interface or VPN guard. The bind IP here may + // be auto-resolved, so only the original profile can classify it. + anyhow::ensure!( + !network.vpn_guard.enabled + && network.vpn_guard.mode.eq_ignore_ascii_case("off") + && network + .p2p_bind_interface + .as_deref() + .is_none_or(|name| name.trim().is_empty()), + "direct route conflicts with configured interface or VPN guard" + ); + } let transfer_root = transfer_root.as_ref().to_path_buf(); let metadata_store = index.metadata_store(); let has_persisted_core_settings = @@ -1741,6 +1757,7 @@ impl EmulebbCore { // Captured per-call into each peer attempt so a queued + UDP-eligible // source can detach onto the reask loop (None when reask is off). let reask_register = self.ed2k_reask_handle.lock().unwrap().clone(); + let egress_route = network.egress_route; let outcome = run_ed2k_direct_downloads( DirectDownloadOptions { bind_ip: network.bind_ip, @@ -1774,6 +1791,7 @@ impl EmulebbCore { download_files_from_peer( Ed2kPeerDownloadOptions { bind_ip, + egress_route, peer: &source, hello_identity, secure_ident: &secure_ident, @@ -6412,7 +6430,7 @@ async fn handle_kad_local_store_packet( .port(); spawn_kad_firewalled_response( dht.clone(), - network.bind_ip, + emulebb_ed2k::networking::EgressBinding::new(network.bind_ip, network.egress_route), runtime.reachability.clone(), Arc::clone(kad_firewall), our_tcp_port, @@ -7088,7 +7106,8 @@ async fn handle_kad_find_buddy_res( // back to us, then ping it at the oracle cadence. When the link drops, clear // our acquired buddy so the buddy-management loop re-searches (oracle // buddy-loss SetFindBuddy). - let bind_ip = network.bind_ip; + let binding = + emulebb_ed2k::networking::EgressBinding::new(network.bind_ip, network.egress_route); let hello_identity = buddy_hello_identity(network, reachability); let buddy_user_hash = res.client_hash.0; let buddy_addr = SocketAddr::new(from.ip(), res.tcp_port); @@ -7108,7 +7127,7 @@ async fn handle_kad_find_buddy_res( *buddy_link_cancel.lock().unwrap() = Some(cancel.clone()); tokio::spawn(async move { if let Err(error) = run_outbound_buddy_link(OutboundBuddyLinkOptions { - bind_ip, + binding, buddy_addr, buddy_user_hash, buddy_connect_options: connect_options, @@ -7409,6 +7428,22 @@ struct ParsedEd2kLink { sources: Vec, } +/// Validate an external file-link handoff without opening a profile or queuing work. +/// Transfer creation remains an explicit, separately authenticated operation. +pub fn validate_ed2k_file_link(link: &str) -> Result<()> { + ensure!( + link.len() <= 2048 && !link.chars().any(|c| c.is_whitespace() || c.is_control()), + "ED2K file link must be bounded and contain no whitespace or controls" + ); + let parsed = parse_ed2k_link(link)?; + let _: Ed2kHash = parsed.file_hash.parse().context("invalid ED2K file hash")?; + ensure!( + !parsed.name.is_empty() && !parsed.name.chars().any(char::is_control), + "invalid ED2K file name" + ); + Ok(()) +} + fn parse_ed2k_link(link: &str) -> Result { let parts = link .strip_prefix("ed2k://|file|") diff --git a/crates/emulebb-core/src/network_api.rs b/crates/emulebb-core/src/network_api.rs index d2c7aadc1..94fb7c7a2 100644 --- a/crates/emulebb-core/src/network_api.rs +++ b/crates/emulebb-core/src/network_api.rs @@ -419,8 +419,9 @@ impl EmulebbCore { let prerequisites = async { let configured_bootstrap_endpoints_text = configured_kad_bootstrap_endpoints_text(&network.kad_bootstrap_endpoints); - let kad_bind_if_index = - emulebb_ed2k::networking::require_bind_if_index(network.bind_ip, "Kad UDP")?; + let kad_bind_if_index = network + .egress_route + .interface_index(network.bind_ip, "Kad UDP")?; let dht = DhtNode::new(DhtConfig { bind_addr: Some(network.kad_bind_addr), obfuscation_enabled: network.ed2k.obfuscation_enabled, @@ -430,7 +431,7 @@ impl EmulebbCore { nodes_text: configured_bootstrap_endpoints_text, class_budgets: kad_rpc_class_budgets(), // Pin Kad UDP egress to the VPN bind interface (IP_UNICAST_IF). - bind_if_index: Some(kad_bind_if_index), + bind_if_index: kad_bind_if_index, ..DhtConfig::default() }) .await @@ -620,6 +621,7 @@ impl EmulebbCore { } let port_test_registry = PortTestRegistry::new(); tasks.push(tokio::spawn(run_ed2k_listener(Ed2kListenerOptions { + egress_route: network.egress_route, listener: Arc::clone(&ed2k_listener), dht: dht.clone(), server_state: Arc::clone(&server_state), @@ -738,7 +740,10 @@ impl EmulebbCore { // path keeps precedence (eMule GetPublicIP order: server, then Kad/STUN). if enable_udp_reask || kad_network_enabled { tasks.push(tokio::spawn(run_ed2k_public_ip_probe( - network.bind_ip, + emulebb_ed2k::networking::EgressBinding::new( + network.bind_ip, + network.egress_route, + ), ed2k_public_ip.clone(), reachability_writer.clone(), Arc::clone(&shutdown), @@ -749,7 +754,10 @@ impl EmulebbCore { // fragile (symmetric). Informational; reask degrades to TCP either way. if enable_udp_reask { tasks.push(tokio::spawn(run_ed2k_nat_type_probe( - network.bind_ip, + emulebb_ed2k::networking::EgressBinding::new( + network.bind_ip, + network.egress_route, + ), Arc::clone(&shutdown), ))); } diff --git a/crates/emulebb-core/src/rest_model.rs b/crates/emulebb-core/src/rest_model.rs index ec6b2b5da..b6a478387 100644 --- a/crates/emulebb-core/src/rest_model.rs +++ b/crates/emulebb-core/src/rest_model.rs @@ -1514,6 +1514,8 @@ pub struct TransferThroughputStats { #[derive(Debug, Clone)] pub struct Ed2kNetworkConfig { + /// Original operator routing intent, independent of the resolved source address. + pub egress_route: emulebb_ed2k::networking::EgressRoute, pub bind_ip: Ipv4Addr, pub kad_bind_addr: SocketAddr, pub listen_port: u16, diff --git a/crates/emulebb-core/src/server_list.rs b/crates/emulebb-core/src/server_list.rs index d9a6f6993..5e7774630 100644 --- a/crates/emulebb-core/src/server_list.rs +++ b/crates/emulebb-core/src/server_list.rs @@ -84,6 +84,11 @@ impl EmulebbCore { base: &Ed2kRuntimeConfig, ) -> Result { let mut config = base.clone(); + // Route intent belongs to the active runtime, not persisted server settings. + config.egress_route = self.ed2k_network.as_ref().map_or( + emulebb_ed2k::networking::EgressRoute::StrictInterface, + |network| network.egress_route, + ); let state = self.state.lock().await; config.server_entries.retain(|entry| { let endpoint = format!("{}:{}", entry.host, entry.port); diff --git a/crates/emulebb-core/src/shared_directories.rs b/crates/emulebb-core/src/shared_directories.rs index 3730a808a..1c38ebc4e 100644 --- a/crates/emulebb-core/src/shared_directories.rs +++ b/crates/emulebb-core/src/shared_directories.rs @@ -1058,6 +1058,22 @@ async fn scan_shared_files(core: &EmulebbCore) -> Result { // runs it off the async executor via spawn_blocking to avoid stalling a tokio // worker thread. let mut result = scan_shared_directory_roots(roots, policy).await?; + // WHY: Interrupted direct-copy outputs can retain ordinary final names. + // Keep them out of both the reload plan and root membership snapshots. + result.files.retain(|file| { + !core + .metadata_store + .delivery_output_is_excluded(&file.path) + .unwrap_or(true) + }); + for (_, files) in &mut result.root_files { + files.retain(|file| { + !core + .metadata_store + .delivery_output_is_excluded(&file.path) + .unwrap_or(true) + }); + } result .files .sort_by(|left, right| left.path.cmp(&right.path)); diff --git a/crates/emulebb-core/src/shared_directories/tests.rs b/crates/emulebb-core/src/shared_directories/tests.rs index dd9ee0a61..0dbef3079 100644 --- a/crates/emulebb-core/src/shared_directories/tests.rs +++ b/crates/emulebb-core/src/shared_directories/tests.rs @@ -35,6 +35,33 @@ async fn configure_test_root(core: &EmulebbCore, root: &Path) { .unwrap(); } +#[tokio::test] +async fn reserved_partial_is_excluded_from_scan_and_direct_share() { + let root = scratch_dir("reserved-partial"); + let core = EmulebbCore::new_in_memory("test", FileIndex::in_memory().unwrap()).unwrap(); + configure_test_root(&core, &root).await; + let partial = root.join("partial.bin"); + let other = root.join("complete.bin"); + core.metadata_store + .reserve_delivery_output(&partial, uuid::Uuid::new_v4()) + .unwrap(); + fs::write(&partial, b"partial bytes").unwrap(); + fs::write(&other, b"complete bytes").unwrap(); + let scan = scan_shared_files(&core).await.unwrap(); + assert_eq!(names(scan.files), vec!["complete.bin"]); + assert!( + core.share_local_file(LocalShareCreate { + path: partial.display().to_string(), + name: None, + }) + .await + .is_err() + ); + let shares = core.reload_shared_directories().await.unwrap(); + assert_eq!(shares.len(), 1); + fs::remove_dir_all(&root).ok(); +} + fn names(mut files: Vec) -> Vec { files.sort_by(|left, right| left.path.cmp(&right.path)); files diff --git a/crates/emulebb-core/src/shared_file_api.rs b/crates/emulebb-core/src/shared_file_api.rs index 550d082c1..9d3af9c23 100644 --- a/crates/emulebb-core/src/shared_file_api.rs +++ b/crates/emulebb-core/src/shared_file_api.rs @@ -88,6 +88,12 @@ impl EmulebbCore { requested_name: Option, progress: Option, ) -> Result { + ensure!( + !self + .metadata_store + .delivery_output_is_excluded(source_path)?, + "local file is an incomplete delivery output" + ); let metadata = tokio::fs::metadata(long_path(source_path)) .await .with_context(|| format!("failed to inspect {}", source_path.display()))?; diff --git a/crates/emulebb-core/src/tests.rs b/crates/emulebb-core/src/tests.rs index 6e9a57863..003d2bd9d 100644 --- a/crates/emulebb-core/src/tests.rs +++ b/crates/emulebb-core/src/tests.rs @@ -270,6 +270,7 @@ fn test_network_config_with_store( kad_snoop_queue: SnoopQueueConfig, ) -> Ed2kNetworkConfig { Ed2kNetworkConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, bind_ip: Ipv4Addr::new(198, 51, 100, 10), kad_bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 10)), 4665), listen_port: 4662, diff --git a/crates/emulebb-core/src/tests/transfer_lifecycle.rs b/crates/emulebb-core/src/tests/transfer_lifecycle.rs index 6c5d69197..6e2691768 100644 --- a/crates/emulebb-core/src/tests/transfer_lifecycle.rs +++ b/crates/emulebb-core/src/tests/transfer_lifecycle.rs @@ -986,7 +986,7 @@ async fn retired_download_does_not_redeliver_replaced_local_output() { ); reloaded.reload_shared_directories().await.unwrap(); - let redelivered_path = incoming_dir.join("Reconcile.Completed (1).bin"); + let redelivered_path = incoming_dir.join("Reconcile.Completed_01.bin"); assert_eq!( std::fs::read(&recorded_path).unwrap(), unrelated_replacement diff --git a/crates/emulebb-core/tests/kad_swarm_support/transfer.rs b/crates/emulebb-core/tests/kad_swarm_support/transfer.rs index 4bb467c2e..64c9e352b 100644 --- a/crates/emulebb-core/tests/kad_swarm_support/transfer.rs +++ b/crates/emulebb-core/tests/kad_swarm_support/transfer.rs @@ -117,6 +117,7 @@ fn test_network_config( }; config.listen_port = Some(listen_port); Ed2kNetworkConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, bind_ip, kad_bind_addr: SocketAddr::new(IpAddr::V4(bind_ip), 0), listen_port, diff --git a/crates/emulebb-core/tests/search_persistence.rs b/crates/emulebb-core/tests/search_persistence.rs index eede87fa6..2ee5f5314 100644 --- a/crates/emulebb-core/tests/search_persistence.rs +++ b/crates/emulebb-core/tests/search_persistence.rs @@ -196,6 +196,7 @@ fn seed_passive_file(metadata_path: &Path, file: IndexedFile) { fn test_network_config(runtime_dir: &Path) -> Ed2kNetworkConfig { let bind_ip = Ipv4Addr::new(198, 51, 100, 10); Ed2kNetworkConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, bind_ip, kad_bind_addr: SocketAddr::new(IpAddr::V4(bind_ip), 4665), listen_port: 4662, diff --git a/crates/emulebb-core/tests/vpn_failclosed.rs b/crates/emulebb-core/tests/vpn_failclosed.rs index 8c7d08de0..6d9c8c818 100644 --- a/crates/emulebb-core/tests/vpn_failclosed.rs +++ b/crates/emulebb-core/tests/vpn_failclosed.rs @@ -106,6 +106,32 @@ async fn data_plane_recovers_when_tunnel_is_confirmed() { ); } +#[test] +fn direct_route_rejects_conflicting_guard_or_named_interface() { + use emulebb_ed2k::networking::EgressRoute; + for case in ["allowed", "guard-enabled", "guard-mode", "named-interface"] { + let root = unique_runtime_dir(case); + let mut network = test_network_config(&root); + // This is an auto-resolved address in the public runtime shape, not + // evidence of an operator-selected bind IP. + network.egress_route = EgressRoute::DirectSystemRoute; + network.vpn_guard.mode = "off".into(); + match case { + "guard-enabled" => network.vpn_guard.enabled = true, + "guard-mode" => network.vpn_guard.mode = "block".into(), + "named-interface" => network.p2p_bind_interface = Some("tunnel".into()), + _ => {} + } + let result = EmulebbCore::new_with_network( + "test", + FileIndex::open(root.join("metadata.sqlite")).unwrap(), + root.join("transfers"), + Some(network), + ); + assert_eq!(result.is_ok(), case == "allowed", "case {case}"); + } +} + fn build_core(transfer_root: &Path, network: Ed2kNetworkConfig) -> EmulebbCore { EmulebbCore::new_with_network( "test", @@ -124,6 +150,7 @@ fn test_network_config(root: &Path) -> Ed2kNetworkConfig { ..Ed2kRuntimeConfig::default() }; Ed2kNetworkConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, bind_ip: Ipv4Addr::new(198, 51, 100, 10), kad_bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 10)), 4665), listen_port: 4662, diff --git a/crates/emulebb-core/tests/vpn_guard_runtime.rs b/crates/emulebb-core/tests/vpn_guard_runtime.rs index a15e9b315..ae2d5b741 100644 --- a/crates/emulebb-core/tests/vpn_guard_runtime.rs +++ b/crates/emulebb-core/tests/vpn_guard_runtime.rs @@ -43,6 +43,7 @@ async fn vpn_guard_uses_runtime_binding_state() { fn test_network_config(root: &Path) -> Ed2kNetworkConfig { Ed2kNetworkConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, bind_ip: Ipv4Addr::new(198, 51, 100, 10), kad_bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 10)), 4665), listen_port: 4662, diff --git a/crates/emulebb-core/tests/vpn_leak_egress.rs b/crates/emulebb-core/tests/vpn_leak_egress.rs index cb501b53b..20e586726 100644 --- a/crates/emulebb-core/tests/vpn_leak_egress.rs +++ b/crates/emulebb-core/tests/vpn_leak_egress.rs @@ -161,6 +161,7 @@ fn tunnel_network_config( ..Ed2kRuntimeConfig::default() }; Ed2kNetworkConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, bind_ip: ip, kad_bind_addr: SocketAddr::new(IpAddr::V4(ip), 0), listen_port: 0, diff --git a/crates/emulebb-daemon/Cargo.toml b/crates/emulebb-daemon/Cargo.toml index e0f14f28e..c6a93e377 100644 --- a/crates/emulebb-daemon/Cargo.toml +++ b/crates/emulebb-daemon/Cargo.toml @@ -41,6 +41,8 @@ packet-diagnostics = [ anyhow.workspace = true axum.workspace = true clap.workspace = true +chrono.workspace = true +emulebb-diagnostics = { path = "../emulebb-diagnostics" } emulebb-core = { path = "../emulebb-core" } emulebb-ed2k = { path = "../emulebb-ed2k" } emulebb-index = { path = "../emulebb-index" } @@ -56,6 +58,7 @@ serde.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } toml.workspace = true +tempfile.workspace = true tracing.workspace = true tracing-appender.workspace = true tracing-subscriber.workspace = true diff --git a/crates/emulebb-daemon/resources/windows/application.ico b/crates/emulebb-daemon/resources/windows/application.ico index 4eae95324..16379f053 100644 Binary files a/crates/emulebb-daemon/resources/windows/application.ico and b/crates/emulebb-daemon/resources/windows/application.ico differ diff --git a/crates/emulebb-daemon/src/bin/emulebb-rust.rs b/crates/emulebb-daemon/src/bin/emulebb-rust.rs index f1f9f675e..2b279823b 100644 --- a/crates/emulebb-daemon/src/bin/emulebb-rust.rs +++ b/crates/emulebb-daemon/src/bin/emulebb-rust.rs @@ -23,7 +23,7 @@ struct Cli { profile: Option, #[arg( long, - help = "Override the REST/WebUI listener for this run (for example, 127.0.0.1:20017); persist it as rest.bind_addr in the bootstrap TOML" + help = "Override the REST/WebUI listener for this run (default 0.0.0.0:20017); persist it as rest.bind_addr in the bootstrap TOML" )] rest_bind_addr: Option, #[arg(long, help = "Override the finished-download directory for this run")] diff --git a/crates/emulebb-daemon/src/bind_config.rs b/crates/emulebb-daemon/src/bind_config.rs index f28cead96..3b9d2b025 100644 --- a/crates/emulebb-daemon/src/bind_config.rs +++ b/crates/emulebb-daemon/src/bind_config.rs @@ -6,7 +6,43 @@ use emulebb_ed2k::{InterfaceAddressFamily, NetworkInterface}; use crate::DaemonProfile; +/// A transient startup prerequisite, distinct from invalid or ambiguous binding settings. +#[derive(Debug, PartialEq, Eq)] +pub enum NetworkStartupError { + DefaultRouteUnavailable, +} + +impl std::fmt::Display for NetworkStartupError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::DefaultRouteUnavailable => formatter.write_str( + "p2pBindIp or p2pBindInterface is required because no default-route IPv4 interface was detected; \ + start with --p2p-bind-interface , then save the P2P bind interface in WebUI Settings > Network", + ), + } + } +} + +impl std::error::Error for NetworkStartupError {} + impl DaemonProfile { + /// Preserve the original operator route selection before resolving its source IP. + pub(crate) fn egress_route(&self) -> emulebb_ed2k::networking::EgressRoute { + use emulebb_ed2k::networking::EgressRoute; + if self.p2p_bind_ip.is_none() + && self + .p2p_bind_interface + .as_deref() + .is_none_or(|name| name.trim().is_empty()) + && !self.vpn_guard.enabled + && self.vpn_guard.mode.eq_ignore_ascii_case("off") + { + EgressRoute::DirectSystemRoute + } else { + EgressRoute::StrictInterface + } + } + pub(crate) fn resolve_p2p_bind_ip_from_interfaces( &self, interfaces: &[NetworkInterface], @@ -67,10 +103,7 @@ fn resolve_default_route_ip(interfaces: &[NetworkInterface]) -> Result .collect::>(); match candidates.as_slice() { [address] => Ok(*address), - [] => bail!( - "p2pBindIp or p2pBindInterface is required because no default-route IPv4 interface was detected; \ - start with --p2p-bind-interface , then save the P2P bind interface in WebUI Settings > Network" - ), + [] => Err(NetworkStartupError::DefaultRouteUnavailable.into()), _ => bail!( "p2pBindIp or p2pBindInterface is required because multiple default-route IPv4 interfaces were detected; \ start with --p2p-bind-interface , then save the P2P bind interface in WebUI Settings > Network" @@ -135,3 +168,30 @@ fn interface_token_matches(iface: &NetworkInterface, bind_interface: &str) -> bo .as_deref() .is_some_and(|description| description.to_ascii_lowercase().contains(&token)) } + +#[cfg(test)] +mod route_intent_tests { + use super::*; + use emulebb_ed2k::networking::EgressRoute; + + #[test] + fn only_unconfigured_and_disabled_off_profile_uses_system_route() { + let mut profile = DaemonProfile::default(); + assert_eq!(profile.egress_route(), EgressRoute::DirectSystemRoute); + profile.p2p_bind_interface = Some(" ".into()); + assert_eq!(profile.egress_route(), EgressRoute::DirectSystemRoute); + profile.p2p_bind_interface = Some("tunnel".into()); + assert_eq!(profile.egress_route(), EgressRoute::StrictInterface); + profile.p2p_bind_interface = None; + profile.p2p_bind_ip = Some(Ipv4Addr::LOCALHOST); + assert_eq!(profile.egress_route(), EgressRoute::StrictInterface); + profile.p2p_bind_ip = None; + profile.vpn_guard.enabled = true; + assert_eq!(profile.egress_route(), EgressRoute::StrictInterface); + profile.vpn_guard.enabled = false; + profile.vpn_guard.mode = "block".into(); + assert_eq!(profile.egress_route(), EgressRoute::StrictInterface); + profile.vpn_guard.mode = "unknown".into(); + assert_eq!(profile.egress_route(), EgressRoute::StrictInterface); + } +} diff --git a/crates/emulebb-daemon/src/host_bootstrap.rs b/crates/emulebb-daemon/src/host_bootstrap.rs new file mode 100644 index 000000000..4c4f8bfff --- /dev/null +++ b/crates/emulebb-daemon/src/host_bootstrap.rs @@ -0,0 +1,130 @@ +//! Canonical bootstrap access for application hosts, serialized by the profile lease. + +use std::{io::Write, path::Path}; + +use anyhow::{Context, Result}; + +use super::{ + DaemonBootstrapSettings, DaemonProfile, PROFILE_SETTINGS_FILE, ProfileLease, + RestBootstrapSettings, +}; + +/// Execute application-owned maintenance only while no daemon owns this profile. +/// The operation must finish before returning; ownership never escapes this call. +pub fn while_stopped(profile_dir: &Path, operation: impl FnOnce() -> Result) -> Result { + let _lease = ProfileLease::acquire(profile_dir, true)?; + operation() +} + +/// Read existing settings without opening or changing the metadata database. +/// Atomic bootstrap publication permits read-only setup/status UI while the +/// daemon owns the profile. Only writes require an exclusive profile lease. +pub fn read(profile_dir: &Path) -> Result> { + let path = profile_dir.join(PROFILE_SETTINGS_FILE); + if !path.exists() { + return Ok(None); + } + super::validate_profile_settings_permissions(&path)?; + let text = std::fs::read_to_string(path)?; + let bootstrap: DaemonBootstrapSettings = + toml::from_str(&text).map_err(|error: toml::de::Error| { + anyhow::anyhow!("invalid bootstrap: {}", error.message()) + })?; + validate(&bootstrap.rest)?; + Ok(Some(bootstrap.rest)) +} + +/// Validate the same bootstrap contract used by daemon startup. +pub fn validate(rest: &RestBootstrapSettings) -> Result<()> { + let profile = DaemonProfile { + rest: rest.clone(), + ..Default::default() + }; + profile.validate_rest_api_key()?; + profile.rest_bind_addr()?; + Ok(()) +} + +/// Atomically replace stopped-host bootstrap settings while retaining exclusive ownership. +pub fn write(profile_dir: &Path, rest: RestBootstrapSettings) -> Result<()> { + validate(&rest)?; + let lease = ProfileLease::acquire(profile_dir, true)?; + let text = toml::to_string(&DaemonBootstrapSettings { rest })?; + let mut temporary = tempfile::NamedTempFile::new_in(lease.directory())?; + temporary.write_all(text.as_bytes())?; + temporary.as_file().sync_all()?; + temporary + .persist(lease.directory().join(PROFILE_SETTINGS_FILE)) + .map_err(|error| error.error) + .context("failed to publish bootstrap settings")?; + #[cfg(unix)] + std::fs::File::open(lease.directory())?.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_settings_keep_one_owner_and_do_not_initialize_metadata() { + let root = tempfile::tempdir().unwrap(); + let rest = RestBootstrapSettings { + bind_addr: Some("127.0.0.1:20017".parse().unwrap()), + api_key: "0123456789abcdef0123456789abcdef".into(), + web_root_dir: None, + }; + assert!(read(root.path()).unwrap().is_none()); + write(root.path(), rest.clone()).unwrap(); + assert_eq!(read(root.path()).unwrap().unwrap().api_key, rest.api_key); + assert!( + !root + .path() + .join(super::super::PROFILE_METADATA_FILE) + .exists() + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(root.path().join(PROFILE_SETTINGS_FILE)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + } + let _lease = ProfileLease::acquire(root.path(), false).unwrap(); + assert!(write(root.path(), rest).is_err()); + assert!(read(root.path()).unwrap().is_some()); + } + + #[test] + fn invalid_settings_do_not_replace_the_existing_bootstrap() { + let root = tempfile::tempdir().unwrap(); + let mut rest = RestBootstrapSettings { + bind_addr: Some("127.0.0.1:20017".parse().unwrap()), + api_key: "0123456789abcdef0123456789abcdef".into(), + web_root_dir: None, + }; + write(root.path(), rest.clone()).unwrap(); + let original = std::fs::read(root.path().join(PROFILE_SETTINGS_FILE)).unwrap(); + rest.api_key = "short".into(); + assert!(write(root.path(), rest).is_err()); + assert_eq!( + std::fs::read(root.path().join(PROFILE_SETTINGS_FILE)).unwrap(), + original + ); + for text in [ + "[rest]\napi_key = '0123456789abcdef0123456789abcdef'", + "[rest]\nbindAddr = '127.0.0.1:20017'", + "[rest]\nbind_addr = 'invalid'", + "[rest]\napi_key = 'short'", + "[unknown]\nvalue=1", + ] { + std::fs::write(root.path().join(PROFILE_SETTINGS_FILE), text).unwrap(); + assert!(read(root.path()).is_err()); + } + } +} diff --git a/crates/emulebb-daemon/src/lib.rs b/crates/emulebb-daemon/src/lib.rs index 06c7b4d5d..5bdca50bd 100644 --- a/crates/emulebb-daemon/src/lib.rs +++ b/crates/emulebb-daemon/src/lib.rs @@ -17,7 +17,9 @@ use emulebb_ed2k::{ ipfilter::IpFilter, }; use emulebb_index::{FileIndex, KadLocalStoreConfig, SnoopQueueConfig}; -use emulebb_metadata::{MetadataLocalIdentity, MetadataStore, SCHEMA_VERSION}; +use emulebb_metadata::{ + InitialProfileStorage, MetadataLocalIdentity, MetadataStore, SCHEMA_VERSION, +}; use emulebb_rest::{RestServerSettings, router_with_shutdown}; use emulebb_settings::{ DaemonSettings, Ed2kSettings, Ed2kUploadQueueSettings, IpFilterSettings, KadSettings, @@ -31,13 +33,16 @@ use tokio::time::MissedTickBehavior; use tracing::{info, warn}; mod bind_config; +pub use bind_config::NetworkStartupError; mod discovery_bootstrap; +pub mod host_bootstrap; pub mod logging; mod profile_import; mod profile_ownership; mod rest_listener; pub mod shared_directory_import; mod shutdown; +pub mod support_diagnostics; mod vpn_guard_monitor; pub use profile_import::{ProfileImportApplySummary, apply_profile_import}; @@ -48,6 +53,7 @@ pub const PROFILE_SETTINGS_FILE: &str = "emulebb-rust-settings.toml"; pub const PROFILE_METADATA_FILE: &str = "emulebb-rust-metadata.db"; pub use emulebb_core::{ BUILD_CHANNEL, BUILD_SOURCE_REVISION, BUILD_VERSION, build_version_description, + validate_ed2k_file_link, }; const REGULAR_DIAGNOSTIC_SUMMARY_INTERVAL: Duration = Duration::from_secs(10); @@ -69,6 +75,9 @@ const UNSAFE_REST_API_KEYS: &[&str] = &[ #[serde(default, rename_all = "camelCase")] pub struct DaemonProfile { pub profile_dir: PathBuf, + /// Application-owned payload storage, separate from private metadata. + #[serde(skip)] + pub host_transfer_root: Option, /// Global finished-file delivery directory (eMule Incoming folder). When a /// completed transfer has no category path, its payload is materialized here /// by its canonical name. Defaults to `/incoming` when unset. @@ -109,6 +118,7 @@ impl Default for DaemonProfile { let daemon = DaemonSettings::default(); Self { profile_dir: PathBuf::from("profile"), + host_transfer_root: None, incoming_dir: daemon.incoming_dir, p2p_bind_ip: daemon.p2p_bind_ip, p2p_bind_interface: daemon.p2p_bind_interface, @@ -141,14 +151,25 @@ impl DaemonProfile { Self::load_from_dir(profile_dir) } + /// Load an application profile, applying folder defaults only to a new database. + /// Existing canonical settings and removed sharing roots are never reseeded. + pub fn load_hosted(profile_dir: PathBuf, incoming: PathBuf, shared: PathBuf) -> Result { + let lease = ProfileLease::acquire(&profile_dir, true)?; + Self::load_owned(profile_dir, lease, Some((incoming, shared))) + } + fn load_from_dir(profile_dir: PathBuf) -> Result { // WHY: bootstrap creation, schema initialization and reads of mutable // state must all belong to the same exclusive writable-profile session. let profile_lease = ProfileLease::acquire(&profile_dir, true)?; - Self::load_owned(profile_dir, profile_lease) + Self::load_owned(profile_dir, profile_lease, None) } - fn load_owned(profile_dir: PathBuf, profile_lease: ProfileLease) -> Result { + fn load_owned( + profile_dir: PathBuf, + profile_lease: ProfileLease, + host_folders: Option<(PathBuf, PathBuf)>, + ) -> Result { let settings_path = profile_lease.directory().join(PROFILE_SETTINGS_FILE); if !settings_path.exists() { create_default_profile_bootstrap(&profile_dir, &settings_path)?; @@ -167,22 +188,42 @@ impl DaemonProfile { ) })?; let metadata_path = profile_lease.directory().join(PROFILE_METADATA_FILE); - // WHY: a packaged bootstrap already exists before its first run. Seed - // network defaults only for a new database, never from TOML existence. - let fresh_metadata = !metadata_path.exists(); - let metadata = MetadataStore::open_with_profile_lease(metadata_path, profile_lease.file()) - .with_context(|| { - format!( - "failed to open metadata store under {}", - profile_dir.display() - ) - })?; - if fresh_metadata { + let opened = match host_folders { + Some((incoming, shared)) => MetadataStore::open_with_profile_lease_and_storage( + metadata_path, + profile_lease.file(), + move || { + anyhow::ensure!( + incoming.is_absolute() && shared.is_absolute(), + "host folders must be absolute" + ); + fs::create_dir_all(&incoming)?; + fs::create_dir_all(&shared)?; + Ok(InitialProfileStorage { + incoming_dir: PathBuf::from(emulebb_core::normal_path_display( + incoming.canonicalize()?, + )), + shared_dir: PathBuf::from(emulebb_core::normal_path_display( + shared.canonicalize()?, + )), + }) + }, + ), + None => MetadataStore::open_with_profile_lease(metadata_path, profile_lease.file()), + }; + let metadata = opened.with_context(|| { + format!( + "failed to open metadata store under {}", + profile_dir.display() + ) + })?; + if metadata.initialized_new_schema() { seed_default_profile_network_settings(&metadata)?; } let settings = load_settings(&metadata)?; let profile = Self { profile_dir, + host_transfer_root: None, incoming_dir: settings.daemon.incoming_dir, p2p_bind_ip: settings.daemon.p2p_bind_ip, p2p_bind_interface: settings.daemon.p2p_bind_interface, @@ -231,7 +272,9 @@ impl DaemonProfile { } pub fn transfer_root(&self) -> PathBuf { - self.profile_io_dir().join("transfers") + self.host_transfer_root + .clone() + .unwrap_or_else(|| self.profile_io_dir().join("transfers")) } /// Resolve the finished-file delivery directory: the configured @@ -275,7 +318,11 @@ impl DaemonProfile { let secure_ident = Arc::new(load_or_create_secure_ident(metadata)?); let ip_filter = self.load_ip_filter()?; let vpn_interface_bound = self.vpn_binding_confirmed(bind_ip, detected_interfaces); + // WHY: a default-route address is resolved for source binding, but it + // must not be mistaken for an operator-selected interface pin. + let egress_route = self.egress_route(); Ok(Some(Ed2kNetworkConfig { + egress_route, bind_ip, kad_bind_addr: self.kad_bind_addr(bind_ip)?, listen_port, @@ -573,6 +620,7 @@ where fn ed2k_runtime_config_from_settings(settings: Ed2kSettings) -> Ed2kRuntimeConfig { Ed2kRuntimeConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, listen_port: settings.listen_port, server_entries: Vec::new(), server_endpoints: Vec::new(), @@ -733,12 +781,6 @@ async fn run_inner(mut profile: DaemonProfile, host: Option) -> metadata_schema_version = SCHEMA_VERSION, "starting emulebb-rust" ); - if !rest_bind_addr.ip().is_loopback() { - warn!( - %rest_bind_addr, - "REST is listening beyond loopback over plaintext HTTP; restrict network exposure or terminate TLS at a trusted reverse proxy" - ); - } // Embedded callers can construct a profile without loading it from disk. // They acquire the same lease before any discovery, DB or runtime mutation. if profile.profile_lease.is_none() { diff --git a/crates/emulebb-daemon/src/logging.rs b/crates/emulebb-daemon/src/logging.rs index f28fbd7ea..86c10844f 100644 --- a/crates/emulebb-daemon/src/logging.rs +++ b/crates/emulebb-daemon/src/logging.rs @@ -110,12 +110,17 @@ pub struct LoggingGuard { error_counter: Option, monitor_shutdown: Option>, monitor_task: Option>, + process_monitor: Option<(std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>)>, shutdown_reported: bool, } impl LoggingGuard { /// Stops drop monitoring, reports any loss, and flushes retained logs. pub async fn shutdown(mut self) { + if let Some((shutdown, task)) = self.process_monitor.take() { + let _ = shutdown.send(()); + let _ = tokio::task::spawn_blocking(move || task.join()).await; + } if let Some(shutdown) = self.monitor_shutdown.take() { let _ = shutdown.send(true); } @@ -138,6 +143,8 @@ impl LoggingGuard { impl Drop for LoggingGuard { fn drop(&mut self) { + // Dropping the sender wakes the optional process monitor immediately. + self.process_monitor.take(); if let Some(task) = self.monitor_task.take() { task.abort(); } @@ -157,6 +164,16 @@ impl Drop for LoggingGuard { /// The profile must already be validated before this function is called. File /// setup failure is non-fatal: console and REST logging remain available. pub fn init(profile_dir: &Path) -> Result { + init_inner(profile_dir, false) +} + +/// Install the canonical subscriber for a host process spanning several Tokio +/// runtimes. Retain its guard for that process's lifetime, not one daemon run. +pub fn init_process(profile_dir: &Path) -> Result { + init_inner(profile_dir, true) +} + +fn init_inner(profile_dir: &Path, process_owned: bool) -> Result { let (filter_spec, filter_warning) = read_filter_spec(); let mut file_warning = None; let file_output = match create_file_output(profile_dir) { @@ -213,7 +230,7 @@ pub fn init(profile_dir: &Path) -> Result { ); } - let (monitor_shutdown, monitor_task) = match error_counter.as_ref() { + let (monitor_shutdown, monitor_task) = match error_counter.as_ref().filter(|_| !process_owned) { Some(counter) => { let (shutdown_tx, shutdown_rx) = watch::channel(false); let task = tokio::spawn(monitor_dropped_lines(counter.clone(), shutdown_rx)); @@ -221,12 +238,37 @@ pub fn init(profile_dir: &Path) -> Result { } None => (None, None), }; + let process_monitor = error_counter + .as_ref() + .filter(|_| process_owned) + .and_then(|counter| { + let (sender, receiver) = std::sync::mpsc::channel(); + let counter = counter.clone(); + match std::thread::Builder::new() + .name("emulebb-log-health".into()) + .spawn(move || { + let mut last_dropped = 0; + while matches!( + receiver.recv_timeout(DROP_MONITOR_INTERVAL), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ) { + report_dropped_lines(&counter, &mut last_dropped); + } + }) { + Ok(task) => Some((sender, task)), + Err(error) => { + tracing::warn!(%error, "persistent log drop monitoring is unavailable"); + None + } + } + }); Ok(LoggingGuard { worker_guard, error_counter, monitor_shutdown, monitor_task, + process_monitor, shutdown_reported: false, }) } @@ -294,20 +336,24 @@ async fn monitor_dropped_lines(counter: ErrorCounter, mut shutdown: watch::Recei } } _ = interval.tick() => { - let dropped = counter.dropped_lines(); - if dropped > last_dropped { - tracing::warn!( - dropped_file_log_lines = dropped, - dropped_since_last_report = dropped - last_dropped, - "persistent log writer is dropping events" - ); - last_dropped = dropped; - } + report_dropped_lines(&counter, &mut last_dropped); } } } } +fn report_dropped_lines(counter: &ErrorCounter, last_dropped: &mut usize) { + let dropped = counter.dropped_lines(); + if dropped > *last_dropped { + tracing::warn!( + dropped_file_log_lines = dropped, + dropped_since_last_report = dropped - *last_dropped, + "persistent log writer is dropping events" + ); + *last_dropped = dropped; + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/emulebb-daemon/src/support_diagnostics.rs b/crates/emulebb-daemon/src/support_diagnostics.rs new file mode 100644 index 000000000..129bdf447 --- /dev/null +++ b/crates/emulebb-daemon/src/support_diagnostics.rs @@ -0,0 +1,224 @@ +//! Share-safe summaries of retained daemon logs. Raw bundles remain private. + +use std::{fs, path::Path}; + +use anyhow::{Result, ensure}; +use chrono::{DateTime, Utc}; +use emulebb_diagnostics::log_tail::read_jsonl_tail; +use serde_json::{Value, json}; + +const MAX_SCAN: usize = 256; +const MAX_SEGMENTS: usize = 2; +const MAX_BYTES: u64 = 256 * 1024; +const MAX_RECORDS: usize = 200; + +/// Read a bounded snapshot without opening settings or metadata, mutating the +/// profile, or exposing arbitrary log strings. Safe while a writer is active. +pub fn capture(profile: &Path) -> Result { + ensure!(profile.is_absolute(), "diagnostic profile must be absolute"); + let mut records = Vec::new(); + let mut bytes_read = 0; + let mut segments_read = 0; + let mut malformed_records = 0; + let mut read_issues = 0; + let mut truncated = false; + let directory = profile.join("logs"); + if directory.exists() { + let profile = profile.canonicalize()?; + ensure!( + !fs::symlink_metadata(&directory)?.file_type().is_symlink(), + "diagnostic log directory must be owned" + ); + ensure!( + directory.canonicalize()?.starts_with(&profile), + "diagnostic logs escaped the profile" + ); + let mut candidates = Vec::new(); + for (index, entry) in fs::read_dir(&directory)?.enumerate() { + if index >= MAX_SCAN { + truncated = true; + break; + } + let Ok(entry) = entry else { + read_issues += 1; + continue; + }; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if !name.starts_with("emulebb-rust.") || !name.ends_with(".jsonl") { + continue; + } + let Ok(kind) = entry.file_type() else { + read_issues += 1; + continue; + }; + if !kind.is_file() { + continue; + } + let Ok(metadata) = entry.metadata() else { + read_issues += 1; + continue; + }; + let Ok(modified) = metadata.modified() else { + read_issues += 1; + continue; + }; + candidates.push((modified, entry.path())); + } + candidates.sort(); + truncated |= candidates.len() > MAX_SEGMENTS; + let mut tails = Vec::new(); + for (_, path) in candidates.into_iter().rev().take(MAX_SEGMENTS) { + let remaining = MAX_BYTES.saturating_sub(bytes_read); + if remaining == 0 { + truncated = true; + break; + } + match read_jsonl_tail(&path, remaining, true) { + Ok(tail) => { + bytes_read += tail.bytes_read as u64; + segments_read += 1; + truncated |= tail.tail_start_offset > 0 || tail.incomplete_final_line_ignored; + tails.push(tail); + } + Err(_) => read_issues += 1, + } + } + for line in tails.into_iter().rev().flat_map(|tail| tail.lines) { + let parsed = serde_json::from_slice::(&line).ok(); + match parsed.as_ref().and_then(project_record) { + Some(record) => records.push(record), + None => malformed_records += 1, + } + } + } + if records.len() > MAX_RECORDS { + truncated = true; + records.drain(..records.len() - MAX_RECORDS); + } + Ok(json!({ + "schema": "emulebb.support-diagnostics/1", "capturedAt": Utc::now(), + "build": {"version": crate::BUILD_VERSION, "channel": crate::BUILD_CHANNEL, "sourceRevision": crate::BUILD_SOURCE_REVISION}, + "included": "Bounded timestamp, severity, component, safe event category and OS error summaries only", + "excluded": ["credentials", "configuration", "paths", "peer identifiers", "content identifiers", "free-form messages"], + "limits": {"maxReadBytes": MAX_BYTES, "maxSegments": MAX_SEGMENTS, "maxRecords": MAX_RECORDS}, + "snapshot": {"bytesRead": bytes_read, "segmentsRead": segments_read, "truncated": truncated, "malformedRecordsOmitted": malformed_records, "readIssues": read_issues}, + "recentLogs": records, + })) +} + +fn project_record(record: &Value) -> Option { + let timestamp = DateTime::parse_from_rfc3339(record.get("timestamp")?.as_str()?).ok()?; + let level = match record.get("level")?.as_str()? { + "ERROR" => "ERROR", + "WARN" => "WARN", + "INFO" => "INFO", + "DEBUG" => "DEBUG", + "TRACE" => "TRACE", + _ => return None, + }; + let component = match record + .get("target") + .and_then(Value::as_str) + .and_then(|target| target.split("::").next()) + { + Some("emulebb_daemon") => "daemon", + Some("emulebb_core") => "core", + Some("emulebb_ed2k") => "ed2k", + Some("emulebb_rest") => "rest", + Some("emulebb_metadata") => "metadata", + Some("emulebb_kad_dht") => "kad", + _ => "dependency", + }; + let fields = record.get("fields")?; + // WHY: arbitrary strings can contain filenames, endpoints and credentials. + // Emit closed categories, never copied message/error/span/config values. + let event = match fields.get("message").and_then(Value::as_str) { + Some("starting emulebb-rust") => "daemon_starting", + Some("persistent logging is unavailable; continuing with console and REST logging") => { + "persistent_logging_unavailable" + } + Some("automatic Kad startup failed; REST start remains available") => "kad_start_failed", + Some("automatic ED2K/Kad startup failed; REST connect remains available") => { + "peer_connect_failed" + } + Some("initial shared-directory scan failed; continuing") => "sharing_scan_failed", + Some("empty server list remains available for manual WebUI import") => { + "discovery_servers_unavailable" + } + _ => "free_form_details_omitted", + }; + let os_error = fields + .get("error") + .and_then(Value::as_str) + .and_then(|error| error.rsplit_once("(os error ").map(|(_, suffix)| suffix)) + .and_then(|suffix| suffix.strip_suffix(')')) + .and_then(|code| code.parse::().ok()) + .filter(|code| (1..=65535).contains(code)); + Some( + json!({"timestamp": timestamp.to_rfc3339(), "level": level, "component": component, + "eventCode": event, "errorPresent": fields.get("error").is_some(), + "osError": os_error.map(|code| json!({"code": code, "kind": format!("{:?}", std::io::Error::from_raw_os_error(code).kind())})), + }), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sharing_omits_credentials_config_paths_and_peer_content_from_all_fields() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("logs")).unwrap(); + let secret = "0123456789abcdef0123456789abcdef"; + let record = json!({"timestamp":"2026-11-13T12:00:00Z", "level":"ERROR", "target":"emulebb_daemon::private_module", + "fields":{"message":format!("{secret} /private/profile Sample.bin 192.0.2.1"), "error":"/private/profile: denied (os error 13)", "api_key":secret}, + "span":{"file":"Sample.bin", "peer":"192.0.2.1"}}); + fs::write( + root.path().join("logs/emulebb-rust.2026-11-13.jsonl"), + format!("{record}\n{{unfinished"), + ) + .unwrap(); + fs::write( + root.path().join("emulebb-rust-settings.toml"), + format!("api_key = '{secret}'"), + ) + .unwrap(); + let report = capture(root.path()).unwrap(); + let encoded = serde_json::to_string(&report).unwrap(); + for private in [ + secret, + "/private/profile", + "Sample.bin", + "192.0.2.1", + "private_module", + ] { + assert!(!encoded.contains(private)); + } + assert_eq!(report["recentLogs"][0]["level"], "ERROR"); + assert_eq!(report["recentLogs"][0]["osError"]["code"], 13); + assert_eq!(report["snapshot"]["truncated"], true); + } + + #[test] + fn sharing_a_missing_profile_creates_nothing_and_bounds_recent_records() { + let root = tempfile::tempdir().unwrap(); + let missing = root.path().join("absent"); + assert_eq!(capture(&missing).unwrap()["recentLogs"], json!([])); + assert!(!missing.exists()); + fs::create_dir(root.path().join("logs")).unwrap(); + let line = "{\"timestamp\":\"2026-11-13T12:00:00Z\",\"level\":\"INFO\",\"target\":\"emulebb_daemon\",\"fields\":{\"message\":\"starting emulebb-rust\"}}\n"; + fs::write( + root.path().join("logs/emulebb-rust.2026-11-13.jsonl"), + line.repeat(300), + ) + .unwrap(); + let report = capture(root.path()).unwrap(); + assert_eq!(report["recentLogs"].as_array().unwrap().len(), MAX_RECORDS); + assert!(report["snapshot"]["bytesRead"].as_u64().unwrap() <= MAX_BYTES); + assert_eq!(report["recentLogs"][0]["eventCode"], "daemon_starting"); + } +} diff --git a/crates/emulebb-daemon/src/tests.rs b/crates/emulebb-daemon/src/tests.rs index dc4b5ca4b..0f5f1b9e9 100644 --- a/crates/emulebb-daemon/src/tests.rs +++ b/crates/emulebb-daemon/src/tests.rs @@ -244,7 +244,7 @@ fn retargeted_profile_alias_keeps_io_in_the_leased_directory() { fs::canonicalize(&other).unwrap() ); - let profile = DaemonProfile::load_owned(alias.clone(), lease).unwrap(); + let profile = DaemonProfile::load_owned(alias.clone(), lease, None).unwrap(); let metadata = profile.open_metadata_store().unwrap(); put_setting(&metadata, "fixture", "leased", serde_json::json!(true)); let leased = fs::canonicalize(leased).unwrap(); @@ -258,7 +258,7 @@ fn retargeted_profile_alias_keeps_io_in_the_leased_directory() { } #[test] -fn default_profile_bootstrap_is_loopback_authenticated_and_seeds_network_ports() { +fn default_profile_bootstrap_listens_on_all_ipv4_interfaces_and_seeds_network_ports() { let temp = tempfile::tempdir().unwrap(); let profile_dir = temp.path().join("profile"); let settings_path = profile_dir.join(PROFILE_SETTINGS_FILE); @@ -269,7 +269,7 @@ fn default_profile_bootstrap_is_loopback_authenticated_and_seeds_network_ports() let bootstrap: DaemonBootstrapSettings = toml::from_str(&text).unwrap(); assert_eq!( bootstrap.rest.bind_addr, - Some("127.0.0.1:20017".parse().unwrap()) + Some("0.0.0.0:20017".parse().unwrap()) ); assert_eq!( bootstrap.rest.api_key, @@ -359,7 +359,7 @@ fn packaged_bootstrap_loads_and_seeds_a_fresh_database() { assert_eq!( profile.rest_bind_addr().unwrap(), - "127.0.0.1:20017".parse().unwrap() + "0.0.0.0:20017".parse().unwrap() ); assert_eq!( profile.rest.api_key, @@ -452,6 +452,100 @@ fn default_profile_candidate_is_the_executable_directory() { ); } +#[test] +fn hosted_folder_defaults_are_canonical_and_never_reseed_operator_changes() { + let temp = tempfile::tempdir().unwrap(); + let directory = temp.path().join("profile"); + let incoming = temp.path().join("Downloads/eMuleBB/Incoming"); + let shared = temp.path().join("Downloads/eMuleBB/Shared"); + let profile = + DaemonProfile::load_hosted(directory.clone(), incoming.clone(), shared.clone()).unwrap(); + assert_eq!( + profile.incoming_dir(), + PathBuf::from(emulebb_core::normal_path_display( + incoming.canonicalize().unwrap() + )) + ); + let mut metadata = profile.open_metadata_store().unwrap(); + assert_eq!(metadata.shared_directory_roots().unwrap().len(), 1); + assert_eq!( + metadata.shared_directory_roots().unwrap()[0].path, + emulebb_core::normal_path_display(shared.canonicalize().unwrap()) + ); + let edited = temp.path().join("operator-incoming"); + metadata + .put_setting_json( + SECTION_DAEMON, + "incomingDir", + &serde_json::to_string(&edited).unwrap(), + ) + .unwrap(); + metadata.replace_shared_directory_roots(&[]).unwrap(); + drop(metadata); + drop(profile); + let profile = DaemonProfile::load_hosted(directory, incoming, shared).unwrap(); + assert_eq!(profile.incoming_dir(), edited); + assert!( + profile + .open_metadata_store() + .unwrap() + .shared_directory_roots() + .unwrap() + .is_empty() + ); +} + +#[test] +fn hosted_defaults_do_not_reinterpret_an_existing_unconfigured_profile() { + let temp = tempfile::tempdir().unwrap(); + let directory = temp.path().join("profile"); + drop(DaemonProfile::load(Some(directory.clone())).unwrap()); + let incoming = temp.path().join("not-created-incoming"); + let shared = temp.path().join("not-created-shared"); + let profile = DaemonProfile::load_hosted(directory, incoming.clone(), shared.clone()).unwrap(); + assert!(profile.incoming_dir.is_none()); + assert!(!incoming.exists() && !shared.exists()); + assert!( + profile + .open_metadata_store() + .unwrap() + .shared_directory_roots() + .unwrap() + .is_empty() + ); +} + +#[test] +fn hosted_folder_initialization_retries_after_a_first_start_filesystem_failure() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("profile"); + let incoming = root.path().join("Incoming"); + let shared = root.path().join("Shared"); + fs::write(&shared, b"owned conflicting fixture").unwrap(); + assert!( + DaemonProfile::load_hosted(directory.clone(), incoming.clone(), shared.clone()).is_err() + ); + assert!(directory.join(PROFILE_METADATA_FILE).exists()); + fs::remove_file(&shared).unwrap(); + let profile = DaemonProfile::load_hosted(directory, incoming.clone(), shared).unwrap(); + assert_eq!( + profile.incoming_dir(), + PathBuf::from(emulebb_core::normal_path_display( + incoming.canonicalize().unwrap() + )) + ); + assert_eq!( + profile + .open_metadata_store() + .unwrap() + .shared_directory_roots() + .unwrap() + .len(), + 1 + ); + assert_eq!(profile.ed2k.listen_port, Some(20019)); +} + #[test] fn load_for_import_creates_an_explicit_profile() { let temp = tempfile::tempdir().unwrap(); @@ -1166,9 +1260,26 @@ fn ed2k_network_config_requires_a_direct_default_route_or_explicit_bind() { &metadata_store(&profile), &[iface("Offline", "192.0.2.10")], ) - .unwrap_err() - .to_string(); - assert!(error.contains("no default-route IPv4 interface was detected")); + .unwrap_err(); + assert!( + error + .to_string() + .contains("no default-route IPv4 interface was detected") + ); + assert_eq!( + error.downcast_ref::(), + Some(&NetworkStartupError::DefaultRouteUnavailable) + ); + + let mut first = iface("First", "192.0.2.10"); + first.has_default_route = true; + let mut second = iface("Second", "198.51.100.10"); + second.has_default_route = true; + let ambiguous = profile + .ed2k_network_config_from_interfaces(&metadata_store(&profile), &[first, second]) + .unwrap_err(); + assert!(ambiguous.to_string().contains("multiple default-route")); + assert!(ambiguous.downcast_ref::().is_none()); } #[test] diff --git a/crates/emulebb-daemon/tests/process_logging.rs b/crates/emulebb-daemon/tests/process_logging.rs new file mode 100644 index 000000000..5a0227418 --- /dev/null +++ b/crates/emulebb-daemon/tests/process_logging.rs @@ -0,0 +1,28 @@ +//! Process-owned logging survives replacement of an application-owned Tokio runtime. + +#[test] +fn process_logging_keeps_the_canonical_writer_across_runtime_restarts() { + let root = tempfile::tempdir().unwrap(); + let first = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let guard = + first.block_on(async { emulebb_daemon::logging::init_process(root.path()).unwrap() }); + drop(first); + tracing::info!("first application runtime released"); + let second = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + second.block_on(async { + tracing::info!("second application runtime active"); + guard.shutdown().await; + }); + let mut retained = String::new(); + for entry in std::fs::read_dir(root.path().join("logs")).unwrap() { + retained.push_str(&std::fs::read_to_string(entry.unwrap().path()).unwrap()); + } + assert!(retained.contains("first application runtime released")); + assert!(retained.contains("second application runtime active")); +} diff --git a/crates/emulebb-diagnostics/src/lib.rs b/crates/emulebb-diagnostics/src/lib.rs index 5bee6c36e..75e9e5980 100644 --- a/crates/emulebb-diagnostics/src/lib.rs +++ b/crates/emulebb-diagnostics/src/lib.rs @@ -11,6 +11,8 @@ //! and `EMULEBB_RUST_DIAG_MIN_FREE_BYTES` values. Scan, deletion, free-space //! probe, and writer I/O faults fail closed and remain latched in writer health. +pub mod log_tail; + #[cfg(any(feature = "enabled", test))] use std::{ collections::HashMap, diff --git a/crates/emulebb-diagnostics/src/log_tail.rs b/crates/emulebb-diagnostics/src/log_tail.rs new file mode 100644 index 000000000..a4be2ff30 --- /dev/null +++ b/crates/emulebb-diagnostics/src/log_tail.rs @@ -0,0 +1,83 @@ +//! Bounded complete-record JSONL reads shared by diagnostic exporters. + +use std::{ + fs::File, + io::{self, Read, Seek, SeekFrom}, + path::Path, +}; + +pub struct JsonlTail { + pub file_size: u64, + pub tail_start_offset: u64, + pub bytes_read: usize, + pub incomplete_final_line_ignored: bool, + pub lines: Vec>, +} + +/// Read at most `max_bytes`, excluding a partial leading record and optionally +/// an active writer's incomplete final record. The caller owns the byte budget. +pub fn read_jsonl_tail( + path: &Path, + max_bytes: u64, + ignore_incomplete_final_line: bool, +) -> io::Result { + let mut file = File::open(path)?; + let file_size = file.metadata()?.len(); + let bytes_to_read = file_size.min(max_bytes); + let tail_start_offset = file_size.saturating_sub(bytes_to_read); + file.seek(SeekFrom::Start(tail_start_offset))?; + let capacity = usize::try_from(bytes_to_read).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "JSONL tail budget exceeds addressable memory", + ) + })?; + let mut bytes = Vec::with_capacity(capacity); + file.take(bytes_to_read).read_to_end(&mut bytes)?; + let bytes_read = bytes.len(); + if tail_start_offset > 0 { + if let Some(newline) = bytes.iter().position(|byte| *byte == b'\n') { + bytes.drain(..=newline); + } else { + bytes.clear(); + } + } + let incomplete_final_line_ignored = + ignore_incomplete_final_line && bytes.last().is_some_and(|last| *last != b'\n'); + if incomplete_final_line_ignored { + if let Some(newline) = bytes.iter().rposition(|byte| *byte == b'\n') { + bytes.truncate(newline + 1); + } else { + bytes.clear(); + } + } + let lines = bytes + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .map(<[u8]>::to_vec) + .collect(); + Ok(JsonlTail { + file_size, + tail_start_offset, + bytes_read, + incomplete_final_line_ignored, + lines, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bounded_tail_excludes_partial_leading_and_active_final_records() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("sample.jsonl"); + std::fs::write(&path, b"old-record\ncomplete\nunfinished").unwrap(); + let tail = read_jsonl_tail(&path, 25, true).unwrap(); + assert!(tail.bytes_read <= 25); + assert!(tail.tail_start_offset > 0); + assert!(tail.incomplete_final_line_ignored); + assert_eq!(tail.lines, vec![b"complete".to_vec()]); + } +} diff --git a/crates/emulebb-ed2k/Cargo.toml b/crates/emulebb-ed2k/Cargo.toml index 9d41bd84d..bfe1269b4 100644 --- a/crates/emulebb-ed2k/Cargo.toml +++ b/crates/emulebb-ed2k/Cargo.toml @@ -72,7 +72,7 @@ windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Globali [target.'cfg(unix)'.dependencies] libc.workspace = true -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] rustix.workspace = true [dev-dependencies] diff --git a/crates/emulebb-ed2k/src/config.rs b/crates/emulebb-ed2k/src/config.rs index 69f088ee6..4d236ea79 100644 --- a/crates/emulebb-ed2k/src/config.rs +++ b/crates/emulebb-ed2k/src/config.rs @@ -12,6 +12,9 @@ const DEFAULT_UPLOAD_SESSION_TIME_LIMIT_SECS: u64 = 7_200; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase")] pub struct Ed2kRuntimeConfig { + /// Internal route intent, derived from the active profile rather than persisted settings. + #[serde(skip)] + pub egress_route: crate::networking::EgressRoute, /// Local ED2K peer TCP listener port, resolved by the daemon from config. pub listen_port: Option, /// Ordered ED2K server bootstrap entries mirrored from `server.met`. @@ -257,6 +260,7 @@ impl Ed2kRuntimeConfig { impl Default for Ed2kRuntimeConfig { fn default() -> Self { Self { + egress_route: crate::networking::EgressRoute::StrictInterface, listen_port: None, server_entries: Vec::new(), server_endpoints: Vec::new(), diff --git a/crates/emulebb-ed2k/src/ed2k_server/active_callback.rs b/crates/emulebb-ed2k/src/ed2k_server/active_callback.rs index 42a90b732..ffeee6398 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/active_callback.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/active_callback.rs @@ -57,7 +57,7 @@ pub async fn request_callback_on_server(options: Ed2kCallbackRequestOptions<'_>) login_identity_for_server_transport(hello_identity, use_server_obfuscation); let transport_endpoint = resolved_server.transport_endpoint(use_server_obfuscation); let mut session = ServerSession::connect( - bind_ip, + crate::networking::EgressBinding::new(bind_ip, config.egress_route), transport_endpoint, Arc::new(RwLock::new(Ed2kServerState::default())), "active_callback", diff --git a/crates/emulebb-ed2k/src/ed2k_server/active_keyword.rs b/crates/emulebb-ed2k/src/ed2k_server/active_keyword.rs index 841507542..5120541e2 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/active_keyword.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/active_keyword.rs @@ -208,7 +208,11 @@ pub async fn search_keyword_udp_servers( return Ok(Vec::new()); } - let socket = bind_server_udp_socket(bind_ip).await?; + let socket = bind_server_udp_socket(crate::networking::EgressBinding::new( + bind_ip, + config.egress_route, + )) + .await?; let mut results = Vec::new(); let mut last_error = None; let mut queried_servers = Vec::new(); @@ -385,7 +389,7 @@ pub async fn search_keyword_servers( ); match search_keyword_on_server( - bind_ip, + crate::networking::EgressBinding::new(bind_ip, config.egress_route), &resolved_server, hello_identity, &nickname, @@ -421,7 +425,7 @@ pub async fn search_keyword_servers( reason = "single-call-site protocol session inputs stay explicit" )] async fn search_keyword_on_server( - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, server: &ResolvedServerEntry, hello_identity: Ed2kHelloIdentity, nickname: &str, @@ -430,13 +434,14 @@ async fn search_keyword_on_server( idle_timeout: Duration, cancel: &CancellationToken, ) -> Result> { + let bind_ip = binding.address; let use_server_obfuscation = should_use_server_obfuscation(hello_identity.connect_options, server); let login_identity = login_identity_for_server_transport(hello_identity, use_server_obfuscation); let transport_endpoint = server.transport_endpoint(use_server_obfuscation); let mut session = ServerSession::connect( - bind_ip, + binding, transport_endpoint, Arc::new(RwLock::new(Ed2kServerState::default())), "active_search", @@ -679,7 +684,10 @@ mod tests { let cancel = CancellationToken::new(); let results = search_keyword_on_server( - crate::test_bind_ip(), + crate::networking::EgressBinding::new( + crate::test_bind_ip(), + crate::networking::EgressRoute::StrictInterface, + ), &active_keyword_server(endpoint), active_keyword_identity(), "eMule", diff --git a/crates/emulebb-ed2k/src/ed2k_server/active_source.rs b/crates/emulebb-ed2k/src/ed2k_server/active_source.rs index 751e49f81..83e13e7b2 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/active_source.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/active_source.rs @@ -132,7 +132,7 @@ pub async fn search_source_servers( file_hash ); match search_sources_on_server(SourceSearchServerOptions { - bind_ip, + binding: crate::networking::EgressBinding::new(bind_ip, config.egress_route), server: &resolved_server, hello_identity, nickname: &nickname, @@ -170,7 +170,7 @@ pub async fn search_source_servers( } struct SourceSearchServerOptions<'a> { - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, server: &'a ResolvedServerEntry, hello_identity: Ed2kHelloIdentity, nickname: &'a str, @@ -185,7 +185,7 @@ async fn search_sources_on_server( options: SourceSearchServerOptions<'_>, ) -> Result> { let SourceSearchServerOptions { - bind_ip, + binding, server, hello_identity, nickname, @@ -195,13 +195,14 @@ async fn search_sources_on_server( idle_timeout, cancel, } = options; + let bind_ip = binding.address; let use_server_obfuscation = should_use_server_obfuscation(hello_identity.connect_options, server); let login_identity = login_identity_for_server_transport(hello_identity, use_server_obfuscation); let transport_endpoint = server.transport_endpoint(use_server_obfuscation); let mut session = ServerSession::connect( - bind_ip, + binding, transport_endpoint, Arc::new(RwLock::new(Ed2kServerState::default())), "active_sources", @@ -459,7 +460,10 @@ mod tests { let server = active_source_server(endpoint); let results = search_sources_on_server(SourceSearchServerOptions { - bind_ip: crate::test_bind_ip(), + binding: crate::networking::EgressBinding::new( + crate::test_bind_ip(), + crate::networking::EgressRoute::StrictInterface, + ), server: &server, hello_identity: active_source_identity(), nickname: "eMule", diff --git a/crates/emulebb-ed2k/src/ed2k_server/active_source_udp.rs b/crates/emulebb-ed2k/src/ed2k_server/active_source_udp.rs index 73bbabde0..8f39c19d5 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/active_source_udp.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/active_source_udp.rs @@ -145,7 +145,11 @@ pub async fn search_source_udp_servers( return Ok(Vec::new()); } - let socket = bind_server_udp_socket(bind_ip).await?; + let socket = bind_server_udp_socket(crate::networking::EgressBinding::new( + bind_ip, + config.egress_route, + )) + .await?; let mut aggregated_results = Vec::new(); let mut last_error = None; let per_server_timeout = timeout.max(Duration::from_secs(5)); @@ -445,7 +449,11 @@ pub async fn search_source_udp_server_batches( file_size: target.file_size, }) .collect::>(); - let socket = bind_server_udp_socket(bind_ip).await?; + let socket = bind_server_udp_socket(crate::networking::EgressBinding::new( + bind_ip, + config.egress_route, + )) + .await?; let mut results_by_hash: HashMap> = HashMap::new(); let mut last_error = None; let per_server_timeout = timeout.max(Duration::from_secs(5)); diff --git a/crates/emulebb-ed2k/src/ed2k_server/loop_runtime.rs b/crates/emulebb-ed2k/src/ed2k_server/loop_runtime.rs index 0e34aa21d..25ec50013 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/loop_runtime.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/loop_runtime.rs @@ -97,6 +97,7 @@ pub async fn run_ed2k_server_loop(options: Ed2kServerLoopOptions) { let initial_config = initial_plan.config(); let session_context_base = ServerSessionContext { bind_ip, + egress_route: initial_config.egress_route, nat, hello_identity, nickname: initial_config.server_nickname(), diff --git a/crates/emulebb-ed2k/src/ed2k_server/server_description_poll.rs b/crates/emulebb-ed2k/src/ed2k_server/server_description_poll.rs index 3ef105dab..42115bb0c 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/server_description_poll.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/server_description_poll.rs @@ -79,13 +79,15 @@ pub(super) async fn poll_server_descriptions( let Some(events) = events else { return; }; - let socket = match bind_server_udp_socket(bind_ip).await { - Ok(socket) => socket, - Err(error) => { - warn!("server description poll disabled: {error}"); - return; - } - }; + let route = server_connect_plan.borrow().config().egress_route; + let socket = + match bind_server_udp_socket(crate::networking::EgressBinding::new(bind_ip, route)).await { + Ok(socket) => socket, + Err(error) => { + warn!("server description poll disabled: {error}"); + return; + } + }; let mut last_polled = None; 'runtime: loop { if task_cancel.is_cancelled() || shutdown.load(Ordering::Relaxed) { diff --git a/crates/emulebb-ed2k/src/ed2k_server/session.rs b/crates/emulebb-ed2k/src/ed2k_server/session.rs index 54c4f8c32..eea5d6b91 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/session.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/session.rs @@ -1,7 +1,7 @@ use std::{ collections::HashSet, io, - net::{IpAddr, Ipv4Addr, SocketAddr}, + net::{IpAddr, SocketAddr}, sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -253,14 +253,14 @@ impl Drop for ServerSession { impl ServerSession { pub(super) async fn connect( - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, endpoint: SocketAddr, state: Arc>, trace_role: &'static str, timeout: Duration, ) -> Result { Self::connect_until( - bind_ip, + binding, endpoint, state, trace_role, @@ -271,7 +271,7 @@ impl ServerSession { } pub(super) async fn connect_until( - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, endpoint: SocketAddr, state: Arc>, trace_role: &'static str, @@ -280,7 +280,7 @@ impl ServerSession { ) -> Result { let trace_id = Self::next_trace_id(); Self::connect_until_with_trace_id( - bind_ip, + binding, endpoint, state, trace_role, @@ -296,7 +296,7 @@ impl ServerSession { } pub(super) async fn connect_until_with_trace_id( - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, endpoint: SocketAddr, state: Arc>, trace_role: &'static str, @@ -304,6 +304,7 @@ impl ServerSession { pending_login_deadline: Instant, trace_id: u64, ) -> Result { + let bind_ip = binding.address; emit_server_connection_state( endpoint, trace_id, @@ -332,19 +333,18 @@ impl ServerSession { "failed to bind ED2K server socket to {bind_ip}: {error}" )) })?; - let bind_if_index = - crate::networking::require_bind_if_index(bind_ip, "ED2K server TCP").map_err( - |error| { - Ed2kServerFailure::local_socket(format!( - "failed to resolve ED2K server bind interface for {bind_ip}: {error}" - )) - }, - )?; + let bind_if_index = binding + .interface_index("ED2K server TCP") + .map_err(|error| { + Ed2kServerFailure::local_socket(format!( + "failed to resolve ED2K server bind interface for {bind_ip}: {error}" + )) + })?; // Egress-pin to the VPN tunnel interface (IP_UNICAST_IF) before connect // so the server session leaves via the tunnel — solid VPN binding. emulebb_kad_dht::socket_opts::pin_egress_to_interface( socket2::SockRef::from(&socket), - Some(bind_if_index), + bind_if_index, ) .map_err(|error| { Ed2kServerFailure::local_socket(format!( @@ -1443,6 +1443,7 @@ mod receive_tests; #[cfg(test)] mod tests { use super::*; + use std::net::Ipv4Addr; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; @@ -1581,13 +1582,40 @@ mod tests { ); } + #[tokio::test] + async fn direct_server_tcp_uses_source_address_without_interface_pin() { + let listener = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = listener.local_addr().unwrap(); + let binding = crate::networking::EgressBinding::new( + Ipv4Addr::LOCALHOST, + crate::networking::EgressRoute::DirectSystemRoute, + ); + let session = ServerSession::connect( + binding, + endpoint, + Arc::new(RwLock::new(Ed2kServerState::default())), + "test", + Duration::from_secs(2), + ) + .await + .unwrap(); + let (_stream, from) = listener.accept().await.unwrap(); + assert_eq!(from.ip(), IpAddr::V4(Ipv4Addr::LOCALHOST)); + drop(session); + } + #[tokio::test] async fn server_connect_rolls_back_connecting_state_on_setup_failure() { let state = Arc::new(RwLock::new(Ed2kServerState::default())); let endpoint = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4661); let err = ServerSession::connect( - Ipv4Addr::new(203, 0, 113, 200), + crate::networking::EgressBinding::new( + Ipv4Addr::new(203, 0, 113, 200), + crate::networking::EgressRoute::StrictInterface, + ), endpoint, Arc::clone(&state), "test", @@ -1629,7 +1657,10 @@ mod tests { let state = Arc::new(RwLock::new(Ed2kServerState::default())); let error = ServerSession::connect( - Ipv4Addr::LOCALHOST, + crate::networking::EgressBinding::new( + Ipv4Addr::LOCALHOST, + crate::networking::EgressRoute::StrictInterface, + ), endpoint, state, "test", diff --git a/crates/emulebb-ed2k/src/ed2k_server/session_driver.rs b/crates/emulebb-ed2k/src/ed2k_server/session_driver.rs index 7bf99e6a7..169300105 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/session_driver.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/session_driver.rs @@ -134,7 +134,7 @@ async fn connect_server_transport_attempt( login_identity.udp_port = ports.udp; let transport_endpoint = server.transport_endpoint(use_server_obfuscation); let mut session = ServerSession::connect_until_with_trace_id( - context.bind_ip, + crate::networking::EgressBinding::new(context.bind_ip, context.egress_route), transport_endpoint, Arc::clone(&context.state), "background", @@ -256,7 +256,12 @@ pub(super) async fn run_one_server_session( // discovery cycle below will refresh it for the current identity. live_server.entry.udp_key = 0; } - let server_udp_socket = match bind_server_udp_socket(context.bind_ip).await { + let server_udp_socket = match bind_server_udp_socket(crate::networking::EgressBinding::new( + context.bind_ip, + context.egress_route, + )) + .await + { Ok(socket) => { debug!( "bound ED2K server UDP helper local={} remote={} trace_id={}", diff --git a/crates/emulebb-ed2k/src/ed2k_server/tests.rs b/crates/emulebb-ed2k/src/ed2k_server/tests.rs index 141204633..3abb6e1db 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/tests.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/tests.rs @@ -220,6 +220,7 @@ async fn connected_id_change_fixture() -> ( }]); let reachability_authority = ReachabilityRuntimeAuthority::new(); let context = ServerSessionContext { + egress_route: crate::networking::EgressRoute::StrictInterface, bind_ip: Ipv4Addr::LOCALHOST, nat: Arc::new(NatManager::default()), hello_identity: Ed2kHelloIdentity { diff --git a/crates/emulebb-ed2k/src/ed2k_server/tests/background.rs b/crates/emulebb-ed2k/src/ed2k_server/tests/background.rs index 74a8c3b4e..5d1f92b5b 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/tests/background.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/tests/background.rs @@ -920,7 +920,10 @@ async fn server_obfuscation_handshake_encrypts_login_request() { let state = Arc::new(RwLock::new(Ed2kServerState::default())); // Bind the client socket to the same loopback address as the listener. let mut session = ServerSession::connect( - crate::test_bind_ip(), + crate::networking::EgressBinding::new( + crate::test_bind_ip(), + crate::networking::EgressRoute::StrictInterface, + ), endpoint, state, "test", @@ -948,6 +951,7 @@ fn background_test_context( crate::ed2k_tcp::ed2k_callback_session_channel(); let reachability_authority = ReachabilityRuntimeAuthority::new(); let session = ServerSessionContext { + egress_route: crate::networking::EgressRoute::StrictInterface, bind_ip: crate::test_bind_ip(), nat: Arc::new(NatManager::default()), hello_identity: Ed2kHelloIdentity { diff --git a/crates/emulebb-ed2k/src/ed2k_server/types.rs b/crates/emulebb-ed2k/src/ed2k_server/types.rs index 51529875d..909d77914 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/types.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/types.rs @@ -295,6 +295,7 @@ impl Ed2kServerState { #[derive(Clone)] pub(super) struct ServerSessionContext { pub(super) bind_ip: Ipv4Addr, + pub(super) egress_route: crate::networking::EgressRoute, pub(super) nat: Arc, pub(super) hello_identity: Ed2kHelloIdentity, pub(super) nickname: String, diff --git a/crates/emulebb-ed2k/src/ed2k_server/udp_callback.rs b/crates/emulebb-ed2k/src/ed2k_server/udp_callback.rs index 0ffa2f169..b3516a105 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/udp_callback.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/udp_callback.rs @@ -68,7 +68,11 @@ pub async fn request_callback_via_udp_server( options.requester_tcp_port, options.target_client_id, ); - let socket = bind_server_udp_socket(options.bind_ip).await?; + let socket = bind_server_udp_socket(crate::networking::EgressBinding::new( + options.bind_ip, + options.config.egress_route, + )) + .await?; send_server_udp_packet(&socket, &server, OP_GLOBCALLBACKREQ, &body).await } diff --git a/crates/emulebb-ed2k/src/ed2k_server/udp_runtime.rs b/crates/emulebb-ed2k/src/ed2k_server/udp_runtime.rs index f59b2b059..17b9e4163 100644 --- a/crates/emulebb-ed2k/src/ed2k_server/udp_runtime.rs +++ b/crates/emulebb-ed2k/src/ed2k_server/udp_runtime.rs @@ -1,5 +1,5 @@ use std::{ - net::{IpAddr, Ipv4Addr, SocketAddr}, + net::{IpAddr, SocketAddr}, sync::atomic::{AtomicU64, Ordering}, }; @@ -41,15 +41,18 @@ fn received_udp_evidence(from: SocketAddr, transport_mode: &str) -> Ed2kServerUd Ed2kServerUdpEvidence::new(next_server_udp_receive_trace_id(), transport, from) } -pub(super) async fn bind_server_udp_socket(bind_ip: Ipv4Addr) -> Result { +pub(super) async fn bind_server_udp_socket( + binding: crate::networking::EgressBinding, +) -> Result { + let bind_ip = binding.address; let socket = UdpSocket::bind(SocketAddr::new(IpAddr::V4(bind_ip), 0)) .await .with_context(|| format!("failed to bind ED2K server UDP helper on {bind_ip}:0"))?; - let bind_if_index = crate::networking::require_bind_if_index(bind_ip, "ED2K server UDP")?; + let bind_if_index = binding.interface_index("ED2K server UDP")?; // Egress-pin to the VPN tunnel interface (IP_UNICAST_IF) — solid VPN binding. emulebb_kad_dht::socket_opts::pin_egress_to_interface( socket2::SockRef::from(&socket), - Some(bind_if_index), + bind_if_index, ) .with_context(|| format!("failed to pin ED2K server UDP egress for {bind_ip}"))?; Ok(socket) @@ -589,3 +592,31 @@ mod tests { assert_eq!(packet.evidence.peer(), server_peer); } } + +#[cfg(test)] +mod route_intent_tests { + use super::*; + use crate::networking::{EgressBinding, EgressRoute}; + use std::net::Ipv4Addr; + + #[tokio::test] + async fn direct_server_udp_keeps_source_address_and_exchanges_datagrams() { + let socket = bind_server_udp_socket(EgressBinding::new( + Ipv4Addr::LOCALHOST, + EgressRoute::DirectSystemRoute, + )) + .await + .expect("direct UDP bind"); + let receiver = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("receiver"); + socket + .send_to(b"test", receiver.local_addr().unwrap()) + .await + .unwrap(); + let mut bytes = [0; 4]; + let (count, from) = receiver.recv_from(&mut bytes).await.unwrap(); + assert_eq!(&bytes[..count], b"test"); + assert_eq!(from.ip(), IpAddr::V4(Ipv4Addr::LOCALHOST)); + } +} diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/buddy_link.rs b/crates/emulebb-ed2k/src/ed2k_tcp/buddy_link.rs index 815f97ebd..13835cb70 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/buddy_link.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/buddy_link.rs @@ -22,7 +22,7 @@ //! (`MIN2MS(10)` cadence). //! - `srchybrid/ListenSocket.cpp` `OP_CALLBACK` relayed to the firewalled client. -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use std::time::Duration; @@ -61,8 +61,8 @@ pub enum BuddyLinkEnd { /// Inputs for the outbound buddy link task. pub struct OutboundBuddyLinkOptions { - /// VPN/tunnel bind IP for the buddy socket. - pub bind_ip: Ipv4Addr, + /// Selected source and routing intent for the buddy socket. + pub binding: crate::networking::EgressBinding, /// The buddy's TCP endpoint (its source IP + the TCP port it returned). pub buddy_addr: SocketAddr, /// The buddy's eD2k user hash (used for registry identity + obfuscation). @@ -104,7 +104,7 @@ pub struct OutboundBuddyLinkOptions { /// the outbound slot is evicted and the termination reason is returned. pub async fn run_outbound_buddy_link(options: OutboundBuddyLinkOptions) -> Result { let OutboundBuddyLinkOptions { - bind_ip, + binding, buddy_addr, buddy_user_hash, buddy_connect_options, @@ -130,7 +130,7 @@ pub async fn run_outbound_buddy_link(options: OutboundBuddyLinkOptions) -> Resul } let mut transport = connect_and_hello_buddy( - bind_ip, + binding, buddy_addr, buddy_user_hash, buddy_connect_options, @@ -425,7 +425,7 @@ async fn handle_buddy_packet( /// Connect out to the buddy and complete the hello side channel so the buddy /// recognizes us as its incoming buddy, returning the held transport. async fn connect_and_hello_buddy( - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, buddy_addr: SocketAddr, buddy_user_hash: [u8; 16], buddy_connect_options: u8, @@ -434,7 +434,7 @@ async fn connect_and_hello_buddy( ) -> Result { let mut transport = Ed2kTransport::connect_outgoing( "kad_buddy", - bind_ip, + binding, buddy_addr, hello_identity.connect_options, Some(buddy_user_hash), @@ -665,7 +665,10 @@ mod tests { )); connect_and_hello_buddy( - bind_ip, + crate::networking::EgressBinding::new( + bind_ip, + crate::networking::EgressRoute::StrictInterface, + ), buddy_addr, expected_hash, 0, @@ -688,7 +691,10 @@ mod tests { )); let error = connect_and_hello_buddy( - bind_ip, + crate::networking::EgressBinding::new( + bind_ip, + crate::networking::EgressRoute::StrictInterface, + ), buddy_addr, [0x44; 16], 0, diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/download/startup.rs b/crates/emulebb-ed2k/src/ed2k_tcp/download/startup.rs index e4033817f..bba866654 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/download/startup.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/download/startup.rs @@ -39,6 +39,7 @@ use super::session::{ /// Inputs for one outbound native ED2K peer download attempt. pub struct Ed2kPeerDownloadOptions<'a> { pub bind_ip: Ipv4Addr, + pub egress_route: crate::networking::EgressRoute, pub peer: &'a Ed2kFoundSource, pub hello_identity: Ed2kHelloIdentity, pub secure_ident: &'a Arc, @@ -114,6 +115,7 @@ pub async fn download_files_from_peer( ) -> Result { let Ed2kPeerDownloadOptions { bind_ip, + egress_route, peer, hello_identity, secure_ident, @@ -172,7 +174,7 @@ pub async fn download_files_from_peer( async { let transport = Ed2kTransport::connect_outgoing( "native_download", - bind_ip, + crate::networking::EgressBinding::new(bind_ip, egress_route), peer_addr, hello_identity.connect_options, peer.user_hash, diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/firewall_helper.rs b/crates/emulebb-ed2k/src/ed2k_tcp/firewall_helper.rs index 4e6846e46..11626bd91 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/firewall_helper.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/firewall_helper.rs @@ -295,7 +295,7 @@ async fn handle_firewall_helper_packet( /// Send one `OP_FWCHECKUDPREQ` to a helper peer over eD2k TCP. pub async fn request_udp_firewall_check( dht: Option, - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, helper_addr: SocketAddr, hello_identity: Ed2kHelloIdentity, secure_ident: Arc, @@ -304,7 +304,7 @@ pub async fn request_udp_firewall_check( ) -> Result<()> { let mut transport = match Ed2kTransport::connect_outgoing( "udp_firewall_check", - bind_ip, + binding, helper_addr, hello_identity.connect_options, None, @@ -533,7 +533,7 @@ pub(crate) async fn complete_authorized_kad_callback( /// eD2k client connection, sends its normal `OP_HELLO`, then reports TCP reachability /// with `OP_KAD_FWTCPCHECK_ACK` on that same connection. pub async fn send_kad_firewall_tcp_ack( - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, peer_addr: SocketAddr, hello_identity: Ed2kHelloIdentity, peer_user_hash: [u8; 16], @@ -542,7 +542,7 @@ pub async fn send_kad_firewall_tcp_ack( ) -> Result { let mut transport = Ed2kTransport::connect_outgoing( "kad_tcp_firewall_ack", - bind_ip, + binding, peer_addr, hello_identity.connect_options, Some(peer_user_hash), diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/listener/callback.rs b/crates/emulebb-ed2k/src/ed2k_tcp/listener/callback.rs index 22f676a20..c967f6920 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/listener/callback.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/listener/callback.rs @@ -145,6 +145,7 @@ pub(in crate::ed2k_tcp) struct Ed2kCallbackSessionRuntime { pub port_test_registry: PortTestRegistry, pub callback_sessions: Ed2kCallbackSessionHandle, pub bind_ip: Ipv4Addr, + pub egress_route: crate::networking::EgressRoute, pub shutdown: Arc, } @@ -211,7 +212,7 @@ async fn serve_callback_session( } let transport = Ed2kTransport::connect_outgoing( "listener_callback", - runtime.bind_ip, + crate::networking::EgressBinding::new(runtime.bind_ip, runtime.egress_route), request.peer_addr, runtime.hello_identity.connect_options, request.expected_user_hash, @@ -333,6 +334,7 @@ mod tests { port_test_registry: PortTestRegistry::new(), callback_sessions, bind_ip: Ipv4Addr::LOCALHOST, + egress_route: crate::networking::EgressRoute::StrictInterface, shutdown: Arc::new(AtomicBool::new(false)), }; (root, runtime) diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/listener/mod.rs b/crates/emulebb-ed2k/src/ed2k_tcp/listener/mod.rs index 596b31898..9d3e80fb8 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/listener/mod.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/listener/mod.rs @@ -43,6 +43,7 @@ pub(in crate::ed2k_tcp) use session::{ /// Inputs for the long-lived ED2K TCP listener task. pub struct Ed2kListenerOptions { + pub egress_route: crate::networking::EgressRoute, pub listener: Arc, pub dht: DhtNode, pub server_state: Arc>, @@ -78,6 +79,7 @@ pub struct Ed2kListenerOptions { /// buddy traffic, and firewall verification. pub async fn run_ed2k_listener(options: Ed2kListenerOptions) { let Ed2kListenerOptions { + egress_route, listener, dht, server_state, @@ -108,15 +110,13 @@ pub async fn run_ed2k_listener(options: Ed2kListenerOptions) { // a per-connection interface lookup. let (bind_ip, bind_if_index) = match listener.local_addr() { Ok(addr) => match addr.ip() { - std::net::IpAddr::V4(v4) => { - match crate::networking::require_bind_if_index(v4, "eD2K listener") { - Ok(index) => (v4, index), - Err(error) => { - warn!("eD2K listener disabled: {error:#}"); - return; - } + std::net::IpAddr::V4(v4) => match egress_route.interface_index(v4, "eD2K listener") { + Ok(index) => (v4, index), + Err(error) => { + warn!("eD2K listener disabled: {error:#}"); + return; } - } + }, _ => { warn!("eD2K listener disabled: non-IPv4 listener addresses are not supported"); return; @@ -141,6 +141,7 @@ pub async fn run_ed2k_listener(options: Ed2kListenerOptions) { buddy_registry: buddy_registry.clone(), port_test_registry: port_test_registry.clone(), callback_sessions: callback_sessions.clone(), + egress_route, bind_ip, shutdown: Arc::clone(&shutdown), }, @@ -162,6 +163,7 @@ pub async fn run_ed2k_listener(options: Ed2kListenerOptions) { buddy_registry: buddy_registry.clone(), port_test_registry: port_test_registry.clone(), callback_sessions: callback_sessions.clone(), + egress_route, bind_ip, callback_timeout, shutdown: Arc::clone(&shutdown), @@ -206,7 +208,7 @@ pub async fn run_ed2k_listener(options: Ed2kListenerOptions) { // inbound connection into an unpinned data-plane leak. if let Err(error) = emulebb_kad_dht::socket_opts::pin_egress_to_interface( socket2::SockRef::from(&stream), - Some(bind_if_index), + bind_if_index, ) { debug!( "dropping inbound eD2k connection from {peer_addr}: failed to pin egress: {error}" diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/queue_only.rs b/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/queue_only.rs index 207aaff78..598f679ad 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/queue_only.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/queue_only.rs @@ -308,6 +308,7 @@ async fn queued_peer_waits_past_read_timeout_for_late_accept_upload() { let (reask_handle, mut reask_rx) = crate::ed2k_client_udp::reask_command_channel(); let result = download_file_from_peer(Ed2kPeerDownloadOptions { + egress_route: crate::networking::EgressRoute::StrictInterface, bind_ip: test_bind_ip(), peer: &Ed2kFoundSource { file_hash, diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/startup_metadata.rs b/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/startup_metadata.rs index b2d35329d..6b4dd82f0 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/startup_metadata.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/tests/download/startup_metadata.rs @@ -338,6 +338,7 @@ async fn a4af_switch_requests_two_files_on_one_peer_connection() { }; let report = download_files_from_peer( Ed2kPeerDownloadOptions { + egress_route: crate::networking::EgressRoute::StrictInterface, bind_ip: test_bind_ip(), peer: &peer, hello_identity: Ed2kHelloIdentity { diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/tests/listener_fixture.rs b/crates/emulebb-ed2k/src/ed2k_tcp/tests/listener_fixture.rs index f2abdd194..dc2e4311b 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/tests/listener_fixture.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/tests/listener_fixture.rs @@ -163,6 +163,7 @@ impl ListenerTestRuntime { &self, ) -> Arc { Arc::new(crate::ed2k_tcp::upload_promote::UploadPromoteDriver { + egress_route: crate::networking::EgressRoute::StrictInterface, dht: self.dht.clone(), server_state: Arc::clone(&self.server_state), server_search: self.server_search.clone(), @@ -195,6 +196,7 @@ impl ListenerTestRuntime { let task = tokio::spawn(crate::ed2k_tcp::listener::run_callback_session_supervisor( inbox, crate::ed2k_tcp::listener::Ed2kCallbackSessionRuntime { + egress_route: crate::networking::EgressRoute::StrictInterface, dht: self.dht.clone(), server_state: Arc::clone(&self.server_state), kad_firewall: Arc::clone(&self.kad_firewall), @@ -414,7 +416,10 @@ pub(super) async fn connect_obfuscated_peer_and_exchange_hello( ) -> Ed2kTransport { let mut transport = Ed2kTransport::connect_outgoing( "test", - test_bind_ip(), + crate::networking::EgressBinding::new( + test_bind_ip(), + crate::networking::EgressRoute::StrictInterface, + ), peer_addr, emule_connect_options(true), Some(listener_user_hash), diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/tests/mod.rs b/crates/emulebb-ed2k/src/ed2k_tcp/tests/mod.rs index f0a6f436d..98e5227e7 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/tests/mod.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/tests/mod.rs @@ -122,6 +122,7 @@ macro_rules! download_file_from_peer_test { established_context: $established_context:expr $(,)? ) => { download_file_from_peer(Ed2kPeerDownloadOptions { + egress_route: crate::networking::EgressRoute::StrictInterface, bind_ip: $bind_ip, peer: $peer, hello_identity: $hello_identity, diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/tests/protocol/callback.rs b/crates/emulebb-ed2k/src/ed2k_tcp/tests/protocol/callback.rs index d30f64dc7..fef49a569 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/tests/protocol/callback.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/tests/protocol/callback.rs @@ -881,7 +881,10 @@ async fn udp_firewall_check_request_completes_hello_exchange_before_request() { request_udp_firewall_check( None, - test_bind_ip(), + crate::networking::EgressBinding::new( + test_bind_ip(), + crate::networking::EgressRoute::StrictInterface, + ), helper_addr, Ed2kHelloIdentity { user_hash: [0x77; 16], @@ -950,7 +953,10 @@ async fn udp_firewall_check_request_skips_silent_helper_before_request() { let error = request_udp_firewall_check( None, - test_bind_ip(), + crate::networking::EgressBinding::new( + test_bind_ip(), + crate::networking::EgressRoute::StrictInterface, + ), helper_addr, Ed2kHelloIdentity { user_hash: [0x77; 16], @@ -1050,7 +1056,10 @@ async fn kad_firewall_tcp_ack_sends_hello_then_modern_ack() { }); let mode = send_kad_firewall_tcp_ack( - test_bind_ip(), + crate::networking::EgressBinding::new( + test_bind_ip(), + crate::networking::EgressRoute::StrictInterface, + ), peer_addr, hello_identity, [0x10; 16], diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/transport.rs b/crates/emulebb-ed2k/src/ed2k_tcp/transport.rs index 86042c052..5e1d72e92 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/transport.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/transport.rs @@ -1,7 +1,7 @@ use std::{ collections::VecDeque, io, - net::{IpAddr, Ipv4Addr, SocketAddr}, + net::{IpAddr, SocketAddr}, time::Duration, }; @@ -356,13 +356,14 @@ impl Ed2kTransport { pub(super) async fn connect_outgoing( flow: &'static str, - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, peer_addr: SocketAddr, local_connect_options: u8, peer_user_hash: Option<[u8; 16]>, peer_connect_options: Option, timeout: Duration, ) -> Result { + let bind_ip = binding.address; let socket = match peer_addr { SocketAddr::V4(_) => { TcpSocket::new_v4().context("failed to create outgoing eD2k TCP socket")? @@ -371,7 +372,7 @@ impl Ed2kTransport { anyhow::bail!("IPv6 callback peer connections are not supported yet: {peer_addr}") } }; - let bind_if_index = crate::networking::require_bind_if_index(bind_ip, "eD2k TCP")?; + let bind_if_index = binding.interface_index("eD2k TCP")?; socket .bind(SocketAddr::new(IpAddr::V4(bind_ip), 0)) .with_context(|| format!("failed to bind outgoing eD2k socket to {bind_ip}"))?; @@ -380,7 +381,7 @@ impl Ed2kTransport { // solid VPN binding (eMule ApplyConfiguredIpv4UnicastInterface). emulebb_kad_dht::socket_opts::pin_egress_to_interface( socket2::SockRef::from(&socket), - Some(bind_if_index), + bind_if_index, ) .with_context(|| { format!("failed to pin eD2k egress to the bind interface for {bind_ip}") diff --git a/crates/emulebb-ed2k/src/ed2k_tcp/upload_promote.rs b/crates/emulebb-ed2k/src/ed2k_tcp/upload_promote.rs index 3f554199c..274fc00e4 100644 --- a/crates/emulebb-ed2k/src/ed2k_tcp/upload_promote.rs +++ b/crates/emulebb-ed2k/src/ed2k_tcp/upload_promote.rs @@ -64,6 +64,7 @@ pub(in crate::ed2k_tcp) struct UploadPromoteDriver { pub(in crate::ed2k_tcp) callback_sessions: super::Ed2kCallbackSessionHandle, /// Local bind address for outbound connects (the listener's VPN-pinned IP). pub(in crate::ed2k_tcp) bind_ip: Ipv4Addr, + pub(in crate::ed2k_tcp) egress_route: crate::networking::EgressRoute, /// Maximum time a granted disconnected waiter may take to attach a TCP /// connection (direct dial or callback) before its revocable slot expires. pub(in crate::ed2k_tcp) callback_timeout: Duration, @@ -283,7 +284,7 @@ impl UploadPromoteDriver { } result = Ed2kTransport::connect_outgoing( "upload_promote", - self.bind_ip, + crate::networking::EgressBinding::new(self.bind_ip, self.egress_route), peer_endpoint, self.hello_identity.connect_options, grant.peer.user_hash, diff --git a/crates/emulebb-ed2k/src/ed2k_transfer/deliver.rs b/crates/emulebb-ed2k/src/ed2k_transfer/deliver.rs index 3022becf3..a2cc07221 100644 --- a/crates/emulebb-ed2k/src/ed2k_transfer/deliver.rs +++ b/crates/emulebb-ed2k/src/ed2k_transfer/deliver.rs @@ -10,11 +10,12 @@ //! headless piece-store model. //! //! MECHANICS. Delivery copies `pieces.bin` to a temp sibling and atomically -//! renames it into place. The operator-facing file must not alias the verified +//! renames it into place when supported. If the filesystem rejects atomic +//! no-replace rename, an exclusively created final file is copied and verified +//! under a durable sharing exclusion. The operator-facing file must not alias the verified //! piece store: normal edits to a completed file cannot be allowed to mutate //! bytes later served under the transfer's ED2K hash. A name collision in the -//! destination is resolved by appending ` (1)`, ` (2)`, … before the extension, -//! matching eMule. +//! destination is resolved by appending `_01`, `_02`, … before the extension. //! //! The destination directory and the final delivered file path are //! operator-facing content paths, so every filesystem operation goes through @@ -49,7 +50,8 @@ use uuid::Uuid; use crate::long_path::long_path; use super::fs_durability::{ - NoReplaceMoveOutcome, move_path_no_replace, sync_file_for_publish, sync_parent_directory, + AtomicMoveUnsupported, NoReplaceMoveOutcome, move_path_no_replace, sync_file_for_publish, + sync_parent_directory, }; use super::{ED2K_PART_SIZE, Ed2kTransferRuntime}; @@ -232,6 +234,30 @@ struct InstalledDelivery { fingerprint: NativeFileFingerprint, } +fn delivery_candidate_name( + file_name: &str, + stem: &str, + extension: Option<&str>, + ordinal: u64, +) -> String { + if ordinal == 0 { + return file_name.to_owned(); + } + let mut index = ordinal; + let mut width = 2usize; + let mut limit = 99u64; + while index > limit { + index -= limit; + width += 1; + limit = limit.saturating_mul(10).saturating_add(9); + } + let suffix = format!("_{index:0width$}"); + match extension { + Some(ext) => format!("{stem}{suffix}.{ext}"), + None => format!("{stem}{suffix}"), + } +} + #[derive(Debug, Clone)] pub(super) struct OwnedDeliveryOutput { pub(super) path: PathBuf, @@ -243,6 +269,11 @@ struct VerifiedStagingFile { fingerprint: NativeFileFingerprint, } +struct DeliveryPayloadIdentity { + size: u64, + hash: [u8; 16], +} + fn validate_delivery_attempt_paths(attempt: &MetadataTransferDeliveryAttempt) -> Result<()> { anyhow::ensure!( attempt.target_path.is_absolute() && attempt.staging_path.is_absolute(), @@ -387,19 +418,22 @@ impl Ed2kTransferRuntime { let file_name = sanitize_file_name(display_name); let (stem, extension) = split_stem_extension(&file_name); - for index in 0..=u32::MAX { - let candidate_name = if index == 0 { - file_name.clone() - } else { - match &extension { - Some(extension) => format!("{stem} ({index}).{extension}"), - None => format!("{stem} ({index})"), - } - }; + for ordinal in 0..=u64::MAX { + let candidate_name = + delivery_candidate_name(&file_name, &stem, extension.as_deref(), ordinal); let target_path = dest_dir.join(candidate_name); if path_exists(&target_path).await { continue; } + if self.metadata.delivery_output_is_excluded(&target_path)? { + std::fs::canonicalize(dest_dir).with_context(|| { + format!( + "failed to resolve delivery directory {}", + dest_dir.display() + ) + })?; + continue; + } let attempt_id = Uuid::new_v4(); let staging_path = dest_dir.join(format!(".emulebb-delivery-{attempt_id}.tmp")); @@ -418,7 +452,34 @@ impl Ed2kTransferRuntime { "delivery attempt {attempt_id} disappeared before staging committed" ); - match install_staged_payload(&staging_path, &target_path).await? { + let installation = match install_staged_payload(&staging_path, &target_path).await { + Ok(outcome) => outcome, + Err(error) if error.downcast_ref::().is_some() => { + let direct = self + .publish_direct_copy( + file_hash, + attempt_id, + &staging_path, + &target_path, + verified_staging, + DeliveryPayloadIdentity { + size: expected_size, + hash: expected_hash, + }, + ) + .await?; + if let Some(fingerprint) = direct { + return Ok(InstalledDelivery { + path: target_path, + fingerprint, + }); + } + self.clear_delivery_attempt(file_hash, attempt_id).await?; + continue; + } + Err(error) => return Err(error), + }; + match installation { InstallOutcome::Installed => { #[cfg(test)] run_after_publish_move_hook(&target_path); @@ -470,6 +531,66 @@ impl Ed2kTransferRuntime { ) } + async fn publish_direct_copy( + &self, + file_hash: &str, + attempt_id: Uuid, + staging_path: &Path, + target_path: &Path, + verified: VerifiedStagingFile, + expected: DeliveryPayloadIdentity, + ) -> Result> { + use tokio::io::AsyncSeekExt; + + // WHY: The reservation must survive cancellation and crashes before a + // partial final-looking file can ever be considered for sharing. + if !self + .metadata + .reserve_delivery_output(target_path, attempt_id)? + { + drop(verified.file); + remove_file_if_fingerprint_matches(staging_path, &verified.fingerprint).await?; + return Ok(None); + } + self.refresh_complete_shared_catalog().await?; + let mut output = match tokio::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(long_path(target_path)) + .await + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + self.metadata.release_delivery_output(attempt_id)?; + drop(verified.file); + remove_file_if_fingerprint_matches(staging_path, &verified.fingerprint).await?; + return Ok(None); + } + Err(error) => return Err(error.into()), + }; + #[cfg(test)] + run_after_direct_create_hook(target_path).await; + let mut input = tokio::fs::File::from_std(verified.file); + input.seek(std::io::SeekFrom::Start(0)).await?; + tokio::io::copy(&mut input, &mut output).await?; + let output = output.into_std().await; + sync_file_for_publish(output.try_clone()?).await?; + sync_parent_directory(target_path).await?; + let fingerprint = + capture_published_fingerprint(output, target_path, expected.size, expected.hash) + .await?; + // Keep the barrier until the published journal record is committed. + anyhow::ensure!( + self.metadata + .publish_reserved_delivery_attempt(file_hash, attempt_id, &fingerprint)?, + "direct delivery reservation disappeared before publication committed" + ); + drop(input); + remove_file_if_fingerprint_matches(staging_path, &verified.fingerprint).await?; + Ok(Some(fingerprint)) + } + async fn begin_delivery_attempt( &self, file_hash: &str, @@ -559,6 +680,13 @@ impl Ed2kTransferRuntime { return Ok(()); }; validate_delivery_attempt_paths(&attempt)?; + if self.metadata.has_delivery_reservation(attempt.attempt_id)? { + // WHY: A crash may have interrupted direct creation or copying. + // Preserve both artifacts; their identity cannot be proven here. + self.clear_delivery_attempt(file_hash, attempt.attempt_id) + .await?; + return Ok(()); + } let manifest = { let _manifest_guard = self.lock_manifest(file_hash).await; self.load_manifest_unlocked(file_hash).await? @@ -654,7 +782,41 @@ impl Ed2kTransferRuntime { "delivery staging identity changed for attempt {}", attempt.attempt_id ); - match install_staged_payload(&attempt.staging_path, &attempt.target_path).await? { + let installation = + match install_staged_payload(&attempt.staging_path, &attempt.target_path).await + { + Ok(outcome) => outcome, + Err(error) if error.downcast_ref::().is_some() => { + if let Some(fingerprint) = self + .publish_direct_copy( + file_hash, + attempt.attempt_id, + &attempt.staging_path, + &attempt.target_path, + verified_staging, + DeliveryPayloadIdentity { + size: manifest.file_size, + hash: expected_hash, + }, + ) + .await? + { + return self + .commit_recovered_delivery( + file_hash, + &attempt, + &fingerprint, + super::Ed2kDeliveryOwnership::Owned, + ) + .await; + } + self.clear_delivery_attempt(file_hash, attempt.attempt_id) + .await?; + return Ok(()); + } + Err(error) => return Err(error), + }; + match installation { InstallOutcome::Installed => { #[cfg(test)] run_after_publish_move_hook(&attempt.target_path); @@ -849,6 +1011,40 @@ impl Ed2kTransferRuntime { } } +#[cfg(test)] +type DirectCreateHook = ( + tokio::sync::oneshot::Sender<()>, + std::sync::Arc, +); + +#[cfg(test)] +fn direct_create_hooks() +-> &'static std::sync::Mutex> { + static HOOKS: std::sync::OnceLock< + std::sync::Mutex>, + > = std::sync::OnceLock::new(); + HOOKS.get_or_init(Default::default) +} + +#[cfg(test)] +pub(super) fn pause_after_direct_create(path: &Path) -> tokio::sync::oneshot::Receiver<()> { + let (sender, receiver) = tokio::sync::oneshot::channel(); + direct_create_hooks().lock().unwrap().insert( + path.to_owned(), + (sender, std::sync::Arc::new(tokio::sync::Notify::new())), + ); + receiver +} + +#[cfg(test)] +async fn run_after_direct_create_hook(path: &Path) { + let hook = direct_create_hooks().lock().unwrap().remove(path); + if let Some((sender, gate)) = hook { + let _ = sender.send(()); + gate.notified().await; + } +} + #[cfg(test)] async fn deliver_payload_file( source: &Path, @@ -862,15 +1058,9 @@ async fn deliver_payload_file( tokio::fs::create_dir_all(long_path(dest_dir)).await?; let file_name = sanitize_file_name(display_name); let (stem, extension) = split_stem_extension(&file_name); - for index in 0..=u32::MAX { - let candidate_name = if index == 0 { - file_name.clone() - } else { - match &extension { - Some(extension) => format!("{stem} ({index}).{extension}"), - None => format!("{stem} ({index})"), - } - }; + for index in 0..=u64::MAX { + let candidate_name = + delivery_candidate_name(&file_name, &stem, extension.as_deref(), index); let target_path = dest_dir.join(candidate_name); if path_exists(&target_path).await { continue; @@ -1407,6 +1597,17 @@ fn split_stem_extension(file_name: &str) -> (String, Option) { #[cfg(test)] mod tests { + #[test] + fn collision_suffixes_widen_after_each_full_group() { + use super::delivery_candidate_name; + let name = |n| delivery_candidate_name("abc.mp3", "abc", Some("mp3"), n); + assert_eq!(name(0), "abc.mp3"); + assert_eq!(name(1), "abc_01.mp3"); + assert_eq!(name(99), "abc_99.mp3"); + assert_eq!(name(100), "abc_001.mp3"); + assert_eq!(name(1098), "abc_999.mp3"); + assert_eq!(name(1099), "abc_0001.mp3"); + } use super::*; fn temp_dir() -> PathBuf { @@ -1503,7 +1704,7 @@ mod tests { .await .unwrap(); - assert_eq!(delivered, dest.join("clip (1).mkv")); + assert_eq!(delivered, dest.join("clip_01.mkv")); assert_eq!(tokio::fs::read(&delivered).await.unwrap(), b"data"); tokio::fs::remove_dir_all(&dir).await.ok(); } @@ -1617,7 +1818,7 @@ mod tests { .await .unwrap(); - assert_eq!(delivered, dest.join("recovered (1).bin")); + assert_eq!(delivered, dest.join("recovered_01.bin")); assert_eq!( tokio::fs::read(&installed).await.unwrap(), b"recovered-payload" diff --git a/crates/emulebb-ed2k/src/ed2k_transfer/fs_durability.rs b/crates/emulebb-ed2k/src/ed2k_transfer/fs_durability.rs index 1576837a7..c2ab74ff2 100644 --- a/crates/emulebb-ed2k/src/ed2k_transfer/fs_durability.rs +++ b/crates/emulebb-ed2k/src/ed2k_transfer/fs_durability.rs @@ -11,23 +11,70 @@ pub(super) enum NoReplaceMoveOutcome { AlreadyExists, } +/// Only the rename syscall itself may trigger the direct-copy publication path. +#[derive(Debug)] +pub(super) struct AtomicMoveUnsupported(io::Error); + +impl std::fmt::Display for AtomicMoveUnsupported { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.0, f) + } +} + +impl std::error::Error for AtomicMoveUnsupported {} + pub(super) async fn move_path_no_replace( source: &Path, destination: &Path, ) -> Result { + #[cfg(test)] + if forced_unsupported_moves() + .lock() + .unwrap() + .remove(destination) + { + return Err(AtomicMoveUnsupported(io::Error::new( + io::ErrorKind::Unsupported, + "forced unsupported rename", + )) + .into()); + } let source = source.to_path_buf(); let destination = destination.to_path_buf(); let sync_path = destination.clone(); - let outcome = + let result = tokio::task::spawn_blocking(move || move_path_no_replace_blocking(&source, &destination)) .await - .context("no-replace move task failed")??; + .context("no-replace move task failed")?; + let outcome = match result { + Err(error) if error.kind() == io::ErrorKind::Unsupported => { + return Err(AtomicMoveUnsupported(error).into()); + } + other => other?, + }; if outcome == NoReplaceMoveOutcome::Moved { sync_parent_directory(&sync_path).await?; } Ok(outcome) } +#[cfg(test)] +fn forced_unsupported_moves() +-> &'static std::sync::Mutex> { + static PATHS: std::sync::OnceLock< + std::sync::Mutex>, + > = std::sync::OnceLock::new(); + PATHS.get_or_init(Default::default) +} + +#[cfg(test)] +pub(super) fn force_unsupported_move(path: &Path) { + forced_unsupported_moves() + .lock() + .unwrap() + .insert(path.to_owned()); +} + pub(super) async fn sync_file_for_publish(file: std::fs::File) -> Result<()> { tokio::task::spawn_blocking(move || sync_file_for_publish_blocking(&file)) .await @@ -104,15 +151,16 @@ fn sync_file_for_publish_blocking(file: &std::fs::File) -> io::Result<()> { file.sync_all() } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "android"))] fn move_path_no_replace_blocking( source: &Path, destination: &Path, ) -> io::Result { use rustix::fs::{CWD, RenameFlags, renameat_with}; - // WHY: Rust's bundled musl can lack the renameat2 libc wrapper. Rustix - // supplies the safe Linux operation while preserving atomic no-clobber publication. + // WHY: Android is a separate Rust target but uses Linux's atomic no-replace + // rename too. Rustix also handles musl builds lacking the libc wrapper; + // never substitute a check-then-rename that could overwrite another file. let error = match renameat_with(CWD, source, CWD, destination, RenameFlags::NOREPLACE) { Ok(()) => return Ok(NoReplaceMoveOutcome::Moved), Err(error) => io::Error::from(error), @@ -200,7 +248,12 @@ fn move_path_no_replace_blocking( } } -#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +#[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + windows +)))] fn move_path_no_replace_blocking( _source: &Path, _destination: &Path, diff --git a/crates/emulebb-ed2k/src/ed2k_transfer/tests/deliver_runtime.rs b/crates/emulebb-ed2k/src/ed2k_transfer/tests/deliver_runtime.rs index aa597a0a8..36742ec06 100644 --- a/crates/emulebb-ed2k/src/ed2k_transfer/tests/deliver_runtime.rs +++ b/crates/emulebb-ed2k/src/ed2k_transfer/tests/deliver_runtime.rs @@ -10,10 +10,12 @@ use emulebb_metadata::MetadataTransferDirectoryDeletionAuthority; use md4::{Digest, Md4}; use uuid::Uuid; +use super::super::deliver::pause_after_direct_create; #[cfg(unix)] use super::super::deliver::set_after_publish_move_hook; use super::super::deliver::set_before_fingerprint_remove_hook; use super::super::delivery_tombstone::set_before_move_hook; +use super::super::fs_durability::force_unsupported_move; use super::super::ingest::native_file_fingerprint; use super::super::metadata::{ transfer_directory_authority_marker_path, transfer_directory_tombstone_path, @@ -146,6 +148,134 @@ fn assert_complete_deletion_recovery( assert_eq!(report.recovered, expected_recovered); } +#[tokio::test] +async fn unsupported_atomic_move_uses_verified_exclusive_copy() { + let root = unique_test_dir("ed2k-direct-delivery"); + let runtime = Ed2kTransferRuntime::load_or_create(&root).unwrap(); + let hash = ingest_completed(&runtime, &root, "Sample.bin").await; + let payload = fs::read(runtime.payload_path(&hash)).unwrap(); + let incoming = root.join("incoming"); + let target = incoming.join("Sample.bin"); + force_unsupported_move(&target); + assert_eq!( + runtime + .materialize_completed_payload(&hash, &incoming) + .await + .unwrap(), + Ed2kDeliveryOutcome::Delivered(target.clone()) + ); + assert_eq!(fs::read(&target).unwrap(), payload); + assert!(runtime.payload_path(&hash).exists()); + assert!( + !runtime + .metadata + .delivery_output_is_excluded(&target) + .unwrap() + ); +} + +#[tokio::test] +async fn interrupted_direct_copy_preserves_and_excludes_partial() { + let root = unique_test_dir("ed2k-direct-interrupted"); + let runtime = Ed2kTransferRuntime::load_or_create(&root).unwrap(); + let hash = ingest_completed(&runtime, &root, "Sample.bin").await; + let payload = fs::read(runtime.payload_path(&hash)).unwrap(); + let incoming = root.join("incoming"); + fs::create_dir_all(&incoming).unwrap(); + let target = incoming.join("Sample.bin"); + let attempt = Uuid::new_v4(); + let staging = incoming.join(format!(".emulebb-delivery-{attempt}.tmp")); + runtime + .metadata + .begin_transfer_delivery_attempt(&hash, attempt, &target, &staging) + .unwrap(); + fs::write(&staging, &payload).unwrap(); + let fingerprint = native_file_fingerprint(&staging).unwrap(); + runtime + .metadata + .mark_transfer_delivery_attempt_staged(&hash, attempt, &fingerprint) + .unwrap(); + assert!( + runtime + .metadata + .reserve_delivery_output(&target, attempt) + .unwrap() + ); + fs::write(&target, b"partial").unwrap(); + let second = incoming.join("Sample_01.bin"); + assert_eq!( + runtime + .materialize_completed_payload(&hash, &incoming) + .await + .unwrap(), + Ed2kDeliveryOutcome::Delivered(second.clone()) + ); + assert_eq!(fs::read(&target).unwrap(), b"partial"); + assert_eq!(fs::read(&second).unwrap(), payload); + assert!( + runtime + .metadata + .delivery_output_is_excluded(&target) + .unwrap() + ); + assert!(runtime.metadata.delete_transfer_manifest(&hash).unwrap()); + assert!( + runtime + .metadata + .delivery_output_is_excluded(&target) + .unwrap() + ); + assert_eq!(fs::read(&target).unwrap(), b"partial"); +} + +#[tokio::test] +async fn cancelled_direct_copy_keeps_partial_excluded_and_retries() { + let root = unique_test_dir("ed2k-direct-cancel"); + let runtime = Arc::new(Ed2kTransferRuntime::load_or_create(&root).unwrap()); + let hash = ingest_completed(&runtime, &root, "Sample.bin").await; + let payload = fs::read(runtime.payload_path(&hash)).unwrap(); + let incoming = root.join("incoming"); + let target = incoming.join("Sample.bin"); + force_unsupported_move(&target); + let created = pause_after_direct_create(&target); + let task_runtime = Arc::clone(&runtime); + let task_hash = hash.clone(); + let task_incoming = incoming.clone(); + let task = tokio::spawn(async move { + task_runtime + .materialize_completed_payload(&task_hash, &task_incoming) + .await + }); + tokio::time::timeout(Duration::from_secs(10), created) + .await + .unwrap() + .unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert!( + runtime + .metadata + .delivery_output_is_excluded(&target) + .unwrap() + ); + assert_eq!(fs::read(&target).unwrap(), b""); + let retry = incoming.join("Sample_01.bin"); + assert_eq!( + runtime + .materialize_completed_payload(&hash, &incoming) + .await + .unwrap(), + Ed2kDeliveryOutcome::Delivered(retry.clone()) + ); + assert_eq!(fs::read(&retry).unwrap(), payload); + assert!( + runtime + .metadata + .delivery_output_is_excluded(&target) + .unwrap() + ); +} + #[tokio::test] async fn materialize_completed_payload_delivers_by_name() { let root = unique_test_dir("ed2k-deliver-by-name"); @@ -305,7 +435,7 @@ async fn staged_attempt_does_not_adopt_byte_identical_foreign_target() { .mark_transfer_delivery_attempt_staged(&hash, attempt_id, &staging_fingerprint) .unwrap(); - let delivered = incoming.join("Recovered.Collision (1).mkv"); + let delivered = incoming.join("Recovered.Collision_01.mkv"); assert_eq!( runtime .materialize_completed_payload(&hash, &incoming) @@ -355,7 +485,7 @@ async fn publication_rehash_rejects_equal_length_mutation_after_move() { assert_eq!(manifest.delivery_state, Ed2kDeliveryState::Pending); assert_eq!(manifest.delivery_ownership, Ed2kDeliveryOwnership::None); - let delivered = incoming.join("Mutated.After.Move (1).mkv"); + let delivered = incoming.join("Mutated.After.Move_01.mkv"); assert_eq!( runtime .materialize_completed_payload(&hash, &incoming) @@ -464,7 +594,7 @@ async fn prepared_attempt_preserves_unproven_staging_occupant() { let legacy_temp = incoming.join(".Prepared.mkv.ed2k-delivering-legacy"); fs::write(&legacy_temp, b"unproven legacy artifact").unwrap(); - let delivered = incoming.join("Prepared (1).mkv"); + let delivered = incoming.join("Prepared_01.mkv"); assert_eq!( runtime .materialize_completed_payload(&hash, &incoming) @@ -640,7 +770,7 @@ async fn materialize_reconciles_replaced_output_without_overwriting_it() { .materialize_completed_payload(&hash, &incoming) .await .unwrap(); - let redelivered = incoming.join("Replaced (1).mkv"); + let redelivered = incoming.join("Replaced_01.mkv"); assert_eq!( reconciled, Ed2kDeliveryOutcome::Delivered(redelivered.clone()) @@ -731,7 +861,7 @@ async fn materialize_preserves_byte_identical_replacement_without_adopting_it() fs::remove_file(&delivered).unwrap(); fs::write(&delivered, &verified_payload).unwrap(); - let redelivered = incoming.join("Identical (1).mkv"); + let redelivered = incoming.join("Identical_01.mkv"); assert_eq!( runtime .materialize_completed_payload(&hash, &incoming) diff --git a/crates/emulebb-ed2k/src/networking.rs b/crates/emulebb-ed2k/src/networking.rs index cfbbeced3..0a5cc272f 100644 --- a/crates/emulebb-ed2k/src/networking.rs +++ b/crates/emulebb-ed2k/src/networking.rs @@ -187,6 +187,50 @@ pub fn resolve_bind_ip( .map(|address| address.address.clone()) } +/// Per-runtime routing intent. Explicit binds and guards always require a pin. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum EgressRoute { + /// The operator selected neither a bind nor a VPN guard; bind the selected source address. + DirectSystemRoute, + /// Resolve and apply the configured interface pin, failing closed on errors. + #[default] + StrictInterface, +} + +/// Source address and immutable interface-pinning intent for one runtime. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EgressBinding { + pub address: std::net::Ipv4Addr, + pub route: EgressRoute, +} + +impl EgressBinding { + /// Construct a source-address binding without inferring the operator's intent. + #[must_use] + pub const fn new(address: std::net::Ipv4Addr, route: EgressRoute) -> Self { + Self { address, route } + } + + /// Resolve an index when strict pinning was requested. + pub fn interface_index(self, purpose: &str) -> Result> { + self.route.interface_index(self.address, purpose) + } +} + +impl EgressRoute { + /// Resolve the required interface only for strict routing; never downgrade on error. + pub fn interface_index( + self, + bind_ip: std::net::Ipv4Addr, + purpose: &str, + ) -> Result> { + match self { + Self::DirectSystemRoute => Ok(None), + Self::StrictInterface => require_bind_if_index(bind_ip, purpose).map(Some), + } + } +} + /// Resolves the OS interface index that owns `bind_ip`, for `IP_UNICAST_IF` /// egress pinning (eMule `ApplyConfiguredIpv4UnicastInterface`). Returns `None` /// when no local interface advertises that IPv4 address. The index lets the P2P @@ -634,6 +678,20 @@ mod tests { } } + #[test] + fn concurrent_route_intents_do_not_share_state_or_downgrade_strict_errors() { + use super::{EgressBinding, EgressRoute}; + let address = std::net::Ipv4Addr::new(203, 0, 113, 234); + let direct = std::thread::spawn(move || { + EgressBinding::new(address, EgressRoute::DirectSystemRoute).interface_index("direct") + }); + let strict = std::thread::spawn(move || { + EgressBinding::new(address, EgressRoute::StrictInterface).interface_index("strict") + }); + assert_eq!(direct.join().unwrap().unwrap(), None); + assert!(strict.join().unwrap().is_err()); + } + #[test] fn build_interface_binding_report_preserves_selection_state() { let binding = ResolvedInterfaceBindingReport { diff --git a/crates/emulebb-ed2k/src/stun.rs b/crates/emulebb-ed2k/src/stun.rs index bcd9697fc..d4aeb81e3 100644 --- a/crates/emulebb-ed2k/src/stun.rs +++ b/crates/emulebb-ed2k/src/stun.rs @@ -24,8 +24,6 @@ use anyhow::{Context, Result, anyhow, bail}; use rand::Rng; use tokio::net::UdpSocket; -use crate::networking::require_bind_if_index; - const STUN_BINDING_REQUEST: u16 = 0x0001; const STUN_BINDING_SUCCESS: u16 = 0x0101; const ATTR_MAPPED_ADDRESS: u16 = 0x0001; @@ -62,8 +60,11 @@ pub struct StunProbeOutcome { /// `bind_ip` should be the resolved data-plane bind address (the VPN tunnel IP) /// so the probe is egress-pinned exactly like the eD2k/Kad sockets; pass an /// unspecified address (`0.0.0.0`) for a default-route baseline probe. -pub async fn stun_probe(bind_ip: Ipv4Addr, timeout: Duration) -> Result { - stun_probe_servers_detailed(DEFAULT_STUN_SERVERS, bind_ip, timeout) +pub async fn stun_probe( + binding: crate::networking::EgressBinding, + timeout: Duration, +) -> Result { + stun_probe_servers_for_binding(DEFAULT_STUN_SERVERS, binding, timeout) .await .map(|outcome| *outcome.endpoint.ip()) } @@ -89,6 +90,22 @@ pub async fn stun_probe_servers_detailed( servers: &[(&'static str, u16)], bind_ip: Ipv4Addr, timeout: Duration, +) -> Result { + stun_probe_servers_for_binding( + servers, + crate::networking::EgressBinding::new( + bind_ip, + crate::networking::EgressRoute::StrictInterface, + ), + timeout, + ) + .await +} + +async fn stun_probe_servers_for_binding( + servers: &[(&'static str, u16)], + binding: crate::networking::EgressBinding, + timeout: Duration, ) -> Result { let mut set = tokio::task::JoinSet::new(); for &(host, port) in servers { @@ -96,7 +113,7 @@ pub async fn stun_probe_servers_detailed( // Bound the whole per-server probe (DNS + connect + send + recv) with // one deadline, consistent with the libtorrent/eMuleBB probes. let label = format!("{host}:{port}"); - let result = match tokio::time::timeout(timeout, probe_one(host, port, bind_ip)).await { + let result = match tokio::time::timeout(timeout, probe_one(host, port, binding)).await { Ok(result) => result, Err(_) => Err(anyhow!("timed out")), }; @@ -153,9 +170,10 @@ pub enum NatMappingBehavior { /// the gateway, not the socket), so it generalizes to the eD2k/Kad socket; it does /// NOT yield an advertisable port (see the parse note). pub async fn stun_probe_mapping_behavior( - bind_ip: Ipv4Addr, + binding: crate::networking::EgressBinding, timeout: Duration, ) -> NatMappingBehavior { + let bind_ip = binding.address; let mut hosts = DEFAULT_STUN_SERVERS.iter(); let Some(&first) = hosts.next() else { return NatMappingBehavior::Inconclusive; @@ -164,7 +182,7 @@ pub async fn stun_probe_mapping_behavior( return NatMappingBehavior::Inconclusive; }; - let bind_if_index = match require_bind_if_index(bind_ip, "STUN NAT mapping probe") { + let bind_if_index = match binding.interface_index("STUN NAT mapping probe") { Ok(index) => index, Err(error) => { tracing::debug!("STUN NAT mapping probe blocked by bind policy: {error:#}"); @@ -177,7 +195,7 @@ pub async fn stun_probe_mapping_behavior( }; if emulebb_kad_dht::socket_opts::pin_egress_to_interface( socket2::SockRef::from(&socket), - Some(bind_if_index), + bind_if_index, ) .is_err() { @@ -238,8 +256,13 @@ async fn probe_port_unconnected( /// One server probe: resolve, bind + egress-pin, `connect()`, single send, parse. /// The caller bounds the whole probe with a timeout (see `stun_probe_servers`). -async fn probe_one(host: &'static str, port: u16, bind_ip: Ipv4Addr) -> Result { - let bind_if_index = require_bind_if_index(bind_ip, "STUN public IP probe")?; +async fn probe_one( + host: &'static str, + port: u16, + binding: crate::networking::EgressBinding, +) -> Result { + let bind_ip = binding.address; + let bind_if_index = binding.interface_index("STUN public IP probe")?; let server = tokio::net::lookup_host((host, port)) .await .with_context(|| format!("STUN DNS lookup failed for {host}:{port}"))? @@ -261,7 +284,7 @@ async fn probe_one(host: &'static str, port: u16, bind_ip: Ipv4Addr) -> Result Result { + let parent = path.parent().context("delivery path has no parent")?; + let name = path.file_name().context("delivery path has no filename")?; + Ok(std::fs::canonicalize(parent)?.join(name)) +} + +pub(crate) fn is_excluded(conn: &Connection, path: &Path) -> Result { + let any: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM delivery_exclusions)", + [], + |row| row.get(0), + )?; + if !any { + return Ok(false); + } + let normalized = match exclusion_path(path) { + Ok(path) => path, + // WHY: If the parent cannot be resolved, an alias of a reserved path + // cannot be ruled out. Do not admit it to the shared catalog. + Err(_) => return Ok(true), + }; + let native = NativePath::from(normalized.as_path()); + Ok(conn + .query_row( + "SELECT 1 FROM delivery_exclusions WHERE platform = ?1 AND native_path = ?2", + params![NativePath::platform(), native.storage_bytes()], + |_| Ok(()), + ) + .optional()? + .is_some()) +} + +impl MetadataStore { + /// Reserve a final name before exclusive creation; exclusions outlive transfers. + pub fn reserve_delivery_output(&self, path: &Path, id: Uuid) -> Result { + let normalized = exclusion_path(path)?; + let native = NativePath::from(normalized.as_path()); + let mut conn = self.connection()?; + let tx = conn.transaction()?; + let inserted = tx.execute( + "INSERT OR IGNORE INTO delivery_exclusions(platform, native_path, reservation_id) VALUES (?1, ?2, ?3)", + params![NativePath::platform(), native.storage_bytes(), id.to_string()], + )? == 1; + if inserted { + // WHY: an old session may still advertise a removed file under an + // alias of this name. Withdraw it before a new partial can exist. + let paths = { + let mut stmt = tx.prepare("SELECT local_paths.id, local_paths.platform, local_paths.native_path + FROM local_file_instances JOIN local_paths ON local_paths.id = local_file_instances.path_id + WHERE EXISTS (SELECT 1 FROM shared_file_memberships m + WHERE m.path_id = local_paths.id AND m.removed_at_ms IS NULL) + OR EXISTS (SELECT 1 FROM temp.session_verified_local_paths v WHERE v.path_id = local_paths.id)")?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + crate::native_path::path_from_row(row, 1, 2)?, + )) + })?; + rows.collect::>>()? + }; + let mut parents: HashMap> = HashMap::new(); + for (path_id, known_path) in paths { + let Some((parent, name)) = known_path.parent().zip(known_path.file_name()) else { + continue; + }; + let resolved = parents + .entry(parent.to_path_buf()) + .or_insert_with(|| std::fs::canonicalize(parent).ok()); + if resolved + .as_ref() + .is_some_and(|parent| parent.join(name) == normalized) + { + tx.execute("UPDATE shared_file_memberships SET removed_at_ms = ?2 WHERE path_id = ?1 AND removed_at_ms IS NULL", params![path_id, crate::store::unix_ms()])?; + tx.execute( + "DELETE FROM temp.session_verified_local_paths WHERE path_id = ?1", + [path_id], + )?; + } + } + } + tx.commit()?; + Ok(inserted) + } + + /// Release only a reservation whose exclusive creation did not succeed. + pub fn release_delivery_output(&self, id: Uuid) -> Result<()> { + self.connection()?.execute( + "DELETE FROM delivery_exclusions WHERE reservation_id = ?1", + [id.to_string()], + )?; + Ok(()) + } + + /// Return whether a path is barred from local sharing. + pub fn delivery_output_is_excluded(&self, path: &Path) -> Result { + let conn = self.connection()?; + is_excluded(&conn, path) + } + + /// A retained reservation indicates an interrupted direct-copy attempt. + pub fn has_delivery_reservation(&self, id: Uuid) -> Result { + Ok(self.connection()?.query_row( + "SELECT EXISTS(SELECT 1 FROM delivery_exclusions WHERE reservation_id = ?1)", + [id.to_string()], + |row| row.get(0), + )?) + } +} + +pub(crate) fn ensure_not_excluded(conn: &Connection, path: &Path) -> Result<()> { + ensure!( + !is_excluded(conn, path)?, + "local file is an incomplete delivery output" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reservations_are_independent_and_prevent_duplicate_claims() { + let root = std::env::temp_dir().join(format!("emulebb-exclusion-{}", Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("sample.bin"); + let store = MetadataStore::in_memory().unwrap(); + let first = Uuid::new_v4(); + let second = Uuid::new_v4(); + assert!(store.reserve_delivery_output(&path, first).unwrap()); + assert!(store.delivery_output_is_excluded(&path).unwrap()); + assert!(!store.reserve_delivery_output(&path, second).unwrap()); + store.release_delivery_output(second).unwrap(); + assert!(store.delivery_output_is_excluded(&path).unwrap()); + store.release_delivery_output(first).unwrap(); + assert!(!store.delivery_output_is_excluded(&path).unwrap()); + std::fs::remove_dir(root).unwrap(); + } + + #[test] + fn reservation_survives_profile_reopen() { + let root = + std::env::temp_dir().join(format!("emulebb-exclusion-reopen-{}", Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + let database = root.join("profile.db"); + let target = root.join("partial.bin"); + let id = Uuid::new_v4(); + { + let store = MetadataStore::open(&database).unwrap(); + assert!(store.reserve_delivery_output(&target, id).unwrap()); + } + let store = MetadataStore::open(&database).unwrap(); + assert!(store.has_delivery_reservation(id).unwrap()); + assert!(store.delivery_output_is_excluded(&target).unwrap()); + drop(store); + std::fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn exclusion_survives_reopen_and_resolves_parent_aliases() { + let root = + std::env::temp_dir().join(format!("emulebb-exclusion-reopen-{}", Uuid::new_v4())); + std::fs::create_dir_all(root.join("real")).unwrap(); + std::os::unix::fs::symlink(root.join("real"), root.join("alias")).unwrap(); + let database = root.join("profile.db"); + let canonical_target = root.join("real").join("partial.bin"); + let alias_target = root.join("alias").join("partial.bin"); + let id = Uuid::new_v4(); + { + let store = MetadataStore::open(&database).unwrap(); + assert!( + store + .reserve_delivery_output(&canonical_target, id) + .unwrap() + ); + assert!(store.delivery_output_is_excluded(&alias_target).unwrap()); + } + let store = MetadataStore::open(&database).unwrap(); + assert!(store.has_delivery_reservation(id).unwrap()); + assert!(store.delivery_output_is_excluded(&alias_target).unwrap()); + drop(store); + std::fs::remove_dir_all(root).unwrap(); + } +} diff --git a/crates/emulebb-metadata/src/lib.rs b/crates/emulebb-metadata/src/lib.rs index 05bd4e144..ba962934f 100644 --- a/crates/emulebb-metadata/src/lib.rs +++ b/crates/emulebb-metadata/src/lib.rs @@ -2,6 +2,7 @@ mod aich_recovery_store; mod content_identity; +mod delivery_exclusion_store; mod download_history_model; mod download_history_store; mod identity_model; @@ -55,7 +56,7 @@ pub use search_model::{ MetadataSearch, MetadataSearchResult, MetadataSearchResultObservation, MetadataSearchSpec, }; pub use search_store::normalized_search_query; -pub use store::MetadataStore; +pub use store::{InitialProfileStorage, MetadataStore}; pub use text::normalize_search_text; pub use transfer_model::{ MetadataFileImportRecord, MetadataSharedSourceFailure, MetadataTransferAichIdentity, diff --git a/crates/emulebb-metadata/src/local_file_store.rs b/crates/emulebb-metadata/src/local_file_store.rs index 8003bbb08..63336debc 100644 --- a/crates/emulebb-metadata/src/local_file_store.rs +++ b/crates/emulebb-metadata/src/local_file_store.rs @@ -239,7 +239,11 @@ impl MetadataStore { fingerprint: &NativeFileFingerprint, ) -> Result { let native = NativePath::from(path); - let changed = self.connection()?.execute( + let conn = self.connection()?; + if crate::delivery_exclusion_store::is_excluded(&conn, path)? { + return Ok(false); + } + let changed = conn.execute( "INSERT OR REPLACE INTO temp.session_verified_local_paths(path_id, verified_fingerprint) SELECT local_file_instances.path_id, local_file_instances.verified_fingerprint FROM local_file_instances @@ -328,6 +332,7 @@ impl MetadataStore { now: i64, origin: &'static str, ) -> Result<()> { + crate::delivery_exclusion_store::ensure_not_excluded(tx, &content.path)?; ensure!( matches!(origin, "local_computed" | "stock_import"), "invalid local AICH origin" @@ -526,6 +531,7 @@ impl MetadataStore { let hash = decode_fixed_hex(file_hash, 16, "ED2K hash")?; let mut conn = self.connection()?; let tx = conn.transaction()?; + crate::delivery_exclusion_store::ensure_not_excluded(&tx, path)?; let (known_file_id, known_size): (i64, i64) = tx .query_row( "SELECT id, size_bytes FROM known_files WHERE ed2k_hash = ?1", diff --git a/crates/emulebb-metadata/src/profile_store.rs b/crates/emulebb-metadata/src/profile_store.rs index 042bd7231..75371c918 100644 --- a/crates/emulebb-metadata/src/profile_store.rs +++ b/crates/emulebb-metadata/src/profile_store.rs @@ -17,6 +17,37 @@ enum CategoryDeletionWrite { ReadyToCommit, } +pub(crate) fn seed_host_storage_defaults( + tx: &rusqlite::Transaction<'_>, + storage: &crate::InitialProfileStorage, +) -> Result<()> { + ensure!( + storage.incoming_dir.is_absolute() && storage.shared_dir.is_absolute(), + "host folders must be absolute" + ); + let incoming = storage + .incoming_dir + .to_str() + .ok_or_else(|| anyhow::anyhow!("host incoming path must be valid Unicode"))?; + let now = unix_ms(); + tx.execute( + "INSERT INTO settings(section, key, value_json, updated_at_ms) VALUES ('daemon', 'incomingDir', json_quote(?1), ?2)", + params![incoming, now], + )?; + let path_id = upsert_local_path(tx, &storage.shared_dir)?; + tx.execute( + "INSERT INTO shared_directory_roots(path_id, monitor_owned, shareable, accessible, enabled, created_at_ms, deleted_at_ms) + VALUES (?1, 0, 1, 1, 1, ?2, NULL)", + params![path_id, now], + )?; + tx.execute( + "INSERT INTO shared_directory_root_origins(root_id, origin, monitor_owned, shareable, accessible) + VALUES ((SELECT id FROM shared_directory_roots WHERE path_id = ?1), 'explicit', 0, 1, 1)", + params![path_id], + )?; + Ok(()) +} + impl super::MetadataStore { pub fn load_setting_json(&self, section: &str, key: &str) -> Result> { ensure!( diff --git a/crates/emulebb-metadata/src/schema.rs b/crates/emulebb-metadata/src/schema.rs index ca6079607..a41af0c1e 100644 --- a/crates/emulebb-metadata/src/schema.rs +++ b/crates/emulebb-metadata/src/schema.rs @@ -16,7 +16,7 @@ //! rejected: this module contains no migration or legacy interpretation. pub const SCHEMA_ID: &str = "emulebb.metadata"; -pub const SCHEMA_VERSION: i64 = 68; +pub const SCHEMA_VERSION: i64 = 69; pub const SCHEMA_SQL: &str = include_str!("schema.sql"); #[cfg(test)] diff --git a/crates/emulebb-metadata/src/schema.sql b/crates/emulebb-metadata/src/schema.sql index 61dc83b9a..d03487b6d 100644 --- a/crates/emulebb-metadata/src/schema.sql +++ b/crates/emulebb-metadata/src/schema.sql @@ -403,6 +403,15 @@ CREATE TABLE transfer_retirements ( ) ) STRICT; +-- WHY: A crash can leave an exclusively created but incomplete final name. +-- Its sharing exclusion must outlive both the attempt and transfer deletion. +CREATE TABLE delivery_exclusions ( + platform TEXT NOT NULL, + native_path BLOB NOT NULL, + reservation_id TEXT NOT NULL UNIQUE, + PRIMARY KEY(platform, native_path) +) STRICT; + CREATE TABLE delivery_attempts ( transfer_id INTEGER PRIMARY KEY REFERENCES transfers(id) ON DELETE CASCADE, attempt_id TEXT NOT NULL UNIQUE CHECK(length(trim(attempt_id)) > 0), diff --git a/crates/emulebb-metadata/src/shared_membership_store.rs b/crates/emulebb-metadata/src/shared_membership_store.rs index 07c2ee451..3efbc8fa8 100644 --- a/crates/emulebb-metadata/src/shared_membership_store.rs +++ b/crates/emulebb-metadata/src/shared_membership_store.rs @@ -44,6 +44,9 @@ impl MetadataStore { let native_root = NativePath::from(root); let native_path = NativePath::from(path); let conn = self.connection()?; + if crate::delivery_exclusion_store::is_excluded(&conn, path)? { + return Ok(false); + } let found = conn.query_row( "SELECT shared_directory_roots.id, local_file_instances.path_id, local_file_instances.verified_fingerprint @@ -192,6 +195,9 @@ impl MetadataStore { DELETE FROM temp.scanned_membership_paths;", )?; for (path, relative, current_fingerprint) in paths { + if crate::delivery_exclusion_store::is_excluded(&tx, path)? { + continue; + } ensure!( !relative.as_os_str().is_empty() && !relative.is_absolute() diff --git a/crates/emulebb-metadata/src/store.rs b/crates/emulebb-metadata/src/store.rs index 2a7bb6134..18b921298 100644 --- a/crates/emulebb-metadata/src/store.rs +++ b/crates/emulebb-metadata/src/store.rs @@ -1,6 +1,6 @@ use std::{ fs::File, - path::Path, + path::{Path, PathBuf}, sync::{Arc, Mutex, MutexGuard, TryLockError}, time::{SystemTime, UNIX_EPOCH}, }; @@ -21,12 +21,21 @@ use crate::{ /// its current sources from the network, so older rows are pruned on startup. const TRANSFER_SOURCE_TTL_MS: i64 = 30 * 24 * 60 * 60 * 1000; +/// Canonical host folder values to commit together with a newly initialized schema. +pub struct InitialProfileStorage { + pub incoming_dir: PathBuf, + pub shared_dir: PathBuf, +} + +type StorageInitializer = Box Result>; + #[derive(Debug, Clone)] pub struct MetadataStore { conn: Arc>, // WHY: close the connection before releasing its process profile lease. // Clones retain both handles; independent profile connections share the lease. _profile_lease: Option>, + initialized_new_schema: bool, } impl MetadataStore { @@ -41,7 +50,26 @@ impl MetadataStore { /// Every cloned store retains it until its connection has finished closing. pub fn open_with_profile_lease(path: impl AsRef, lease: Arc) -> Result { let conn = Connection::open(path)?; - Self::from_connection_with_profile_lease(conn, Some(lease)) + Self::from_connection_with_profile_lease(conn, Some(lease), None) + } + + /// Apply storage defaults only within fresh-schema creation. The factory is + /// never called for an existing schema, including an empty operator root list. + pub fn open_with_profile_lease_and_storage( + path: impl AsRef, + lease: Arc, + initialize: impl FnOnce() -> Result + 'static, + ) -> Result { + Self::from_connection_with_profile_lease( + Connection::open(path)?, + Some(lease), + Some(Box::new(initialize)), + ) + } + + /// Whether this connection initialized the profile rather than opening one. + pub fn initialized_new_schema(&self) -> bool { + self.initialized_new_schema } pub fn in_memory() -> Result { @@ -49,18 +77,20 @@ impl MetadataStore { } pub fn from_connection(conn: Connection) -> Result { - Self::from_connection_with_profile_lease(conn, None) + Self::from_connection_with_profile_lease(conn, None, None) } fn from_connection_with_profile_lease( conn: Connection, profile_lease: Option>, + initialize_storage: Option, ) -> Result { // The guarded constructor retains ownership before any pragma/schema // initialization, including all early-error exits below. let mut store = Self { conn: Arc::new(Mutex::new(conn)), _profile_lease: profile_lease, + initialized_new_schema: false, }; { let conn = store.connection()?; @@ -80,7 +110,7 @@ impl MetadataStore { // SQLITE_BUSY (the persistence audit flagged the absent busy_timeout). conn.pragma_update(None, "busy_timeout", 5000)?; } - store.ensure_schema()?; + store.initialized_new_schema = store.ensure_schema(initialize_storage)?; store.connection()?.execute_batch( "CREATE TEMP TABLE session_verified_local_paths ( path_id INTEGER PRIMARY KEY, @@ -113,7 +143,7 @@ impl MetadataStore { /// database, and otherwise requires the stored schema marker to match exactly. /// An incompatible product profile must be replaced with an entirely fresh /// profile; this crate never migrates, repairs, or resets it. - fn ensure_schema(&mut self) -> Result<()> { + fn ensure_schema(&mut self, initialize_storage: Option) -> Result { if self.table_exists("metadata_schema")? { let stored_version = self .stored_schema_version()? @@ -124,13 +154,14 @@ impl MetadataStore { ); crate::schema_validation::validate(&*self.connection()?)?; self.prune_transfer_sources()?; - return Ok(()); + return Ok(false); } ensure!( !self.has_user_tables()?, "metadata database has existing user objects but no metadata_schema marker; refusing to create or repair schema in Rust" ); - self.create_schema() + self.create_schema(initialize_storage)?; + Ok(true) } fn stored_schema_version(&self) -> Result> { @@ -194,7 +225,7 @@ impl MetadataStore { Ok(count != 0) } - fn create_schema(&mut self) -> Result<()> { + fn create_schema(&mut self, initialize_storage: Option) -> Result<()> { let now = unix_ms(); let profile_uuid = Uuid::new_v4().to_string(); let mut conn = self.connection()?; @@ -211,6 +242,13 @@ impl MetadataStore { "#, params![profile_uuid, now], )?; + // WHY: publishing the schema marker before host defaults can make a + // failed first start permanently skip its folders on retry. Both belong + // to the same transaction; failed initialization leaves no current schema. + if let Some(initialize) = initialize_storage { + let storage = initialize()?; + crate::profile_store::seed_host_storage_defaults(&tx, &storage)?; + } tx.commit()?; Ok(()) } @@ -720,6 +758,65 @@ pub(crate) fn unix_ms() -> i64 { #[cfg(test)] mod tests { use super::*; + + #[test] + fn fresh_storage_failure_rolls_back_schema_and_retry_preserves_later_edits() { + let path = + std::env::temp_dir().join(format!("emulebb-initial-storage-{}.db", Uuid::new_v4())); + let rejected = MetadataStore::from_connection_with_profile_lease( + Connection::open(&path).unwrap(), + None, + Some(Box::new(|| anyhow::bail!("injected initializer failure"))), + ); + assert!(rejected.is_err()); + let inspect = Connection::open(&path).unwrap(); + assert_eq!(inspect.query_row("SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'", [], |row| row.get::<_, i64>(0)).unwrap(), 0); + drop(inspect); + let incoming = path.with_extension("incoming"); + let shared = path.with_extension("shared"); + let wanted = incoming.clone(); + let mut store = MetadataStore::from_connection_with_profile_lease( + Connection::open(&path).unwrap(), + None, + Some(Box::new(move || { + Ok(InitialProfileStorage { + incoming_dir: incoming, + shared_dir: shared, + }) + })), + ) + .unwrap(); + assert!(store.initialized_new_schema()); + let actual: String = store.connection().unwrap().query_row( + "SELECT json_extract(value_json, '$') FROM settings WHERE section = 'daemon' AND key = 'incomingDir'", [], |row| row.get(0), + ).unwrap(); + assert_eq!(actual, wanted.to_str().unwrap()); + assert_eq!(store.shared_directory_roots().unwrap().len(), 1); + store + .put_setting_json("daemon", "incomingDir", "null") + .unwrap(); + store.replace_shared_directory_roots(&[]).unwrap(); + drop(store); + let store = MetadataStore::from_connection_with_profile_lease( + Connection::open(&path).unwrap(), + None, + Some(Box::new(|| { + panic!("existing profile must not invoke its initializer") + })), + ) + .unwrap(); + assert!(!store.initialized_new_schema()); + assert_eq!( + store + .load_setting_json("daemon", "incomingDir") + .unwrap() + .as_deref(), + Some("null") + ); + assert!(store.shared_directory_roots().unwrap().is_empty()); + drop(store); + std::fs::remove_file(path).unwrap(); + } use rusqlite::Connection; #[test] diff --git a/crates/emulebb-metadata/src/transfer_store.rs b/crates/emulebb-metadata/src/transfer_store.rs index 5d3e89651..419d50864 100644 --- a/crates/emulebb-metadata/src/transfer_store.rs +++ b/crates/emulebb-metadata/src/transfer_store.rs @@ -1212,6 +1212,45 @@ impl super::MetadataStore { Ok(changed != 0) } + /// Commit verified direct publication and release its sharing barrier together. + pub fn publish_reserved_delivery_attempt( + &self, + file_hash: &str, + attempt_id: Uuid, + fingerprint: &NativeFileFingerprint, + ) -> Result { + let hash = decode_fixed_hex(file_hash, 16, "ED2K hash")?; + let mut conn = self.connection()?; + let tx = conn.transaction()?; + let id = attempt_id.to_string(); + let reserved: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM delivery_exclusions WHERE reservation_id = ?1)", + [&id], + |row| row.get(0), + )?; + if !reserved { + return Ok(false); + } + let changed = tx.execute( + "UPDATE delivery_attempts SET phase = 'published', published_fingerprint = ?3, updated_at_ms = ?4 + WHERE attempt_id = ?1 AND phase = 'staged' AND transfer_id = ( + SELECT transfers.id FROM transfers JOIN known_files ON known_files.id = transfers.known_file_id + WHERE known_files.ed2k_hash = ?2 AND transfers.completed != 0 + AND transfers.final_rehash_pending = 0 AND transfers.delivery_state = 'pending' + AND transfers.removed_at_ms IS NULL AND transfers.delete_requested_at_ms IS NULL)", + params![id, hash, fingerprint.storage_bytes(), unix_ms()], + )?; + if changed == 0 { + return Ok(false); + } + tx.execute( + "DELETE FROM delivery_exclusions WHERE reservation_id = ?1", + [&id], + )?; + tx.commit()?; + Ok(true) + } + pub fn clear_transfer_delivery_attempt( &self, file_hash: &str, diff --git a/crates/emulebb-rest/Cargo.toml b/crates/emulebb-rest/Cargo.toml index 27e182839..0a92490b3 100644 --- a/crates/emulebb-rest/Cargo.toml +++ b/crates/emulebb-rest/Cargo.toml @@ -18,6 +18,7 @@ packet-diagnostics = ["emulebb-core/packet-diagnostics"] anyhow.workspace = true axum.workspace = true chrono.workspace = true +emulebb-diagnostics = { path = "../emulebb-diagnostics" } emulebb-core = { path = "../emulebb-core" } futures-util.workspace = true http-body-util.workspace = true @@ -29,6 +30,12 @@ tokio = { workspace = true, features = ["sync", "time"] } tower.workspace = true tower-http.workspace = true +[target.'cfg(unix)'.dependencies] +libc.workspace = true + +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_System_ProcessStatus", "Win32_System_Threading"] } + [dev-dependencies] rusqlite.workspace = true tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/emulebb-rest/src/diagnostic_bundle.rs b/crates/emulebb-rest/src/diagnostic_bundle.rs index 5c0fa2d0d..69b835276 100644 --- a/crates/emulebb-rest/src/diagnostic_bundle.rs +++ b/crates/emulebb-rest/src/diagnostic_bundle.rs @@ -1,8 +1,7 @@ use std::{ collections::{BTreeSet, HashSet}, fmt::Write as _, - fs::{self, File}, - io::{Read, Seek, SeekFrom}, + fs, path::PathBuf, time::SystemTime, }; @@ -771,42 +770,18 @@ fn read_artifact_tail( max_bytes: u64, ignore_incomplete_final_line: bool, ) -> std::io::Result { - let mut file = File::open(&candidate.path)?; - let file_size = file.metadata()?.len(); - let bytes_to_read = file_size.min(max_bytes); - let tail_start_offset = file_size.saturating_sub(bytes_to_read); - file.seek(SeekFrom::Start(tail_start_offset))?; - let mut bytes = Vec::with_capacity(bytes_to_read as usize); - file.take(bytes_to_read).read_to_end(&mut bytes)?; - let bytes_read = bytes.len(); - if tail_start_offset > 0 { - if let Some(newline) = bytes.iter().position(|byte| *byte == b'\n') { - bytes.drain(..=newline); - } else { - bytes.clear(); - } - } - let incomplete_final_line_ignored = - ignore_incomplete_final_line && bytes.last().is_some_and(|last| *last != b'\n'); - if incomplete_final_line_ignored { - if let Some(newline) = bytes.iter().rposition(|byte| *byte == b'\n') { - bytes.truncate(newline + 1); - } else { - bytes.clear(); - } - } - let lines = bytes - .split(|byte| *byte == b'\n') - .filter(|line| !line.is_empty()) - .map(<[u8]>::to_vec) - .collect(); + let tail = emulebb_diagnostics::log_tail::read_jsonl_tail( + &candidate.path, + max_bytes, + ignore_incomplete_final_line, + )?; Ok(TailRead { file_name: candidate.file_name.clone(), - file_size, - tail_start_offset, - bytes_read, - incomplete_final_line_ignored, - lines, + file_size: tail.file_size, + tail_start_offset: tail.tail_start_offset, + bytes_read: tail.bytes_read, + incomplete_final_line_ignored: tail.incomplete_final_line_ignored, + lines: tail.lines, }) } diff --git a/crates/emulebb-rest/src/lib.rs b/crates/emulebb-rest/src/lib.rs index 5bb71af3c..d76066ce9 100644 --- a/crates/emulebb-rest/src/lib.rs +++ b/crates/emulebb-rest/src/lib.rs @@ -13,6 +13,7 @@ mod diagnostic_bundle; mod dto; mod envelope; mod handlers; +mod process_resources; mod request_body_policy; mod responses; mod route_body_metadata; @@ -47,6 +48,7 @@ pub struct RestState { core: Arc, api_key: Arc, shutdown: Option>, + process_resources: Arc, } #[cfg(all(test, feature = "packet-diagnostics"))] #[path = "tests/protocol_fixtures.rs"] diff --git a/crates/emulebb-rest/src/process_resources.rs b/crates/emulebb-rest/src/process_resources.rs new file mode 100644 index 000000000..64d5e904c --- /dev/null +++ b/crates/emulebb-rest/src/process_resources.rs @@ -0,0 +1,312 @@ +//! Bounded, on-demand metrics for this daemon, including native Android builds. + +use std::time::{Duration, Instant}; + +use serde::Serialize; +use tokio::sync::Mutex; + +const MIN_SAMPLE_INTERVAL: Duration = Duration::from_secs(1); + +#[derive(Debug, Default)] +pub(crate) struct ProcessResourceMonitor { + sampler: Mutex, +} + +#[derive(Clone, Copy, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProcessResources { + sampled_at_ms: Option, + resident_memory_bytes: Option, + /// Total process CPU use: 100% means one occupied core, not the whole host. + cpu_usage_percent: Option, + cpu_sample_interval_ms: Option, +} + +impl ProcessResourceMonitor { + /// Share a sample across REST consumers for at least one second. OS reads + /// stay off the async executor; missing counters remain null, never zero. + pub(crate) async fn snapshot(&self) -> ProcessResources { + let mut sampler = self.sampler.lock().await; + if sampler + .previous + .as_ref() + .is_some_and(|sample| sample.at.elapsed() < MIN_SAMPLE_INTERVAL) + { + return sampler.current; + } + match tokio::task::spawn_blocking(RawSample::read).await { + Ok(sample) => sampler.record(sample), + Err(_) => { + *sampler = Sampler::default(); + ProcessResources::default() + } + } + } +} + +#[derive(Debug, Default)] +struct Sampler { + previous: Option, + current: ProcessResources, +} + +#[derive(Debug)] +struct RawSample { + at: Instant, + sampled_at_ms: i64, + cpu_time: Option, + resident_memory_bytes: Option, +} + +impl RawSample { + fn read() -> Self { + let cpu_time = cpu_time(); + let at = Instant::now(); + Self { + at, + sampled_at_ms: chrono::Utc::now().timestamp_millis(), + cpu_time, + resident_memory_bytes: resident_memory_bytes(), + } + } +} + +impl Sampler { + fn record(&mut self, sample: RawSample) -> ProcessResources { + let cpu_sample = self.previous.as_ref().and_then(|previous| { + let elapsed = sample.at.checked_duration_since(previous.at)?; + let cpu = sample.cpu_time?.checked_sub(previous.cpu_time?)?; + (elapsed >= MIN_SAMPLE_INTERVAL).then(|| { + ( + cpu.as_secs_f64() / elapsed.as_secs_f64() * 100.0, + elapsed.as_millis().min(u64::MAX as u128) as u64, + ) + }) + }); + self.current = ProcessResources { + sampled_at_ms: Some(sample.sampled_at_ms), + resident_memory_bytes: sample.resident_memory_bytes, + cpu_usage_percent: cpu_sample.map(|(percent, _)| percent), + cpu_sample_interval_ms: cpu_sample.map(|(_, interval)| interval), + }; + self.previous = Some(sample); + self.current + } +} + +#[cfg(unix)] +fn cpu_time() -> Option { + let mut usage = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrusage initializes the writable output on success; RUSAGE_SELF + // includes every thread in this process and excludes child processes. + if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } != 0 { + return None; + } + // SAFETY: the successful call initialized `usage`. + let usage = unsafe { usage.assume_init() }; + let duration = |time: libc::timeval| { + Some(Duration::new( + time.tv_sec.try_into().ok()?, + u32::try_from(time.tv_usec).ok()?.checked_mul(1_000)?, + )) + }; + duration(usage.ru_utime)?.checked_add(duration(usage.ru_stime)?) +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn resident_memory_bytes() -> Option { + use std::io::Read; + + // WHY: Android restricts global /proc counters. The current process's statm + // remains readable without JVM/root access; its RSS is in runtime pages, + // which must not be assumed to be 4 KiB on Android's 16 KiB builds. + let mut statm = String::new(); + std::fs::File::open("/proc/self/statm") + .ok()? + .take(256) + .read_to_string(&mut statm) + .ok()?; + // SAFETY: sysconf has no pointer arguments; nonpositive results are rejected. + let page_size = u64::try_from(unsafe { libc::sysconf(libc::_SC_PAGESIZE) }).ok()?; + resident_bytes_from_statm(&statm, page_size) +} + +#[cfg(any(target_os = "linux", target_os = "android", test))] +fn resident_bytes_from_statm(statm: &str, page_size: u64) -> Option { + if page_size == 0 { + return None; + } + statm + .split_whitespace() + .nth(1)? + .parse::() + .ok()? + .checked_mul(page_size) +} + +#[cfg(target_os = "macos")] +fn resident_memory_bytes() -> Option { + let mut info = std::mem::MaybeUninit::::uninit(); + let size = std::mem::size_of::().try_into().ok()?; + // SAFETY: the flavor and writable buffer size describe proc_taskinfo for + // this process. The output is read only after a complete structure returns. + let result = unsafe { + libc::proc_pidinfo( + libc::getpid(), + libc::PROC_PIDTASKINFO, + 0, + info.as_mut_ptr().cast(), + size, + ) + }; + if result != size { + return None; + } + // SAFETY: the successful call initialized the complete requested structure. + Some(unsafe { info.assume_init() }.pti_resident_size) +} + +#[cfg(windows)] +fn cpu_time() -> Option { + use windows_sys::Win32::{ + Foundation::FILETIME, + System::Threading::{GetCurrentProcess, GetProcessTimes}, + }; + + let mut created = FILETIME::default(); + let mut exited = FILETIME::default(); + let mut kernel = FILETIME::default(); + let mut user = FILETIME::default(); + // SAFETY: GetCurrentProcess returns a borrowed pseudo-handle and all four + // output pointers address writable FILETIME values for this call. + if unsafe { + GetProcessTimes( + GetCurrentProcess(), + &mut created, + &mut exited, + &mut kernel, + &mut user, + ) + } == 0 + { + return None; + } + let ticks = + |time: FILETIME| (u64::from(time.dwHighDateTime) << 32) | u64::from(time.dwLowDateTime); + let total = ticks(kernel).checked_add(ticks(user))?; + Some(Duration::new( + total / 10_000_000, + ((total % 10_000_000) * 100) as u32, + )) +} + +#[cfg(windows)] +fn resident_memory_bytes() -> Option { + use windows_sys::Win32::System::{ + ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}, + Threading::GetCurrentProcess, + }; + + let size = std::mem::size_of::() as u32; + let mut counters = PROCESS_MEMORY_COUNTERS { + cb: size, + ..Default::default() + }; + // SAFETY: the current-process pseudo-handle is borrowed, and counters is a + // writable structure whose size matches the buffer length supplied. + if unsafe { K32GetProcessMemoryInfo(GetCurrentProcess(), &mut counters, size) } == 0 { + return None; + } + Some(counters.WorkingSetSize as u64) +} + +#[cfg(not(any(unix, windows)))] +fn cpu_time() -> Option { + None +} + +#[cfg(not(any( + windows, + target_os = "linux", + target_os = "android", + target_os = "macos" +)))] +fn resident_memory_bytes() -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cpu_uses_interval_deltas_and_can_exceed_one_core() { + let start = Instant::now(); + let mut sampler = Sampler::default(); + let sample = |seconds, cpu: Option| RawSample { + at: start + Duration::from_secs(seconds), + sampled_at_ms: 1_000 + seconds as i64 * 1_000, + cpu_time: cpu.map(Duration::from_millis), + resident_memory_bytes: Some(42), + }; + let first = sampler.record(sample(0, Some(500))); + assert_eq!(first.resident_memory_bytes, Some(42)); + assert_eq!(first.cpu_usage_percent, None); + assert_eq!(first.cpu_sample_interval_ms, None); + let busy = sampler.record(sample(2, Some(3_000))); + assert_eq!(busy.cpu_usage_percent, Some(125.0)); + assert_eq!(busy.cpu_sample_interval_ms, Some(2_000)); + assert_eq!( + sampler.record(sample(4, Some(3_000))).cpu_usage_percent, + Some(0.0) + ); + assert_eq!(sampler.record(sample(6, None)).cpu_usage_percent, None); + assert_eq!( + sampler.record(sample(8, Some(4_000))).cpu_usage_percent, + None + ); + assert_eq!(sampler.record(sample(10, Some(1))).cpu_usage_percent, None); + } + + #[test] + fn resident_memory_uses_runtime_page_size_and_rejects_bad_counters() { + assert_eq!( + resident_bytes_from_statm("200 100 50 0 0 10 0\n", 16_384), + Some(1_638_400) + ); + assert_eq!(resident_bytes_from_statm("200 100", 4_096), Some(409_600)); + for invalid in [ + "", + "200", + "200 invalid", + "200 -1", + "200 18446744073709551615", + ] { + assert_eq!(resident_bytes_from_statm(invalid, 16_384), None); + } + assert_eq!(resident_bytes_from_statm("200 100", 0), None); + } + + #[tokio::test] + async fn native_metrics_are_live_and_repeated_requests_reuse_the_sample() { + let monitor = ProcessResourceMonitor::default(); + let first = monitor.snapshot().await; + assert!(first.sampled_at_ms.is_some()); + assert!(first.resident_memory_bytes.is_some_and(|bytes| bytes > 0)); + assert!( + monitor + .sampler + .lock() + .await + .previous + .as_ref() + .unwrap() + .cpu_time + .is_some() + ); + let second = monitor.snapshot().await; + assert_eq!(first.sampled_at_ms, second.sampled_at_ms); + assert_eq!(first.resident_memory_bytes, second.resident_memory_bytes); + assert_eq!(second.cpu_usage_percent, None); + } +} diff --git a/crates/emulebb-rest/src/responses.rs b/crates/emulebb-rest/src/responses.rs index 4dfc7f588..ea686bb65 100644 --- a/crates/emulebb-rest/src/responses.rs +++ b/crates/emulebb-rest/src/responses.rs @@ -121,7 +121,8 @@ pub(crate) async fn status_response(state: &RestState) -> Value { shared_hashing_count, shared_directory_reload_progress.clone(), &upload_policy, - ); + ) + .await; json!({ "lifecycle": lifecycle_response(&status.lifecycle), "stats": stats_response(&status, &upload_policy, &throughput, shared_hashing_count), @@ -161,9 +162,10 @@ pub(crate) async fn runtime_diagnostics_response(state: &RestState) -> Value { shared_directories.reload_progress, &upload_policy, ) + .await } -fn runtime_diagnostics_response_with( +async fn runtime_diagnostics_response_with( state: &RestState, shared_file_count: usize, download_file_count: usize, @@ -171,8 +173,10 @@ fn runtime_diagnostics_response_with( shared_directory_reload_progress: impl serde::Serialize, upload_policy: &UploadPolicyMetrics, ) -> Value { + let process_resources = state.process_resources.snapshot().await; json!({ "processId": std::process::id(), + "processResources": process_resources, "knownFileCount": shared_file_count, "sharedFileCount": shared_file_count, "sharedHashingCount": shared_hashing_count, diff --git a/crates/emulebb-rest/src/routes.rs b/crates/emulebb-rest/src/routes.rs index ebd7fba29..351737b3f 100644 --- a/crates/emulebb-rest/src/routes.rs +++ b/crates/emulebb-rest/src/routes.rs @@ -46,6 +46,7 @@ pub fn router_with_shutdown( core, api_key: Arc::new(config.api_key), shutdown, + process_resources: Arc::default(), }; let health_router = Router::new() .route("/healthz", get(health)) diff --git a/crates/emulebb-rest/src/tests/app.rs b/crates/emulebb-rest/src/tests/app.rs index fca044268..f8911012d 100644 --- a/crates/emulebb-rest/src/tests/app.rs +++ b/crates/emulebb-rest/src/tests/app.rs @@ -783,6 +783,42 @@ async fn diagnostics_returns_runtime_diagnostics_directly() { assert_eq!(value["data"]["geolocation"], Value::Null); } +#[tokio::test] +async fn process_resources_are_exposed_on_every_runtime_diagnostics_surface() { + let router = test_router(); + for (path, pointer) in [ + ("/api/v1/diagnostics", "/data/processResources"), + ( + "/api/v1/status", + "/data/runtimeDiagnostics/processResources", + ), + ( + "/api/v1/snapshot", + "/data/status/runtimeDiagnostics/processResources", + ), + ] { + let response = router + .clone() + .oneshot( + Request::builder() + .uri(path) + .header("X-API-Key", "secret") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let value: Value = serde_json::from_slice(&body).unwrap(); + let resources = value.pointer(pointer).unwrap(); + assert!(resources["sampledAtMs"].as_i64().unwrap() > 0); + assert!(resources["residentMemoryBytes"].as_u64().unwrap() > 0); + assert!(resources.get("cpuUsagePercent").is_some()); + assert!(resources.get("cpuSampleIntervalMs").is_some()); + } +} + #[tokio::test] async fn diagnostics_reports_transfer_event_subscribers() { let core = diff --git a/crates/emulebb-rest/src/tests/support.rs b/crates/emulebb-rest/src/tests/support.rs index d74a832e1..427cd215e 100644 --- a/crates/emulebb-rest/src/tests/support.rs +++ b/crates/emulebb-rest/src/tests/support.rs @@ -66,6 +66,7 @@ pub(crate) fn test_search_core(files: &[IndexedFile]) -> Arc { fn test_network_config(runtime_dir: &std::path::Path) -> Ed2kNetworkConfig { let bind_ip = Ipv4Addr::new(198, 51, 100, 10); Ed2kNetworkConfig { + egress_route: emulebb_ed2k::networking::EgressRoute::StrictInterface, bind_ip, kad_bind_addr: SocketAddr::new(IpAddr::V4(bind_ip), 4665), listen_port: 4662, diff --git a/emulebb-rust-settings.example.toml b/emulebb-rust-settings.example.toml index b5dc841da..0aeb22200 100644 --- a/emulebb-rust-settings.example.toml +++ b/emulebb-rust-settings.example.toml @@ -1,8 +1,7 @@ [rest] -bind_addr = "127.0.0.1:20017" +bind_addr = "0.0.0.0:20017" api_key = "change_me_123change_me_123change_me_123" -# Ready for local use. Replace this shared starter key before exposing the API -# outside this computer; use at least 32 printable ASCII characters, no spaces. +# API keys require at least 32 printable ASCII characters, without spaces. # Keep this file private to the account running the daemon (mode 0600 on Unix). # Optional. When unset, the daemon serves `webui` beside its executable if # that directory exists. Relative paths are resolved from the profile directory. diff --git a/packaging/docker/compose.gluetun.example.yaml b/packaging/docker/compose.gluetun.example.yaml index e15c31975..34f6e8a65 100644 --- a/packaging/docker/compose.gluetun.example.yaml +++ b/packaging/docker/compose.gluetun.example.yaml @@ -15,7 +15,7 @@ services: - ${EMULEBB_TEST_VPN_PRIVATE_ROOT:?provide an existing read-only VPN private path}/StaticKey.pem:/gluetun/StaticKey.pem:ro secrets: [openvpn_user, openvpn_password] ports: - - "127.0.0.1:20017:20017/tcp" + - "0.0.0.0:20017:20017/tcp" restart: unless-stopped emulebb-rust: image: ghcr.io/emulebb/emulebb-rust:0.1.0-beta.2 diff --git a/resources/branding.json b/resources/branding.json new file mode 100644 index 000000000..dc935b222 --- /dev/null +++ b/resources/branding.json @@ -0,0 +1,82 @@ +{ + "schema": "emulebb.branding/1", + "selection": "proposal-5", + "source_manifest_url": "https://raw.githubusercontent.com/emulebb/emulebb-tooling/main/manifests/branding/proposal-5-source.json", + "source_manifest_sha256": "2473a87ae2c76ffcf1110d73477c9e3739060d50412c2199624a617c3fcd947d", + "approval": "Operator-approved proposal 5: exact small icons and unchanged proposal-1 large artwork.", + "android_monochrome": "Android uses the approved adjoining blocks as a solid alpha-only square; the launcher uses unchanged large donkey art within the adaptive safe region.", + "assets": [ + { + "path": "resources/icons/donkey-16.png", + "source_file": "icon-16.png", + "bytes": 216, + "sha256": "851b8eb75f0081eb0cb69d95d5ec90e43d9a0747e8fd1fe799853639b9ad0e1a" + }, + { + "path": "resources/icons/donkey-24.png", + "source_file": "icon-24.png", + "bytes": 264, + "sha256": "5304b784fcd2242278efcf10f3715b892c6a502a26569f1bbe4f07bb37146f64" + }, + { + "path": "resources/icons/donkey-32.png", + "source_file": "icon-32.png", + "bytes": 308, + "sha256": "b3ce51a35147830a93d3b200666b7fd844e0f117e172273edf3049904e4d1f5e" + }, + { + "path": "resources/icons/donkey-48.png", + "source_file": "icon-48.png", + "bytes": 414, + "sha256": "3f89fab34d0c7e6aced8c06f7cc91f7f605295ec823a1a7f2a7baf336d8121af" + }, + { + "path": "resources/icons/donkey-64.png", + "source_file": "donkey-64.png", + "bytes": 7944, + "sha256": "a95e1c89f0a0b2866cca714b652e5dae4ffabb4614144d352b8cb79fb8a9f5f8" + }, + { + "path": "resources/icons/donkey-128.png", + "source_file": "donkey-128.png", + "bytes": 26053, + "sha256": "7d6df0fbf48993c1c677f4c75e488079c017f608373b08b9e750e91311663eab" + }, + { + "path": "resources/icons/donkey-256.png", + "source_file": "donkey-256.png", + "bytes": 84445, + "sha256": "7f26b62bac49daa0e52c362cfa2cd777c4a6990d3bcdaf3a63f4d88153967b2b" + }, + { + "path": "resources/icons/donkey-512.png", + "source_file": "donkey-512.png", + "bytes": 282308, + "sha256": "e79dc7628376307cdb646d09b6966e75ef3716e720ba3cdee3814c2e3a67ae25" + }, + { + "path": "webui/public/favicon.ico", + "source_file": "favicon.ico", + "bytes": 992, + "sha256": "58144a359a6bc5def936047e42605d058be7c7f1b21dd2f27ac9643bb723c79a" + }, + { + "path": "webui/public/apple-touch-icon.png", + "source_file": "donkey-180.png", + "bytes": 46087, + "sha256": "4b269a328482ada12976aa90ca86a7da4b5559c2d225b9e579143f15b46df1ac" + }, + { + "path": "crates/emulebb-daemon/resources/windows/application.ico", + "source_file": "application.ico", + "bytes": 119762, + "sha256": "7603b7ef9bfa7f75ebdc0afc83aef317385e04a3559e5bd368c8025e5dd8360a" + }, + { + "path": "android/app/src/main/res/drawable-nodpi/donkey_launcher.png", + "source_file": "donkey-512.png", + "bytes": 282308, + "sha256": "e79dc7628376307cdb646d09b6966e75ef3716e720ba3cdee3814c2e3a67ae25" + } + ] +} diff --git a/resources/icons/donkey-16.png b/resources/icons/donkey-16.png index 54a7f9343..876fcf184 100644 Binary files a/resources/icons/donkey-16.png and b/resources/icons/donkey-16.png differ diff --git a/resources/icons/donkey-24.png b/resources/icons/donkey-24.png index b7a1307b6..4499ded27 100644 Binary files a/resources/icons/donkey-24.png and b/resources/icons/donkey-24.png differ diff --git a/resources/icons/donkey-32.png b/resources/icons/donkey-32.png index f5cf62e78..74f7d7172 100644 Binary files a/resources/icons/donkey-32.png and b/resources/icons/donkey-32.png differ diff --git a/resources/icons/donkey-48.png b/resources/icons/donkey-48.png index 3245d2de2..018f1fd65 100644 Binary files a/resources/icons/donkey-48.png and b/resources/icons/donkey-48.png differ diff --git a/tools/check_rust_client_policy.py b/tools/check_rust_client_policy.py index 615170b04..31513f9cf 100644 --- a/tools/check_rust_client_policy.py +++ b/tools/check_rust_client_policy.py @@ -1521,20 +1521,69 @@ def check_release_android_packages(workflow_text: str | None = None) -> list[str errors.append("Android developer packaging must not waive failures") if any("continue-on-error" in step for step in steps): errors.append("Android developer packaging steps must not waive failures") - expected_needs = ["prepare", "native-package-smoke", "musl-package", "image-native-validation", "android-package"] + expected_needs = ["prepare", "native-package-smoke", "musl-package", "image-native-validation", "android-package", "android-apk-package", "android-apk-qualification"] if assemble.get("needs") != expected_needs: errors.append("Candidate assembly must retain every native/image/Android prerequisite") expected_condition = """${{ !cancelled() && needs.prepare.result == 'success' && needs.native-package-smoke.result == 'success' && needs.musl-package.result == 'success' && needs.image-native-validation.result == 'success' && (needs.android-package.result == 'success' || - (inputs.channel != 'nightly' && needs.android-package.result == 'skipped')) }}""" + (inputs.channel != 'nightly' && needs.android-package.result == 'skipped')) && + (needs.android-apk-package.result == 'success' || + (inputs.channel != 'nightly' && needs.android-apk-package.result == 'skipped')) && + (needs.android-apk-qualification.result == 'success' || + (inputs.channel != 'nightly' && needs.android-apk-qualification.result == 'skipped')) }}""" if " ".join(assemble.get("if", "").split()) != " ".join(expected_condition.split()): errors.append("Candidate assembly must require Android success at nightly and allow skipped Android only outside nightly") downloads = [step for step in assemble.get("steps", []) if step.get("with", {}).get("pattern") == "emulebb-rust-android-developer-payload-*"] if len(downloads) != 1 or downloads[0].get("if") != "${{ inputs.channel == 'nightly' }}": errors.append("Candidate assembly must download Android developer bundles exactly for nightly") + authorities = yaml.safe_load(text).get("env", {}) + for key in ("EMULEBB_RELEASE_VALIDATOR_REF", "EMULEBB_APK_BUILD_REF", "EMULEBB_APK_BUILD_TESTS_REF"): + if re.fullmatch(r"[0-9a-f]{40}", str(authorities.get(key, ""))) is None: + errors.append(f"APK release requires explicit immutable {key}") + if re.fullmatch(r"[0-9a-f]{64}", str(authorities.get("EMULEBB_ANDROID_SIGNER_SHA256", ""))) is None: + errors.append("APK release requires explicit trusted signer identity") + package = jobs.get("android-apk-package", {}) + qualify = jobs.get("android-apk-qualification", {}) + if package.get("needs") != ["prepare", "verify-source"]: + errors.append("Signed APK packaging must require exact-source qualification") + if qualify.get("needs") != ["prepare", "android-apk-package", "android-package"]: + errors.append("APK runtime qualification must require the same signed APK and source-matched CLI") + if qualify.get("strategy", {}).get("matrix") != {"image": ["apk31", "apk32", "apk33", "apk34", "apk35", "apk36", "apk37"]}: + errors.append("APK runtime qualification must cover exactly Android APIs 31 through 37") + for job in (package, qualify): + if job.get("if") != "${{ inputs.channel == 'nightly' }}" or "continue-on-error" in job: + errors.append("APK jobs must be nightly-required without failure waivers") + if any("continue-on-error" in step for step in job.get("steps", [])): + errors.append("APK steps must not waive failures") + for job, command, fragments in ( + (package, "package-rust-android-apk-ci", ('--expected-source-sha "$EXPECTED_SOURCE_SHA"', + '--expected-cert-sha256 "$EMULEBB_ANDROID_SIGNER_SHA256"', + '--version-code "$EMULEBB_ANDROID_VERSION_CODE"')), + (qualify, "qualify-rust-android-apk-ci", ('--apk-source-sha "$EXPECTED_SOURCE_SHA"', + '--apk-manifest-sha256 "$APK_MANIFEST_SHA256"', + '--expected-cert-sha256 "$EMULEBB_ANDROID_SIGNER_SHA256"', + '--apk-version-code "$EMULEBB_ANDROID_VERSION_CODE"', + '--apk-build-ref "$EMULEBB_APK_BUILD_REF"', '--apk-harness-ref "$EMULEBB_APK_BUILD_TESTS_REF"')), + ): + commands = [step for step in job.get("steps", []) if command in step.get("run", "")] + if len(commands) != 1 or any(fragment not in commands[0]["run"] for fragment in fragments): + errors.append(f"{command} must bind all trusted APK authorities") + elif commands[0].get("env", {}).get("EXPECTED_SOURCE_SHA") != "${{ needs.prepare.outputs.source_sha }}": + errors.append(f"{command} must use the qualified source") + if qualify.get("env", {}).get("EMULEBB_ANDROID_VERSION_CODE") != "${{ needs.prepare.outputs.apk_version_code }}": + errors.append("APK runtime qualification must use the single prepared version code") + for name in ("assemble-candidate", "publish-image", "publish-native"): + steps = jobs.get(name, {}).get("steps", []) + readers = [step for step in steps if step.get("with", {}).get("path") == "emulebb-build"] + if len(readers) != 1 or readers[0]["with"].get("ref") != "${{ env.EMULEBB_RELEASE_VALIDATOR_REF }}": + errors.append(f"{name} must use independently pinned release reader authority") + if not any(step.get("with", {}).get("path") == "emulebb-apk-tests" and + step["with"].get("ref") == "${{ env.EMULEBB_APK_BUILD_TESTS_REF }}" for step in steps): + errors.append(f"{name} must separately check out the APK evidence reader") + return errors @@ -1560,7 +1609,7 @@ def check_release_output_paths(workflow_text: str | None = None) -> list[str]: "${{ runner.temp }}/emulebb-rust-out/release/": "external release assets", "assemble-emulebb-rust-release-ci": "verified release assembly", "native-package-smoke:": "isolated package lifecycle job", - "needs: [prepare, native-package-smoke, musl-package, image-native-validation, android-package]": "candidate dependency on all native lifecycle and nightly Android evidence", + "needs: [prepare, native-package-smoke, musl-package, image-native-validation, android-package, android-apk-package, android-apk-qualification]": "candidate dependency on all native lifecycle and nightly Android evidence", "assemble-candidate:": "non-publishing candidate assembly job", "ref: ${{ github.workflow_sha }}": "trusted release-control checkout", "candidate-manifest": "candidate revision and evidence manifest", diff --git a/tools/ci-toolchain.json b/tools/ci-toolchain.json index 13f487eda..01ca44695 100644 --- a/tools/ci-toolchain.json +++ b/tools/ci-toolchain.json @@ -31,10 +31,10 @@ "inputs": { "emulebb-miniupnp": "4e7a10948004ca2085ac8103e14fc77edb898b4c", "emulebb-libpcpnatpmp": "a6e787f837e507336a8774522c33b54264ef4a4b", - "emulebb-build-tests": "7c680ed68e478c2f22c56c65f21f12c3790d3826", - "emulebb-tooling": "26abd1eab97b28977f2d55adba17e6ba28821498" + "emulebb-build-tests": "281ccbcdf9f4c9960c5efda4705df966418b6bb2", + "emulebb-tooling": "f88a8ce608bc75a619aea6909f54a9cdecdddb6e" }, "node": "26.13.0", "npm": "11.20.0", - "openapi_sha256": "4c133d3efa69d433c2dc76e77a0e64dde2c9a56838567f674ae950a029ba7c9a" + "openapi_sha256": "ad1e77ce0675e818b2d68ce1d9fc10ebb9afe9b37df3763e090f4ea7c47ed13e" } diff --git a/tools/nightly_release.py b/tools/nightly_release.py index 59b3216a7..7d7dfebd9 100644 --- a/tools/nightly_release.py +++ b/tools/nightly_release.py @@ -402,7 +402,7 @@ def render_notes( if channel == "nightly": if not parsed: raise ValueError(f"invalid nightly version: {version}") - introduction = "Automated nightly beta from `main`. It is unsigned, may be unstable, and is not a formal beta release." + introduction = "Automated nightly beta from the verified source commit. Native archives are unsigned; the Android APK is signed. This is not a formal beta release." baseline_label = "Previous successful nightly" build_label = "nightly beta" elif channel in {"beta", "candidate"} and PROMOTED_BETA_RE.fullmatch(version): diff --git a/tools/package_release_zip.py b/tools/package_release_zip.py index 38a57339f..15f8a0192 100644 --- a/tools/package_release_zip.py +++ b/tools/package_release_zip.py @@ -203,7 +203,7 @@ def package_readme(version: str) -> str: "Run `emulebb-rust.exe`: its executable directory is the default profile, regardless of the shell working directory.\n" "Use `--profile ` to override it; relative paths use the shell working directory and missing profiles are created.\n" "Open http://127.0.0.1:20017/ and enter change_me_123change_me_123change_me_123.\n" - "Replace the shared starter key before exposing the API outside this computer.\n" + "The REST/WebUI listener defaults to 0.0.0.0:20017.\n" "The daemon serves the embedded browser WebUI from the packaged `webui` directory beside the executable.\n\n" ) diff --git a/tools/release_publish.py b/tools/release_publish.py index 14f90d2ef..dd055343c 100644 --- a/tools/release_publish.py +++ b/tools/release_publish.py @@ -18,6 +18,7 @@ import tempfile import tomllib from contextlib import contextmanager +from datetime import UTC, datetime from pathlib import Path from typing import Sequence from urllib.parse import quote @@ -50,6 +51,14 @@ MAX_INDEX_BYTES = 1024 * 1024 +def android_version_code(now: datetime | None = None) -> int: + """One UTC authority across manual and scheduled APK entrypoints.""" + code = int(((now or datetime.now(UTC)) - datetime(2020, 1, 1, tzinfo=UTC)).total_seconds()) + if not 1 <= code <= 2_100_000_000: + raise RuntimeError("Android version code is outside its supported UTC range") + return code + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -273,7 +282,7 @@ def candidate_owner(args: argparse.Namespace): ): raise RuntimeError("Candidate control requires the actual GitHub run and workflow identities") checked_checkout(ROOT, args.workflow_sha, label="Release control") - build_root = checked_checkout(args.build_repo, args.emulebb_build_ref, label="Packaging owner") + build_root = checked_checkout(args.build_repo, args.release_validator_build_ref, label="Release reader") for name, module in tuple(sys.modules.items()): if name == "emule_workspace" or name.startswith("emule_workspace."): filename = getattr(module, "__file__", None) @@ -292,6 +301,14 @@ def candidate_owner(args: argparse.Namespace): def candidate_context(args: argparse.Namespace, owner): + apk = {} + if args.channel == "nightly": + if not all((args.apk_build_ref, args.apk_harness_ref, args.apk_signer_sha256, args.apk_harness_repo, args.apk_version_code)): + raise RuntimeError("Nightly APK validation requires explicit source, signer, version code and Harness authorities") + harness = checked_checkout(args.apk_harness_repo, args.apk_harness_ref, label="APK Harness") + apk = {"apk_sources": owner.ApkSourceContext(args.apk_build_ref, args.apk_harness_ref), + "apk_signer_sha256": args.apk_signer_sha256, "apk_harness_root": harness, + "apk_version_code": args.apk_version_code} return owner.CandidateContext( version=args.release_version, channel=args.channel, source={"emulebbRust": args.source_sha, "emulebbBuild": args.emulebb_build_ref, @@ -300,6 +317,8 @@ def candidate_context(args: argparse.Namespace, owner): workflow={"repository": args.repository, "runId": args.run_id, "runAttempt": args.run_attempt, "workflowSha": args.workflow_sha}, platform_profile=args.platform_profile, + release_validator_build_sha=args.release_validator_build_ref, + **apk, ).checked() @@ -342,6 +361,10 @@ def link(name: str, label: str = "") -> str: (item["abi"], link(item["asset"])) for item in android ] payloads.update(item["asset"] for item in android) + apk = data.get("experimentalAndroidApk") + if isinstance(apk, dict) and apk.get("asset") in names: + groups["Android application"] = [("Android 12+ · ARM64 and x86_64", link(apk["asset"], "Signed universal APK"))] + payloads.add(apk["asset"]) lines = ["", "## Downloads", ""] for heading, rows in groups.items(): @@ -349,9 +372,8 @@ def link(name: str, label: str = "") -> str: lines.extend((f"### {heading}", "")) if heading == "Experimental Android native packages": lines.extend(( - "Experimental native CLI developer archives. APK packaging and signing remain deferred.", - "Evidence covers compilation and ELF checks. Device runtime, transfers and VPN behavior remain unqualified.", - "Completed-file delivery and route discovery have known Android limitations. Read the bundled `emulebb-rust/README.txt` before use.", + "Experimental native CLI developer archives. Their evidence covers compilation and ELF checks; APK runtime qualification is recorded separately.", + "Read the bundled `emulebb-rust/README.txt` before use.", "", )) lines.extend(("| Platform | Download |", "|---|---|")) @@ -633,6 +655,8 @@ def verify_release_assets(args: argparse.Namespace, data: dict, candidate: Path) def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) + version_code = subparsers.add_parser("android-version-code", help="derive one cross-workflow monotonic APK version code") + version_code.add_argument("--github-output", required=True, type=Path) digest = subparsers.add_parser( "oci-digest", help="validate an OCI archive and write its manifest digest", @@ -671,6 +695,7 @@ def build_parser() -> argparse.ArgumentParser: expected_arguments = { "release-version": "RELEASE_VERSION", "channel": "RELEASE_CHANNEL", "source-sha": "EMULEBB_SOURCE_REVISION", "emulebb-build-ref": "EMULEBB_BUILD_REF", + "release-validator-build-ref": "EMULEBB_RELEASE_VALIDATOR_REF", "emulebb-build-tests-ref": "EMULEBB_BUILD_TESTS_REF", "emulebb-tooling-ref": "EMULEBB_TOOLING_REF", "emulebb-miniupnp-ref": "MINIUPNP_REF", "emulebb-libpcpnatpmp-ref": "PCPNATPMP_REF", "repository": "GITHUB_REPOSITORY", "run-id": "GITHUB_RUN_ID", @@ -681,6 +706,11 @@ def build_parser() -> argparse.ArgumentParser: candidate.add_argument("--" + flag, default=default, required=not default, help=f"Expected identity; defaults to the caller's {environment}.") candidate.add_argument("--build-repo", required=True, type=Path) + for flag, environment in {"apk-build-ref": "EMULEBB_APK_BUILD_REF", "apk-harness-ref": "EMULEBB_APK_BUILD_TESTS_REF", + "apk-signer-sha256": "EMULEBB_ANDROID_SIGNER_SHA256"}.items(): + candidate.add_argument("--" + flag, default=os.environ.get(environment)) + candidate.add_argument("--apk-version-code", type=int, default=os.environ.get("EMULEBB_ANDROID_VERSION_CODE")) + candidate.add_argument("--apk-harness-repo", type=Path) candidate.add_argument("--github-output", type=Path) if command in {"candidate-manifest", "verify-candidate", "publish-images"}: candidate.add_argument("--archives-dir", required=True, type=Path) @@ -707,6 +737,11 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: Sequence[str] | None = None) -> int: args = build_parser().parse_args(argv) + if args.command == "android-version-code": + code = android_version_code() + with args.github_output.open("a", encoding="utf-8", newline="\n") as stream: + stream.write(f"apk_version_code={code}\n") + return 0 if args.command == "platform-profile": profile = resolve_platform_profile(args.profile, event_name=args.event_name, channel=args.channel, publish=args.publish) diff --git a/tools/test_webui.py b/tools/test_webui.py index b9ee8359b..2047aba7d 100644 --- a/tools/test_webui.py +++ b/tools/test_webui.py @@ -32,7 +32,10 @@ def main(argv: list[str] | None = None) -> int: run([npm, "run", "typecheck"], workspace) return 0 run([npm, "run", "test:unit"], workspace) - run([npm, "exec", "--", "playwright", "install", "chromium"], workspace) + # WHY: Linux WebKit needs native libraries absent from the hosted base image; + # Windows browser installation does not need the media-feature-pack path. + dependencies = ["--with-deps"] if sys.platform == "linux" else [] + run([npm, "exec", "--", "playwright", "install", *dependencies, "chromium", "webkit"], workspace) run([npm, "run", "test:e2e"], workspace) run( [npm, "run", "build", "--", "--outDir", str(workspace.dist), "--emptyOutDir"], diff --git a/tools/tests/fixtures/android_launcher_alpha.json b/tools/tests/fixtures/android_launcher_alpha.json new file mode 100644 index 000000000..0e09ae27b --- /dev/null +++ b/tools/tests/fixtures/android_launcher_alpha.json @@ -0,0 +1,45 @@ +{ + "source_path": "android/app/src/main/res/drawable-nodpi/donkey_launcher.png", + "sha256": "e79dc7628376307cdb646d09b6966e75ef3716e720ba3cdee3814c2e3a67ae25", + "width": 512, + "height": 512, + "method": "Convex hull of corners of every pixel with alpha greater than zero, decoded with Pillow. Each row contributes its leftmost and rightmost occupied pixel corners; this includes all interior pixel corners in the hull.", + "opaque_pixel_corner_hull": [ + [ + 17, + 47 + ], + [ + 23, + 46 + ], + [ + 468, + 27 + ], + [ + 504, + 27 + ], + [ + 504, + 492 + ], + [ + 67, + 492 + ], + [ + 43, + 470 + ], + [ + 18, + 90 + ], + [ + 17, + 70 + ] + ] +} diff --git a/tools/tests/test_android_release_workflow.py b/tools/tests/test_android_release_workflow.py index 81d9f2ff0..5d41e3191 100644 --- a/tools/tests/test_android_release_workflow.py +++ b/tools/tests/test_android_release_workflow.py @@ -26,7 +26,7 @@ def test_each_failed_prerequisite_remains_blocking_and_nightly_skip_is_not_succe condition = self.jobs["assemble-candidate"]["if"] mutations = [condition.replace(f"needs.{job}.result == 'success'", "true", 1) for job in ("prepare", "native-package-smoke", "musl-package", - "image-native-validation", "android-package")] + "image-native-validation", "android-package", "android-apk-package", "android-apk-qualification")] mutations.extend((condition.replace("inputs.channel != 'nightly' && ", ""), condition.replace("!cancelled()", "always()"), condition.replace("'skipped'", "'failure'"))) @@ -36,6 +36,30 @@ def test_each_failed_prerequisite_remains_blocking_and_nightly_skip_is_not_succe workflow["jobs"]["assemble-candidate"]["if"] = mutated self.assertTrue(POLICY.check_release_android_packages(yaml.safe_dump(workflow))) + def test_apk_api_and_authority_mutations_are_rejected(self): + for mutation in ("api", "source", "certificate", "version", "reader", "authority", "waiver"): + workflow = deepcopy(self.workflow) + job = workflow["jobs"]["android-apk-qualification"] + step = next(step for step in job["steps"] if "qualify-rust-android-apk-ci" in step.get("run", "")) + if mutation == "api": + job["strategy"]["matrix"]["image"].pop() + elif mutation == "source": + step["env"]["EXPECTED_SOURCE_SHA"] = "${{ github.sha }}" + elif mutation == "certificate": + step["run"] = step["run"].replace('--expected-cert-sha256 "$EMULEBB_ANDROID_SIGNER_SHA256"', "") + elif mutation == "version": + job["env"]["EMULEBB_ANDROID_VERSION_CODE"] = "${{ github.run_number }}" + elif mutation == "reader": + reader = next(step for step in workflow["jobs"]["assemble-candidate"]["steps"] + if step.get("with", {}).get("path") == "emulebb-build") + reader["with"]["ref"] = "${{ env.EMULEBB_BUILD_REF }}" + elif mutation == "authority": + workflow["env"].pop("EMULEBB_APK_BUILD_REF") + else: + job["continue-on-error"] = True + with self.subTest(mutation=mutation): + self.assertTrue(POLICY.check_release_android_packages(yaml.safe_dump(workflow))) + def test_source_identity_and_exact_two_abis_cannot_be_removed(self): for mutation in ("source", "abi", "gate", "step-waiver", "job-waiver"): workflow = deepcopy(self.workflow) @@ -49,6 +73,8 @@ def test_source_identity_and_exact_two_abis_cannot_be_removed(self): job["needs"].remove("verify-source") elif mutation == "step-waiver": job["steps"][0]["continue-on-error"] = True + elif mutation == "authority": + workflow["env"].pop("EMULEBB_APK_BUILD_REF") else: job["continue-on-error"] = True with self.subTest(mutation=mutation): diff --git a/tools/tests/test_apk_diagnostic_workflow.py b/tools/tests/test_apk_diagnostic_workflow.py new file mode 100644 index 000000000..7e45e4d09 --- /dev/null +++ b/tools/tests/test_apk_diagnostic_workflow.py @@ -0,0 +1,190 @@ +"""Keep original-byte diagnostic replay isolated from the Nightly release graph.""" +from __future__ import annotations + +import ast +import os +from pathlib import Path +import re +import subprocess +import sys +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +FIELDS = ("diagnostic_run_id", "diagnostic_source_sha", "diagnostic_manifest_sha256", "diagnostic_image") +VALID = dict(zip(FIELDS, ("43524927475", "a" * 40, "b" * 64, "apk34"))) + + +def evaluate(expression, values, event="workflow_dispatch"): + """Evaluate only the boolean/string subset used by the actual routing guards.""" + source = expression.removeprefix("${{").removesuffix("}}").strip() + source = source.replace("github.event_name", repr(event)) + source = source.replace("github.ref", repr("refs/heads/fix/android-delivery-diagnostics")) + source = re.sub(r"inputs\.([a-z0-9_]+)", lambda match: repr(values.get(match[1], "")), source) + source = re.sub(r"!(?!=)", " not ", source).replace("&&", " and ").replace("||", " or ") + + def visit(node): + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not visit(node.operand) + if isinstance(node, ast.BoolOp): + result = visit(node.values[0]) + for value in node.values[1:]: + if (isinstance(node.op, ast.And) and result) or (isinstance(node.op, ast.Or) and not result): + result = visit(value) + return result + if isinstance(node, ast.Compare) and len(node.ops) == 1: + left, right = visit(node.left), visit(node.comparators[0]) + if isinstance(node.ops[0], ast.Eq): + return left == right + if isinstance(node.ops[0], ast.NotEq): + return left != right + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "format": + template, *arguments = [visit(argument) for argument in node.args] + return template.format(*arguments) + raise AssertionError("Unsupported routing expression") + + return visit(ast.parse("(" + source.strip() + ")", mode="eval").body) + + +def selected(expression, values, event="workflow_dispatch"): + return bool(evaluate(expression, values, event)) + + +class ApkDiagnosticWorkflowTests(unittest.TestCase): + def setUp(self): + self.workflow = yaml.safe_load((ROOT / ".github/workflows/nightly.yml").read_text(encoding="utf-8")) + self.jobs = self.workflow["jobs"] + self.diagnostic = self.jobs["apk-diagnostic"] + + def test_absent_inputs_preserve_normal_manual_and_scheduled_nightlies(self): + for event in ("schedule", "workflow_dispatch"): + for values in ({}, {"diagnostic_image": "none"}): + with self.subTest(event=event, values=values): + self.assertTrue(selected(self.jobs["prepare"]["if"], values, event)) + self.assertFalse(selected(self.diagnostic["if"], values, event)) + self.assertEqual(self.jobs["verify-source"]["needs"], "prepare") + self.assertEqual(self.jobs["package"]["needs"], ["prepare", "verify-source"]) + self.assertEqual(self.jobs["package"]["if"], + "${{ needs.prepare.outputs.should_build == 'true' && needs.verify-source.result == 'success' }}") + self.assertEqual(self.jobs["package"]["uses"], "./.github/workflows/release.yml") + + def test_diagnostics_do_not_hold_the_normal_publication_concurrency_lock(self): + concurrency = self.workflow["concurrency"] + self.assertIs(concurrency["cancel-in-progress"], False) + for event in ("schedule", "workflow_dispatch"): + for values in ({}, {"diagnostic_image": "none"}, {"publish": True}): + self.assertEqual(evaluate(concurrency["group"], values, event), "rust-nightly") + for field in FIELDS: + self.assertEqual(evaluate(concurrency["group"], {field: VALID[field]}), + "rust-nightly-android-diagnostic-refs/heads/fix/android-delivery-diagnostics-" + + (VALID[field] if field == "diagnostic_image" else "")) + self.assertNotEqual(evaluate(concurrency["group"], {**VALID, "diagnostic_image": "apk35"}), + evaluate(concurrency["group"], {**VALID, "diagnostic_image": "apk36"})) + + def test_every_diagnostic_field_routes_partial_requests_away_from_publication(self): + for field in FIELDS: + for publish in (False, True): + values = {field: VALID[field], "publish": publish} + with self.subTest(field=field, publish=publish): + self.assertFalse(selected(self.jobs["prepare"]["if"], values)) + self.assertTrue(selected(self.diagnostic["if"], values)) + self.assertFalse(selected(self.jobs["prepare"]["if"], VALID)) + self.assertTrue(selected(self.diagnostic["if"], VALID)) + + def run_guard(self, values, publish="false"): + first = self.diagnostic["steps"][0] + self.assertEqual(first["shell"], "python") + self.assertEqual(first["env"], { + "DIAGNOSTIC_PUBLISH": "${{ inputs.publish }}", "PRODUCER_RUN_ID": "${{ inputs.diagnostic_run_id }}", + "PRODUCER_SOURCE_SHA": "${{ inputs.diagnostic_source_sha }}", + "APK_MANIFEST_SHA256": "${{ inputs.diagnostic_manifest_sha256 }}", + "DIAGNOSTIC_IMAGE": "${{ inputs.diagnostic_image }}", + }) + names = {"DIAGNOSTIC_PUBLISH": publish, "PRODUCER_RUN_ID": values.get(FIELDS[0], ""), + "PRODUCER_SOURCE_SHA": values.get(FIELDS[1], ""), "APK_MANIFEST_SHA256": values.get(FIELDS[2], ""), + "DIAGNOSTIC_IMAGE": values.get(FIELDS[3], "")} + return subprocess.run([sys.executable, "-c", first["run"]], env={**os.environ, **names}, + capture_output=True, text=True, timeout=10) + + def test_actual_first_step_rejects_publication_partial_and_malformed_requests(self): + for api in (31, 32, 33, 34, 35, 36, 37): + self.assertEqual(self.run_guard({**VALID, FIELDS[3]: f"apk{api}"}).returncode, 0) + self.assertNotEqual(self.run_guard(VALID, publish="true").returncode, 0) + for field in FIELDS: + with self.subTest(missing=field): + self.assertNotEqual(self.run_guard({key: value for key, value in VALID.items() if key != field}).returncode, 0) + for field, value in ((FIELDS[0], "0"), (FIELDS[0], "12;echo unsafe"), (FIELDS[1], "refs/heads/main"), + (FIELDS[1], "A" * 40), (FIELDS[2], "b" * 63), (FIELDS[3], "none"), (FIELDS[3], "current"), + (FIELDS[3], "apk30"), (FIELDS[3], "apk38")): + with self.subTest(field=field, value=value): + self.assertNotEqual(self.run_guard({**VALID, field: value}).returncode, 0) + + def test_replay_has_only_read_permissions_and_no_publication_dependency(self): + self.assertEqual(self.diagnostic["permissions"], {"actions": "read", "contents": "read"}) + self.assertNotIn("needs", self.diagnostic) + self.assertNotIn("secrets", self.diagnostic) + self.assertNotIn("uses", self.diagnostic) + self.assertEqual(set(self.jobs), {"prepare", "verify-source", "package", "apk-diagnostic"}) + for step in self.diagnostic["steps"]: + self.assertNotIn("continue-on-error", step) + self.assertNotIn("EMULEBB_ANDROID_KEYSTORE", str(step)) + self.assertNotIn("secrets.", str(step)) + # Existing release jobs depend solely on the original prepare/source gates. + self.assertNotIn("apk-diagnostic", str(self.jobs["package"])) + + def test_executor_and_original_source_authorities_are_separate(self): + constants = self.workflow["env"] + for name in ("EMULEBB_APK_DIAGNOSTIC_BUILD_REF", "EMULEBB_APK_DIAGNOSTIC_HARNESS_REF", + "EMULEBB_APK_DIAGNOSTIC_TOOLING_REF", "EMULEBB_APK_DIAGNOSTIC_MINIUPNP_REF", + "EMULEBB_APK_DIAGNOSTIC_PCPNATPMP_REF"): + self.assertRegex(constants[name], r"^[0-9a-f]{40}$") + self.assertRegex(constants["EMULEBB_APK_DIAGNOSTIC_SIGNER_SHA256"], r"^[0-9a-f]{64}$") + checkouts = {step["with"]["path"]: step["with"] for step in self.diagnostic["steps"] + if step.get("uses", "").startswith("actions/checkout@")} + self.assertEqual(checkouts[".ci/original-rust"]["ref"], "${{ inputs.diagnostic_source_sha }}") + self.assertEqual(checkouts[".ci/emulebb-build"]["ref"], "${{ env.EMULEBB_APK_DIAGNOSTIC_BUILD_REF }}") + self.assertEqual(checkouts[".ci/emulebb-build-tests"]["ref"], "${{ env.EMULEBB_APK_DIAGNOSTIC_HARNESS_REF }}") + self.assertTrue(all(value["persist-credentials"] is False for value in checkouts.values())) + replay = next(step for step in self.diagnostic["steps"] if step.get("id") == "replay") + self.assertIn("diagnose-rust-android-apk-ci", replay["run"]) + for flag in ("--producer-run-id", "--producer-source-sha", "--apk-manifest-sha256", "--image", + "--diagnostic-build-ref", "--diagnostic-harness-ref", "--expected-cert-sha256"): + self.assertIn(flag, replay["run"]) + self.assertEqual(replay["env"]["GH_TOKEN"], "${{ github.token }}") + + def test_user_selected_source_is_admitted_only_after_host_setup(self): + steps = self.diagnostic["steps"] + checkout_index = next(index for index, step in enumerate(steps) + if step.get("with", {}).get("ref") == "${{ inputs.diagnostic_source_sha }}") + setup_prefixes = ("actions/setup-python@", "docker/setup-docker-action@", "docker/setup-buildx-action@") + for prefix in setup_prefixes: + setup_index = next(index for index, step in enumerate(steps) + if step.get("uses", "").startswith(prefix)) + self.assertLess(setup_index, checkout_index) + for index, step in enumerate(steps): + if "pip install" in step.get("run", "") or "setfacl" in step.get("run", ""): + self.assertLess(index, checkout_index) + # WHY: no host setup/cache action may execute after the untrusted tree + # appears; only the pinned replay owner and its sanitized upload remain. + remaining = steps[checkout_index + 1:] + self.assertEqual(len(remaining), 2) + self.assertEqual(remaining[0]["id"], "replay") + self.assertEqual(remaining[0]["working-directory"], ".ci/emulebb-build") + self.assertTrue(remaining[1]["uses"].startswith("actions/upload-artifact@")) + + def test_retained_evidence_is_only_the_ineligible_outer_public_projection(self): + upload = self.diagnostic["steps"][-1] + self.assertTrue(upload["uses"].startswith("actions/upload-artifact@")) + self.assertEqual(upload["if"], "always()") + self.assertEqual(upload["with"]["path"], + "${{ runner.temp }}/emulebb-rust-out/reports/android-emulator-boot/*/validation.json") + self.assertIn("diagnostic", upload["with"]["name"]) + self.assertEqual(upload["with"]["if-no-files-found"], "warn") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_branding.py b/tools/tests/test_branding.py new file mode 100644 index 000000000..23b2ae1f7 --- /dev/null +++ b/tools/tests/test_branding.py @@ -0,0 +1,92 @@ +"""Keep approved PNG bytes, ICO payloads and platform resource wiring aligned.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import struct +import unittest +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[2] +ANDROID = ROOT / "android/app/src/main" +ATTR = "{http://schemas.android.com/apk/res/android}" + + +class BrandingTests(unittest.TestCase): + def test_all_platform_assets_match_approved_inventory(self): + manifest = json.loads((ROOT / "resources/branding.json").read_text(encoding="utf-8")) + self.assertEqual(manifest["selection"], "proposal-5") + for asset in manifest["assets"]: + with self.subTest(path=asset["path"]): + data = (ROOT / asset["path"]).read_bytes() + self.assertEqual(len(data), asset["bytes"]) + self.assertEqual(hashlib.sha256(data).hexdigest(), asset["sha256"]) + + def test_ico_frames_are_exact_native_png_payloads(self): + cases = (("webui/public/favicon.ico", (16, 32, 48)), + ("crates/emulebb-daemon/resources/windows/application.ico", (16, 24, 32, 48, 64, 128, 256))) + for path, sizes in cases: + data = (ROOT / path).read_bytes() + self.assertEqual(struct.unpack_from(" None: validate-emulebb-rust-image-ci path: ${{ runner.temp }}/emulebb-rust-out/release/rust-v0.1.0-beta.2 assemble-emulebb-rust-release-ci -needs: [prepare, native-package-smoke, musl-package, image-native-validation, android-package] +needs: [prepare, native-package-smoke, musl-package, image-native-validation, android-package, android-apk-package, android-apk-qualification] assemble-candidate: ref: ${{ github.workflow_sha }} candidate-manifest diff --git a/tools/tests/test_nightly_release.py b/tools/tests/test_nightly_release.py index 719ea0b0b..d1b112c4d 100644 --- a/tools/tests/test_nightly_release.py +++ b/tools/tests/test_nightly_release.py @@ -241,7 +241,7 @@ def test_commit_subjects_are_grouped_and_linked(self) -> None: self.assertIn("## Fixed", rendered) self.assertIn("## Changed", rendered) self.assertIn("## Engineering", rendered) - self.assertIn("Automated nightly beta from `main`", rendered) + self.assertIn("Automated nightly beta from the verified source commit", rendered) self.assertIn("not a formal beta release", rendered) self.assertIn("Metadata schema: `25` (changed from `24`", rendered) self.assertIn("Profile schema status: incompatible", rendered) diff --git a/tools/tests/test_platform_gate_graph.py b/tools/tests/test_platform_gate_graph.py index 00fa162bd..c844f1baa 100644 --- a/tools/tests/test_platform_gate_graph.py +++ b/tools/tests/test_platform_gate_graph.py @@ -37,7 +37,7 @@ def test_selected_package_and_smoke_graphs_share_the_actual_profile_matrix(self) "windows-x64", "windows-arm64", "linux-x64", "linux-arm64", }) self.assertEqual(set(release["assemble-candidate"]["needs"]), { - "prepare", "native-package-smoke", "musl-package", "image-native-validation", "android-package", + "prepare", "native-package-smoke", "musl-package", "image-native-validation", "android-package", "android-apk-package", "android-apk-qualification", }) self.assertEqual(set(release["image-candidate"]["needs"]), {"prepare", "native-package", "musl-package"}) self.assertEqual(release["native-package-smoke"]["needs"], ["prepare", "native-package"]) diff --git a/tools/tests/test_release_publish.py b/tools/tests/test_release_publish.py index 939f0bed0..ed443fc29 100644 --- a/tools/tests/test_release_publish.py +++ b/tools/tests/test_release_publish.py @@ -479,8 +479,8 @@ def test_downloads_and_supporting_links_use_present_inventory_and_actual_digests self.assertNotIn("b" * 64, rendered) self.assertIn(candidate, rendered) self.assertIn("/actions/runs/101/attempts/2", rendered) - self.assertIn("Device runtime, transfers and VPN behavior remain unqualified", rendered) - self.assertIn("APK packaging and signing remain deferred", rendered) + self.assertIn("compilation and ELF checks", rendered) + self.assertIn("APK runtime qualification is recorded separately", rendered) self.assertNotIn("android-x86_64", rendered) self.assertNotIn("Alpine", rendered) @@ -808,11 +808,45 @@ def test_registry_writes_require_actual_archive_verification_and_current_source_ if fault != "source": source.assert_not_called() + def test_android_code_uses_utc_seconds_across_entrypoints_and_rejects_bounds(self): + from datetime import UTC, datetime, timedelta + epoch = datetime(2020, 1, 1, tzinfo=UTC) + self.assertEqual(PUBLISH.android_version_code(epoch + timedelta(seconds=216_000_001)), 216_000_001) + self.assertEqual(PUBLISH.android_version_code(epoch + timedelta(seconds=2_100_000_000)), 2_100_000_000) + for seconds in (0, -1, 2_100_000_001): + with self.subTest(seconds=seconds), self.assertRaises(RuntimeError): + PUBLISH.android_version_code(epoch + timedelta(seconds=seconds)) + + def test_apk_authorities_are_independent_of_native_sources_and_reader(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + args = PUBLISH.build_parser().parse_args([ + "verify-public-candidate", *self.common(root), "--verified-candidate-sha256", "a" * 64, + "--apk-build-ref", "9" * 40, "--apk-harness-ref", "a" * 40, + "--apk-signer-sha256", "b" * 64, "--apk-version-code", "216000001", + "--apk-harness-repo", str(root / "apk-harness"), + ]) + args.channel = "nightly" + owner = mock.Mock() + with mock.patch.object(PUBLISH, "checked_checkout", return_value=root / "apk-harness") as checked: + PUBLISH.candidate_context(args, owner) + checked.assert_called_once_with(root / "apk-harness", "a" * 40, label="APK Harness") + values = owner.CandidateContext.call_args.kwargs + self.assertEqual(values["source"]["emulebbBuild"], "2" * 40) + self.assertEqual(values["release_validator_build_sha"], "8" * 40) + owner.ApkSourceContext.assert_called_once_with("9" * 40, "a" * 40) + self.assertEqual(values["apk_version_code"], 216000001) + self.assertEqual(values["apk_signer_sha256"], "b" * 64) + args.apk_harness_ref = "" + with self.assertRaisesRegex(RuntimeError, "explicit source"): + PUBLISH.candidate_context(args, owner) + def test_expected_identities_can_come_from_the_workflow_environment(self) -> None: environment = { "RELEASE_VERSION": "0.1.0-beta.2", "RELEASE_CHANNEL": "candidate", "EMULEBB_SOURCE_REVISION": "1" * 40, "EMULEBB_BUILD_REF": "2" * 40, "EMULEBB_BUILD_TESTS_REF": "3" * 40, "EMULEBB_TOOLING_REF": "4" * 40, + "EMULEBB_RELEASE_VALIDATOR_REF": "8" * 40, "MINIUPNP_REF": "5" * 40, "PCPNATPMP_REF": "6" * 40, "GITHUB_REPOSITORY": "example/project", "GITHUB_RUN_ID": "101", "GITHUB_RUN_ATTEMPT": "2", "GITHUB_WORKFLOW_SHA": "7" * 40, @@ -833,7 +867,7 @@ def common(root: Path) -> list[str]: return [ "--release-dir", str(root), "--release-version", "0.1.0-beta.2", "--channel", "candidate", "--source-sha", "1" * 40, "--emulebb-build-ref", "2" * 40, - "--emulebb-build-tests-ref", "3" * 40, "--emulebb-tooling-ref", "4" * 40, + "--release-validator-build-ref", "8" * 40, "--emulebb-build-tests-ref", "3" * 40, "--emulebb-tooling-ref", "4" * 40, "--emulebb-miniupnp-ref", "5" * 40, "--emulebb-libpcpnatpmp-ref", "6" * 40, "--repository", "example/project", "--run-id", "101", "--run-attempt", "2", "--workflow-sha", "7" * 40, "--build-repo", str(root / "build"), diff --git a/tools/tests/test_webui_workspace.py b/tools/tests/test_webui_workspace.py index aaf062858..b3e47b03f 100644 --- a/tools/tests/test_webui_workspace.py +++ b/tools/tests/test_webui_workspace.py @@ -23,10 +23,11 @@ class TestWebUiExecutionModes(unittest.TestCase): - def exercise(self, arguments, *, failure=None): + def exercise(self, arguments, *, failure=None, platform="win32"): workspace = SimpleNamespace(dist=Path("synthetic-dist")) pinned = Mock() with patch.dict(sys.modules, {"ci_environment": SimpleNamespace(prepare_command=pinned)}), \ + patch.object(WEBUI_TEST.sys, "platform", platform), \ patch.object(WEBUI_TEST, "npm_command", return_value="pinned-npm"), \ patch.object(WEBUI_TEST, "prepare_webui_workspace", return_value=workspace) as stage, \ patch.object(WEBUI_TEST, "run", side_effect=failure) as run: @@ -38,16 +39,27 @@ def exercise(self, arguments, *, failure=None): return pinned, stage, [call.args[0] for call in run.call_args_list] def test_default_keeps_complete_test_browser_and_build_sequence(self): - pinned, stage, commands = self.exercise([]) - pinned.assert_not_called() - stage.assert_called_once_with("webui-test") - self.assertEqual(commands, [ - ["pinned-npm", "ci", "--ignore-scripts"], - ["pinned-npm", "run", "test:unit"], - ["pinned-npm", "exec", "--", "playwright", "install", "chromium"], - ["pinned-npm", "run", "test:e2e"], - ["pinned-npm", "run", "build", "--", "--outDir", "synthetic-dist", "--emptyOutDir"], - ]) + for platform in ("win32", "linux", "darwin"): + with self.subTest(platform=platform): + pinned, stage, commands = self.exercise([], platform=platform) + pinned.assert_not_called() + stage.assert_called_once_with("webui-test") + dependencies = ["--with-deps"] if platform == "linux" else [] + self.assertEqual(commands, [ + ["pinned-npm", "ci", "--ignore-scripts"], + ["pinned-npm", "run", "test:unit"], + ["pinned-npm", "exec", "--", "playwright", "install", *dependencies, "chromium", "webkit"], + ["pinned-npm", "run", "test:e2e"], + ["pinned-npm", "run", "build", "--", "--outDir", "synthetic-dist", "--emptyOutDir"], + ]) + + def test_failed_browser_dependencies_prevent_browser_tests_and_build(self): + for platform in ("win32", "linux"): + with self.subTest(platform=platform): + _, _, commands = self.exercise([], platform=platform, + failure=[None, None, subprocess.CalledProcessError(1, "synthetic-browser-install")]) + self.assertEqual(len(commands), 3) + self.assertEqual(commands[-1][4], "install") def test_typecheck_only_uses_separate_stage_pins_and_locked_install(self): pinned, stage, commands = self.exercise(["--typecheck-only"]) diff --git a/webui/playwright.config.ts b/webui/playwright.config.ts index 3a807f122..64d26541a 100644 --- a/webui/playwright.config.ts +++ b/webui/playwright.config.ts @@ -16,6 +16,12 @@ export default defineConfig({ { name: "chromium", use: { ...devices["Desktop Chrome"] } + }, + { + name: "webkit-mobile", + use: { ...devices["iPhone 13"] }, + testMatch: "mobile-appearance.spec.ts", + grep: /mobile Safari smoke/ } ], webServer: { diff --git a/webui/public/favicon.ico b/webui/public/favicon.ico index d31fe07cd..b987c678a 100644 Binary files a/webui/public/favicon.ico and b/webui/public/favicon.ico differ diff --git a/webui/src/api.ts b/webui/src/api.ts index 4f6a0e0aa..b6a20df34 100644 --- a/webui/src/api.ts +++ b/webui/src/api.ts @@ -114,6 +114,7 @@ export type ServerStatus = { export type RuntimeDiagnostics = { processId?: number; + processResources?: ProcessResources; knownFileCount?: number; sharedFileCount?: number; sharedHashingCount?: number; @@ -128,6 +129,13 @@ export type RuntimeDiagnostics = { [key: string]: unknown; }; +export type ProcessResources = { + sampledAtMs: number | null; + residentMemoryBytes: number | null; + cpuUsagePercent: number | null; + cpuSampleIntervalMs: number | null; +}; + export type TransferEventRuntimeDiagnostics = { enabled?: boolean; stream?: string; @@ -996,6 +1004,17 @@ export class RestClient { return this.request("POST", path, body); } + async captureDiagnosticBundle(maxBytes: number): Promise> { + // This endpoint deliberately returns the bundle itself, not a data envelope. + const bundle = await this.request>( + "POST", "diagnostics/bundles", { confirmCapture: true, maxBytes }, "json" + ); + if (!isRecord(bundle) || bundle.schema !== "emulebb.diagnostic-bundle.v6" || typeof bundle.capturedAt !== "string") { + throw new Error("Diagnostic bundle returned an unexpected response"); + } + return bundle; + } + async patch(path: string, body: unknown): Promise { return this.request("PATCH", path, body); } @@ -1036,7 +1055,7 @@ export class RestClient { await readTransferEventStream(response.body, onEvent, options.signal); } - private async request(method: string, path: string, body?: unknown): Promise { + private async request(method: string, path: string, body?: unknown, responseKind: "envelope" | "json" = "envelope"): Promise { const headers: Record = {}; if (this.apiKey) { headers["X-API-Key"] = this.apiKey; @@ -1054,7 +1073,7 @@ export class RestClient { const message = value?.error?.message ?? `${method} ${requestPath} failed`; throw new RestError(message, response.status, value?.error?.code, value?.error?.details); } - return value?.data as T; + return (responseKind === "json" ? value : value?.data) as T; } } diff --git a/webui/src/app.tsx b/webui/src/app.tsx index 6ae243acb..159be5146 100644 --- a/webui/src/app.tsx +++ b/webui/src/app.tsx @@ -1,6 +1,7 @@ import type { ComponentChildren } from "preact"; import { AppIdentity } from "./AppIdentity"; -import { useCallback, useEffect, useState } from "preact/hooks"; +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "preact/hooks"; +import { type Appearance, type AppearancePreference } from "./appearance"; import { Activity, FolderTree, @@ -125,7 +126,7 @@ function currentRoute(): AppRoute { if (route) { const canonicalPath = pathForRoute(route); if (window.location.pathname !== canonicalPath) { - window.history.replaceState(null, "", canonicalPath); + window.history.replaceState(null, "", canonicalPath + window.location.search + window.location.hash); } return route; } @@ -134,7 +135,32 @@ function currentRoute(): AppRoute { return fallback; } -export function App() { +export function App({ appearance }: { appearance: Appearance }) { + const [appearancePreference, setAppearancePreference] = useState(appearance.getPreference); + const [mobileMenuExpanded, setMobileMenuExpanded] = useState(false); + const menuButton = useRef(null); + useEffect(() => appearance.subscribe(setAppearancePreference), [appearance]); + useLayoutEffect(() => { + if (!mobileMenuExpanded) return; + const ownerDocument = menuButton.current?.ownerDocument; + if (!ownerDocument) return; + // WHY: Capture Escape immediately, including body focus and controls that consume bubbling key events. + const close = (event: KeyboardEvent) => { + if (event.key === "Escape") { + event.preventDefault(); + setMobileMenuExpanded(false); + menuButton.current?.focus(); + } + }; + ownerDocument.addEventListener("keydown", close, true); + return () => ownerDocument.removeEventListener("keydown", close, true); + }, [mobileMenuExpanded]); + useEffect(() => { + const mobile = window.matchMedia("(max-width: 639px)"); + const reset = () => { if (!mobile.matches) setMobileMenuExpanded(false); }; + mobile.addEventListener("change", reset); + return () => mobile.removeEventListener("change", reset); + }, []); const [apiKey, setApiKey] = useState(readStoredApiKey); const [apiKeyInput, setApiKeyInput] = useState(apiKey); const [authState, setAuthState] = useState("checking"); @@ -200,6 +226,7 @@ export function App() { }, []); const navigate = useCallback((nextRoute: AppRoute, replace = false) => { + setMobileMenuExpanded(false); const path = pathForRoute(nextRoute); if (window.location.pathname !== path) { window.history[replace ? "replaceState" : "pushState"](null, "", path); @@ -216,7 +243,10 @@ export function App() { }, [navigate]); useEffect(() => { - const syncRoute = () => setRoute(currentRoute()); + const syncRoute = () => { + setMobileMenuExpanded(false); + setRoute(currentRoute()); + }; window.addEventListener("popstate", syncRoute); return () => window.removeEventListener("popstate", syncRoute); }, []); @@ -725,6 +755,15 @@ export function App() { + {authState === "authenticated" && ( <> {authState === "authenticated" && ( -